From 8cdb7d38d766827bf665aa9dc7eb7eeb85578055 Mon Sep 17 00:00:00 2001
From: David Hassell <david.hassell@ncas.ac.uk>
Date: Wed, 13 Mar 2024 23:47:03 +0000
Subject: [PATCH 1/4] dev

---
 cf/cfimplementation.py              | 27 +++++++++++++++++
 cf/read_write/netcdf/netcdfwrite.py | 47 +++++++++++++++++++++++------
 2 files changed, 65 insertions(+), 9 deletions(-)

diff --git a/cf/cfimplementation.py b/cf/cfimplementation.py
index 9a4c2090d8..5253497634 100644
--- a/cf/cfimplementation.py
+++ b/cf/cfimplementation.py
@@ -48,6 +48,33 @@ class CFImplementation(cfdm.CFDMImplementation):
 
     """
 
+    def nc_set_hdf5_chunksizes(self, data, sizes, override=False):
+        """Set the data HDF5 chunksizes.
+
+        .. versionadded:: NEXTVERSION
+
+        :Parameters:
+
+            data: `Data`
+                The data.
+
+            sizes: sequence of `int`
+                The new HDF5 chunk sizes.
+
+            override: `bool`, optional
+                If True then set the HDF5 chunks sizes even if some
+                have already been specified. If False, the default,
+                then only set the HDF5 chunks sizes if some none have
+                already been specified.
+
+        :Returns:
+
+            `None`
+
+        """
+        if override or not data.nc_hdf5_chunksizes():
+            data.nc_set_hdf5_chunksizes(sizes)
+
     def set_construct(self, parent, construct, axes=None, copy=True, **kwargs):
         """Insert a construct into a field or domain.
 
diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py
index ad7da4c9cd..12f89736a3 100644
--- a/cf/read_write/netcdf/netcdfwrite.py
+++ b/cf/read_write/netcdf/netcdfwrite.py
@@ -431,10 +431,12 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
             `None`
 
         """
+        import time # TODO
+        print (f"\n{ncvar}") # TODO
+        
         g = self.write_vars
 
         ndim = data.ndim
-
         cfa = self._cfa_aggregation_instructions(data, cfvar)
 
         # ------------------------------------------------------------
@@ -482,16 +484,21 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         aggregated_data_attr = []
 
         # Location
+        start = time.time() # TODO
         term = "location"
+        data = cfa[term]
+        self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            cfa[term],
+            data ,#cfa[term],
             aggregated_data.get(term, f"cfa_{term}"),
             location_ncdimensions,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
-
+        print (f"{term:<10}: {time.time() - start:.3}")
+        
         # File
         term = "file"
+        start = time.time() # TODO
         if substitutions:
             # Create the "substitutions" netCDF attribute
             subs = []
@@ -502,16 +509,20 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         else:
             attributes = None
 
+        data = cfa[term]
+        self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            cfa[term],
+            data, #cfa[term],
             aggregated_data.get(term, f"cfa_{term}"),
             fragment_ncdimensions,
             attributes=attributes,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
+        print (f"{term:<10}: {time.time() - start:.3}")
 
         # Address
         term = "address"
+        start = time.time() # TODO
 
         # Attempt to reduce addresses to a common scalar value
         u = cfa[term].unique().compressed().persist()
@@ -521,15 +532,19 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         else:
             dimensions = fragment_ncdimensions
 
+        data = cfa[term]
+        self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            cfa[term],
+            data, #         cfa[term],
             aggregated_data.get(term, f"cfa_{term}"),
             dimensions,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
+        print (f"{term:<10}: {time.time() - start:.3}")
 
         # Format
         term = "format"
+        start = time.time() # TODO
 
         # Attempt to reduce addresses to a common scalar value
         u = cfa[term].unique().compressed().persist()
@@ -539,13 +554,16 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         else:
             dimensions = fragment_ncdimensions
 
+        data = cfa[term]
+        self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            cfa[term],
+            data, #cfa[term],
             aggregated_data.get(term, f"cfa_{term}"),
             dimensions,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
 
+        print (f"{term:<10}: {time.time() - start:.3}")
         # ------------------------------------------------------------
         # Look for non-standard CFA terms stored as field ancillaries
         # on a field and write them to the CFA-netCDF file
@@ -809,8 +827,10 @@ def _cfa_write_non_standard_terms(
             terms.append(term)
 
             # Create the new CFA term variable
+            data = type(data)(dx)
+            self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
             term_ncvar = self._cfa_write_term_variable(
-                data=type(data)(dx),
+                data=data, #type(data)(dx),
                 ncvar=aggregated_data.get(term, f"cfa_{term}"),
                 ncdimensions=fragment_ncdimensions,
             )
@@ -884,7 +904,9 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         from os.path import abspath, join, relpath
         from pathlib import PurePath
         from urllib.parse import urlparse
-
+        import time # TODO
+        start = time.time() # TODO
+        
         g = self.write_vars
 
         # Define the CFA file susbstitutions, giving precedence over
@@ -899,11 +921,15 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         # Size of the trailing dimension
         n_trailing = 0
 
+        start1 = time.time() # TODO
         aggregation_file = []
         aggregation_address = []
         aggregation_format = []
+        nnn = 0
         for indices in data.chunk_indices():
+            nnn += 1
             file_details = self._cfa_get_file_details(data[indices])
+       
             if len(file_details) != 1:
                 if file_details:
                     raise ValueError(
@@ -948,7 +974,9 @@ def _cfa_aggregation_instructions(self, data, cfvar):
             aggregation_file.append(tuple(filenames2))
             aggregation_address.append(addresses)
             aggregation_format.append(formats)
-
+        print ('len(data.chunk_indices()) =',nnn)
+        print (f"loop 1: {time.time() - start1:.3}")
+       
         # Pad each value of the aggregation instruction arrays so that
         # it has 'n_trailing' elements
         a_shape = data.numblocks
@@ -1005,6 +1033,7 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         # Return Data objects
         # ------------------------------------------------------------
         data = type(data)
+        print (f"_cfa_aggregation_instructions: {time.time() - start:.3}")
         return {
             "location": data(aggregation_location),
             "file": data(aggregation_file),

From 538e7f49fb7fef8f855d97f0fbde3a8c1447d99c Mon Sep 17 00:00:00 2001
From: David Hassell <david.hassell@ncas.ac.uk>
Date: Thu, 14 Mar 2024 09:17:27 +0000
Subject: [PATCH 2/4] dev

---
 cf/read_write/netcdf/netcdfwrite.py | 14 ++------------
 1 file changed, 2 insertions(+), 12 deletions(-)

diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py
index 12f89736a3..b248cfd7fa 100644
--- a/cf/read_write/netcdf/netcdfwrite.py
+++ b/cf/read_write/netcdf/netcdfwrite.py
@@ -430,10 +430,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
 
             `None`
 
-        """
-        import time # TODO
-        print (f"\n{ncvar}") # TODO
-        
+        """        
         g = self.write_vars
 
         ndim = data.ndim
@@ -484,7 +481,6 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         aggregated_data_attr = []
 
         # Location
-        start = time.time() # TODO
         term = "location"
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
@@ -494,11 +490,9 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
             location_ncdimensions,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
-        print (f"{term:<10}: {time.time() - start:.3}")
         
         # File
         term = "file"
-        start = time.time() # TODO
         if substitutions:
             # Create the "substitutions" netCDF attribute
             subs = []
@@ -518,11 +512,9 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
             attributes=attributes,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
-        print (f"{term:<10}: {time.time() - start:.3}")
 
         # Address
         term = "address"
-        start = time.time() # TODO
 
         # Attempt to reduce addresses to a common scalar value
         u = cfa[term].unique().compressed().persist()
@@ -540,11 +532,9 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
             dimensions,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
-        print (f"{term:<10}: {time.time() - start:.3}")
 
         # Format
         term = "format"
-        start = time.time() # TODO
 
         # Attempt to reduce addresses to a common scalar value
         u = cfa[term].unique().compressed().persist()
@@ -563,7 +553,6 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
 
-        print (f"{term:<10}: {time.time() - start:.3}")
         # ------------------------------------------------------------
         # Look for non-standard CFA terms stored as field ancillaries
         # on a field and write them to the CFA-netCDF file
@@ -905,6 +894,7 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         from pathlib import PurePath
         from urllib.parse import urlparse
         import time # TODO
+        print (f"\n{cfvar!r}") # TODO
         start = time.time() # TODO
         
         g = self.write_vars

From d53e228284ca8f083d139b925687cd62fd53aa77 Mon Sep 17 00:00:00 2001
From: David Hassell <david.hassell@ncas.ac.uk>
Date: Fri, 15 Mar 2024 11:05:54 +0000
Subject: [PATCH 3/4] dev

---
 Changelog.rst                       |  3 +++
 cf/read_write/netcdf/netcdfwrite.py | 40 ++++++++++++-----------------
 2 files changed, 19 insertions(+), 24 deletions(-)

diff --git a/Changelog.rst b/Changelog.rst
index 7b6dbd7576..7ee8ae9523 100644
--- a/Changelog.rst
+++ b/Changelog.rst
@@ -9,6 +9,9 @@ version NEXT
   to regrid the vertical axis in logarithmic coordinates to
   `cf.Field.regrids` and `cf.Field.regridc`
   (https://github.com/NCAS-CMS/cf-python/issues/715)
+* Reduce output CFA netCDF file size by setting the HDF5 chunksizes of
+  CFA variables to be no larger than required
+  (https://github.com/NCAS-CMS/cf-python/issues/739)
 * Fix misleading error message when it is not possible to create area
   weights requested from `cf.Field.collapse`
   (https://github.com/NCAS-CMS/cf-python/issues/731)
diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py
index b248cfd7fa..ff3d8554d5 100644
--- a/cf/read_write/netcdf/netcdfwrite.py
+++ b/cf/read_write/netcdf/netcdfwrite.py
@@ -430,7 +430,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
 
             `None`
 
-        """        
+        """
         g = self.write_vars
 
         ndim = data.ndim
@@ -485,12 +485,12 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data ,#cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             location_ncdimensions,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
-        
+
         # File
         term = "file"
         if substitutions:
@@ -506,7 +506,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data, #cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             fragment_ncdimensions,
             attributes=attributes,
@@ -527,7 +527,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data, #         cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             dimensions,
         )
@@ -547,7 +547,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data, #cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             dimensions,
         )
@@ -819,7 +819,7 @@ def _cfa_write_non_standard_terms(
             data = type(data)(dx)
             self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
             term_ncvar = self._cfa_write_term_variable(
-                data=data, #type(data)(dx),
+                data=data,
                 ncvar=aggregated_data.get(term, f"cfa_{term}"),
                 ncdimensions=fragment_ncdimensions,
             )
@@ -893,10 +893,7 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         from os.path import abspath, join, relpath
         from pathlib import PurePath
         from urllib.parse import urlparse
-        import time # TODO
-        print (f"\n{cfvar!r}") # TODO
-        start = time.time() # TODO
-        
+
         g = self.write_vars
 
         # Define the CFA file susbstitutions, giving precedence over
@@ -911,15 +908,12 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         # Size of the trailing dimension
         n_trailing = 0
 
-        start1 = time.time() # TODO
         aggregation_file = []
         aggregation_address = []
         aggregation_format = []
-        nnn = 0
         for indices in data.chunk_indices():
-            nnn += 1
             file_details = self._cfa_get_file_details(data[indices])
-       
+
             if len(file_details) != 1:
                 if file_details:
                     raise ValueError(
@@ -964,9 +958,7 @@ def _cfa_aggregation_instructions(self, data, cfvar):
             aggregation_file.append(tuple(filenames2))
             aggregation_address.append(addresses)
             aggregation_format.append(formats)
-        print ('len(data.chunk_indices()) =',nnn)
-        print (f"loop 1: {time.time() - start1:.3}")
-       
+
         # Pad each value of the aggregation instruction arrays so that
         # it has 'n_trailing' elements
         a_shape = data.numblocks
@@ -980,6 +972,8 @@ def _cfa_aggregation_instructions(self, data, cfvar):
             ):
                 n = n_trailing - len(filenames)
                 if n:
+                    # This chunk has fewer fragment files than some
+                    # others, so some padding is required.
                     pad = ("",) * n
                     aggregation_file[i] = filenames + pad
                     aggregation_format[i] = formats + pad
@@ -1023,7 +1017,6 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         # Return Data objects
         # ------------------------------------------------------------
         data = type(data)
-        print (f"_cfa_aggregation_instructions: {time.time() - start:.3}")
         return {
             "location": data(aggregation_location),
             "file": data(aggregation_file),
@@ -1074,13 +1067,12 @@ def _cfa_get_file_details(self, data):
         {(('/home/file.pp',), (34556,), ('um',))}
 
         """
-        out = set()
+        out = []
+        append = out.append
         for a in data.todict().values():
             try:
-                out.update(
-                    ((a.get_filenames(), a.get_addresses(), a.get_formats()),)
-                )
+                append((a.get_filenames(), a.get_addresses(), a.get_formats()))
             except AttributeError:
                 pass
 
-        return out
+        return set(out)

From e7c6f8eb322d05b05dd446228e66292161f5a251 Mon Sep 17 00:00:00 2001
From: David Hassell <david.hassell@ncas.ac.uk>
Date: Thu, 21 Mar 2024 17:57:42 +0000
Subject: [PATCH 4/4] dev

---
 Changelog.rst                       |  3 +++
 cf/read_write/netcdf/netcdfwrite.py | 38 ++++++++++++-----------------
 2 files changed, 18 insertions(+), 23 deletions(-)

diff --git a/Changelog.rst b/Changelog.rst
index c09f1d0802..3ec1572b6a 100644
--- a/Changelog.rst
+++ b/Changelog.rst
@@ -11,6 +11,9 @@ version NEXT
   (https://github.com/NCAS-CMS/cf-python/issues/715)
 * Improve `cf.Field.collapse` performance by lazily computing reduced
   axis coordinates (https://github.com/NCAS-CMS/cf-python/issues/741)
+* Reduce output CFA file size with by tailoring the HDF5 chunk sizes
+  to fit exactly the CFA instruction variables
+  (https://github.com/NCAS-CMS/cf-python/issues/739)
 * Fix misleading error message when it is not possible to create area
   weights requested from `cf.Field.collapse`
   (https://github.com/NCAS-CMS/cf-python/issues/731)
diff --git a/cf/read_write/netcdf/netcdfwrite.py b/cf/read_write/netcdf/netcdfwrite.py
index b248cfd7fa..746ca73229 100644
--- a/cf/read_write/netcdf/netcdfwrite.py
+++ b/cf/read_write/netcdf/netcdfwrite.py
@@ -430,10 +430,11 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
 
             `None`
 
-        """        
+        """
         g = self.write_vars
 
         ndim = data.ndim
+
         cfa = self._cfa_aggregation_instructions(data, cfvar)
 
         # ------------------------------------------------------------
@@ -485,12 +486,12 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data ,#cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             location_ncdimensions,
         )
         aggregated_data_attr.append(f"{term}: {term_ncvar}")
-        
+
         # File
         term = "file"
         if substitutions:
@@ -506,7 +507,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data, #cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             fragment_ncdimensions,
             attributes=attributes,
@@ -527,7 +528,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data, #         cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             dimensions,
         )
@@ -547,7 +548,7 @@ def _create_cfa_data(self, ncvar, ncdimensions, data, cfvar):
         data = cfa[term]
         self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
         term_ncvar = self._cfa_write_term_variable(
-            data, #cfa[term],
+            data,
             aggregated_data.get(term, f"cfa_{term}"),
             dimensions,
         )
@@ -819,7 +820,7 @@ def _cfa_write_non_standard_terms(
             data = type(data)(dx)
             self.implementation.nc_set_hdf5_chunksizes(data, data.shape)
             term_ncvar = self._cfa_write_term_variable(
-                data=data, #type(data)(dx),
+                data=data,
                 ncvar=aggregated_data.get(term, f"cfa_{term}"),
                 ncdimensions=fragment_ncdimensions,
             )
@@ -893,10 +894,7 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         from os.path import abspath, join, relpath
         from pathlib import PurePath
         from urllib.parse import urlparse
-        import time # TODO
-        print (f"\n{cfvar!r}") # TODO
-        start = time.time() # TODO
-        
+
         g = self.write_vars
 
         # Define the CFA file susbstitutions, giving precedence over
@@ -911,15 +909,11 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         # Size of the trailing dimension
         n_trailing = 0
 
-        start1 = time.time() # TODO
         aggregation_file = []
         aggregation_address = []
         aggregation_format = []
-        nnn = 0
         for indices in data.chunk_indices():
-            nnn += 1
             file_details = self._cfa_get_file_details(data[indices])
-       
             if len(file_details) != 1:
                 if file_details:
                     raise ValueError(
@@ -964,9 +958,7 @@ def _cfa_aggregation_instructions(self, data, cfvar):
             aggregation_file.append(tuple(filenames2))
             aggregation_address.append(addresses)
             aggregation_format.append(formats)
-        print ('len(data.chunk_indices()) =',nnn)
-        print (f"loop 1: {time.time() - start1:.3}")
-       
+
         # Pad each value of the aggregation instruction arrays so that
         # it has 'n_trailing' elements
         a_shape = data.numblocks
@@ -1023,7 +1015,6 @@ def _cfa_aggregation_instructions(self, data, cfvar):
         # Return Data objects
         # ------------------------------------------------------------
         data = type(data)
-        print (f"_cfa_aggregation_instructions: {time.time() - start:.3}")
         return {
             "location": data(aggregation_location),
             "file": data(aggregation_file),
@@ -1074,13 +1065,14 @@ def _cfa_get_file_details(self, data):
         {(('/home/file.pp',), (34556,), ('um',))}
 
         """
-        out = set()
+        out = []
+        out_append = out.append
         for a in data.todict().values():
             try:
-                out.update(
-                    ((a.get_filenames(), a.get_addresses(), a.get_formats()),)
+                out_append(
+                    (a.get_filenames(), a.get_addresses(), a.get_formats())
                 )
             except AttributeError:
                 pass
 
-        return out
+        return set(out)
