[IGNORE]

.github/workflows/dask-migration-testing.yml

@@ -0,0 +1,113 @@
+# A GitHub Action to run test_Data.py only for the 'lama-to-dask' branch
+name: Test `cf.Data` during the replacement of LAMA with Dask
+
+on:
+  push:
+    branches:
+      - lama-to-dask
+  pull_request:
+    types: [opened, reopened, ready_for_review]
+    branches:
+      - lama-to-dask
+
+jobs:
+  test-suite-job-0:
+
+    # Set-up the build matrix. We run on different distros and Python versions.
+    strategy:
+      matrix:
+        # Skip older ubuntu-16.04 & macos-10.15 to save usage resource
+        os: [ubuntu-latest, macos-latest]
+        python-version: [3.7, 3.8, 3.9]
+
+    # Run on new and old(er) versions of the distros we support (Linux, Mac OS)
+    runs-on: ${{ matrix.os }}
+
+    # The sequence of tasks that will be executed as part of this job:
+    steps:
+
+    - name: Checkout cf-python
+      uses: actions/checkout@v2
+      with:
+        path: main
+
+    # Provide a notification message
+    - name: Notify about setup
+      run: echo Now setting up the environment for the cf-python test suite...
+
+    - name: Checkout the current cfdm master to use as the dependency
+      uses: actions/checkout@v2
+      with:
+        repository: NCAS-CMS/cfdm
+        path: cfdm
+
+    # Prepare to run the test-suite on different versions of Python 3:
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v1
+      with:
+        python-version: ${{ matrix.python-version }}
+
+    # Setup conda, which is the simplest way to access all dependencies,
+    # especially as some are C-based so otherwise difficult to setup.
+    - name: Setup Miniconda
+      uses: conda-incubator/setup-miniconda@v2
+      with:
+        auto-update-conda: true
+        miniconda-version: 'latest'
+        activate-environment: cf-latest
+        python-version: ${{ matrix.python-version }}
+        channels: ncas, conda-forge
+
+    # Ensure shell is configured with conda activated:
+    - name: Check conda config
+      shell: bash -l {0}
+      run: |
+        conda info
+        conda list
+        conda config --show-sources
+        conda config --show
+    # Install cf-python dependencies, excluding cfdm, pre-testing
+    # We do so with conda which was setup in a previous step.
+    - name: Install dependencies
+      shell: bash -l {0}
+      run: |
+        conda install -c ncas -c conda-forge udunits2=2.2.25
+        conda install -c conda-forge mpich esmpy
+        conda install scipy matplotlib dask
+        pip install pycodestyle
+    # Install cfdm from master branch, then the cf-python development version
+    # We do so with conda which was setup in a previous step.
+    - name: Install development cfdm and cf-python
+      shell: bash -l {0}
+      run: |
+        cd ${{ github.workspace }}/cfdm
+        pip install -e .
+        cd ${{ github.workspace }}/main
+        pip install -e .
+    # Make UMRead library
+    - name: Make UMRead
+      shell: bash -l {0}
+      run: |
+        cd ${{ github.workspace }}/main/cf/umread_lib/c-lib
+        make
+    # Install the coverage library
+    # We do so with conda which was setup in a previous step.
+    - name: Install coverage
+      shell: bash -l {0}
+      run: |
+        conda install coverage
+    # Provide another notification message
+    - name: Notify about starting testing
+      run: echo Setup complete. Now starting to run the cf-python test suite...
+
+    # Finally run test_Data.py!
+    - name: Run the test_Data test module
+      shell: bash -l {0}
+      run: |
+        cd ${{ github.workspace }}/main/cf/test
+        python test_Data.py
+
+    # End with a message indicating the suite has completed its run
+    - name: Notify about a completed run
+      run: |
+        echo The test_Data module has run and you can inspect the results.

cf/__init__.py

@@ -81,17 +81,11 @@
 """
 
 __Conventions__ = "CF-1.8"
-__date__ = "2021-06-10"
-__version__ = "3.10.0"
-
-_requires = (
-    "numpy",
-    "netCDF4",
-    "cftime",
-    "cfunits",
-    "cfdm",
-    "psutil",
-)
+__author__ = "David Hassell"
+__date__ = "2021-??-??"
+__version__ = "4.0.0"
+
+_requires = ("numpy", "netCDF4", "cftime", "cfunits", "cfdm", "psutil")
 
 x = ", ".join(_requires)
 _error0 = f"cf v{ __version__} requires the modules {x}. "
@@ -193,8 +187,8 @@
     )
 
 # Check the version of cfdm
-_minimum_vn = "1.8.9.0"
-_maximum_vn = "1.8.10.0"
+_minimum_vn = "1.9.0.1"
+_maximum_vn = "1.9.1.0"
 _cfdm_version = LooseVersion(cfdm.__version__)
 if not LooseVersion(_minimum_vn) <= _cfdm_version < LooseVersion(_maximum_vn):
     raise RuntimeError(
@@ -243,6 +237,7 @@
     RaggedContiguousArray,
     RaggedIndexedArray,
     RaggedIndexedContiguousArray,
+    SubsampledArray,
 )
 
 from .aggregate import aggregate

cf/aggregate.py

@@ -2510,7 +2510,7 @@ def _get_hfl(
     if d._pmsize == 1:
         partition = d.partitions.matrix.item()
         if not partition.part:
-            key = getattr(partition.subarray, "file_pointer", None)
+            key = getattr(partition.subarray, "file_address", None)
             if key is not None:
                 hash_value = hfl_cache.hash.get(key, None)
                 create_hash = hash_value is None

cf/bounds.py

@@ -212,9 +212,7 @@ def contiguous(self, overlap=True, direction=None, period=None, verbose=1):
         else:
             if direction is None:
                 b = data[(0,) * ndim].array
-                direction = b.item(0,) < b.item(
-                    1,
-                )
+                direction = b.item(0) < b.item(1)
 
             if direction:
                 return (data[1:, 0] <= data[:-1, 1]).all()

cf/cfdatetime.py

@@ -385,15 +385,7 @@ def st2elements(date_string):
     if utc_offset:
         raise ValueError("Can't specify a time offset from UTC")
 
-    return (
-        year,
-        month,
-        day,
-        hour,
-        minute,
-        second,
-        microsecond,
-    )
+    return (year, month, day, hour, minute, second, microsecond)
 
 
 def rt2dt(array, units_in, units_out=None, dummy1=None):

cf/constants.py

@@ -537,60 +537,37 @@
         "orog": {
             "surface_altitude": "altitude",
             "surface_height_above_geopotential_datum": "height_above_geopotential_datum",
-        },
+        }
     },
     "atmosphere_sleve_coordinate": {
         "ztop": {
             "altitude_at_top_of_atmosphere_model": "altitude",
             "height_above_geopotential_datum_at_top_of_atmosphere_model": "height_above_geopotential_datum",
-        },
-    },
-    "ocean_sigma_coordinate": {
-        "depth": _D1_depth_mapping,
-    },
-    "ocean_s_coordinate": {
-        "depth": _D1_depth_mapping,
-    },
-    "ocean_s_coordinate_g1": {
-        "depth": _D1_depth_mapping,
-    },
-    "ocean_s_coordinate_g2": {
-        "depth": _D1_depth_mapping,
-    },
-    "ocean_sigma_z_coordinate": {
-        "depth": _D1_depth_mapping,
-    },
-    "ocean_double_sigma_coordinate": {
-        "depth": _D1_depth_mapping,
-    },
+        }
+    },
+    "ocean_sigma_coordinate": {"depth": _D1_depth_mapping},
+    "ocean_s_coordinate": {"depth": _D1_depth_mapping},
+    "ocean_s_coordinate_g1": {"depth": _D1_depth_mapping},
+    "ocean_s_coordinate_g2": {"depth": _D1_depth_mapping},
+    "ocean_sigma_z_coordinate": {"depth": _D1_depth_mapping},
+    "ocean_double_sigma_coordinate": {"depth": _D1_depth_mapping},
 }
 
 # --------------------------------------------------------------------
 # Define the canonical units of formula terms, as described in
 # Appendix D: Parametric Vertical Coordinates of the CF conventions.
 # --------------------------------------------------------------------
 formula_terms_units = {
-    "atmosphere_ln_pressure_coordinate": {
-        "p0": "Pa",
-        "lev": "",
-    },
-    "atmosphere_sigma_coordinate": {
-        "sigma": "",
-        "ptop": "Pa",
-        "ps": "Pa",
-    },
+    "atmosphere_ln_pressure_coordinate": {"p0": "Pa", "lev": ""},
+    "atmosphere_sigma_coordinate": {"sigma": "", "ptop": "Pa", "ps": "Pa"},
     "atmosphere_hybrid_sigma_pressure_coordinate": {
         "p0": "Pa",
         "ps": "Pa",
         "ap": "Pa",
         "a": "",
         "b": "",
     },
-    "atmosphere_hybrid_height_coordinate": {
-        "a": "m",
-        "b": "",
-        "orog": "m",
-    },
+    "atmosphere_hybrid_height_coordinate": {"a": "m", "b": "", "orog": "m"},
     "atmosphere_sleve_coordinate": {
         "ztop": "m",
         "a": "",
@@ -599,11 +576,7 @@
         "zsurf1": "m",
         "zsurf2": "m",
     },
-    "ocean_sigma_coordinate": {
-        "eta": "m",
-        "depth": "m",
-        "sigma": "",
-    },
+    "ocean_sigma_coordinate": {"eta": "m", "depth": "m", "sigma": ""},
     "ocean_s_coordinate": {
         "eta": "m",
         "depth": "m",

cf/constructlist.py

@@ -96,14 +96,7 @@ def __docstring_method_exclusions__(self):
         See `_docstring_method_exclusions` for details.
 
         """
-        return (
-            "append",
-            "extend",
-            "insert",
-            "pop",
-            "reverse",
-            "clear",
-        )
+        return ("append", "extend", "insert", "pop", "reverse", "clear")
 
     # ----------------------------------------------------------------
     # Overloaded list methods

cf/data/QUESTIONS.rst

@@ -0,0 +1,25 @@
+Questions and answers
+=====================
+
+A place to record random thoughts about the daskification of
+`cf.Data`, possibly prior to starting an issue on GitHub.
+
+----
+
+Q. When we run something that executes all of the lazy operations
+   (like `cf.Data.is_masked`), should/could we replace the dask array
+   with a "persisted" version of the computed data? If we did this, we
+   would want to have the ability to cache persisted chunks to disk,
+   as they came into being on each thread (see, for instance,
+   `chest`). To do this or not do this could be controlled by a
+   configuation setting.
+
+A. ?
+
+----
+
+Q.
+
+A. ?
+
+----

cf/data/README.rst

@@ -0,0 +1,24 @@
+`cf.Data` developer notes
+=========================
+
+Hardness of the mask
+--------------------
+
+Any `cf.Data` method that changes the dask array should consider
+whether or not the mask hardness needs resetting before
+returning. This will be necessary if there is the possibility that the
+operation being applied to the dask array could lose the "memory" on
+its chunks of whether or not the mask is hard.
+
+A common situation that causes a chunk to lose its memory of whether
+or not the mask is hard is when a chunk could have contained a
+unmasked `numpy` array prior to the operation, but the operation could
+convert it to a masked `numpy` array. The new masked array will always
+have the `numpy` default hardness (i.e. soft), which may be
+incorrect.
+
+The mask hardness is most easily reset with the
+`cf.Data._reset_mask_hardness` method.
+
+`cf.Data.__setitem__` and `cf.Data.where` are examples of methods that
+need to reset the mask in this manner.

cf/data/__init__.py

@@ -1,16 +1,18 @@
 from .cachedarray import CachedArray
 from .netcdfarray import NetCDFArray
-from .filledarray import FilledArray
 from .umarray import UMArray
 
+from .filledarray import FilledArray
+
 from .gatheredarray import GatheredArray
 from .raggedcontiguousarray import RaggedContiguousArray
 from .raggedindexedarray import RaggedIndexedArray
 from .raggedindexedcontiguousarray import RaggedIndexedContiguousArray
+from .subsampledarray import SubsampledArray
 
-from .gatheredsubarray import GatheredSubarray
-from .raggedcontiguoussubarray import RaggedContiguousSubarray
-from .raggedindexedsubarray import RaggedIndexedSubarray
-from .raggedindexedcontiguoussubarray import RaggedIndexedContiguousSubarray
+# from .gatheredsubarray import GatheredSubarray
+# from .raggedcontiguoussubarray import RaggedContiguousSubarray
+# from .raggedindexedsubarray import RaggedIndexedSubarray
+# from .raggedindexedcontiguoussubarray import RaggedIndexedContiguousSubarray
 
 from .data import Data

cf/data/abstract/__init__.py

@@ -1,3 +1,2 @@
 from .array import Array
-from .compressedsubarray import CompressedSubarray
 from .filearray import FileArray