From 914a2210b40e8ba9dcacda9cb46cdc01b956c244 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sat, 2 Feb 2019 15:49:18 -0800 Subject: [PATCH 1/4] add h5netcdf+dask tests --- xarray/tests/test_backends.py | 39 +++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 55e4eb7c8db..31881070c5f 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1916,6 +1916,45 @@ def test_dump_encodings_h5py(self): assert actual.x.encoding['compression_opts'] is None +@requires_h5netcdf +@requires_dask +@pytest.mark.filterwarnings('ignore:deallocating CachingFileManager') +class TestH5NetCDFViaDaskData(TestH5NetCDFData): + + @contextlib.contextmanager + def roundtrip(self, data, save_kwargs={}, open_kwargs={}, + allow_cleanup_failure=False): + with TestH5NetCDFData.roundtrip( + self, data, save_kwargs, open_kwargs, + allow_cleanup_failure) as ds: + yield ds.chunk() + + def test_dataset_caching(self): + # caching behavior differs for dask + pass + + @pytest.mark.xfail(reason="Failing to round trip unlimited dims") + def test_encoding_unlimited_dims(self): + # TODO: this should pass + super(TestH5NetCDFViaDaskData, self).test_encoding_unlimited_dims() + + def test_write_inconsistent_chunks(self): + # Construct two variables with the same dimensions, but different + # chunk sizes. + x = da.zeros((100, 100), dtype='f4', chunks=(50, 100)) + x = DataArray(data=x, dims=('lat', 'lon'), name='x') + x.encoding['chunksizes'] = (50, 100) + x.encoding['original_shape'] = (100, 100) + y = da.ones((100, 100), dtype='f4', chunks=(100, 50)) + y = DataArray(data=y, dims=('lat', 'lon'), name='y') + y.encoding['chunksizes'] = (100, 50) + y.encoding['original_shape'] = (100, 100) + # Put them both into the same dataset + ds = Dataset({'x': x, 'y': y}) + with self.roundtrip(ds) as actual: + assert actual['x'].encoding['chunksizes'] == (50, 100) + assert actual['y'].encoding['chunksizes'] == (100, 50) + @pytest.fixture(params=['scipy', 'netcdf4', 'h5netcdf', 'pynio']) def readengine(request): return request.param From 98850eee8245b66b29be79cd1b2e81a2758fe9a8 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sat, 2 Feb 2019 15:50:56 -0800 Subject: [PATCH 2/4] pep8 --- xarray/tests/test_backends.py | 1 + 1 file changed, 1 insertion(+) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 31881070c5f..2665691dfda 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1955,6 +1955,7 @@ def test_write_inconsistent_chunks(self): assert actual['x'].encoding['chunksizes'] == (50, 100) assert actual['y'].encoding['chunksizes'] == (100, 50) + @pytest.fixture(params=['scipy', 'netcdf4', 'h5netcdf', 'pynio']) def readengine(request): return request.param From d2d55ef7e608bc013936b4261ebb2509c81d97b3 Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Sat, 2 Feb 2019 19:41:32 -0800 Subject: [PATCH 3/4] reactivate pynio/rasterio/iris in py36 test builds --- ci/requirements-py36.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/requirements-py36.yml b/ci/requirements-py36.yml index 7523b14608b..311e4a275a8 100644 --- a/ci/requirements-py36.yml +++ b/ci/requirements-py36.yml @@ -20,14 +20,14 @@ dependencies: - scipy - seaborn - toolz - # - rasterio # xref #2683 + - rasterio - bottleneck - zarr - pseudonetcdf>=3.0.1 - eccodes - cdms2 -# - pynio # xref #2683 -# - iris>=1.10 # xref #2683 + - pynio + - iris>=1.10 - pydap - lxml - pip: From dcec9adeb50c52f93119bb5adff903f8751d470a Mon Sep 17 00:00:00 2001 From: Joseph Hamman Date: Wed, 6 Feb 2019 15:14:21 -0800 Subject: [PATCH 4/4] revert changes to test_backends.py -- unrelated to this PR --- xarray/tests/test_backends.py | 40 ----------------------------------- 1 file changed, 40 deletions(-) diff --git a/xarray/tests/test_backends.py b/xarray/tests/test_backends.py index 2665691dfda..55e4eb7c8db 100644 --- a/xarray/tests/test_backends.py +++ b/xarray/tests/test_backends.py @@ -1916,46 +1916,6 @@ def test_dump_encodings_h5py(self): assert actual.x.encoding['compression_opts'] is None -@requires_h5netcdf -@requires_dask -@pytest.mark.filterwarnings('ignore:deallocating CachingFileManager') -class TestH5NetCDFViaDaskData(TestH5NetCDFData): - - @contextlib.contextmanager - def roundtrip(self, data, save_kwargs={}, open_kwargs={}, - allow_cleanup_failure=False): - with TestH5NetCDFData.roundtrip( - self, data, save_kwargs, open_kwargs, - allow_cleanup_failure) as ds: - yield ds.chunk() - - def test_dataset_caching(self): - # caching behavior differs for dask - pass - - @pytest.mark.xfail(reason="Failing to round trip unlimited dims") - def test_encoding_unlimited_dims(self): - # TODO: this should pass - super(TestH5NetCDFViaDaskData, self).test_encoding_unlimited_dims() - - def test_write_inconsistent_chunks(self): - # Construct two variables with the same dimensions, but different - # chunk sizes. - x = da.zeros((100, 100), dtype='f4', chunks=(50, 100)) - x = DataArray(data=x, dims=('lat', 'lon'), name='x') - x.encoding['chunksizes'] = (50, 100) - x.encoding['original_shape'] = (100, 100) - y = da.ones((100, 100), dtype='f4', chunks=(100, 50)) - y = DataArray(data=y, dims=('lat', 'lon'), name='y') - y.encoding['chunksizes'] = (100, 50) - y.encoding['original_shape'] = (100, 100) - # Put them both into the same dataset - ds = Dataset({'x': x, 'y': y}) - with self.roundtrip(ds) as actual: - assert actual['x'].encoding['chunksizes'] == (50, 100) - assert actual['y'].encoding['chunksizes'] == (100, 50) - - @pytest.fixture(params=['scipy', 'netcdf4', 'h5netcdf', 'pynio']) def readengine(request): return request.param