diff --git a/docs/conf.py b/docs/conf.py index 0c978ac2..b2a89606 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ def __getattr__(cls, name): return Mock() -MOCK_MODULES = ['msgpack', 'zfpy'] +MOCK_MODULES = ['msgpack'] sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) diff --git a/docs/release.rst b/docs/release.rst index a483ed64..d9473f65 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -5,6 +5,10 @@ Release notes Unreleased ---------- +* Fix a flatten array error for ZFPY, ZFPY codec is supported on Python 3.9 + and 3.10 on Linux and MacOS, the docs about ZFPY is also available. + By :user:`Haiying Xu `, `John Kirkham `, `Ryan Abernathey ` : + issue:`303`. .. _release_0.9.1: diff --git a/numcodecs/compat.py b/numcodecs/compat.py index f414470b..b1187838 100644 --- a/numcodecs/compat.py +++ b/numcodecs/compat.py @@ -50,7 +50,7 @@ def ensure_ndarray(buf): return arr -def ensure_contiguous_ndarray(buf, max_buffer_size=None): +def ensure_contiguous_ndarray(buf, max_buffer_size=None, flatten=True): """Convenience function to coerce `buf` to a numpy array, if it is not already a numpy array. Also ensures that the returned value exports fully contiguous memory, and supports the new-style buffer interface. If the optional max_buffer_size is @@ -91,8 +91,10 @@ def ensure_contiguous_ndarray(buf, max_buffer_size=None): # check memory is contiguous, if so flatten if arr.flags.c_contiguous or arr.flags.f_contiguous: - # can flatten without copy - arr = arr.reshape(-1, order='A') + # check if flatten flag is on or not + if flatten: + # can flatten without copy + arr = arr.reshape(-1, order='A') else: raise ValueError('an array with contiguous memory is required') diff --git a/numcodecs/tests/test_zfpy.py b/numcodecs/tests/test_zfpy.py index 15ae12ea..7766b059 100644 --- a/numcodecs/tests/test_zfpy.py +++ b/numcodecs/tests/test_zfpy.py @@ -38,7 +38,6 @@ np.random.normal(loc=1000, scale=1, size=(100, 10)), np.random.normal(loc=1000, scale=1, size=(10, 10, 10)), np.random.normal(loc=1000, scale=1, size=(2, 5, 10, 10)), - np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20))), np.random.randint(-(2 ** 31), -(2 ** 31) + 20, size=1000, dtype="i4").reshape( 100, 10 ), @@ -84,3 +83,26 @@ def test_err_decode_object_buffer(): def test_err_encode_object_buffer(): check_err_encode_object_buffer(ZFPY()) + + +def test_err_encode_list(): + data = ['foo', 'bar', 'baz'] + for codec in codecs: + with pytest.raises(TypeError): + codec.encode(data) + + +def test_err_encode_non_contiguous(): + # non-contiguous memory + arr = np.arange(1000, dtype='i4')[::2] + for codec in codecs: + with pytest.raises(ValueError): + codec.encode(arr) + + +def test_err_encode_fortran_array(): + # fortran array + arr = np.asfortranarray(np.random.normal(loc=1000, scale=1, size=(5, 10, 20))) + for codec in codecs: + with pytest.raises(ValueError): + codec.encode(arr) diff --git a/numcodecs/zfpy.py b/numcodecs/zfpy.py index 51940399..899dd330 100644 --- a/numcodecs/zfpy.py +++ b/numcodecs/zfpy.py @@ -9,6 +9,7 @@ from .abc import Codec from .compat import ndarray_copy, ensure_contiguous_ndarray, ensure_bytes + import numpy as np # noinspection PyShadowingBuiltins class ZFPY(Codec): @@ -54,8 +55,16 @@ def __init__( def encode(self, buf): - # normalise inputs - buf = ensure_contiguous_ndarray(buf) + # not flatten c-order array and raise exception for f-order array + if not isinstance(buf, np.ndarray): + raise TypeError("The zfp codec does not support none numpy arrays." + f" Your buffers were {type(buf)}.") + if buf.flags.c_contiguous: + flatten = False + else: + raise ValueError("The zfp codec does not support F order arrays. " + f"Your arrays flags were {buf.flags}.") + buf = ensure_contiguous_ndarray(buf, flatten=flatten) # do compression return _zfpy.compress_numpy( diff --git a/requirements.txt b/requirements.txt index 543bfb43..3fe95a08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,3 +2,4 @@ cython numpy msgpack pytest +zfpy diff --git a/requirements_dev.txt b/requirements_dev.txt index fa8d5d7b..1717cc90 100644 --- a/requirements_dev.txt +++ b/requirements_dev.txt @@ -1,5 +1,6 @@ Cython==0.29.21 msgpack==1.0.2 numpy==1.21.0 -zfpy==0.5.5; python_version < '3.9' +zfpy==0.5.5 + diff --git a/requirements_rtfd.txt b/requirements_rtfd.txt index 72bb7fe5..8854bded 100644 --- a/requirements_rtfd.txt +++ b/requirements_rtfd.txt @@ -6,4 +6,4 @@ numpydoc mock numpy cython -zfpy==0.5.5; python_version < '3.9' +zfpy==0.5.5