Some read tests fail on big-endian machines

[QuLogic]

Zarr version

2.14.2

Numcodecs version

0.11.0

Python Version

3.11.2

Operating System

Fedora Rawhide

Installation

from source

Description

When running tests on s390x (a big-endian machine), several read tests fail. They all either raise ValueError: read length must be non-negative or -1 in read, or a system error OSError: [Errno 22] Invalid argument in seek. So it appears that a file size or block size or similar thing in these files is being read with the native endianness instead of explicitly little endian, and produces invalid values.

________________ TestArrayWithFSStorePartialRead.test_append_2d ________________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_append_2d>
    def test_append_2d(self):
    
        a = np.arange(105*105, dtype='i4').reshape((105, 105))
        z = self.create_array(shape=a.shape, chunks=(10, 10), dtype=a.dtype)
        z[:] = a
        assert a.shape == z.shape
        assert a.dtype == z.dtype
        assert (10, 10) == z.chunks
>       actual = z[:]
zarr/tests/test_core.py:746: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea1435f60>
args = (-402653020,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
_____________ TestArrayWithFSStorePartialRead.test_append_2d_axis ______________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_append_2d_axis>
    def test_append_2d_axis(self):
    
        a = np.arange(105*105, dtype='i4').reshape((105, 105))
        z = self.create_array(shape=a.shape, chunks=(10, 10), dtype=a.dtype)
        z[:] = a
        assert a.shape == z.shape
        assert a.dtype == z.dtype
        assert (10, 10) == z.chunks
>       assert_array_equal(a, z[:])
zarr/tests/test_core.py:768: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea00b0dc0>
args = (-402653020,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
________________ TestArrayWithFSStorePartialRead.test_array_1d _________________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_array_1d>
    def test_array_1d(self):
        a = np.arange(1050)
        z = self.create_array(shape=a.shape, chunks=100, dtype=a.dtype)
    
        # check properties
        assert len(a) == len(z)
        assert a.ndim == z.ndim
        assert a.shape == z.shape
        assert a.dtype == z.dtype
        assert (100,) == z.chunks
        assert a.nbytes == z.nbytes
        assert 11 == z.nchunks
        assert 0 == z.nchunks_initialized
        assert (11,) == z.cdata_shape
    
        # check empty
        b = z[:]
        assert isinstance(b, np.ndarray)
        assert a.shape == b.shape
        assert a.dtype == b.dtype
    
        # check attributes
        z.attrs['foo'] = 'bar'
        assert 'bar' == z.attrs['foo']
    
        # set data
        z[:] = a
    
        # check properties
        assert a.nbytes == z.nbytes
        assert 11 == z.nchunks
        assert 11 == z.nchunks_initialized
    
        # check slicing
>       assert_array_equal(a, np.array(z))
zarr/tests/test_core.py:237: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:603: in __array__
    a = self[...]
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea0eb74c0>
args = (-536870748,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
___________ TestArrayWithFSStorePartialRead.test_array_1d_fill_value ___________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_array_1d_fill_value>
    def test_array_1d_fill_value(self):
        for fill_value in -1, 0, 1, 10:
    
            a = np.arange(1050)
            f = np.empty_like(a)
            f.fill(fill_value)
            z = self.create_array(shape=a.shape, chunks=100, dtype=a.dtype,
                                  fill_value=fill_value)
            z[190:310] = a[190:310]
    
>           assert_array_equal(f[:190], z[:190])
zarr/tests/test_core.py:295: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea064b070>
args = (-536870803,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
___________ TestArrayWithFSStorePartialRead.test_array_1d_selections ___________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_array_1d_selections>
    def test_array_1d_selections(self):
        # light test here, full tests in test_indexing
    
        # setup
        a = np.arange(1050)
        z = self.create_array(shape=a.shape, chunks=100, dtype=a.dtype)
        z[:] = a
    
        # get
>       assert_array_equal(a[50:150], z.get_orthogonal_selection(slice(50, 150)))
zarr/tests/test_core.py:329: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:1101: in get_orthogonal_selection
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea0c694b0>
args = (-1241513809,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
________________ TestArrayWithFSStorePartialRead.test_array_2d _________________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_array_2d>
    def test_array_2d(self):
        a = np.arange(10000).reshape((1000, 10))
        z = self.create_array(shape=a.shape, chunks=(100, 2), dtype=a.dtype)
    
        # check properties
        assert len(a) == len(z)
        assert a.ndim == z.ndim
        assert a.shape == z.shape
        assert a.dtype == z.dtype
        assert (100, 2) == z.chunks
        assert 0 == z.nchunks_initialized
        assert (10, 5) == z.cdata_shape
    
        # set data
        z[:] = a
    
        # check properties
        assert a.nbytes == z.nbytes
        assert 50 == z.nchunks_initialized
    
        # check array-like
>       assert_array_equal(a, np.array(z))
zarr/tests/test_core.py:386: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:603: in __array__
    a = self[...]
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fe9ab29b10>
args = (-1526726301,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
___________ TestArrayWithFSStorePartialRead.test_array_2d_edge_case ____________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_array_2d_edge_case>
    def test_array_2d_edge_case(self):
        # this fails with filters - chunks extend beyond edge of array, messes with delta
        # filter if no fill value?
        shape = 1000, 10
        chunks = 300, 30
        dtype = 'i8'
        z = self.create_array(shape=shape, dtype=dtype, chunks=chunks)
        z[:] = 0
        expect = np.zeros(shape, dtype=dtype)
>       actual = z[:]
zarr/tests/test_core.py:483: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea091b280>
args = (-2013522690,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
____________ TestArrayWithFSStorePartialRead.test_array_2d_partial _____________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_array_2d_partial>
    def test_array_2d_partial(self):
        z = self.create_array(shape=(1000, 10), chunks=(100, 2), dtype='i4',
                              fill_value=0)
    
        # check partial assignment, single row
        c = np.arange(z.shape[1])
        z[0, :] = c
        with pytest.raises(ValueError):
            # N.B., NumPy allows this, but we'll be strict for now
            z[2:3] = c
        with pytest.raises(ValueError):
            # N.B., NumPy allows this, but we'll be strict for now
            z[-1:] = c
        z[2:3] = c[None, :]
        z[-1:] = c[None, :]
>       assert_array_equal(c, z[0, :])
zarr/tests/test_core.py:503: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea0ef54b0>
args = (-536870812,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
_______________ TestArrayWithFSStorePartialRead.test_array_order _______________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_array_order>
    def test_array_order(self):
    
        # 1D
        a = np.arange(1050)
        for order in 'C', 'F':
            z = self.create_array(shape=a.shape, chunks=100, dtype=a.dtype,
                                  order=order)
            assert order == z.order
            if order == 'F':
                assert z[:].flags.f_contiguous
            else:
                assert z[:].flags.c_contiguous
            z[:] = a
>           assert_array_equal(a, z[:])
zarr/tests/test_core.py:543: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea0918550>
args = (-536870748,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
_________________ TestArrayWithFSStorePartialRead.test_islice __________________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_islice>
    def test_islice(self):
        params = (
            ((1,), (1,), 0, 1),
            ((2,), (1,), 0, 1),
            ((1,), (2,), 0, 1),
            ((3,), (3,), 1, 2),
            ((1000,), (100,), 150, 1050),
            ((100,), (1000,), 25, 75),
            ((1, 100), (1, 1), 0, 1),
            ((100, 1), (3, 1), 56, 100),
            ((100, 100), (10, 10), 13, 99),
            ((10, 10, 10), (3, 3, 3), 2, 4),
        )
        for shape, chunks, start, end in params:
            z = self.create_array(shape=shape, chunks=chunks, dtype=int)
            a = np.arange(np.product(shape)).reshape(shape)
            z[:] = a
            end_array = min(end, a.shape[0])
>           for expect, actual in zip_longest(a[start:end_array],
                                              z.islice(start, end)):
zarr/tests/test_core.py:1506: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:668: in islice
    chunk = self[chunk_start:chunk_end]
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:266: in read_block
    f.seek(offset)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fe9b173a00>
args = (-1879048192,), kwargs = {}
    def seek(self, *args, **kwargs):
>       return self.f.seek(*args, **kwargs)
E       OSError: [Errno 22] Invalid argument
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:358: OSError
__________________ TestArrayWithFSStorePartialRead.test_iter ___________________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_iter>
    def test_iter(self):
        params = (
            ((1,), (1,)),
            ((2,), (1,)),
            ((1,), (2,)),
            ((3,), (3,)),
            ((1000,), (100,)),
            ((100,), (1000,)),
            ((1, 100), (1, 1)),
            ((1, 0), (1, 1)),
            ((0, 1), (1, 1)),
            ((0, 1), (2, 1)),
            ((100, 1), (3, 1)),
            ((100, 100), (10, 10)),
            ((10, 10, 10), (3, 3, 3)),
        )
        for shape, chunks in params:
            z = self.create_array(shape=shape, chunks=chunks, dtype=int)
            a = np.arange(np.product(shape)).reshape(shape)
            z[:] = a
>           for expect, actual in zip_longest(a, z):
zarr/tests/test_core.py:1484: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:663: in islice
    chunk = self[j: j + chunk_size]
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:266: in read_block
    f.seek(offset)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fe9ac1d570>
args = (-1879048192,), kwargs = {}
    def seek(self, *args, **kwargs):
>       return self.f.seek(*args, **kwargs)
E       OSError: [Errno 22] Invalid argument
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:358: OSError
________________ TestArrayWithFSStorePartialRead.test_non_cont _________________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_non_cont>
    def test_non_cont(self):
        z = self.create_array(shape=(500, 500, 500), chunks=(50, 50, 50), dtype="<i4")
        z[:, :, :] = 1
        # actually go through the partial read by accessing a single item
>       assert z[0, :, 0].any()
zarr/tests/test_core.py:2612: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:266: in read_block
    f.seek(offset)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea0ef6470>
args = (-1742864384,), kwargs = {}
    def seek(self, *args, **kwargs):
>       return self.f.seek(*args, **kwargs)
E       OSError: [Errno 22] Invalid argument
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:358: OSError
________________ TestArrayWithFSStorePartialRead.test_np_ufuncs ________________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_np_ufuncs>
    def test_np_ufuncs(self):
        z = self.create_array(shape=(100, 100), chunks=(10, 10))
        a = np.arange(10000).reshape(100, 100)
        z[:] = a
    
>       assert np.sum(a) == np.sum(z)
zarr/tests/test_core.py:859: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
<__array_function__ internals>:200: in sum
    ???
/usr/lib64/python3.11/site-packages/numpy/core/fromnumeric.py:2324: in sum
    return _wrapreduction(a, np.add, 'sum', axis, dtype, out, keepdims=keepdims,
/usr/lib64/python3.11/site-packages/numpy/core/fromnumeric.py:86: in _wrapreduction
    return ufunc.reduce(obj, axis, dtype, out, **passkwargs)
zarr/core.py:603: in __array__
    a = self[...]
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea1e786a0>
args = (-2030042805,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
__________ TestArrayWithFSStorePartialRead.test_read_from_all_blocks ___________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_read_from_all_blocks>
    def test_read_from_all_blocks(self):
        '''Tests to make sure `PartialReadBuffer.read_part` doesn't fail when
        stop isn't in the `start_points` array
        '''
        z = self.create_array(shape=1000000, chunks=100_000)
        z[2:99_000] = 1
        path = None if self.version == 2 else z.path
        b = Array(z.store, path=path, read_only=True, partial_decompress=True)
>       assert (b[2:99_000] == 1).all()
zarr/tests/test_core.py:2632: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:266: in read_block
    f.seek(offset)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea12df8e0>
args = (-466616320,), kwargs = {}
    def seek(self, *args, **kwargs):
>       return self.f.seek(*args, **kwargs)
E       OSError: [Errno 22] Invalid argument
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:358: OSError
_ TestArrayWithFSStorePartialRead.test_read_nitems_less_than_blocksize_from_multiple_chunks _
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_read_nitems_less_than_blocksize_from_multiple_chunks>
    def test_read_nitems_less_than_blocksize_from_multiple_chunks(self):
        '''Tests to make sure decompression doesn't fail when `nitems` is
        less than a compressed block size, but covers multiple blocks
        '''
        z = self.create_array(shape=1000000, chunks=100_000)
        z[40_000:80_000] = 1
        path = None if self.version == 2 else z.path
        b = Array(z.store, path=path, read_only=True, partial_decompress=True)
>       assert (b[40_000:80_000] == 1).all()
zarr/tests/test_core.py:2622: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fea1436080>
args = (-2054874925,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError
________________ TestArrayWithFSStorePartialRead.test_resize_2d ________________
self = <zarr.tests.test_core.TestArrayWithFSStorePartialRead testMethod=test_resize_2d>
    def test_resize_2d(self):
    
        z = self.create_array(shape=(105, 105), chunks=(10, 10), dtype='i4',
                              fill_value=0)
        a = np.arange(105*105, dtype='i4').reshape((105, 105))
        z[:] = a
        assert (105, 105) == z.shape
>       assert (105, 105) == z[:].shape
zarr/tests/test_core.py:661: 
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
zarr/core.py:821: in __getitem__
    result = self.get_basic_selection(pure_selection, fields=fields)
zarr/core.py:947: in get_basic_selection
    return self._get_basic_selection_nd(selection=selection, out=out,
zarr/core.py:990: in _get_basic_selection_nd
    return self._get_selection(indexer=indexer, out=out, fields=fields)
zarr/core.py:1290: in _get_selection
    self._chunk_getitems(lchunk_coords, lchunk_selection, out, lout_selection,
zarr/core.py:2063: in _chunk_getitems
    self._process_chunk(
zarr/core.py:1937: in _process_chunk
    cdata.read_part(start, nitems)
zarr/util.py:634: in read_part
    data_buff = self.fs.read_block(self.key_path, start_byte, length)
/usr/lib/python3.11/site-packages/fsspec/spec.py:1235: in read_block
    return read_block(f, offset, length, delimiter)
/usr/lib/python3.11/site-packages/fsspec/utils.py:267: in read_block
    b = f.read(length)
_ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ 
self = <fsspec.implementations.local.LocalFileOpener object at 0x3fe9ac95360>
args = (-402653020,), kwargs = {}
    def read(self, *args, **kwargs):
>       return self.f.read(*args, **kwargs)
E       ValueError: read length must be non-negative or -1
/usr/lib/python3.11/site-packages/fsspec/implementations/local.py:349: ValueError

Steps to reproduce

$ pytest

Additional output

No response

[joshmoore]

Thanks, @QuLogic.

See also: #869