diff --git a/.travis.yml b/.travis.yml index 13618a50f5..49e9fdad76 100644 --- a/.travis.yml +++ b/.travis.yml @@ -24,7 +24,6 @@ services: matrix: include: - - python: 2.7 - python: 3.5 - python: 3.6 - python: 3.7 diff --git a/appveyor.yml b/appveyor.yml index 45c5a8af7a..903abb8df8 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -18,10 +18,6 @@ environment: matrix: - - PYTHON: "C:\\Python27-x64" - PYTHON_VERSION: "2.7" - DISTUTILS_USE_SDK: "1" - - PYTHON: "C:\\Python35-x64" PYTHON_VERSION: "3.5" diff --git a/bench/compress_normal.py b/bench/compress_normal.py index 5dd8113790..e9b3945f4a 100644 --- a/bench/compress_normal.py +++ b/bench/compress_normal.py @@ -1,11 +1,14 @@ -import numpy as np import sys -sys.path.insert(0, '..') -import zarr -import line_profiler import timeit + +import numpy as np + +import line_profiler +import zarr from zarr import blosc +sys.path.insert(0, '..') + # setup a = np.random.normal(2000, 1000, size=200000000).astype('u2') z = zarr.empty_like(a, chunks=1000000, compression='blosc', compression_opts=dict(cname='lz4', clevel=5, shuffle=2)) diff --git a/build.cmd b/build.cmd index ef92badb41..053894d249 100644 --- a/build.cmd +++ b/build.cmd @@ -23,9 +23,7 @@ SET COMMAND_TO_RUN=%* SET WIN_SDK_ROOT=C:\Program Files\Microsoft SDKs\Windows SET MAJOR_PYTHON_VERSION="%PYTHON_VERSION:~0,1%" -IF %MAJOR_PYTHON_VERSION% == "2" ( - SET WINDOWS_SDK_VERSION="v7.0" -) ELSE IF %MAJOR_PYTHON_VERSION% == "3" ( +IF %MAJOR_PYTHON_VERSION% == "3" ( SET WINDOWS_SDK_VERSION="v7.1" ) ELSE ( ECHO Unsupported Python version: "%MAJOR_PYTHON_VERSION%" diff --git a/docs/conf.py b/docs/conf.py index 9c128f4c9c..673f4b439b 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,27 +14,15 @@ # serve to show the default. -import sys import os -from mock import Mock as MagicMock - - -PY2 = sys.version_info[0] == 2 - - -class Mock(MagicMock): - @classmethod - def __getattr__(cls, name): - return Mock() - - -MOCK_MODULES = [] -if PY2: - MOCK_MODULES.append('lzma') - - -sys.modules.update((mod_name, Mock()) for mod_name in MOCK_MODULES) +import sys +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +import zarr # If extensions (or modules to document with autodoc) are in another directory, # add these directories to sys.path here. If the directory is relative to the @@ -81,12 +69,6 @@ def __getattr__(cls, name): copyright = '2018, Zarr Developers' author = 'Zarr Developers' -# The version info for the project you're documenting, acts as replacement for -# |version| and |release|, also used in various other places throughout the -# built documents. -# -# The short X.Y version. -import zarr version = zarr.__version__ # The full version, including alpha/beta/rc tags. release = zarr.__version__ diff --git a/docs/release.rst b/docs/release.rst index 84a6e0853d..4749502344 100644 --- a/docs/release.rst +++ b/docs/release.rst @@ -15,6 +15,9 @@ Upcoming Release compatibility issue with testing against the Azure Storage Emulator. By :user:`alimanfoo`; :issue:`468`, :issue:`467`. +* Removed support for Python 2. + By :user:`jhamman`; :issue:`393`, :issue:`470`. + .. _release_2.3.2: 2.3.2 diff --git a/setup.py b/setup.py index e2367233f5..e586f13aca 100644 --- a/setup.py +++ b/setup.py @@ -1,8 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -from setuptools import setup import sys +from setuptools import setup DESCRIPTION = 'An implementation of chunked, compressed, ' \ 'N-dimensional arrays for Python.' @@ -17,11 +16,6 @@ 'numcodecs>=0.6.2', ] -if sys.version_info < (3, 5): - dependencies.append('scandir') -if sys.version_info < (3, 3) and sys.platform == "win32": - dependencies.append('pyosreplace') - setup( name='zarr', description=DESCRIPTION, @@ -35,6 +29,7 @@ 'setuptools>18.0', 'setuptools-scm>1.5.4' ], + python_requires='>=3.5', install_requires=dependencies, package_dir={'': '.'}, packages=['zarr', 'zarr.tests'], @@ -47,8 +42,6 @@ 'Programming Language :: Python', 'Topic :: Software Development :: Libraries :: Python Modules', 'Operating System :: Unix', - 'Programming Language :: Python :: 2', - 'Programming Language :: Python :: 2.7', 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', diff --git a/tox.ini b/tox.ini index 2a466b8731..9a296708bf 100644 --- a/tox.ini +++ b/tox.ini @@ -4,15 +4,12 @@ # and then run "tox" from this directory. [tox] -envlist = py27, py35, py36, py37-npy{115,116,latest}, docs +envlist = py35, py36, py37-npy{115,116,latest}, docs [testenv] install_command = pip install --no-binary=numcodecs {opts} {packages} setenv = PYTHONHASHSEED = 42 - # hooks for coverage exclusions based on Python major version - py35,py36,py37: PY_MAJOR_VERSION = py3 - py27: PY_MAJOR_VERSION = py2 passenv = ZARR_TEST_ABS ZARR_TEST_MONGO @@ -21,13 +18,13 @@ commands = # clear out any data files generated during tests python -c 'import glob; import shutil; import os; [(shutil.rmtree(d) if os.path.isdir(d) else os.remove(d) if os.path.isfile(d) else None) for d in glob.glob("./example*")]' # main unit test runner - py27,py35,py36: pytest -v --cov=zarr --cov-config=.coveragerc zarr + py35,py36: pytest -v --cov=zarr --cov-config=.coveragerc zarr # don't collect coverage when running older numpy versions py37-{npy115,npy116}: pytest -v zarr # collect coverage and run doctests under py37 py37-npylatest: pytest -v --cov=zarr --cov-config=.coveragerc --doctest-plus zarr --remote-data # generate a coverage report - py27,py35,py36,py37-npylatest: coverage report -m + py35,py36,py37-npylatest: coverage report -m # run doctests in the tutorial and spec py37-npylatest: python -m doctest -o NORMALIZE_WHITESPACE -o ELLIPSIS docs/tutorial.rst docs/spec/v2.rst # pep8 checks @@ -35,10 +32,9 @@ commands = # print environment for debugging pip freeze deps = - py27: backports.lzma py37-npy115: numpy==1.15.4 py37-npy116: numpy==1.16.4 - py27,py35,py36,py37-npylatest: -rrequirements_dev_numpy.txt + py35,py36,py37-npylatest: -rrequirements_dev_numpy.txt -rrequirements_dev_minimal.txt -rrequirements_dev_optional.txt diff --git a/zarr/__init__.py b/zarr/__init__.py index 55b5016b6d..a35beafa6c 100644 --- a/zarr/__init__.py +++ b/zarr/__init__.py @@ -1,20 +1,19 @@ # -*- coding: utf-8 -*- # flake8: noqa -from __future__ import absolute_import, print_function, division - - -from zarr.core import Array -from zarr.creation import (empty, zeros, ones, full, array, empty_like, zeros_like, - ones_like, full_like, open_array, open_like, create) -from zarr.storage import (DictStore, MemoryStore, DirectoryStore, ZipStore, TempStore, - NestedDirectoryStore, DBMStore, LMDBStore, SQLiteStore, - LRUStoreCache, ABSStore, RedisStore, MongoDBStore) -from zarr.hierarchy import group, open_group, Group -from zarr.sync import ThreadSynchronizer, ProcessSynchronizer from zarr.codecs import * -from zarr.convenience import (open, save, save_array, save_group, load, copy_store, - copy, copy_all, tree, consolidate_metadata, - open_consolidated) +from zarr.convenience import (consolidate_metadata, copy, copy_all, copy_store, + load, open, open_consolidated, save, save_array, + save_group, tree) +from zarr.core import Array +from zarr.creation import (array, create, empty, empty_like, full, full_like, + ones, ones_like, open_array, open_like, zeros, + zeros_like) +from zarr.errors import CopyError, MetadataError +from zarr.hierarchy import Group, group, open_group from zarr.n5 import N5Store -from zarr.errors import CopyError, MetadataError, PermissionError +from zarr.storage import (ABSStore, DBMStore, DictStore, DirectoryStore, + LMDBStore, LRUStoreCache, MemoryStore, MongoDBStore, + NestedDirectoryStore, RedisStore, SQLiteStore, + TempStore, ZipStore) +from zarr.sync import ProcessSynchronizer, ThreadSynchronizer from zarr.version import version as __version__ diff --git a/zarr/attrs.py b/zarr/attrs.py index f103650ed6..6c02940c4d 100644 --- a/zarr/attrs.py +++ b/zarr/attrs.py @@ -1,9 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division +from collections.abc import MutableMapping - -from zarr.compat import MutableMapping -from zarr.errors import PermissionError from zarr.meta import parse_metadata from zarr.util import json_dumps diff --git a/zarr/codecs.py b/zarr/codecs.py index 34230de03d..2ccc58f2a6 100644 --- a/zarr/codecs.py +++ b/zarr/codecs.py @@ -1,7 +1,4 @@ # -*- coding: utf-8 -*- # flake8: noqa -from __future__ import absolute_import, print_function, division - - from numcodecs import * from numcodecs.registry import codec_registry diff --git a/zarr/compat.py b/zarr/compat.py deleted file mode 100644 index c17142cec3..0000000000 --- a/zarr/compat.py +++ /dev/null @@ -1,39 +0,0 @@ -# -*- coding: utf-8 -*- -# flake8: noqa -from __future__ import absolute_import, print_function, division -import sys - - -PY2 = sys.version_info[0] == 2 - - -if PY2: # pragma: py3 no cover - - text_type = unicode - binary_type = str - reduce = reduce - from itertools import izip_longest as zip_longest - - class PermissionError(Exception): - pass - - def OrderedDict_move_to_end(od, key): - od[key] = od.pop(key) - - from collections import Mapping, MutableMapping - from scandir import scandir - - -else: # pragma: py2 no cover - - text_type = str - binary_type = bytes - from functools import reduce - from itertools import zip_longest - PermissionError = PermissionError - - def OrderedDict_move_to_end(od, key): - od.move_to_end(key) - - from collections.abc import Mapping, MutableMapping - from os import scandir diff --git a/zarr/convenience.py b/zarr/convenience.py index 42354d25c3..6e269a10ee 100644 --- a/zarr/convenience.py +++ b/zarr/convenience.py @@ -1,20 +1,20 @@ # -*- coding: utf-8 -*- """Convenience functions for storing and loading data.""" -from __future__ import absolute_import, print_function, division import io -import re import itertools - +import re +from collections.abc import Mapping from zarr.core import Array -from zarr.creation import (open_array, normalize_store_arg, - array as _create_array) -from zarr.hierarchy import open_group, group as _create_group, Group -from zarr.storage import contains_array, contains_group -from zarr.errors import err_path_not_found, CopyError -from zarr.util import normalize_storage_path, TreeViewer, buffer_size -from zarr.compat import Mapping, PY2, text_type +from zarr.creation import array as _create_array +from zarr.creation import normalize_store_arg, open_array +from zarr.errors import CopyError, err_path_not_found +from zarr.hierarchy import Group +from zarr.hierarchy import group as _create_group +from zarr.hierarchy import open_group from zarr.meta import json_dumps, json_loads +from zarr.storage import contains_array, contains_group +from zarr.util import TreeViewer, buffer_size, normalize_storage_path # noinspection PyShadowingBuiltins @@ -451,9 +451,6 @@ def __exit__(self, *args): def __call__(self, *args, **kwargs): if self.log_file is not None: kwargs['file'] = self.log_file - if PY2: # pragma: py3 no cover - # expect file opened in text mode, need to adapt message - args = [text_type(a) for a in args] print(*args, **kwargs) if hasattr(self.log_file, 'flush'): # get immediate feedback diff --git a/zarr/core.py b/zarr/core.py index 29d54e4750..d64392c264 100644 --- a/zarr/core.py +++ b/zarr/core.py @@ -1,29 +1,28 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division import binascii -import operator -import itertools import hashlib +import itertools +import operator import re - +from functools import reduce import numpy as np from numcodecs.compat import ensure_bytes, ensure_ndarray - -from zarr.util import (is_total_slice, human_readable_size, normalize_resize_args, - normalize_storage_path, normalize_shape, normalize_chunks, - InfoReporter, check_array_shape, nolock) -from zarr.storage import array_meta_key, attrs_key, listdir, getsize -from zarr.meta import decode_array_metadata, encode_array_metadata from zarr.attrs import Attributes -from zarr.errors import PermissionError, err_read_only, err_array_not_found -from zarr.compat import reduce from zarr.codecs import AsType, get_codec -from zarr.indexing import (OIndex, OrthogonalIndexer, BasicIndexer, VIndex, - CoordinateIndexer, MaskIndexer, check_fields, pop_fields, - ensure_tuple, is_scalar, is_contiguous_selection, - err_too_many_indices, check_no_multi_fields) +from zarr.errors import err_array_not_found, err_read_only +from zarr.indexing import (BasicIndexer, CoordinateIndexer, MaskIndexer, + OIndex, OrthogonalIndexer, VIndex, check_fields, + check_no_multi_fields, ensure_tuple, + err_too_many_indices, is_contiguous_selection, + is_scalar, pop_fields) +from zarr.meta import decode_array_metadata, encode_array_metadata +from zarr.storage import array_meta_key, attrs_key, getsize, listdir +from zarr.util import (InfoReporter, check_array_shape, human_readable_size, + is_total_slice, nolock, normalize_chunks, + normalize_resize_args, normalize_shape, + normalize_storage_path) # noinspection PyUnresolvedReferences @@ -1786,7 +1785,7 @@ def _encode_chunk(self, chunk): def __repr__(self): t = type(self) - r = '<%s.%s' % (t.__module__, t.__name__) + r = '<{}.{}'.format(t.__module__, t.__name__) if self.name: r += ' %r' % self.name r += ' %s' % str(self.shape) @@ -1827,11 +1826,11 @@ def info_items(self): def _info_items_nosync(self): def typestr(o): - return '%s.%s' % (type(o).__module__, type(o).__name__) + return '{}.{}'.format(type(o).__module__, type(o).__name__) def bytestr(n): if n > 2**10: - return '%s (%s)' % (n, human_readable_size(n)) + return '{} ({})'.format(n, human_readable_size(n)) else: return str(n) @@ -1872,7 +1871,7 @@ def bytestr(n): ('Storage ratio', '%.1f' % (self.nbytes / self.nbytes_stored)), ] items += [ - ('Chunks initialized', '%s/%s' % (self.nchunks_initialized, self.nchunks)) + ('Chunks initialized', '{}/{}'.format(self.nchunks_initialized, self.nchunks)) ] return items @@ -1930,7 +1929,7 @@ def hexdigest(self, hashname="sha1"): checksum = binascii.hexlify(self.digest(hashname=hashname)) # This is a bytes object on Python 3 and we want a str. - if type(checksum) is not str: # pragma: py2 no cover + if type(checksum) is not str: checksum = checksum.decode('utf8') return checksum diff --git a/zarr/creation.py b/zarr/creation.py index 082f4fd3dd..632dd24915 100644 --- a/zarr/creation.py +++ b/zarr/creation.py @@ -1,17 +1,16 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division from warnings import warn - import numpy as np - +from numcodecs.registry import codec_registry from zarr.core import Array -from zarr.storage import (DirectoryStore, init_array, contains_array, contains_group, - default_compressor, normalize_storage_path, ZipStore) +from zarr.errors import (err_array_not_found, err_contains_array, + err_contains_group) from zarr.n5 import N5Store -from numcodecs.registry import codec_registry -from zarr.errors import err_contains_array, err_contains_group, err_array_not_found +from zarr.storage import (DirectoryStore, ZipStore, contains_array, + contains_group, default_compressor, init_array, + normalize_storage_path) def create(shape, chunks=True, dtype=None, compressor='default', diff --git a/zarr/errors.py b/zarr/errors.py index a59318377a..fa3cff04d5 100644 --- a/zarr/errors.py +++ b/zarr/errors.py @@ -1,8 +1,4 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division - - -from zarr.compat import PermissionError class MetadataError(Exception): diff --git a/zarr/hierarchy.py b/zarr/hierarchy.py index d0ea600010..6200eafcaa 100644 --- a/zarr/hierarchy.py +++ b/zarr/hierarchy.py @@ -1,23 +1,22 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division +from collections.abc import MutableMapping from itertools import islice import numpy as np - -from zarr.compat import PY2, MutableMapping from zarr.attrs import Attributes from zarr.core import Array -from zarr.storage import (contains_array, contains_group, init_group, - MemoryStore, group_meta_key, attrs_key, listdir, rename, rmdir) -from zarr.creation import (array, create, empty, zeros, ones, full, - empty_like, zeros_like, ones_like, full_like, - normalize_store_arg) -from zarr.util import (normalize_storage_path, normalize_shape, InfoReporter, TreeViewer, - is_valid_python_name, instance_dir, nolock) -from zarr.errors import (err_contains_array, err_contains_group, err_group_not_found, - err_read_only) +from zarr.creation import (array, create, empty, empty_like, full, full_like, + normalize_store_arg, ones, ones_like, zeros, + zeros_like) +from zarr.errors import (err_contains_array, err_contains_group, + err_group_not_found, err_read_only) from zarr.meta import decode_group_metadata +from zarr.storage import (MemoryStore, attrs_key, contains_array, + contains_group, group_meta_key, init_group, listdir, + rename, rmdir) +from zarr.util import (InfoReporter, TreeViewer, is_valid_python_name, nolock, + normalize_shape, normalize_storage_path) class Group(MutableMapping): @@ -218,7 +217,7 @@ def __len__(self): def __repr__(self): t = type(self) - r = '<%s.%s' % (t.__module__, t.__name__) + r = '<{}.{}'.format(t.__module__, t.__name__) if self.name: r += ' %r' % self.name if self._read_only: @@ -229,7 +228,7 @@ def __repr__(self): def info_items(self): def typestr(o): - return '%s.%s' % (type(o).__module__, type(o).__name__) + return '{}.{}'.format(type(o).__module__, type(o).__name__) items = [] @@ -353,11 +352,8 @@ def __getattr__(self, item): raise AttributeError def __dir__(self): - if PY2: # pragma: py3 no cover - base = instance_dir(self) - else: # pragma: py2 no cover - # noinspection PyUnresolvedReferences - base = super().__dir__() + # noinspection PyUnresolvedReferences + base = super().__dir__() keys = sorted(set(base + list(self))) keys = [k for k in keys if is_valid_python_name(k)] return keys diff --git a/zarr/indexing.py b/zarr/indexing.py index 988489e0bb..ff4595d124 100644 --- a/zarr/indexing.py +++ b/zarr/indexing.py @@ -1,15 +1,12 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -import numbers -import itertools import collections - +import itertools +import numbers import numpy as np - -from zarr.errors import (err_too_many_indices, err_boundscheck, err_negative_step, - err_vindex_invalid_selection) +from zarr.errors import (err_boundscheck, err_negative_step, + err_too_many_indices, err_vindex_invalid_selection) def is_integer(x): diff --git a/zarr/meta.py b/zarr/meta.py index 8b5012c612..181c6efa69 100644 --- a/zarr/meta.py +++ b/zarr/meta.py @@ -1,16 +1,12 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division import base64 - +from collections.abc import Mapping import numpy as np - -from zarr.compat import PY2, Mapping from zarr.errors import MetadataError from zarr.util import json_dumps, json_loads - ZARR_FORMAT = 2 @@ -89,11 +85,7 @@ def _decode_dtype_descr(d): # need to convert list of lists to list of tuples if isinstance(d, list): # recurse to handle nested structures - if PY2: # pragma: py3 no cover - # under PY2 numpy rejects unicode field names - d = [(k[0].encode("ascii"), _decode_dtype_descr(k[1])) + tuple(k[2:]) for k in d] - else: # pragma: py2 no cover - d = [(k[0], _decode_dtype_descr(k[1])) + tuple(k[2:]) for k in d] + d = [(k[0], _decode_dtype_descr(k[1])) + tuple(k[2:]) for k in d] return d @@ -191,9 +183,7 @@ def encode_fill_value(v, dtype): encode_fill_value(v.imag, dtype.type().imag.dtype)) return v elif dtype.kind in 'SV': - v = base64.standard_b64encode(v) - if not PY2: # pragma: py2 no cover - v = str(v, 'ascii') + v = str(base64.standard_b64encode(v), 'ascii') return v elif dtype.kind == 'U': return v diff --git a/zarr/meta_v1.py b/zarr/meta_v1.py index 50474359cd..9f40e84397 100644 --- a/zarr/meta_v1.py +++ b/zarr/meta_v1.py @@ -1,17 +1,13 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division import json - import numpy as np - -from zarr.compat import PY2, text_type from zarr.errors import MetadataError def decode_metadata(b): - s = text_type(b, 'ascii') + s = str(b, 'ascii') meta = json.loads(s) zarr_format = meta.get('zarr_format', None) if zarr_format != 1: @@ -60,12 +56,7 @@ def _decode_dtype_descr(d): # need to convert list of lists to list of tuples if isinstance(d, list): # recurse to handle nested structures - if PY2: - # under PY2 numpy rejects unicode field names - d = [(f.encode('ascii'), _decode_dtype_descr(v)) - for f, v in d] - else: - d = [(f, _decode_dtype_descr(v)) for f, v in d] + d = [(f, _decode_dtype_descr(v)) for f, v in d] return d diff --git a/zarr/n5.py b/zarr/n5.py index 34018098ed..df44409565 100644 --- a/zarr/n5.py +++ b/zarr/n5.py @@ -1,23 +1,21 @@ # -*- coding: utf-8 -*- """This module contains a storage class and codec to support the N5 format. """ -from __future__ import absolute_import, division -from .meta import ZARR_FORMAT, json_dumps, json_loads -from .storage import ( - NestedDirectoryStore, - group_meta_key as zarr_group_meta_key, - array_meta_key as zarr_array_meta_key, - attrs_key as zarr_attrs_key, - _prog_ckey, _prog_number) -from numcodecs.abc import Codec -from numcodecs.compat import ndarray_copy -from numcodecs.registry import register_codec, get_codec -import numpy as np +import os import struct import sys -import os import warnings +import numpy as np +from numcodecs.abc import Codec +from numcodecs.compat import ndarray_copy +from numcodecs.registry import get_codec, register_codec + +from .meta import ZARR_FORMAT, json_dumps, json_loads +from .storage import NestedDirectoryStore, _prog_ckey, _prog_number +from .storage import array_meta_key as zarr_array_meta_key +from .storage import attrs_key as zarr_attrs_key +from .storage import group_meta_key as zarr_group_meta_key zarr_to_n5_keys = [ ('chunks', 'blockSize'), @@ -571,7 +569,7 @@ def decode(self, chunk, out=None): # out should only be used if we read a complete chunk assert chunk_shape == self.chunk_shape, ( - "Expected chunk of shape %s, found %s" % ( + "Expected chunk of shape {}, found {}".format( self.chunk_shape, chunk_shape)) diff --git a/zarr/storage.py b/zarr/storage.py index 03ebdefeac..4f4e9c7a09 100644 --- a/zarr/storage.py +++ b/zarr/storage.py @@ -15,34 +15,34 @@ path) and a `getsize` method (return the size in bytes of a given value). """ -from __future__ import absolute_import, print_function, division -from collections import OrderedDict -import os -import operator -import tempfile -import zipfile -import shutil import atexit import errno +import glob +import multiprocessing +import operator +import os import re +import shutil import sys -import multiprocessing +import tempfile +import warnings +import zipfile +from collections import OrderedDict +from collections.abc import MutableMapping +from os import scandir from pickle import PicklingError from threading import Lock, RLock -import glob -import warnings - -from zarr.util import (json_loads, normalize_shape, normalize_chunks, normalize_order, - normalize_storage_path, buffer_size, - normalize_fill_value, nolock, normalize_dtype) -from zarr.meta import encode_array_metadata, encode_group_metadata -from zarr.compat import PY2, MutableMapping, OrderedDict_move_to_end, scandir -from numcodecs.registry import codec_registry from numcodecs.compat import ensure_bytes, ensure_contiguous_ndarray -from zarr.errors import (err_contains_group, err_contains_array, err_bad_compressor, - err_fspath_exists_notdir, err_read_only, MetadataError) +from numcodecs.registry import codec_registry +from zarr.errors import (MetadataError, err_bad_compressor, err_contains_array, + err_contains_group, err_fspath_exists_notdir, + err_read_only) +from zarr.meta import encode_array_metadata, encode_group_metadata +from zarr.util import (buffer_size, json_loads, nolock, normalize_chunks, + normalize_dtype, normalize_fill_value, normalize_order, + normalize_shape, normalize_storage_path) __doctest_requires__ = { ('RedisStore', 'RedisStore.*'): ['redis'], @@ -1420,12 +1420,8 @@ class DBMStore(MutableMapping): def __init__(self, path, flag='c', mode=0o666, open=None, write_lock=True, **open_kwargs): if open is None: - if PY2: # pragma: py3 no cover - import anydbm - open = anydbm.open - else: # pragma: py2 no cover - import dbm - open = dbm.open + import dbm + open = dbm.open path = os.path.abspath(path) # noinspection PyArgumentList self.db = open(path, flag, mode, **open_kwargs) @@ -1444,7 +1440,7 @@ def __init__(self, path, flag='c', mode=0o666, open=None, write_lock=True, def __getstate__(self): try: - self.flush() # needed for py2 and ndbm + self.flush() # needed for ndbm except Exception: # flush may fail if db has already been closed pass @@ -1523,24 +1519,14 @@ def __contains__(self, key): return key in self.db -if PY2: # pragma: py3 no cover - - def _lmdb_decode_key_buffer(key): - # assume buffers=True - return str(key) - - def _lmdb_decode_key_bytes(key): - return key - -else: # pragma: py2 no cover +def _lmdb_decode_key_buffer(key): + # assume buffers=True + return key.tobytes().decode('ascii') - def _lmdb_decode_key_buffer(key): - # assume buffers=True - return key.tobytes().decode('ascii') - def _lmdb_decode_key_bytes(key): - # assume buffers=False - return key.decode('ascii') +def _lmdb_decode_key_bytes(key): + # assume buffers=False + return key.decode('ascii') class LMDBStore(MutableMapping): @@ -1871,7 +1857,7 @@ def __getitem__(self, key): # cache hit if no KeyError is raised self.hits += 1 # treat the end as most recently used - OrderedDict_move_to_end(self._values_cache, key) + self._values_cache.move_to_end(key) except KeyError: # cache miss, retrieve value from the store @@ -2187,14 +2173,7 @@ def update(self, *args, **kwargs): kv_list = [] for dct in args: for k, v in dct.items(): - # Python 2 cannot store `memoryview`s, but it can store - # `buffer`s. However Python 2 won't return `bytes` then. So we - # coerce to `bytes`, which are handled correctly. Python 3 - # doesn't have these issues. - if PY2: # pragma: py3 no cover - v = ensure_bytes(v) - else: # pragma: py2 no cover - v = ensure_contiguous_ndarray(v) + v = ensure_contiguous_ndarray(v) # Accumulate key-value pairs for storage kv_list.append((k, v)) diff --git a/zarr/sync.py b/zarr/sync.py index fe936e13e2..1498678fb4 100644 --- a/zarr/sync.py +++ b/zarr/sync.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -from threading import Lock -from collections import defaultdict import os - +from collections import defaultdict +from threading import Lock import fasteners diff --git a/zarr/tests/__init__.py b/zarr/tests/__init__.py index 1ee762b218..40a96afc6f 100644 --- a/zarr/tests/__init__.py +++ b/zarr/tests/__init__.py @@ -1,2 +1 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division diff --git a/zarr/tests/test_attrs.py b/zarr/tests/test_attrs.py index f546224c73..5ad39ef1a9 100644 --- a/zarr/tests/test_attrs.py +++ b/zarr/tests/test_attrs.py @@ -1,15 +1,10 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division import json import unittest - import pytest - from zarr.attrs import Attributes -from zarr.compat import binary_type, text_type -from zarr.errors import PermissionError from zarr.tests.util import CountingDict @@ -29,8 +24,8 @@ def test_storage(self): a['foo'] = 'bar' a['baz'] = 42 assert 'attrs' in store - assert isinstance(store['attrs'], binary_type) - d = json.loads(text_type(store['attrs'], 'ascii')) + assert isinstance(store['attrs'], bytes) + d = json.loads(str(store['attrs'], 'ascii')) assert dict(foo='bar', baz=42) == d def test_get_set_del_contains(self): diff --git a/zarr/tests/test_convenience.py b/zarr/tests/test_convenience.py index d33d23ecb8..3833c67d17 100644 --- a/zarr/tests/test_convenience.py +++ b/zarr/tests/test_convenience.py @@ -1,24 +1,22 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -import tempfile import atexit import os +import tempfile import unittest from numbers import Integral - import numpy as np -from numpy.testing import assert_array_equal -from numcodecs import Zlib, Adler32 import pytest +from numcodecs import Adler32, Zlib +from numpy.testing import assert_array_equal - -from zarr.convenience import (open, save, save_group, load, copy_store, copy, - consolidate_metadata, open_consolidated) -from zarr.storage import atexit_rmtree, MemoryStore, getsize, ConsolidatedMetadataStore +from zarr.convenience import (consolidate_metadata, copy, copy_store, load, + open, open_consolidated, save, save_group) from zarr.core import Array +from zarr.errors import CopyError from zarr.hierarchy import Group, group -from zarr.errors import CopyError, PermissionError +from zarr.storage import (ConsolidatedMetadataStore, MemoryStore, + atexit_rmtree, getsize) def test_open_array(): diff --git a/zarr/tests/test_core.py b/zarr/tests/test_core.py index 43daca9f03..00bf04f0ab 100644 --- a/zarr/tests/test_core.py +++ b/zarr/tests/test_core.py @@ -1,17 +1,28 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -import unittest -from tempfile import mkdtemp, mktemp import atexit -import shutil -import pickle import os -import warnings - +import pickle +import shutil +import unittest +from itertools import zip_longest +from tempfile import mkdtemp, mktemp import numpy as np -from numpy.testing import assert_array_equal, assert_array_almost_equal import pytest +from numcodecs import (BZ2, JSON, LZ4, Blosc, Categorize, Delta, + FixedScaleOffset, GZip, MsgPack, Pickle, VLenArray, + VLenBytes, VLenUTF8, Zlib) +from numcodecs.compat import ensure_bytes, ensure_ndarray +from numcodecs.tests.common import greetings +from numpy.testing import assert_array_almost_equal, assert_array_equal + +from zarr.core import Array +from zarr.meta import json_loads +from zarr.n5 import N5Store, n5_keywords +from zarr.storage import (ABSStore, DBMStore, DirectoryStore, LMDBStore, + LRUStoreCache, NestedDirectoryStore, SQLiteStore, + atexit_rmglob, atexit_rmtree, init_array, init_group) +from zarr.util import buffer_size try: import azure.storage.blob as asb @@ -19,32 +30,11 @@ asb = None -from zarr.storage import (DirectoryStore, init_array, init_group, NestedDirectoryStore, - DBMStore, LMDBStore, SQLiteStore, ABSStore, atexit_rmtree, - atexit_rmglob, LRUStoreCache) -from zarr.core import Array -from zarr.errors import PermissionError -from zarr.compat import PY2, text_type, binary_type, zip_longest -from zarr.meta import json_loads -from zarr.util import buffer_size -from zarr.n5 import n5_keywords, N5Store -from numcodecs import (Delta, FixedScaleOffset, LZ4, GZip, Zlib, Blosc, BZ2, MsgPack, Pickle, - Categorize, JSON, VLenUTF8, VLenBytes, VLenArray) -from numcodecs.compat import ensure_bytes, ensure_ndarray -from numcodecs.tests.common import greetings - - # also check for environment variables indicating whether tests requiring # services should be run ZARR_TEST_ABS = os.environ.get('ZARR_TEST_ABS', '0') -# needed for PY2/PY3 consistent behaviour -if PY2: # pragma: py3 no cover - warnings.resetwarnings() - warnings.simplefilter('always') - - # noinspection PyMethodMayBeStatic class TestArray(unittest.TestCase): @@ -102,10 +92,7 @@ def test_store_has_text_keys(self): z = self.create_array(shape=(1050,), chunks=100, dtype='f8', compressor=[]) z[:] = np.random.random(z.shape) - if PY2: # pragma: py3 no cover - expected_type = (str, text_type) - else: # pragma: py2 no cover - expected_type = text_type + expected_type = str for k in z.chunk_store.keys(): if not isinstance(k, expected_type): # pragma: no cover @@ -134,7 +121,7 @@ def test_store_has_bytes_values(self): z[:] = np.random.random(z.shape) # Check in-memory array only contains `bytes` - assert all([isinstance(v, binary_type) for v in z.chunk_store.values()]) + assert all([isinstance(v, bytes) for v in z.chunk_store.values()]) def test_nbytes_stored(self): @@ -1102,7 +1089,7 @@ def test_object_arrays_vlen_text(self): assert_array_equal(data, a) # convenience API - z = self.create_array(shape=data.shape, dtype=text_type) + z = self.create_array(shape=data.shape, dtype=str) assert z.dtype == object assert isinstance(z.filters[0], VLenUTF8) z[:] = data @@ -1143,7 +1130,7 @@ def test_object_arrays_vlen_bytes(self): assert_array_equal(data, a) # convenience API - z = self.create_array(shape=data.shape, dtype=binary_type) + z = self.create_array(shape=data.shape, dtype=bytes) assert z.dtype == object assert isinstance(z.filters[0], VLenBytes) z[:] = data @@ -1640,7 +1627,7 @@ def test_object_arrays_vlen_text(self): # convenience API with pytest.raises(ValueError): - self.create_array(shape=data.shape, dtype=text_type) + self.create_array(shape=data.shape, dtype=str) def test_object_arrays_vlen_bytes(self): @@ -1652,7 +1639,7 @@ def test_object_arrays_vlen_bytes(self): # convenience API with pytest.raises(ValueError): - self.create_array(shape=data.shape, dtype=binary_type) + self.create_array(shape=data.shape, dtype=bytes) def test_object_arrays_vlen_array(self): diff --git a/zarr/tests/test_creation.py b/zarr/tests/test_creation.py index 65a0caff63..d28ef401b6 100644 --- a/zarr/tests/test_creation.py +++ b/zarr/tests/test_creation.py @@ -1,26 +1,22 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -import tempfile -import shutil import atexit import os.path - +import shutil +import tempfile import numpy as np -from numpy.testing import assert_array_equal import pytest +from numpy.testing import assert_array_equal - -from zarr.creation import (array, empty, zeros, ones, full, open_array, empty_like, - zeros_like, ones_like, full_like, open_like, create) -from zarr.sync import ThreadSynchronizer +from zarr.codecs import Zlib from zarr.core import Array -from zarr.storage import DirectoryStore -from zarr.n5 import N5Store +from zarr.creation import (array, create, empty, empty_like, full, full_like, + ones, ones_like, open_array, open_like, zeros, + zeros_like) from zarr.hierarchy import open_group -from zarr.errors import PermissionError -from zarr.codecs import Zlib -from zarr.compat import PY2 +from zarr.n5 import N5Store +from zarr.storage import DirectoryStore +from zarr.sync import ThreadSynchronizer # something bcolz-like @@ -162,17 +158,8 @@ def test_full(): # bytes fill value / unicode dtype v = b'xxx' - if PY2: # pragma: py3 no cover - # allow this on PY2 - z = full(100, chunks=10, fill_value=v, dtype='U3') - a = z[...] - assert z.dtype == a.dtype - assert v == a[0] - assert np.all(a == v) - else: # pragma: py2 no cover - # be strict on PY3 - with pytest.raises(ValueError): - full(100, chunks=10, fill_value=v, dtype='U3') + with pytest.raises(ValueError): + full(100, chunks=10, fill_value=v, dtype='U3') def test_open_array(): @@ -476,15 +463,12 @@ def test_compression_args(): assert 'zlib' == z.compressor.codec_id assert 9 == z.compressor.level - # cannot get warning tests to work on PY2 - if not PY2: # pragma: py2 no cover - - with pytest.warns(UserWarning): - # 'compressor' overrides 'compression' - create(100, compressor=Zlib(9), compression='bz2', compression_opts=1) - with pytest.warns(UserWarning): - # 'compressor' ignores 'compression_opts' - create(100, compressor=Zlib(9), compression_opts=1) + with pytest.warns(UserWarning): + # 'compressor' overrides 'compression' + create(100, compressor=Zlib(9), compression='bz2', compression_opts=1) + with pytest.warns(UserWarning): + # 'compressor' ignores 'compression_opts' + create(100, compressor=Zlib(9), compression_opts=1) def test_create_read_only(): diff --git a/zarr/tests/test_filters.py b/zarr/tests/test_filters.py index 4ac256a438..7abe6d960a 100644 --- a/zarr/tests/test_filters.py +++ b/zarr/tests/test_filters.py @@ -1,16 +1,11 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division - - import numpy as np -from numpy.testing import assert_array_equal, assert_array_almost_equal +from numcodecs import (BZ2, AsType, Blosc, Categorize, Delta, FixedScaleOffset, + PackBits, Quantize, Zlib) +from numpy.testing import assert_array_almost_equal, assert_array_equal - -from numcodecs import (AsType, Delta, FixedScaleOffset, PackBits, Categorize, Zlib, Blosc, - BZ2, Quantize) from zarr.creation import array - compressors = [ None, Zlib(), diff --git a/zarr/tests/test_hierarchy.py b/zarr/tests/test_hierarchy.py index 546e2ab93f..6e2292ab4c 100644 --- a/zarr/tests/test_hierarchy.py +++ b/zarr/tests/test_hierarchy.py @@ -1,18 +1,27 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -import unittest -import tempfile import atexit -import shutil -import textwrap import os import pickle -import warnings - +import shutil +import tempfile +import textwrap +import unittest import numpy as np -from numpy.testing import assert_array_equal import pytest +from numcodecs import Zlib +from numpy.testing import assert_array_equal + +from zarr.attrs import Attributes +from zarr.core import Array +from zarr.creation import open_array +from zarr.hierarchy import Group, group, open_group +from zarr.storage import (ABSStore, DBMStore, DirectoryStore, LMDBStore, + LRUStoreCache, MemoryStore, NestedDirectoryStore, + SQLiteStore, ZipStore, array_meta_key, atexit_rmglob, + atexit_rmtree, group_meta_key, init_array, + init_group) +from zarr.util import InfoReporter try: import azure.storage.blob as asb @@ -20,31 +29,11 @@ asb = None -from zarr.storage import (MemoryStore, DirectoryStore, ZipStore, init_group, init_array, - array_meta_key, group_meta_key, atexit_rmtree, - NestedDirectoryStore, DBMStore, LMDBStore, SQLiteStore, - ABSStore, atexit_rmglob, LRUStoreCache) -from zarr.core import Array -from zarr.compat import PY2, text_type -from zarr.hierarchy import Group, group, open_group -from zarr.attrs import Attributes -from zarr.errors import PermissionError -from zarr.creation import open_array -from zarr.util import InfoReporter -from numcodecs import Zlib - - # also check for environment variables indicating whether tests requiring # services should be run ZARR_TEST_ABS = os.environ.get('ZARR_TEST_ABS', '0') -# needed for PY2/PY3 consistent behaviour -if PY2: # pragma: py3 no cover - warnings.resetwarnings() - warnings.simplefilter('always') - - # noinspection PyStatementEffect class TestGroup(unittest.TestCase): @@ -1191,10 +1180,8 @@ def test_group_key_completions(): def _check_tree(g, expect_bytes, expect_text): assert expect_bytes == bytes(g.tree()) - assert expect_text == text_type(g.tree()) + assert expect_text == str(g.tree()) expect_repr = expect_text - if PY2: # pragma: py3 no cover - expect_repr = expect_bytes assert expect_repr == repr(g.tree()) # test _repr_html_ lightly # noinspection PyProtectedMember diff --git a/zarr/tests/test_indexing.py b/zarr/tests/test_indexing.py index 8aeeab51b9..a68223a5f0 100644 --- a/zarr/tests/test_indexing.py +++ b/zarr/tests/test_indexing.py @@ -1,15 +1,11 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division - - import numpy as np -from numpy.testing import assert_array_equal import pytest +from numpy.testing import assert_array_equal - -from zarr.indexing import (normalize_integer_selection, replace_ellipsis, oindex, - oindex_set) import zarr +from zarr.indexing import (normalize_integer_selection, oindex, oindex_set, + replace_ellipsis) def test_normalize_integer_selection(): diff --git a/zarr/tests/test_info.py b/zarr/tests/test_info.py index e0d2330d6b..1488f59275 100644 --- a/zarr/tests/test_info.py +++ b/zarr/tests/test_info.py @@ -1,9 +1,7 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division - +import numcodecs import zarr -import numcodecs def test_info(): diff --git a/zarr/tests/test_meta.py b/zarr/tests/test_meta.py index 7ade1b95c2..5051ff4379 100644 --- a/zarr/tests/test_meta.py +++ b/zarr/tests/test_meta.py @@ -1,25 +1,22 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -import json import base64 - +import json import numpy as np import pytest - -from zarr.compat import binary_type, text_type, PY2 -from zarr.meta import (decode_array_metadata, encode_dtype, decode_dtype, ZARR_FORMAT, - decode_group_metadata, encode_array_metadata) +from zarr.codecs import Blosc, Delta, Zlib from zarr.errors import MetadataError -from zarr.codecs import Delta, Zlib, Blosc +from zarr.meta import (ZARR_FORMAT, decode_array_metadata, decode_dtype, + decode_group_metadata, encode_array_metadata, + encode_dtype) def assert_json_equal(expect, actual): - if isinstance(expect, binary_type): # pragma: py3 no cover - expect = text_type(expect, 'ascii') - if isinstance(actual, binary_type): - actual = text_type(actual, 'ascii') + if isinstance(expect, bytes): # pragma: py3 no cover + expect = str(expect, 'ascii') + if isinstance(actual, bytes): + actual = str(actual, 'ascii') ej = json.loads(expect) aj = json.loads(actual) assert ej == aj @@ -368,8 +365,7 @@ def test_encode_decode_fill_values_bytes(): # define expected metadata encoded as JSON s = base64.standard_b64encode(v) - if not PY2: - s = s.decode() + s = s.decode() meta_json = '''{ "chunks": [10], "compressor": {"id": "zlib", "level": 1}, diff --git a/zarr/tests/test_storage.py b/zarr/tests/test_storage.py index a713acaf8a..0b517c4399 100644 --- a/zarr/tests/test_storage.py +++ b/zarr/tests/test_storage.py @@ -1,20 +1,34 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -from contextlib import contextmanager -import unittest -import tempfile +import array import atexit -import pickle import json -import array -import shutil import os +import pickle +import shutil +import tempfile +import unittest +from contextlib import contextmanager from pickle import PicklingError - import numpy as np -from numpy.testing import assert_array_equal, assert_array_almost_equal import pytest +from numpy.testing import assert_array_almost_equal, assert_array_equal + +from zarr.codecs import BZ2, AsType, Blosc, Zlib +from zarr.errors import MetadataError +from zarr.hierarchy import group +from zarr.meta import (ZARR_FORMAT, decode_array_metadata, + decode_group_metadata, encode_array_metadata, + encode_group_metadata) +from zarr.n5 import N5Store +from zarr.storage import (ABSStore, ConsolidatedMetadataStore, DBMStore, + DictStore, DirectoryStore, LMDBStore, LRUStoreCache, + MemoryStore, MongoDBStore, NestedDirectoryStore, + RedisStore, SQLiteStore, TempStore, ZipStore, + array_meta_key, atexit_rmglob, atexit_rmtree, + attrs_key, default_compressor, getsize, + group_meta_key, init_array, init_group, migrate_1to2) +from zarr.tests.util import CountingDict try: import sqlite3 @@ -36,20 +50,6 @@ except ImportError: # pragma: no cover redis = None -from zarr.storage import (init_array, array_meta_key, attrs_key, DictStore, MemoryStore, - DirectoryStore, ZipStore, init_group, group_meta_key, - getsize, migrate_1to2, TempStore, atexit_rmtree, - NestedDirectoryStore, default_compressor, DBMStore, - LMDBStore, SQLiteStore, ABSStore, atexit_rmglob, LRUStoreCache, - ConsolidatedMetadataStore, MongoDBStore, RedisStore) -from zarr.meta import (decode_array_metadata, encode_array_metadata, ZARR_FORMAT, - decode_group_metadata, encode_group_metadata) -from zarr.compat import PY2 -from zarr.codecs import AsType, Zlib, Blosc, BZ2 -from zarr.errors import PermissionError, MetadataError -from zarr.hierarchy import group -from zarr.n5 import N5Store -from zarr.tests.util import CountingDict try: from zarr.codecs import LZMA @@ -988,19 +988,14 @@ class TestDBMStoreDumb(TestDBMStore): def create_store(self): path = tempfile.mktemp(suffix='.dumbdbm') atexit.register(atexit_rmglob, path + '*') - if PY2: # pragma: py3 no cover - import dumbdbm - else: # pragma: py2 no cover - import dbm.dumb as dumbdbm + + import dbm.dumb as dumbdbm store = DBMStore(path, flag='n', open=dumbdbm.open) return store try: - if PY2: # pragma: py3 no cover - import gdbm - else: # pragma: py2 no cover - import dbm.gnu as gdbm + import dbm.gnu as gdbm except ImportError: # pragma: no cover gdbm = None @@ -1015,20 +1010,20 @@ def create_store(self): return store -if not PY2: # pragma: py2 no cover - try: - import dbm.ndbm as ndbm - except ImportError: # pragma: no cover - ndbm = None +try: + import dbm.ndbm as ndbm +except ImportError: # pragma: no cover + ndbm = None + - @unittest.skipIf(ndbm is None, reason='ndbm is not installed') - class TestDBMStoreNDBM(TestDBMStore): +@unittest.skipIf(ndbm is None, reason='ndbm is not installed') +class TestDBMStoreNDBM(TestDBMStore): - def create_store(self): - path = tempfile.mktemp(suffix='.ndbm') - atexit.register(atexit_rmglob, path + '*') - store = DBMStore(path, flag='n', open=ndbm.open) - return store + def create_store(self): + path = tempfile.mktemp(suffix='.ndbm') + atexit.register(atexit_rmglob, path + '*') + store = DBMStore(path, flag='n', open=ndbm.open) + return store try: @@ -1059,12 +1054,7 @@ class TestLMDBStore(StoreTests, unittest.TestCase): def create_store(self): path = tempfile.mktemp(suffix='.lmdb') atexit.register(atexit_rmtree, path) - if PY2: # pragma: py3 no cover - # don't use buffers, otherwise would have to rewrite tests as bytes and - # buffer don't compare equal in PY2 - buffers = False - else: # pragma: py2 no cover - buffers = True + buffers = True store = LMDBStore(path, buffers=buffers) return store diff --git a/zarr/tests/test_sync.py b/zarr/tests/test_sync.py index 4ee62e68e9..8af28c152d 100644 --- a/zarr/tests/test_sync.py +++ b/zarr/tests/test_sync.py @@ -1,26 +1,23 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -from tempfile import mkdtemp import atexit import shutil -from multiprocessing.pool import ThreadPool +import tempfile from multiprocessing import Pool as ProcessPool from multiprocessing import cpu_count -import tempfile - +from multiprocessing.pool import ThreadPool +from tempfile import mkdtemp import numpy as np from numpy.testing import assert_array_equal - +from zarr.attrs import Attributes +from zarr.core import Array +from zarr.hierarchy import Group +from zarr.storage import DirectoryStore, atexit_rmtree, init_array, init_group +from zarr.sync import ProcessSynchronizer, ThreadSynchronizer from zarr.tests.test_attrs import TestAttributes from zarr.tests.test_core import TestArray from zarr.tests.test_hierarchy import TestGroup -from zarr.sync import ThreadSynchronizer, ProcessSynchronizer -from zarr.core import Array -from zarr.attrs import Attributes -from zarr.storage import init_array, DirectoryStore, init_group, atexit_rmtree -from zarr.hierarchy import Group class TestAttributesWithThreadSynchronizer(TestAttributes): diff --git a/zarr/tests/test_util.py b/zarr/tests/test_util.py index 836b494b46..809ef8a6fa 100644 --- a/zarr/tests/test_util.py +++ b/zarr/tests/test_util.py @@ -1,15 +1,11 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division - - import numpy as np import pytest - -from zarr.util import (normalize_shape, normalize_chunks, is_total_slice, - normalize_resize_args, human_readable_size, normalize_order, - guess_chunks, info_html_report, info_text_report, - normalize_fill_value) +from zarr.util import (guess_chunks, human_readable_size, info_html_report, + info_text_report, is_total_slice, normalize_chunks, + normalize_fill_value, normalize_order, + normalize_resize_args, normalize_shape) def test_normalize_shape(): diff --git a/zarr/tests/util.py b/zarr/tests/util.py index b86e1a94c3..2b6a809d8b 100644 --- a/zarr/tests/util.py +++ b/zarr/tests/util.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division import collections -from zarr.compat import MutableMapping +from collections.abc import MutableMapping class CountingDict(MutableMapping): diff --git a/zarr/util.py b/zarr/util.py index 6447445c00..cc0ef1e729 100644 --- a/zarr/util.py +++ b/zarr/util.py @@ -1,33 +1,27 @@ # -*- coding: utf-8 -*- -from __future__ import absolute_import, print_function, division -from textwrap import TextWrapper, dedent import codecs +import inspect import json import numbers import uuid -import inspect - +from textwrap import TextWrapper, dedent +import numpy as np from asciitree import BoxStyle, LeftAligned from asciitree.traversal import Traversal -import numpy as np -from numcodecs.compat import ensure_ndarray, ensure_contiguous_ndarray +from numcodecs.compat import ensure_contiguous_ndarray, ensure_ndarray from numcodecs.registry import codec_registry - -from zarr.compat import PY2, text_type, binary_type - - # codecs to use for object dtype convenience API object_codecs = { - text_type.__name__: 'vlen-utf8', - binary_type.__name__: 'vlen-bytes', + str.__name__: 'vlen-utf8', + bytes.__name__: 'vlen-bytes', 'array': 'vlen-array', } -def ensure_text_type(s): - if not isinstance(s, text_type): +def ensure_str(s): + if not isinstance(s, str): s = ensure_contiguous_ndarray(s) s = codecs.decode(s, 'ascii') return s @@ -41,7 +35,7 @@ def json_dumps(o): def json_loads(s): """Read JSON in a consistent way.""" - return json.loads(ensure_text_type(s)) + return json.loads(ensure_str(s)) def normalize_shape(shape): @@ -265,17 +259,13 @@ def normalize_fill_value(fill_value, dtype): # special case unicode because of encoding issues on Windows if passed through numpy # https://github.com/alimanfoo/zarr/pull/172#issuecomment-343782713 - if PY2 and isinstance(fill_value, binary_type): # pragma: py3 no cover - # this is OK on PY2, can be written as JSON - pass - - elif not isinstance(fill_value, text_type): + if not isinstance(fill_value, str): raise ValueError('fill_value {!r} is not valid for dtype {}; must be a ' 'unicode string'.format(fill_value, dtype)) else: try: - if isinstance(fill_value, binary_type) and dtype.kind == 'V': + if isinstance(fill_value, bytes) and dtype.kind == 'V': # special case for numpy 1.14 compatibility fill_value = np.array(fill_value, dtype=dtype.str).view(dtype)[()] else: @@ -292,7 +282,7 @@ def normalize_fill_value(fill_value, dtype): def normalize_storage_path(path): # handle bytes - if not PY2 and isinstance(path, bytes): # pragma: py2 no cover + if isinstance(path, bytes): path = str(path, 'ascii') # ensure str @@ -533,8 +523,7 @@ def __bytes__(self): # Unicode characters slip in on Python 3. # So we need to straighten that out first. - if not PY2: - result = result.encode() + result = result.encode() return result @@ -547,10 +536,7 @@ def __unicode__(self): return drawer(root) def __repr__(self): - if PY2: # pragma: py3 no cover - return self.__bytes__() - else: # pragma: py2 no cover - return self.__unicode__() + return self.__unicode__() def _repr_html_(self): return tree_html(self.group, expand=self.expand, level=self.level) @@ -566,28 +552,8 @@ def check_array_shape(param, array, shape): def is_valid_python_name(name): - if PY2: # pragma: py3 no cover - import ast - # noinspection PyBroadException - try: - ast.parse('"".{};'.format(name)) - except Exception: - return False - else: - return True - else: # pragma: py2 no cover - from keyword import iskeyword - return name.isidentifier() and not iskeyword(name) - - -def instance_dir(obj): # pragma: py3 no cover - """Vanilla implementation of built-in dir() for PY2 to help with overriding __dir__. - Based on implementation of dir() in pypy.""" - d = dict() - d.update(obj.__dict__) - d.update(class_dir(obj.__class__)) - result = sorted(d.keys()) - return result + from keyword import iskeyword + return name.isidentifier() and not iskeyword(name) def class_dir(klass): # pragma: py3 no cover