diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index bedf66383e..9673ff4360 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -36,7 +36,7 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('requirements{0}.txt', matrix.spark-version-suffix))) }} - name: Install dependencies run: | - python -m pip install --upgrade pip==21.2.4 setuptools wheel + python -m pip install --upgrade pip==22.0.3 setuptools wheel make setup${{ matrix.spark-version-suffix }} pip freeze - name: Test with coverage @@ -99,7 +99,7 @@ jobs: key: ${{ format('{0}-pip-{1}', runner.os, hashFiles('dev-requirements.txt', format('plugins/{0}/requirements.txt', matrix.plugin-names ))) }} - name: Install dependencies run: | - python -m pip install --upgrade pip==21.2.4 setuptools wheel + python -m pip install --upgrade pip==22.0.3 setuptools wheel make setup cd plugins/${{ matrix.plugin-names }} pip install -e . diff --git a/dev-requirements.txt b/dev-requirements.txt index 7c88c15106..15c78e1002 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,13 +2,13 @@ # This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile dev-requirements.in +# make dev-requirements.txt # -e file:.#egg=flytekit # via # -c requirements.txt # pytest-flyte -arrow==1.2.1 +arrow==1.2.2 # via # -c requirements.txt # jinja2-time @@ -18,17 +18,13 @@ attrs==20.3.0 # jsonschema # pytest # pytest-docker -backports.entry-points-selectable==1.1.1 - # via virtualenv bcrypt==3.2.0 - # via - # -c requirements.txt - # paramiko + # via paramiko binaryornot==0.4.4 # via # -c requirements.txt # cookiecutter -cachetools==4.2.4 +cachetools==5.0.0 # via google-auth certifi==2021.10.8 # via @@ -46,7 +42,7 @@ chardet==4.0.0 # via # -c requirements.txt # binaryornot -charset-normalizer==2.0.9 +charset-normalizer==2.0.11 # via # -c requirements.txt # requests @@ -54,7 +50,7 @@ checksumdir==1.2.0 # via # -c requirements.txt # flytekit -click==7.1.2 +click==8.0.3 # via # -c requirements.txt # cookiecutter @@ -69,9 +65,9 @@ cookiecutter==1.7.3 # via # -c requirements.txt # flytekit -coverage[toml]==6.2 +coverage[toml]==6.3.1 # via -r dev-requirements.in -croniter==1.1.0 +croniter==1.2.0 # via # -c requirements.txt # flytekit @@ -79,11 +75,12 @@ cryptography==36.0.1 # via # -c requirements.txt # paramiko + # secretstorage dataclasses-json==0.5.6 # via # -c requirements.txt # flytekit -decorator==5.1.0 +decorator==5.1.1 # via # -c requirements.txt # retry @@ -91,7 +88,7 @@ deprecated==1.2.13 # via # -c requirements.txt # flytekit -diskcache==5.3.0 +diskcache==5.4.0 # via # -c requirements.txt # flytekit @@ -117,18 +114,18 @@ docstring-parser==0.13 # via # -c requirements.txt # flytekit -filelock==3.4.0 +filelock==3.4.2 # via virtualenv -flyteidl==0.21.17 +flyteidl==0.21.25 # via # -c requirements.txt # flytekit -google-api-core[grpc]==2.4.0 +google-api-core[grpc]==2.5.0 # via # google-cloud-bigquery # google-cloud-bigquery-storage # google-cloud-core -google-auth==2.3.3 +google-auth==2.6.0 # via # google-api-core # google-cloud-core @@ -155,18 +152,23 @@ grpcio==1.43.0 # grpcio-status grpcio-status==1.43.0 # via google-api-core -identify==2.4.0 +identify==2.4.8 # via pre-commit idna==3.3 # via # -c requirements.txt # requests -importlib-metadata==4.10.0 +importlib-metadata==4.10.1 # via # -c requirements.txt # keyring iniconfig==1.1.1 # via pytest +jeepney==0.7.1 + # via + # -c requirements.txt + # keyring + # secretstorage jinja2==3.0.3 # via # -c requirements.txt @@ -183,11 +185,11 @@ jsonschema==3.2.0 # via # -c requirements.txt # docker-compose -keyring==23.4.0 +keyring==23.5.0 # via # -c requirements.txt # flytekit -libcst==0.4.0 +libcst==0.4.1 # via google-cloud-bigquery-storage markupsafe==2.0.1 # via @@ -209,14 +211,14 @@ marshmallow-jsonschema==0.13.0 # flytekit mock==4.0.3 # via -r dev-requirements.in -mypy==0.930 +mypy==0.931 # via -r dev-requirements.in mypy-extensions==0.4.3 # via # -c requirements.txt # mypy # typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via # -c requirements.txt # flytekit @@ -229,34 +231,29 @@ numpy==1.21.5 # pyarrow packaging==21.3 # via - # -c requirements.txt # google-cloud-bigquery # pytest pandas==1.3.5 # via # -c requirements.txt # flytekit -paramiko==2.8.1 - # via - # -c requirements.txt - # docker -platformdirs==2.4.0 - # via - # -c requirements.txt - # virtualenv +paramiko==2.9.2 + # via docker +platformdirs==2.4.1 + # via virtualenv pluggy==1.0.0 # via pytest poyo==0.5.0 # via # -c requirements.txt # cookiecutter -pre-commit==2.16.0 +pre-commit==2.17.0 # via -r dev-requirements.in -proto-plus==1.19.8 +proto-plus==1.19.9 # via # google-cloud-bigquery # google-cloud-bigquery-storage -protobuf==3.19.1 +protobuf==3.19.4 # via # -c requirements.txt # flyteidl @@ -285,15 +282,11 @@ pycparser==2.21 # via # -c requirements.txt # cffi -pynacl==1.4.0 - # via - # -c requirements.txt - # paramiko -pyparsing==3.0.6 - # via - # -c requirements.txt - # packaging -pyrsistent==0.18.0 +pynacl==1.5.0 + # via paramiko +pyparsing==3.0.7 + # via packaging +pyrsistent==0.18.1 # via # -c requirements.txt # jsonschema @@ -306,7 +299,7 @@ pytest-docker==0.10.3 # via pytest-flyte pytest-flyte @ git+https://github.com/flyteorg/pytest-flyte@main # via -r dev-requirements.in -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # -c requirements.txt # arrow @@ -339,11 +332,11 @@ pyyaml==5.4.1 # docker-compose # libcst # pre-commit -regex==2021.11.10 +regex==2022.1.18 # via # -c requirements.txt # docker-image-py -requests==2.26.0 +requests==2.27.1 # via # -c requirements.txt # cookiecutter @@ -353,7 +346,7 @@ requests==2.26.0 # google-api-core # google-cloud-bigquery # responses -responses==0.16.0 +responses==0.18.0 # via # -c requirements.txt # flytekit @@ -363,6 +356,10 @@ retry==0.9.2 # flytekit rsa==4.8 # via google-auth +secretstorage==3.3.1 + # via + # -c requirements.txt + # keyring six==1.16.0 # via # -c requirements.txt @@ -372,9 +369,7 @@ six==1.16.0 # google-auth # grpcio # jsonschema - # pynacl # python-dateutil - # responses # virtualenv # websocket-client sortedcontainers==2.4.0 @@ -395,14 +390,14 @@ toml==0.10.2 # via # pre-commit # pytest -tomli==1.2.3 +tomli==2.0.0 # via - # -c requirements.txt # coverage # mypy typing-extensions==4.0.1 # via # -c requirements.txt + # flytekit # libcst # mypy # typing-inspect @@ -411,13 +406,13 @@ typing-inspect==0.7.1 # -c requirements.txt # dataclasses-json # libcst -urllib3==1.26.7 +urllib3==1.26.8 # via # -c requirements.txt # flytekit # requests # responses -virtualenv==20.10.0 +virtualenv==20.13.0 # via pre-commit websocket-client==0.59.0 # via @@ -432,7 +427,7 @@ wrapt==1.13.3 # -c requirements.txt # deprecated # flytekit -zipp==3.6.0 +zipp==3.7.0 # via # -c requirements.txt # importlib-metadata diff --git a/doc-requirements.txt b/doc-requirements.txt index 9590c1face..5769373a43 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make doc-requirements.txt @@ -8,26 +8,12 @@ # via -r doc-requirements.in alabaster==0.7.12 # via sphinx -ansiwrap==0.8.4 - # via papermill -appnope==0.1.2 - # via - # ipykernel - # ipython -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time astroid==2.9.3 # via sphinx-autoapi -asttokens==2.0.5 - # via stack-data -attrs==21.4.0 - # via jsonschema babel==2.9.1 # via sphinx -backcall==0.2.0 - # via ipython -bcrypt==3.2.0 - # via paramiko beautifulsoup4==4.10.0 # via # furo @@ -35,58 +21,36 @@ beautifulsoup4==4.10.0 # sphinx-material binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via - # ipython - # papermill -bleach==4.1.0 - # via nbconvert -boto3==1.20.34 - # via sagemaker-training -botocore==1.23.34 - # via - # boto3 - # s3transfer certifi==2021.10.8 # via requests cffi==1.15.0 - # via - # bcrypt - # cryptography - # pynacl + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via - # black # cookiecutter # flytekit - # hmsclient - # papermill cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.1.0 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via # -r doc-requirements.in - # paramiko + # secretstorage css-html-js-minify==2.5.5 # via sphinx-material dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 - # via - # ipython - # retry -defusedxml==0.7.1 - # via nbconvert + # via retry deprecated==1.2.13 # via flytekit diskcache==5.4.0 @@ -99,75 +63,34 @@ docutils==0.17.1 # via # sphinx # sphinx-panels -entrypoints==0.3 - # via - # jupyter-client - # nbconvert - # papermill -executing==0.8.2 - # via stack-data -flyteidl==0.21.22 +flyteidl==0.21.25 # via flytekit furo @ git+https://github.com/flyteorg/furo@main # via -r doc-requirements.in -gevent==21.12.0 - # via sagemaker-training -greenlet==1.1.2 - # via gevent grpcio==1.43.0 # via # -r doc-requirements.in # flytekit -hmsclient==0.1.1 - # via flytekit idna==3.3 # via requests imagesize==1.3.0 # via sphinx -importlib-metadata==4.10.0 - # via keyring -importlib-resources==5.4.0 - # via jsonschema -inotify_simple==1.2.1 - # via sagemaker-training -ipykernel==5.5.6 - # via flytekit -ipython==8.0.0 - # via ipykernel -ipython-genutils==0.2.0 +importlib-metadata==4.10.1 # via - # ipykernel - # nbformat -jedi==0.18.1 - # via ipython + # keyring + # sphinx +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time - # nbconvert # sphinx # sphinx-autoapi jinja2-time==0.2.0 # via cookiecutter -jmespath==0.10.0 - # via - # boto3 - # botocore -jsonschema==4.4.0 - # via nbformat -jupyter-client==7.1.0 - # via - # ipykernel - # nbclient -jupyter-core==4.9.1 - # via - # jupyter-client - # nbconvert - # nbformat -jupyterlab-pygments==0.1.2 - # via nbconvert -k8s-proto==0.0.3 - # via flytekit keyring==23.5.0 # via flytekit lazy-object-proxy==1.7.1 @@ -185,106 +108,41 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib-inline==0.1.3 - # via ipython -mistune==0.8.4 - # via nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 - # via flytekit -nbclient==0.5.9 - # via - # nbconvert - # papermill -nbconvert==6.4.0 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbformat==5.1.3 +numpy==1.22.2 # via - # nbclient - # nbconvert - # papermill -nest-asyncio==1.5.4 - # via - # jupyter-client - # nbclient -numpy==1.22.0 - # via - # flytekit # pandas # pyarrow - # sagemaker-training - # scipy packaging==21.3 - # via - # bleach - # sphinx -pandas==1.3.5 - # via flytekit -pandocfilters==1.5.0 - # via nbconvert -papermill==2.3.3 + # via sphinx +pandas==1.4.0 # via flytekit -paramiko==2.9.2 - # via sagemaker-training -parso==0.8.3 - # via jedi -pathspec==0.9.0 - # via black -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -platformdirs==2.4.1 - # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.24 - # via ipython -protobuf==3.19.3 +protobuf==3.19.4 # via # flyteidl # flytekit - # k8s-proto - # sagemaker-training -psutil==5.9.0 - # via sagemaker-training -ptyprocess==0.7.0 - # via pexpect -pure-eval==0.2.1 - # via stack-data py==1.11.0 # via retry -py4j==0.10.9.2 - # via pyspark pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi pygments==2.11.2 # via - # ipython - # jupyterlab-pygments - # nbconvert # sphinx # sphinx-prompt -pynacl==1.5.0 - # via paramiko -pyparsing==3.0.6 +pyparsing==3.0.7 # via packaging -pyrsistent==0.18.0 - # via jsonschema -pyspark==3.2.0 - # via flytekit -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow - # botocore # croniter # flytekit - # jupyter-client # pandas python-json-logger==2.0.2 # via flytekit @@ -300,53 +158,34 @@ pytz==2021.3 # flytekit # pandas pyyaml==6.0 - # via - # papermill - # sphinx-autoapi -pyzmq==22.3.0 - # via jupyter-client -regex==2021.11.10 + # via sphinx-autoapi +regex==2022.1.18 # via docker-image-py requests==2.27.1 # via # cookiecutter # flytekit - # papermill # responses # sphinx -responses==0.17.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -retrying==1.3.3 - # via sagemaker-training -s3transfer==0.5.0 - # via boto3 -sagemaker-training==3.9.2 - # via flytekit -scipy==1.7.3 - # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via - # asttokens - # bcrypt - # bleach # cookiecutter - # flytekit # grpcio # python-dateutil - # responses - # retrying - # sagemaker-training # sphinx-code-include - # thrift snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via flytekit soupsieve==2.3.1 # via beautifulsoup4 -sphinx==4.3.2 +sphinx==4.4.0 # via # -r doc-requirements.in # furo @@ -389,42 +228,14 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-yt==0.2.2 # via -r doc-requirements.in -stack-data==0.1.3 - # via ipython statsd==3.3.0 # via flytekit -tenacity==8.0.1 - # via papermill -testpath==0.5.0 - # via nbconvert text-unidecode==1.3 # via python-slugify -textwrap3==0.9.2 - # via ansiwrap -thrift==0.15.0 - # via hmsclient -tomli==1.2.3 - # via black -tornado==6.1 - # via - # ipykernel - # jupyter-client -tqdm==4.62.3 - # via papermill -traitlets==5.1.1 - # via - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat typing-extensions==4.0.1 # via # astroid - # black + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json @@ -434,16 +245,9 @@ unidecode==1.3.2 # sphinx-autoapi urllib3==1.26.8 # via - # botocore # flytekit # requests # responses -wcwidth==0.2.5 - # via prompt-toolkit -webencodings==0.5.1 - # via bleach -werkzeug==2.0.2 - # via sagemaker-training wheel==0.37.1 # via flytekit wrapt==1.13.3 @@ -452,14 +256,7 @@ wrapt==1.13.3 # deprecated # flytekit zipp==3.7.0 - # via - # importlib-metadata - # importlib-resources -zope.event==4.5.0 - # via gevent -zope.interface==5.4.0 - # via gevent + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# pip # setuptools diff --git a/requirements-spark2.txt b/requirements-spark2.txt index bdef3764f0..62f27ce465 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -8,107 +8,56 @@ # via # -r requirements-spark2.in # -r requirements.in -ansiwrap==0.8.4 - # via papermill -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time attrs==20.3.0 # via # -r requirements.in # jsonschema -backcall==0.2.0 - # via ipython -bcrypt==3.2.0 - # via paramiko binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via papermill -bleach==4.1.0 - # via nbconvert -boto3==1.20.26 - # via sagemaker-training -botocore==1.23.26 - # via - # boto3 - # s3transfer certifi==2021.10.8 # via requests cffi==1.15.0 - # via - # bcrypt - # cryptography - # pynacl + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.9 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via - # black # cookiecutter # flytekit - # hmsclient - # papermill cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.1.0 +croniter==1.2.0 # via flytekit cryptography==36.0.1 - # via - # paramiko - # secretstorage + # via secretstorage dataclasses-json==0.5.6 # via flytekit -decorator==5.1.0 - # via - # ipython - # retry -defusedxml==0.7.1 - # via nbconvert +decorator==5.1.1 + # via retry deprecated==1.2.13 # via flytekit -diskcache==5.3.0 +diskcache==5.4.0 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 - # via - # jupyter-client - # nbconvert - # papermill -flyteidl==0.21.17 +flyteidl==0.21.25 # via flytekit -gevent==21.12.0 - # via sagemaker-training -greenlet==1.1.2 - # via gevent grpcio==1.43.0 # via flytekit -hmsclient==0.1.1 - # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.0 +importlib-metadata==4.10.1 # via keyring -inotify_simple==1.2.1 - # via sagemaker-training -ipykernel==5.5.6 - # via flytekit -ipython==7.31.1 - # via ipykernel -ipython-genutils==0.2.0 - # via - # ipykernel - # nbformat -jedi==0.18.1 - # via ipython jeepney==0.7.1 # via # keyring @@ -117,31 +66,11 @@ jinja2==3.0.3 # via # cookiecutter # jinja2-time - # nbconvert jinja2-time==0.2.0 # via cookiecutter -jmespath==0.10.0 - # via - # boto3 - # botocore jsonschema==3.2.0 - # via - # -r requirements.in - # nbformat -jupyter-client==7.1.0 - # via - # ipykernel - # nbclient -jupyter-core==4.9.1 - # via - # jupyter-client - # nbconvert - # nbformat -jupyterlab-pygments==0.1.2 - # via nbconvert -k8s-proto==0.0.3 - # via flytekit -keyring==23.4.0 + # via -r requirements.in +keyring==23.5.0 # via flytekit markupsafe==2.0.1 # via jinja2 @@ -154,100 +83,38 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib-inline==0.1.3 - # via ipython -mistune==0.8.4 - # via nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 - # via flytekit -nbclient==0.5.9 - # via - # nbconvert - # papermill -nbconvert==6.3.0 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbformat==5.1.3 - # via - # nbclient - # nbconvert - # papermill -nest-asyncio==1.5.4 - # via - # jupyter-client - # nbclient numpy==1.21.5 # via - # flytekit + # -r requirements.in # pandas # pyarrow - # sagemaker-training - # scipy -packaging==21.3 - # via bleach pandas==1.3.5 - # via flytekit -pandocfilters==1.5.0 - # via nbconvert -papermill==2.3.3 - # via flytekit -paramiko==2.8.1 - # via sagemaker-training -parso==0.8.3 - # via jedi -pathspec==0.9.0 - # via black -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -platformdirs==2.4.0 - # via black + # via + # -r requirements.in + # flytekit poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.24 - # via ipython -protobuf==3.19.1 +protobuf==3.19.4 # via # flyteidl # flytekit - # k8s-proto - # sagemaker-training -psutil==5.8.0 - # via sagemaker-training -ptyprocess==0.7.0 - # via pexpect py==1.11.0 # via retry -py4j==0.10.9.2 - # via pyspark pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi -pygments==2.10.0 - # via - # ipython - # jupyterlab-pygments - # nbconvert -pynacl==1.4.0 - # via paramiko -pyparsing==3.0.6 - # via packaging -pyrsistent==0.18.0 +pyrsistent==0.18.1 # via jsonschema -pyspark==3.2.0 - # via flytekit -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow - # botocore # croniter # flytekit - # jupyter-client # pandas python-json-logger==2.0.2 # via flytekit @@ -260,110 +127,51 @@ pytz==2021.3 # flytekit # pandas pyyaml==5.4.1 - # via - # -r requirements.in - # papermill -pyzmq==22.3.0 - # via jupyter-client -regex==2021.11.10 + # via -r requirements.in +regex==2022.1.18 # via docker-image-py -requests==2.26.0 +requests==2.27.1 # via # cookiecutter # flytekit - # papermill # responses -responses==0.16.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -retrying==1.3.3 - # via sagemaker-training -s3transfer==0.5.0 - # via boto3 -sagemaker-training==3.9.2 - # via flytekit -scipy==1.7.3 - # via sagemaker-training secretstorage==3.3.1 # via keyring six==1.16.0 # via - # bcrypt - # bleach # cookiecutter - # flytekit # grpcio # jsonschema - # pynacl # python-dateutil - # responses - # retrying - # sagemaker-training - # thrift sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit -tenacity==8.0.1 - # via papermill -testpath==0.5.0 - # via nbconvert text-unidecode==1.3 # via python-slugify -textwrap3==0.9.2 - # via ansiwrap -thrift==0.15.0 - # via hmsclient -tomli==1.2.3 - # via black -tornado==6.1 - # via - # ipykernel - # jupyter-client -tqdm==4.62.3 - # via papermill -traitlets==5.1.1 - # via - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat typing-extensions==4.0.1 # via - # black + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.7 +urllib3==1.26.8 # via - # botocore # flytekit # requests # responses -wcwidth==0.2.5 - # via prompt-toolkit -webencodings==0.5.1 - # via bleach -werkzeug==2.0.2 - # via sagemaker-training wheel==0.37.1 # via flytekit wrapt==1.13.3 # via # deprecated # flytekit -zipp==3.6.0 +zipp==3.7.0 # via importlib-metadata -zope.event==4.5.0 - # via gevent -zope.interface==5.4.0 - # via gevent # The following packages are considered to be unsafe in a requirements file: -# pip # setuptools diff --git a/requirements.in b/requirements.in index 408185fca8..0cf01a6f13 100644 --- a/requirements.in +++ b/requirements.in @@ -7,3 +7,7 @@ attrs<21 # https://github.com/flyteorg/flyte/issues/1732. jsonschema<4 pyyaml<6 +# A number of dependencies, including pandas and numpy, started dropping support for python 3.7 in their latest +# releases (pandas>=1.4.0 and numpy>=1.22.0). More details in https://github.com/flyteorg/flyte/issues/2115. +pandas<1.4.0 +numpy<1.22.0 diff --git a/requirements.txt b/requirements.txt index 38f885cec8..335e5c2acd 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,107 +6,56 @@ # -e file:.#egg=flytekit # via -r requirements.in -ansiwrap==0.8.4 - # via papermill -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time attrs==20.3.0 # via # -r requirements.in # jsonschema -backcall==0.2.0 - # via ipython -bcrypt==3.2.0 - # via paramiko binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via papermill -bleach==4.1.0 - # via nbconvert -boto3==1.20.26 - # via sagemaker-training -botocore==1.23.26 - # via - # boto3 - # s3transfer certifi==2021.10.8 # via requests cffi==1.15.0 - # via - # bcrypt - # cryptography - # pynacl + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.9 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via - # black # cookiecutter # flytekit - # hmsclient - # papermill cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.1.0 +croniter==1.2.0 # via flytekit cryptography==36.0.1 - # via - # paramiko - # secretstorage + # via secretstorage dataclasses-json==0.5.6 # via flytekit -decorator==5.1.0 - # via - # ipython - # retry -defusedxml==0.7.1 - # via nbconvert +decorator==5.1.1 + # via retry deprecated==1.2.13 # via flytekit -diskcache==5.3.0 +diskcache==5.4.0 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 - # via - # jupyter-client - # nbconvert - # papermill -flyteidl==0.21.17 +flyteidl==0.21.25 # via flytekit -gevent==21.12.0 - # via sagemaker-training -greenlet==1.1.2 - # via gevent grpcio==1.43.0 # via flytekit -hmsclient==0.1.1 - # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.0 +importlib-metadata==4.10.1 # via keyring -inotify_simple==1.2.1 - # via sagemaker-training -ipykernel==5.5.6 - # via flytekit -ipython==7.31.1 - # via ipykernel -ipython-genutils==0.2.0 - # via - # ipykernel - # nbformat -jedi==0.18.1 - # via ipython jeepney==0.7.1 # via # keyring @@ -115,31 +64,11 @@ jinja2==3.0.3 # via # cookiecutter # jinja2-time - # nbconvert jinja2-time==0.2.0 # via cookiecutter -jmespath==0.10.0 - # via - # boto3 - # botocore jsonschema==3.2.0 - # via - # -r requirements.in - # nbformat -jupyter-client==7.1.0 - # via - # ipykernel - # nbclient -jupyter-core==4.9.1 - # via - # jupyter-client - # nbconvert - # nbformat -jupyterlab-pygments==0.1.2 - # via nbconvert -k8s-proto==0.0.3 - # via flytekit -keyring==23.4.0 + # via -r requirements.in +keyring==23.5.0 # via flytekit markupsafe==2.0.1 # via jinja2 @@ -152,100 +81,38 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib-inline==0.1.3 - # via ipython -mistune==0.8.4 - # via nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 - # via flytekit -nbclient==0.5.9 - # via - # nbconvert - # papermill -nbconvert==6.3.0 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbformat==5.1.3 - # via - # nbclient - # nbconvert - # papermill -nest-asyncio==1.5.4 - # via - # jupyter-client - # nbclient numpy==1.21.5 # via - # flytekit + # -r requirements.in # pandas # pyarrow - # sagemaker-training - # scipy -packaging==21.3 - # via bleach pandas==1.3.5 - # via flytekit -pandocfilters==1.5.0 - # via nbconvert -papermill==2.3.3 - # via flytekit -paramiko==2.8.1 - # via sagemaker-training -parso==0.8.3 - # via jedi -pathspec==0.9.0 - # via black -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -platformdirs==2.4.0 - # via black + # via + # -r requirements.in + # flytekit poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.24 - # via ipython -protobuf==3.19.1 +protobuf==3.19.4 # via # flyteidl # flytekit - # k8s-proto - # sagemaker-training -psutil==5.8.0 - # via sagemaker-training -ptyprocess==0.7.0 - # via pexpect py==1.11.0 # via retry -py4j==0.10.9.2 - # via pyspark pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi -pygments==2.10.0 - # via - # ipython - # jupyterlab-pygments - # nbconvert -pynacl==1.4.0 - # via paramiko -pyparsing==3.0.6 - # via packaging -pyrsistent==0.18.0 +pyrsistent==0.18.1 # via jsonschema -pyspark==3.2.0 - # via flytekit -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow - # botocore # croniter # flytekit - # jupyter-client # pandas python-json-logger==2.0.2 # via flytekit @@ -258,110 +125,51 @@ pytz==2021.3 # flytekit # pandas pyyaml==5.4.1 - # via - # -r requirements.in - # papermill -pyzmq==22.3.0 - # via jupyter-client -regex==2021.11.10 + # via -r requirements.in +regex==2022.1.18 # via docker-image-py -requests==2.26.0 +requests==2.27.1 # via # cookiecutter # flytekit - # papermill # responses -responses==0.16.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -retrying==1.3.3 - # via sagemaker-training -s3transfer==0.5.0 - # via boto3 -sagemaker-training==3.9.2 - # via flytekit -scipy==1.7.3 - # via sagemaker-training secretstorage==3.3.1 # via keyring six==1.16.0 # via - # bcrypt - # bleach # cookiecutter - # flytekit # grpcio # jsonschema - # pynacl # python-dateutil - # responses - # retrying - # sagemaker-training - # thrift sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit -tenacity==8.0.1 - # via papermill -testpath==0.5.0 - # via nbconvert text-unidecode==1.3 # via python-slugify -textwrap3==0.9.2 - # via ansiwrap -thrift==0.15.0 - # via hmsclient -tomli==1.2.3 - # via black -tornado==6.1 - # via - # ipykernel - # jupyter-client -tqdm==4.62.3 - # via papermill -traitlets==5.1.1 - # via - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat typing-extensions==4.0.1 # via - # black + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.7 +urllib3==1.26.8 # via - # botocore # flytekit # requests # responses -wcwidth==0.2.5 - # via prompt-toolkit -webencodings==0.5.1 - # via bleach -werkzeug==2.0.2 - # via sagemaker-training wheel==0.37.1 # via flytekit wrapt==1.13.3 # via # deprecated # flytekit -zipp==3.6.0 +zipp==3.7.0 # via importlib-metadata -zope.event==4.5.0 - # via gevent -zope.interface==5.4.0 - # via gevent # The following packages are considered to be unsafe in a requirements file: -# pip # setuptools diff --git a/setup.py b/setup.py index 740a1d81e5..db8d71fe6d 100644 --- a/setup.py +++ b/setup.py @@ -39,7 +39,7 @@ "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=4.0.0,<7.0.0", - "click>=6.6,<8.0", + "click>=6.6,<9.0", "croniter>=0.3.20,<4.0.0", "deprecated>=1.0,<2.0", "python-dateutil>=2.1", @@ -67,6 +67,10 @@ "checksumdir>=1.2.0", "cloudpickle>=2.0.0", "cookiecutter>=1.7.3", + # numpy and pandas dropped support for python 3.7. + # More details in https://github.com/flyteorg/flyte/issues/2115. + "numpy<1.22.0; python_version < '3.7.0'", + "pandas<1.4.0; python_version < '3.7.0'", ], extras_require=extras_require, scripts=[ diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 8fcc781de3..4c505ef870 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -4,7 +4,7 @@ # # make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt # -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter @@ -14,7 +14,7 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.9 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit @@ -26,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.1.0 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -34,27 +34,27 @@ cycler==0.11.0 # via matplotlib dataclasses-json==0.5.6 # via flytekit -decorator==5.1.0 +decorator==5.1.1 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.3.0 +diskcache==5.4.0 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.21.17 +flyteidl==0.21.25 # via flytekit -flytekit==0.25.0 +flytekit==0.30.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -fonttools==4.28.5 +fonttools==4.29.1 # via matplotlib grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.0 +importlib-metadata==4.10.1 # via keyring jeepney==0.7.1 # via @@ -68,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter joblib==1.1.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -keyring==23.4.0 +keyring==23.5.0 # via flytekit kiwisolver==1.3.2 # via matplotlib @@ -87,25 +87,25 @@ matplotlib==3.5.1 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via flytekit -numpy==1.21.5 +numpy==1.22.2 # via # matplotlib # opencv-python # pandas # pyarrow -opencv-python==4.5.4.60 +opencv-python==4.5.5.62 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in packaging==21.3 # via matplotlib -pandas==1.3.5 +pandas==1.4.0 # via flytekit -pillow==8.4.0 +pillow==9.0.1 # via matplotlib poyo==0.5.0 # via cookiecutter -protobuf==3.19.1 +protobuf==3.19.4 # via # flyteidl # flytekit @@ -115,11 +115,11 @@ pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi -pyparsing==3.0.6 +pyparsing==3.0.7 # via # matplotlib # packaging -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow # croniter @@ -132,18 +132,18 @@ python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2018.4 +pytz==2021.3 # via # flytekit # pandas -regex==2021.11.10 +regex==2022.1.18 # via docker-image-py -requests==2.26.0 +requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.16.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit @@ -152,10 +152,8 @@ secretstorage==3.3.1 six==1.16.0 # via # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 @@ -163,10 +161,12 @@ statsd==3.3.0 text-unidecode==1.3 # via python-slugify typing-extensions==4.0.1 - # via typing-inspect + # via + # flytekit + # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.7 +urllib3==1.26.8 # via # flytekit # requests @@ -179,5 +179,5 @@ wrapt==1.13.3 # via # deprecated # flytekit -zipp==3.6.0 +zipp==3.7.0 # via importlib-metadata