diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 822fa71eae..1e91d21bd1 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -22,3 +22,9 @@ repos: rev: 0.0.9 hooks: - id: check_pdb_hook + - repo: https://github.com/codespell-project/codespell + rev: v2.2.6 + hooks: + - id: codespell + additional_dependencies: + - tomli diff --git a/Dockerfile.dev b/Dockerfile.dev index 97277b6837..8db65d8b9c 100644 --- a/Dockerfile.dev +++ b/Dockerfile.dev @@ -27,7 +27,9 @@ COPY . /flytekit # 3. Clean up the apt cache to reduce image size. Reference: https://gist.github.com/marvell/7c812736565928e602c4 # 4. Create a non-root user 'flytekit' and set appropriate permissions for directories. RUN apt-get update && apt-get install build-essential vim libmagic1 git -y \ - && pip install --no-cache-dir -e /flytekit \ + && pip install --no-cache-dir -U --pre \ + flyteidl \ + -e /flytekit \ -e /flytekit/plugins/flytekit-k8s-pod \ -e /flytekit/plugins/flytekit-deck-standard \ -e /flytekit/plugins/flytekit-flyin \ diff --git a/Makefile b/Makefile index 5d672c21f5..95e6d538ca 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,7 @@ update_boilerplate: .PHONY: setup setup: install-piptools ## Install requirements + pip install flyteidl --pre pip install -r dev-requirements.in .PHONY: fmt @@ -43,7 +44,8 @@ lint: ## Run linters .PHONY: spellcheck spellcheck: ## Runs a spellchecker over all code and documentation - codespell -L "te,raison,fo" --skip="./docs/build,./.git" + # Configuration is in pyproject.toml + codespell .PHONY: test test: lint unit_test diff --git a/dev-requirements.txt b/dev-requirements.txt index a354b70ea4..0634217be7 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -10,7 +10,7 @@ absl-py==2.0.0 # via # tensorboard # tensorflow -adlfs==2023.10.0 +adlfs==2023.9.0 # via flytekit aiobotocore==2.5.4 # via s3fs @@ -24,8 +24,6 @@ aioitertools==0.11.0 # via aiobotocore aiosignal==1.3.1 # via aiohttp -appnope==0.1.3 - # via ipython arrow==1.3.0 # via cookiecutter asttokens==2.4.1 @@ -97,6 +95,7 @@ cryptography==41.0.5 # azure-storage-blob # msal # pyjwt + # secretstorage dataclasses-json==0.5.9 # via flytekit decorator==5.1.1 @@ -224,7 +223,11 @@ jaraco-classes==3.3.0 # keyrings-alt jedi==0.19.1 # via ipython -jinja2==3.1.2 +jeepney==0.8.0 + # via + # keyring + # secretstorage +jinja2==3.1.3 # via cookiecutter jmespath==1.0.1 # via botocore @@ -297,7 +300,7 @@ nodeenv==1.8.0 # via pre-commit numpy==1.23.5 # via - # flytekit + # -r dev-requirements.in # h5py # keras-preprocessing # opt-einsum @@ -321,7 +324,7 @@ packaging==23.2 # pytest # setuptools-scm pandas==1.5.3 - # via flytekit + # via -r dev-requirements.in parso==0.8.3 # via jedi pexpect==4.8.0 @@ -411,6 +414,8 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.7 # via flytekit +python-magic==0.4.27 ; platform_system == "Darwin" or platform_system == "Linux" + # via -r dev-requirements.in python-slugify==8.0.1 # via cookiecutter pytimeparse==1.1.8 @@ -459,6 +464,8 @@ scikit-learn==1.3.2 # via -r dev-requirements.in scipy==1.11.3 # via scikit-learn +secretstorage==3.3.3 + # via keyring setuptools-scm==8.0.4 # via -r dev-requirements.in six==1.16.0 @@ -473,8 +480,7 @@ six==1.16.0 # python-dateutil # tensorflow sortedcontainers==2.4.0 - # via - # hypothesis + # via hypothesis stack-data==0.6.3 # via ipython statsd==3.3.0 @@ -485,7 +491,7 @@ tensorboard-data-server==0.6.1 # via tensorboard tensorboard-plugin-wit==1.8.1 # via tensorboard -tensorflow==2.8.1 ; python_version < "3.11" and (platform_machine != "arm64" or platform_system != "Darwin") +tensorflow==2.8.1 ; python_version < "3.12" # via -r dev-requirements.in tensorflow-estimator==2.8.0 # via tensorflow @@ -503,8 +509,8 @@ tomli==2.0.1 # coverage # mypy # pytest - # . setuptools-scm -torch==1.12.1 ; python_version >= "3.11" or platform_system != "Windows" + # setuptools-scm +torch==1.12.1 ; python_version < "3.12" # via -r dev-requirements.in traitlets==5.13.0 # via @@ -530,7 +536,7 @@ typing-extensions==4.8.0 # mashumaro # mypy # rich-click - # . setuptools-scm + # setuptools-scm # tensorflow # torch # typing-inspect diff --git a/doc-requirements.in b/doc-requirements.in index 4a30b8afef..3b602e7e52 100644 --- a/doc-requirements.in +++ b/doc-requirements.in @@ -11,6 +11,7 @@ sphinx-autoapi sphinx-copybutton sphinx_fontawesome sphinx-panels +sphinx-reredirects sphinxcontrib-youtube==1.2.0 cryptography google-api-core[grpc] @@ -19,37 +20,8 @@ sphinx-tags sphinx-click retry mashumaro +pydantic<2.0.0 # Packages for Plugin docs # Package name Plugin needing it -botocore # fsspec -fsspec # fsspec -google-cloud # bigquery -google-cloud-bigquery # bigquery -markdown # deck -plotly # deck -pandas_profiling # deck -dolt_integrations # dolt -great-expectations # greatexpectations -datasets # huggingface -kubernetes # k8s-pod -modin # modin -pandera # pandera -papermill # papermill -jupyter # papermill -polars # polars -pyspark # spark -sqlalchemy # sqlalchemy -torch # pytorch -skl2onnx # onnxscikitlearn -tf2onnx # onnxtensorflow -tensorflow # onnxtensorflow -whylogs==1.3.3 # whylogs -whylabs-client # whylogs -ray==2.6.3 # ray -scikit-learn # scikit-learn -dask[distributed] # dask vaex # vaex -mlflow==2.7.0 # mlflow -duckdb # duckdb -snowflake-connector-python # snowflake diff --git a/doc-requirements.txt b/doc-requirements.txt index ef8f3cfdcd..52cc2a5d29 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -6,228 +6,139 @@ # -e file:.#egg=flytekit # via -r doc-requirements.in -absl-py==1.4.0 - # via - # tensorboard - # tensorflow -adlfs==2023.8.0 +adlfs==2023.9.0 # via flytekit aiobotocore==2.5.4 # via s3fs -aiohttp==3.8.5 +aiohttp==3.9.1 # via # adlfs # aiobotocore - # datasets - # fsspec # gcsfs # s3fs aioitertools==0.11.0 # via aiobotocore aiosignal==1.3.1 - # via - # aiohttp - # ray -alabaster==0.7.13 + # via aiohttp +alabaster==0.7.16 # via sphinx -alembic==1.12.0 - # via mlflow -altair==4.2.2 - # via great-expectations -ansiwrap==0.8.4 - # via papermill -anyio==3.7.1 +anyio==4.2.0 # via - # fastapi - # jupyter-server + # azure-core # starlette # watchfiles aplus==0.11.0 # via vaex-core -appnope==0.1.3 - # via - # ipykernel - # ipython -argon2-cffi==23.1.0 - # via jupyter-server -argon2-cffi-bindings==21.2.0 - # via argon2-cffi -arrow==1.2.3 - # via - # cookiecutter - # isoduration -asn1crypto==1.5.1 - # via - # oscrypto - # snowflake-connector-python -astroid==2.15.6 +arrow==1.3.0 + # via cookiecutter +astroid==3.0.2 # via sphinx-autoapi -astropy==5.3.3 +astropy==6.0.0 # via vaex-astro -asttokens==2.4.0 +astropy-iers-data==0.2024.1.8.0.30.55 + # via astropy +asttokens==2.4.1 # via stack-data -astunparse==1.6.3 - # via tensorflow -async-lru==2.0.4 - # via jupyterlab async-timeout==4.0.3 # via aiohttp -attrs==23.1.0 - # via - # aiohttp - # jsonschema - # referencing - # visions -azure-core==1.29.1 +attrs==23.2.0 + # via aiohttp +azure-core==1.29.6 # via # adlfs # azure-identity # azure-storage-blob azure-datalake-store==0.0.53 # via adlfs -azure-identity==1.14.0 +azure-identity==1.15.0 # via adlfs -azure-storage-blob==12.18.1 +azure-storage-blob==12.19.0 # via adlfs -babel==2.12.1 - # via - # jupyterlab-server - # sphinx -backcall==0.2.0 - # via ipython +babel==2.14.0 + # via sphinx beautifulsoup4==4.12.2 # via # furo - # nbconvert # sphinx-code-include # sphinx-material binaryornot==0.4.4 # via cookiecutter -blake3==0.3.3 +blake3==0.3.4 # via vaex-core -bleach==6.0.0 - # via nbconvert -blinker==1.6.2 - # via flask botocore==1.31.17 - # via - # -r doc-requirements.in - # aiobotocore -bqplot==0.12.40 + # via aiobotocore +bqplot==0.12.42 # via # ipyvolume # vaex-jupyter -branca==0.6.0 +branca==0.7.0 # via ipyleaflet -cachetools==5.3.1 +cachetools==5.3.2 # via # google-auth # vaex-server -certifi==2023.7.22 +certifi==2023.11.17 # via # kubernetes # requests - # snowflake-connector-python -cffi==1.15.1 +cffi==1.16.0 # via - # argon2-cffi-bindings # azure-datalake-store # cryptography - # snowflake-connector-python cfgv==3.4.0 # via pre-commit chardet==5.2.0 # via binaryornot -charset-normalizer==3.2.0 - # via - # aiohttp - # requests - # snowflake-connector-python +charset-normalizer==3.3.2 + # via requests click==8.1.7 # via # cookiecutter # dask - # databricks-cli - # distributed - # flask # flytekit - # great-expectations - # mlflow - # papermill - # ray # rich-click # sphinx-click # uvicorn -cloudpickle==2.2.1 +cloudpickle==3.0.0 # via # dask - # distributed # flytekit - # mlflow # vaex-core -colorama==0.4.6 - # via great-expectations -comm==0.1.4 - # via - # ipykernel - # ipywidgets -contourpy==1.1.1 +comm==0.2.1 + # via ipywidgets +contourpy==1.2.0 # via matplotlib -cookiecutter==2.3.0 +cookiecutter==2.5.0 # via flytekit -croniter==1.4.1 +croniter==2.0.1 # via flytekit -cryptography==41.0.3 +cryptography==41.0.7 # via # -r doc-requirements.in # azure-identity # azure-storage-blob - # great-expectations # msal # pyjwt - # snowflake-connector-python + # secretstorage css-html-js-minify==2.5.5 # via sphinx-material -cycler==0.11.0 +cycler==0.12.1 # via matplotlib -dacite==1.8.1 - # via ydata-profiling -dask[distributed]==2023.9.2 - # via - # -r doc-requirements.in - # distributed - # vaex-core -databricks-cli==0.17.7 - # via mlflow +dask==2023.12.1 + # via vaex-core dataclasses-json==0.5.9 - # via - # dolt-integrations - # flytekit -datasets==2.14.5 - # via -r doc-requirements.in -debugpy==1.8.0 - # via ipykernel + # via flytekit decorator==5.1.1 # via # gcsfs # ipython # retry -defusedxml==0.7.1 - # via nbconvert -dill==0.3.7 - # via - # datasets - # multiprocess diskcache==5.6.3 # via flytekit -distlib==0.3.7 +distlib==0.3.8 # via virtualenv -distributed==2023.9.2 - # via dask docker==6.1.3 - # via - # flytekit - # mlflow + # via flytekit docstring-parser==0.15 # via flytekit docutils==0.17.1 @@ -235,83 +146,47 @@ docutils==0.17.1 # sphinx # sphinx-click # sphinx-panels -dolt-integrations==0.1.5 - # via -r doc-requirements.in -doltcli==0.1.18 - # via dolt-integrations -duckdb==0.8.1 - # via -r doc-requirements.in -entrypoints==0.4 - # via - # altair - # mlflow - # papermill -exceptiongroup==1.1.3 +exceptiongroup==1.2.0 # via # anyio # ipython -executing==1.2.0 +executing==2.0.1 # via stack-data -fastapi==0.103.1 +fastapi==0.108.0 # via vaex-server -fastjsonschema==2.18.0 - # via nbformat -filelock==3.12.4 - # via - # huggingface-hub - # ray - # snowflake-connector-python - # torch +filelock==3.13.1 + # via # vaex-core # virtualenv -flask==2.3.3 - # via mlflow -flatbuffers==23.5.26 - # via tensorflow flyteidl==1.10.6 # via flytekit -fonttools==4.42.1 +fonttools==4.47.0 # via matplotlib -fqdn==1.5.1 - # via jsonschema -frozendict==2.3.8 +frozendict==2.4.0 # via vaex-core -frozenlist==1.4.0 +frozenlist==1.4.1 # via # aiohttp # aiosignal - # ray -fsspec[http]==2023.6.0 +fsspec==2023.9.2 # via - # -r doc-requirements.in # adlfs # dask - # datasets # flytekit # gcsfs - # huggingface-hub - # modin # s3fs furo @ git+https://github.com/flyteorg/furo@main # via -r doc-requirements.in future==0.18.3 # via vaex-core -gast==0.4.0 - # via tensorflow -gcsfs==2023.6.0 +gcsfs==2023.9.2 # via flytekit -gitdb==4.0.10 - # via gitpython -gitpython==3.1.36 - # via - # mlflow -google-api-core[grpc]==2.11.1 +google-api-core[grpc]==2.15.0 # via # -r doc-requirements.in - # google-cloud-bigquery # google-cloud-core # google-cloud-storage -google-auth==2.23.0 +google-auth==2.26.1 # via # gcsfs # google-api-core @@ -319,119 +194,66 @@ google-auth==2.23.0 # google-cloud-core # google-cloud-storage # kubernetes - # tensorboard -google-auth-oauthlib==1.0.0 - # via - # gcsfs - # tensorboard -google-cloud==0.34.0 - # via -r doc-requirements.in -google-cloud-bigquery==3.11.4 - # via -r doc-requirements.in -google-cloud-core==2.3.3 - # via - # google-cloud-bigquery - # google-cloud-storage -google-cloud-storage==2.10.0 +google-auth-oauthlib==1.2.0 + # via gcsfs +google-cloud-core==2.4.1 + # via google-cloud-storage +google-cloud-storage==2.14.0 # via gcsfs google-crc32c==1.5.0 - # via google-resumable-media -google-pasta==0.2.0 - # via tensorflow -google-resumable-media==2.6.0 # via - # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos==1.60.0 + # google-resumable-media +google-resumable-media==2.7.0 + # via google-cloud-storage +googleapis-common-protos==1.62.0 # via # flyteidl # flytekit # google-api-core # grpcio-status -great-expectations==0.17.16 - # via -r doc-requirements.in -grpcio==1.53.0 +grpcio==1.60.0 # via # -r doc-requirements.in # flytekit # google-api-core - # google-cloud-bigquery # grpcio-status - # ray - # tensorboard - # tensorflow -grpcio-status==1.53.0 +grpcio-status==1.60.0 # via # flytekit # google-api-core -gunicorn==21.2.0 - # via mlflow h11==0.14.0 # via uvicorn -h5py==3.9.0 - # via - # tensorflow - # vaex-hdf5 -htmlmin==0.1.12 - # via ydata-profiling -httptools==0.6.0 +h5py==3.10.0 + # via vaex-hdf5 +httptools==0.6.1 # via uvicorn -huggingface-hub==0.17.1 - # via datasets -identify==2.5.29 +identify==2.5.33 # via pre-commit -idna==3.4 +idna==3.6 # via # anyio - # jsonschema # requests - # snowflake-connector-python # yarl -imagehash==4.3.1 - # via - # visions - # ydata-profiling imagesize==1.4.1 # via sphinx -importlib-metadata==6.8.0 +importlib-metadata==7.0.1 # via # dask - # flask # flytekit - # jupyter-client - # jupyter-lsp - # jupyterlab - # jupyterlab-server # keyring - # markdown - # mlflow - # nbconvert - # sphinx -importlib-resources==6.0.1 - # via matplotlib ipydatawidgets==4.3.5 # via pythreejs -ipykernel==6.25.2 - # via - # jupyter - # jupyter-console - # jupyterlab - # qtconsole -ipyleaflet==0.17.4 +ipyleaflet==0.18.1 # via vaex-jupyter ipympl==0.9.3 # via vaex-jupyter -ipython==8.15.0 +ipython==8.18.1 # via - # great-expectations - # ipykernel # ipympl # ipywidgets - # jupyter-console ipython-genutils==0.2.0 - # via - # ipympl - # qtconsole + # via ipympl ipyvolume==0.6.3 # via vaex-jupyter ipyvue==1.10.1 @@ -447,159 +269,58 @@ ipywebrtc==0.6.0 ipywidgets==8.1.1 # via # bqplot - # great-expectations # ipydatawidgets # ipyleaflet # ipympl # ipyvolume # ipyvue - # jupyter # pythreejs isodate==0.6.1 # via azure-storage-blob -isoduration==20.11.0 - # via jsonschema -itsdangerous==2.1.2 - # via flask jaraco-classes==3.3.0 # via keyring -jedi==0.19.0 +jedi==0.19.1 # via ipython -jinja2==3.1.2 +jeepney==0.8.0 + # via + # keyring + # secretstorage +jinja2==3.1.3 # via - # altair # branca # cookiecutter - # distributed - # flask - # great-expectations - # jupyter-server - # jupyterlab - # jupyterlab-server - # mlflow - # nbconvert # sphinx # sphinx-autoapi - # torch # vaex-ml - # ydata-profiling jmespath==1.0.1 # via botocore joblib==1.3.2 # via # flytekit - # phik # scikit-learn -json5==0.9.14 - # via jupyterlab-server -jsonpatch==1.33 - # via great-expectations jsonpickle==3.0.2 # via flytekit -jsonpointer==2.4 - # via - # jsonpatch - # jsonschema -jsonschema[format-nongpl]==4.19.0 - # via - # altair - # great-expectations - # jupyter-events - # jupyterlab-server - # nbformat - # ray -jsonschema-specifications==2023.7.1 - # via jsonschema -jupyter==1.0.0 - # via -r doc-requirements.in -jupyter-client==8.3.1 - # via - # ipykernel - # jupyter-console - # jupyter-server - # nbclient - # qtconsole -jupyter-console==6.6.3 - # via jupyter -jupyter-core==5.3.1 - # via - # ipykernel - # jupyter-client - # jupyter-console - # jupyter-server - # jupyterlab - # nbclient - # nbconvert - # nbformat - # qtconsole -jupyter-events==0.7.0 - # via jupyter-server -jupyter-lsp==2.2.0 - # via jupyterlab -jupyter-server==2.7.3 - # via - # jupyter-lsp - # jupyterlab - # jupyterlab-server - # notebook - # notebook-shim -jupyter-server-terminals==0.4.4 - # via jupyter-server -jupyterlab==4.0.6 - # via notebook -jupyterlab-pygments==0.2.2 - # via nbconvert -jupyterlab-server==2.25.0 - # via - # jupyterlab - # notebook jupyterlab-widgets==3.0.9 # via ipywidgets -keras==2.13.1 - # via tensorflow -keyring==24.2.0 +keyring==24.3.0 # via flytekit kiwisolver==1.4.5 # via matplotlib -kubernetes==27.2.0 - # via - # -r doc-requirements.in - # flytekit -lazy-object-proxy==1.9.0 - # via astroid -libclang==16.0.6 - # via tensorflow -lit==16.0.6 - # via triton -llvmlite==0.40.1 +kubernetes==29.0.0 + # via flytekit +llvmlite==0.41.1 # via numba locket==1.0.0 - # via - # distributed - # partd -lxml==4.9.3 + # via partd +lxml==5.1.0 # via sphinx-material -makefun==1.15.1 - # via great-expectations -mako==1.2.4 - # via alembic -markdown==3.4.4 - # via - # -r doc-requirements.in - # mlflow - # tensorboard markdown-it-py==3.0.0 # via rich markupsafe==2.1.3 - # via - # jinja2 - # mako - # nbconvert - # werkzeug -marshmallow==3.20.1 + # via jinja2 +marshmallow==3.20.2 # via # dataclasses-json - # great-expectations # marshmallow-enum # marshmallow-jsonschema marshmallow-enum==1.5.1 @@ -608,299 +329,122 @@ marshmallow-enum==1.5.1 # flytekit marshmallow-jsonschema==0.13.0 # via flytekit -mashumaro==3.10 +mashumaro==3.11 # via # -r doc-requirements.in # flytekit -matplotlib==3.8.0 +matplotlib==3.8.2 # via # ipympl # ipyvolume - # mlflow - # phik - # seaborn # vaex-viz - # wordcloud - # ydata-profiling matplotlib-inline==0.1.6 - # via - # ipykernel - # ipython + # via ipython mdurl==0.1.2 # via markdown-it-py -mistune==3.0.1 - # via - # great-expectations - # nbconvert -mlflow==2.7.0 - # via -r doc-requirements.in -modin==0.22.3 - # via -r doc-requirements.in -more-itertools==10.1.0 +more-itertools==10.2.0 # via jaraco-classes -mpmath==1.3.0 - # via sympy -msal==1.24.0 +msal==1.26.0 # via # azure-datalake-store # azure-identity # msal-extensions -msal-extensions==1.0.0 +msal-extensions==1.1.0 # via azure-identity -msgpack==1.0.5 - # via - # distributed - # ray multidict==6.0.4 # via # aiohttp # yarl -multimethod==1.9.1 - # via - # pandera - # visions - # ydata-profiling -multiprocess==0.70.15 - # via datasets mypy-extensions==1.0.0 # via typing-inspect -nbclient==0.8.0 - # via - # nbconvert - # papermill -nbconvert==7.8.0 - # via - # jupyter - # jupyter-server -nbformat==5.9.2 - # via - # great-expectations - # jupyter-server - # nbclient - # nbconvert - # papermill nest-asyncio==1.5.8 - # via - # ipykernel - # vaex-core -networkx==3.1 - # via - # torch - # visions + # via vaex-core nodeenv==1.8.0 # via pre-commit -notebook==7.0.3 - # via - # great-expectations - # jupyter -notebook-shim==0.2.3 - # via - # jupyterlab - # notebook -numba==0.57.1 +numba==0.58.1 # via vaex-ml -numpy==1.23.5 +numpy==1.26.3 # via - # altair # astropy # bqplot # contourpy - # datasets - # flytekit - # great-expectations # h5py - # imagehash # ipydatawidgets # ipympl # ipyvolume # matplotlib - # mlflow - # modin # numba - # onnx - # onnxconverter-common - # opt-einsum # pandas - # pandera - # patsy - # phik # pyarrow # pyerfa # pythreejs - # pywavelets - # ray # scikit-learn # scipy - # seaborn - # statsmodels - # tensorboard - # tensorflow - # tf2onnx # vaex-core - # visions - # wordcloud # xarray - # ydata-profiling oauthlib==3.2.2 # via - # databricks-cli # kubernetes # requests-oauthlib -onnx==1.14.1 - # via - # onnxconverter-common - # skl2onnx - # tf2onnx -onnxconverter-common==1.13.0 - # via skl2onnx -opt-einsum==3.3.0 - # via tensorflow -oscrypto==1.3.0 - # via snowflake-connector-python -overrides==7.4.0 - # via jupyter-server -packaging==23.1 +packaging==23.2 # via # astropy # dask - # datasets - # distributed # docker - # google-cloud-bigquery - # great-expectations - # gunicorn - # huggingface-hub - # ipykernel - # jupyter-server - # jupyterlab - # jupyterlab-server # marshmallow # matplotlib - # mlflow - # modin - # nbconvert - # onnxconverter-common - # pandera - # plotly - # qtconsole - # qtpy - # ray - # snowflake-connector-python + # msal-extensions # sphinx - # statsmodels - # tensorflow # xarray -pandas==1.5.3 +pandas==2.1.4 # via - # altair # bqplot - # datasets - # dolt-integrations - # flytekit - # great-expectations - # mlflow - # modin - # pandera - # phik - # seaborn - # statsmodels # vaex-core - # visions # xarray - # ydata-profiling -pandas-profiling==3.6.6 - # via -r doc-requirements.in -pandera==0.14.5 - # via -r doc-requirements.in -pandocfilters==1.5.0 - # via nbconvert -papermill==2.4.0 - # via -r doc-requirements.in parso==0.8.3 # via jedi -partd==1.4.0 +partd==1.4.1 # via dask -patsy==0.5.3 - # via statsmodels -pexpect==4.8.0 +pexpect==4.9.0 # via ipython -phik==0.12.3 - # via ydata-profiling -pickleshare==0.7.5 - # via ipython -pillow==10.0.1 +pillow==10.2.0 # via - # imagehash # ipympl # ipyvolume # matplotlib + # sphinx-gallery # vaex-viz - # visions - # wordcloud -platformdirs==3.8.1 - # via - # jupyter-core - # snowflake-connector-python - # virtualenv - # whylogs -plotly==5.17.0 - # via -r doc-requirements.in -polars==0.19.3 - # via -r doc-requirements.in +platformdirs==4.1.0 + # via virtualenv portalocker==2.8.2 # via msal-extensions -pre-commit==3.4.0 +pre-commit==3.6.0 # via sphinx-tags -progressbar2==4.2.0 +progressbar2==4.3.2 # via vaex-core -prometheus-client==0.17.1 - # via jupyter-server -prompt-toolkit==3.0.39 - # via - # ipython - # jupyter-console -proto-plus==1.22.3 - # via google-cloud-bigquery -protobuf==4.24.3 +prompt-toolkit==3.0.43 + # via ipython +protobuf==4.24.4 # via # flyteidl + # flytekit # google-api-core - # google-cloud-bigquery # googleapis-common-protos # grpcio-status - # mlflow - # onnx - # onnxconverter-common - # proto-plus # protoc-gen-swagger - # ray - # tensorboard - # tensorflow - # whylogs protoc-gen-swagger==0.1.0 # via flyteidl -psutil==5.9.5 - # via - # distributed - # ipykernel - # modin ptyprocess==0.7.0 - # via - # pexpect - # terminado + # via pexpect pure-eval==0.2.2 # via stack-data py==1.11.0 # via retry -py4j==0.10.9.7 - # via pyspark -pyarrow==10.0.1 +pyarrow==14.0.2 # via - # datasets # flytekit - # mlflow # vaex-core -pyasn1==0.5.0 +pyasn1==0.5.1 # via # pyasn1-modules # rsa @@ -908,64 +452,43 @@ pyasn1-modules==0.3.0 # via google-auth pycparser==2.21 # via cffi -pycryptodomex==3.19.0 - # via snowflake-connector-python -pydantic==1.10.12 +pydantic==1.10.13 # via + # -r doc-requirements.in # fastapi - # great-expectations - # pandera # vaex-core - # ydata-profiling -pyerfa==2.0.0.3 +pyerfa==2.0.1.1 # via astropy -pygments==2.16.1 +pygments==2.17.2 # via # furo # ipython - # jupyter-console - # nbconvert - # qtconsole # rich # sphinx # sphinx-prompt pyjwt[crypto]==2.8.0 # via - # databricks-cli # msal - # snowflake-connector-python -pyopenssl==23.2.0 - # via - # snowflake-connector-python + # pyjwt pyparsing==3.1.1 - # via - # great-expectations - # matplotlib -pyspark==3.4.1 - # via -r doc-requirements.in + # via matplotlib python-dateutil==2.8.2 # via # arrow # botocore # croniter - # google-cloud-bigquery - # great-expectations - # jupyter-client # kubernetes # matplotlib # pandas - # whylabs-client python-dotenv==1.0.0 # via uvicorn python-json-logger==2.0.7 - # via - # flytekit - # jupyter-events + # via flytekit python-slugify[unidecode]==8.0.1 # via # cookiecutter # sphinx-material -python-utils==3.7.0 +python-utils==3.8.1 # via progressbar2 pythreejs==2.4.2 # via ipyvolume @@ -973,167 +496,73 @@ pytimeparse==1.1.8 # via flytekit pytz==2023.3.post1 # via - # great-expectations - # mlflow + # croniter # pandas - # snowflake-connector-python -pywavelets==1.4.1 - # via imagehash pyyaml==6.0.1 # via # astropy # cookiecutter # dask - # datasets - # distributed # flytekit - # huggingface-hub - # jupyter-events # kubernetes - # mlflow - # papermill # pre-commit - # ray # sphinx-autoapi # uvicorn # vaex-core - # ydata-profiling -pyzmq==25.1.1 - # via - # ipykernel - # jupyter-client - # jupyter-console - # jupyter-server - # qtconsole -qtconsole==5.4.4 - # via jupyter -qtpy==2.4.0 - # via qtconsole -querystring-parser==1.2.4 - # via mlflow -ray==2.6.3 - # via -r doc-requirements.in -referencing==0.30.2 - # via - # jsonschema - # jsonschema-specifications - # jupyter-events requests==2.31.0 # via # azure-core # azure-datalake-store # cookiecutter - # databricks-cli - # datasets # docker # flytekit - # fsspec # gcsfs # google-api-core - # google-cloud-bigquery # google-cloud-storage - # great-expectations - # huggingface-hub # ipyvolume - # jupyterlab-server # kubernetes - # mlflow # msal - # papermill - # ray # requests-oauthlib - # snowflake-connector-python # sphinx # sphinxcontrib-youtube - # tensorboard - # tf2onnx # vaex-core - # whylogs - # ydata-profiling requests-oauthlib==1.3.1 # via # google-auth-oauthlib # kubernetes retry==0.9.2 # via -r doc-requirements.in -rfc3339-validator==0.1.4 - # via - # jsonschema - # jupyter-events -rfc3986-validator==0.1.1 - # via - # jsonschema - # jupyter-events -rich==13.5.3 +rich==13.7.0 # via # cookiecutter # flytekit # rich-click # vaex-core -rich-click==1.6.1 +rich-click==1.7.3 # via flytekit -rpds-py==0.10.3 - # via - # jsonschema - # referencing rsa==4.9 # via google-auth -ruamel-yaml==0.17.17 - # via great-expectations -ruamel-yaml-clib==0.2.7 - # via ruamel-yaml -s3fs==2023.6.0 +s3fs==2023.9.2 # via flytekit -scikit-learn==1.3.0 - # via - # -r doc-requirements.in - # mlflow - # skl2onnx -scipy==1.11.2 - # via - # great-expectations - # imagehash - # mlflow - # phik - # scikit-learn - # statsmodels - # ydata-profiling -seaborn==0.12.2 - # via ydata-profiling -send2trash==1.8.2 - # via jupyter-server +scikit-learn==1.3.2 + # via -r doc-requirements.in +scipy==1.11.4 + # via scikit-learn +secretstorage==3.3.3 + # via keyring six==1.16.0 # via # asttokens - # astunparse # azure-core - # bleach - # databricks-cli - # google-pasta # isodate # kubernetes - # patsy # python-dateutil - # querystring-parser - # rfc3339-validator # sphinx-code-include - # tensorflow - # tf2onnx # vaex-core -skl2onnx==1.15.0 - # via -r doc-requirements.in -smmap==5.0.1 - # via gitdb sniffio==1.3.0 # via anyio snowballstemmer==2.2.0 # via sphinx -snowflake-connector-python==3.2.0 - # via -r doc-requirements.in -sortedcontainers==2.4.0 - # via - # distributed - # snowflake-connector-python soupsieve==2.5 # via beautifulsoup4 sphinx==4.5.0 @@ -1150,13 +579,14 @@ sphinx==4.5.0 # sphinx-material # sphinx-panels # sphinx-prompt + # sphinx-reredirects # sphinx-tags # sphinxcontrib-youtube sphinx-autoapi==2.0.1 # via -r doc-requirements.in sphinx-basic-ng==1.0.0b2 # via furo -sphinx-click==5.0.1 +sphinx-click==5.1.0 # via -r doc-requirements.in sphinx-code-include==1.1.1 # via -r doc-requirements.in @@ -1164,7 +594,7 @@ sphinx-copybutton==0.5.2 # via -r doc-requirements.in sphinx-fontawesome==0.0.6 # via -r doc-requirements.in -sphinx-gallery==0.14.0 +sphinx-gallery==0.15.0 # via -r doc-requirements.in sphinx-material==0.0.36 # via -r doc-requirements.in @@ -1172,6 +602,8 @@ sphinx-panels==0.6.0 # via -r doc-requirements.in sphinx-prompt==1.5.0 # via -r doc-requirements.in +sphinx-reredirects==0.1.3 + # via -r doc-requirements.in sphinx-tags==0.2.1 # via -r doc-requirements.in sphinxcontrib-applehelp==1.0.4 @@ -1188,111 +620,34 @@ sphinxcontrib-serializinghtml==1.1.5 # via sphinx sphinxcontrib-youtube==1.2.0 # via -r doc-requirements.in -sqlalchemy==2.0.20 - # via - # -r doc-requirements.in - # alembic - # mlflow -sqlparse==0.4.4 - # via mlflow -stack-data==0.6.2 +stack-data==0.6.3 # via ipython -starlette==0.27.0 +starlette==0.32.0.post1 # via fastapi statsd==3.3.0 # via flytekit -statsmodels==0.14.0 - # via ydata-profiling -sympy==1.12 - # via torch tabulate==0.9.0 - # via - # databricks-cli - # vaex-core -tangled-up-in-unicode==0.2.0 - # via visions -tblib==2.0.0 - # via distributed -tenacity==8.2.3 - # via - # papermill - # plotly -tensorboard==2.13.0 - # via tensorflow -tensorboard-data-server==0.7.1 - # via tensorboard -tensorflow==2.13.0 - # via -r doc-requirements.in -tensorflow-estimator==2.13.0 - # via tensorflow -tensorflow==2.13.0 - # via tensorflow -termcolor==2.3.0 - # via tensorflow -terminado==0.17.1 - # via - # jupyter-server - # jupyter-server-terminals + # via vaex-core text-unidecode==1.3 # via python-slugify -textwrap3==0.9.2 - # via ansiwrap -tf2onnx==1.8.4 - # via -r doc-requirements.in threadpoolctl==3.2.0 # via scikit-learn -tinycss2==1.2.1 - # via nbconvert -tomli==2.0.1 - # via jupyterlab -tomlkit==0.12.1 - # via snowflake-connector-python toolz==0.12.0 # via - # altair # dask - # distributed # partd -torch==2.0.1 - # via -r doc-requirements.in -tornado==6.3.3 - # via - # distributed - # ipykernel - # jupyter-client - # jupyter-server - # jupyterlab - # notebook - # terminado - # vaex-server -tqdm==4.66.1 - # via - # datasets - # great-expectations - # huggingface-hub - # papermill - # ydata-profiling -traitlets==5.10.0 +tornado==6.4 + # via vaex-server +traitlets==5.14.1 # via # bqplot # comm - # ipykernel # ipympl # ipython # ipyvolume # ipywidgets - # jupyter-client - # jupyter-console - # jupyter-core - # jupyter-events - # jupyter-server - # jupyterlab # matplotlib-inline - # nbclient - # nbconvert - # nbformat # pythreejs - # qtconsole # traittypes # vaex-ml traittypes==0.2.1 @@ -1301,67 +656,40 @@ traittypes==0.2.1 # ipydatawidgets # ipyleaflet # ipyvolume -typed-ast==1.5.5 - # via doltcli -typeguard==2.13.3 - # via ydata-profiling -types-requests==2.31.0.2 - # via whylogs -types-urllib3==1.26.25.14 - # via types-requests -typing-extensions==4.5.0 - # via - # aioitertools - # alembic +types-python-dateutil==2.8.19.20240106 + # via arrow +typing-extensions==4.9.0 + # via + # anyio # astroid - # async-lru # azure-core # azure-storage-blob # fastapi # flytekit - # great-expectations - # huggingface-hub - # ipython # mashumaro - # onnx # pydantic # python-utils - # snowflake-connector-python - # sqlalchemy - # starlette - # tensorflow - # torch + # rich-click # typing-inspect # uvicorn - # whylogs typing-inspect==0.9.0 - # via - # dataclasses-json - # pandera -tzlocal==5.0.1 - # via great-expectations -unidecode==1.3.6 + # via dataclasses-json +tzdata==2023.4 + # via pandas +unidecode==1.3.7 # via # python-slugify # sphinx-autoapi -uri-template==1.3.0 - # via jsonschema -urllib3==1.26.16 +urllib3==1.26.18 # via # botocore - # databricks-cli - # distributed # docker # flytekit - # google-auth - # great-expectations # kubernetes # requests - # snowflake-connector-python - # whylabs-client -uvicorn[standard]==0.23.2 +uvicorn[standard]==0.25.0 # via vaex-server -uvloop==0.17.0 +uvloop==0.19.0 # via uvicorn vaex==4.17.0 # via -r doc-requirements.in @@ -1388,69 +716,30 @@ vaex-viz==0.5.4 # via # vaex # vaex-jupyter -virtualenv==20.24.1 +virtualenv==20.25.0 # via pre-commit -visions[type_image_path]==0.7.5 - # via ydata-profiling -watchfiles==0.20.0 +watchfiles==0.21.0 # via uvicorn -wcwidth==0.2.6 +wcwidth==0.2.13 # via prompt-toolkit -webcolors==1.13 - # via jsonschema -webencodings==0.5.1 - # via - # bleach - # tinycss2 -websocket-client==1.6.3 +websocket-client==1.7.0 # via # docker - # jupyter-server # kubernetes -websockets==11.0.3 +websockets==12.0 # via uvicorn -werkzeug==2.3.7 - # via - # flask - # tensorboard -wheel==0.41.2 - # via - # astunparse - # tensorboard -whylabs-client==0.5.7 - # via - # -r doc-requirements.in - # whylogs -whylogs==1.3.3 - # via -r doc-requirements.in -whylogs-sketching==3.4.1.dev3 - # via whylogs widgetsnbextension==4.0.9 # via ipywidgets -wordcloud==1.9.2 - # via ydata-profiling -wrapt==1.15.0 - # via - # aiobotocore - # astroid - # pandera - # tensorflow -xarray==2023.8.0 +wrapt==1.16.0 + # via aiobotocore +xarray==2023.12.0 # via vaex-jupyter -xxhash==3.3.0 - # via datasets -xyzservices==2023.7.0 +xyzservices==2023.10.1 # via ipyleaflet -yarl==1.9.2 +yarl==1.9.4 # via aiohttp -ydata-profiling==4.5.1 - # via pandas-profiling -zict==3.0.0 - # via distributed -zipp==3.16.2 - # via - # importlib-metadata - # importlib-resources +zipp==3.17.0 + # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: # setuptools diff --git a/docs/source/conf.py b/docs/source/conf.py index b12e355845..f86a8d2c06 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -57,6 +57,7 @@ "sphinx-prompt", "sphinx_copybutton", "sphinx_panels", + "sphinx_reredirects", "sphinxcontrib.youtube", "sphinx_tags", "sphinx_click", @@ -96,7 +97,7 @@ # List of patterns, relative to source directory, that match files and # directories to ignore when looking for source files. # This pattern also affects html_static_path and html_extra_path . -exclude_patterns = [] +exclude_patterns = ["docs_index.rst"] # -- Options for HTML output ------------------------------------------------- @@ -106,20 +107,29 @@ html_theme = "furo" html_title = "Flyte" +announcement = """ +📢 This is the old documentation for Flyte. +Please visit the new documentation here. +""" + html_theme_options = { "light_css_variables": { "color-brand-primary": "#4300c9", "color-brand-content": "#4300c9", + "color-announcement-background": "#FEE7B8", + "color-announcement-text": "#535353", }, "dark_css_variables": { "color-brand-primary": "#9D68E4", "color-brand-content": "#9D68E4", + "color-announcement-background": "#493100", }, # custom flyteorg furo theme options "github_repo": "flytekit", "github_username": "flyteorg", "github_commit": "master", "docs_path": "docs/source", # path to documentation source + "announcement": announcement, } templates_path = ["_templates"] @@ -230,3 +240,27 @@ tags_create_tags = True tags_page_title = "Tag" tags_overview_title = "All Tags" + + +# Sphinx redirects to the monodocs +page_pattern = "https://docs.flyte.org/en/latest/api/flytekit/$source.html" + +if int(os.environ.get("ENABLE_SPHINX_REDIRECTS", 0)): + redirects = { + "generated/*": page_pattern, + "design/*": page_pattern, + "plugins/*": page_pattern, + "flytekit.html": "https://docs.flyte.org/en/latest/api/flytekit/flytekit.html", + "configuration.html": "https://docs.flyte.org/en/latest/api/flytekit/configuration.html", + "remote.html": "https://docs.flyte.org/en/latest/api/flytekit/remote.html", + "clients.html": "https://docs.flyte.org/en/latest/api/flytekit/clients.html", + "testing.html": "https://docs.flyte.org/en/latest/api/flytekit/testing.html", + "extend.html": "https://docs.flyte.org/en/latest/api/flytekit/extend.html", + "deck.html": "https://docs.flyte.org/en/latest/api/flytekit/deck.html", + "tasks*": page_pattern, + "types*": page_pattern, + "extras*": page_pattern, + "experimental.html": "https://docs.flyte.org/en/latest/api/flytekit/experimental.html", + "pyflyte.html": "https://docs.flyte.org/en/latest/api/flytekit/pyflyte.html", + "contributing.html": "https://docs.flyte.org/en/latest/api/flytekit/contributing.html", + } diff --git a/docs/source/design/control_plane.rst b/docs/source/design/control_plane.rst index 4c5357cb87..e05be52129 100644 --- a/docs/source/design/control_plane.rst +++ b/docs/source/design/control_plane.rst @@ -248,7 +248,7 @@ Additional arguments include: * ``project``: the project on which to execute the entity. * ``domain``: the domain on which to execute the entity. * ``type_hints``: a dictionary mapping Python types to their corresponding Flyte types. -* ``options``: options can be configured for a launch plan during registration or overriden during execution. Refer to :py:class:`~flytekit.remote.remote.Options` to know all the acceptable parameters. +* ``options``: options can be configured for a launch plan during registration or overridden during execution. Refer to :py:class:`~flytekit.remote.remote.Options` to know all the acceptable parameters. The following is an example demonstrating how to use the :py:class:`~flytekit.remote.remote.Options` class to configure a Flyte entity: diff --git a/docs/source/docs_index.rst b/docs/source/docs_index.rst new file mode 100644 index 0000000000..dbbf95af83 --- /dev/null +++ b/docs/source/docs_index.rst @@ -0,0 +1,22 @@ +********************** +Flytekit API Reference +********************** + +.. toctree:: + :maxdepth: 2 + + design/index + flytekit + configuration + remote + clients + testing + extras.accelerators + extend + deck + plugins/index + tasks.extend + types.extend + experimental + pyflyte + contributing diff --git a/docs/source/extras.accelerators.rst b/docs/source/extras.accelerators.rst new file mode 100644 index 0000000000..2655200a23 --- /dev/null +++ b/docs/source/extras.accelerators.rst @@ -0,0 +1,4 @@ +.. automodule:: flytekit.extras.accelerators + :members: + :undoc-members: + :show-inheritance: diff --git a/docs/source/index.rst b/docs/source/index.rst index f123248cec..d4d0730236 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -1,3 +1,8 @@ +.. DO NOT EDIT THIS FILE! + This file is the index for the old flytekit documentation. The index for the monodocs is now + at `docs_index.rst`. Please edit that file if you want to add new entries to the flytekit api + documentation. + .. simpleble documentation master file, created by sphinx-quickstart on Fri Mar 9 04:07:53 2018. You can adapt this file completely to your liking, but it should at least @@ -80,6 +85,7 @@ Expected output: testing extend deck + extras.accelerators plugins/index tasks.extend types.extend diff --git a/flytekit/clients/auth/auth_client.py b/flytekit/clients/auth/auth_client.py index 8001bf8885..4735a446be 100644 --- a/flytekit/clients/auth/auth_client.py +++ b/flytekit/clients/auth/auth_client.py @@ -172,7 +172,7 @@ def __call__(cls, *args, **kwargs): class AuthorizationClient(metaclass=_SingletonPerEndpoint): """ Authorization client that stores the credentials in keyring and uses oauth2 standard flow to retrieve the - credentials. NOTE: This will open an web browser to retreive the credentials. + credentials. NOTE: This will open an web browser to retrieve the credentials. """ def __init__( diff --git a/flytekit/clients/auth/authenticator.py b/flytekit/clients/auth/authenticator.py index 6fc08c7140..fdf1d13eae 100644 --- a/flytekit/clients/auth/authenticator.py +++ b/flytekit/clients/auth/authenticator.py @@ -167,7 +167,7 @@ def refresh_credentials(self): class CommandAuthenticator(Authenticator): """ - This Authenticator retreives access_token using the provided command + This Authenticator retrieves access_token using the provided command """ def __init__(self, command: typing.List[str], header_key: str = None): @@ -305,7 +305,7 @@ def refresh_credentials(self): text = f"To Authenticate, navigate in a browser to the following URL: {click.style(resp.verification_uri, fg='blue', underline=True)} and enter code: {click.style(resp.user_code, fg='blue')}" click.secho(text) try: - # Currently the refresh token is not retreived. We may want to add support for refreshTokens so that + # Currently the refresh token is not retrieved. We may want to add support for refreshTokens so that # access tokens can be refreshed for once authenticated machines token, expires_in = token_client.poll_token_endpoint( resp, diff --git a/flytekit/clients/auth_helper.py b/flytekit/clients/auth_helper.py index 5b53856863..04028bc10a 100644 --- a/flytekit/clients/auth_helper.py +++ b/flytekit/clients/auth_helper.py @@ -116,7 +116,7 @@ def get_proxy_authenticator(cfg: PlatformConfig) -> Authenticator: def upgrade_channel_to_proxy_authenticated(cfg: PlatformConfig, in_channel: grpc.Channel) -> grpc.Channel: """ - If activated in the platform config, given a grpc.Channel, preferrably a secure channel, it returns a composed + If activated in the platform config, given a grpc.Channel, preferably a secure channel, it returns a composed channel that uses Interceptor to perform authentication with a proxy infront of Flyte :param cfg: PlatformConfig :param in_channel: grpc.Channel Precreated channel @@ -131,7 +131,7 @@ def upgrade_channel_to_proxy_authenticated(cfg: PlatformConfig, in_channel: grpc def upgrade_channel_to_authenticated(cfg: PlatformConfig, in_channel: grpc.Channel) -> grpc.Channel: """ - Given a grpc.Channel, preferrably a secure channel, it returns a composed channel that uses Interceptor to + Given a grpc.Channel, preferably a secure channel, it returns a composed channel that uses Interceptor to perform an Oauth2.0 Auth flow :param cfg: PlatformConfig :param in_channel: grpc.Channel Precreated channel diff --git a/flytekit/clis/flyte_cli/main.py b/flytekit/clis/flyte_cli/main.py index b190243e6e..75386896ce 100644 --- a/flytekit/clis/flyte_cli/main.py +++ b/flytekit/clis/flyte_cli/main.py @@ -1069,7 +1069,7 @@ def update_launch_plan(state, host, insecure, urn=None): else: # If the commandline parameter urn is not supplied, and neither # the input comes from a pipe, it means the user is not using - # this command approperiately + # this command appropriately raise _click.UsageError('Missing option "-u" / "--urn" or missing pipe inputs') except KeyboardInterrupt: _sys.stdout.flush() @@ -1185,7 +1185,7 @@ def terminate_execution(host, insecure, cause, urn=None): def list_executions(project, domain, host, insecure, token, limit, show_all, filter, sort_by, urns_only): """ List the key information of all the executions under the scope specified by {project, domain}. - Users can supply additional filter arguments to show only the desired exeuctions. + Users can supply additional filter arguments to show only the desired executions. Note that, when the ``--urns-only`` flag is not set, this command prints out the complete tabular output with key pieces of information such as the URN, the Name and the Status of the executions; diff --git a/flytekit/clis/sdk_in_container/fetch.py b/flytekit/clis/sdk_in_container/fetch.py index 8c83b5d548..841ca3f840 100644 --- a/flytekit/clis/sdk_in_container/fetch.py +++ b/flytekit/clis/sdk_in_container/fetch.py @@ -17,7 +17,7 @@ "--recursive", "-r", is_flag=True, - help="Fetch recursively, all variables in the URI. This is not needed for directrories as they" + help="Fetch recursively, all variables in the URI. This is not needed for directories as they" " are automatically recursively downloaded.", ) @click.argument("flyte-data-uri", type=str, required=True, metavar="FLYTE-DATA-URI (format flyte://...)") diff --git a/flytekit/clis/sdk_in_container/init.py b/flytekit/clis/sdk_in_container/init.py index 627b393578..23df6ab62d 100644 --- a/flytekit/clis/sdk_in_container/init.py +++ b/flytekit/clis/sdk_in_container/init.py @@ -5,7 +5,7 @@ @click.command("init") @click.option( "--template", - default="simple-example", + default="basic-template-imagespec", help="cookiecutter template folder name to be used in the repo - https://github.com/flyteorg/flytekit-python-template.git", ) @click.argument("project-name") diff --git a/flytekit/clis/sdk_in_container/run.py b/flytekit/clis/sdk_in_container/run.py index 9c2bb2c17a..d5a96e069a 100644 --- a/flytekit/clis/sdk_in_container/run.py +++ b/flytekit/clis/sdk_in_container/run.py @@ -32,6 +32,7 @@ from flytekit.core.workflow import PythonFunctionWorkflow, WorkflowBase from flytekit.exceptions.system import FlyteSystemException from flytekit.interaction.click_types import FlyteLiteralConverter, key_value_callback +from flytekit.interaction.string_literals import literal_string_repr from flytekit.loggers import logger from flytekit.models import security from flytekit.models.common import RawOutputDataConfig @@ -69,7 +70,7 @@ class RunLevelParams(PyFlyteParams): param_decls=["--destination-dir", "destination_dir"], required=False, type=str, - default="/root", + default=".", show_default=True, help="Directory inside the image where the tar file containing the code will be copied to", ) @@ -240,10 +241,10 @@ class RunLevelParams(PyFlyteParams): param_decls=["--limit", "limit"], required=False, type=int, - default=10, + default=50, + hidden=True, show_default=True, - help="Use this to limit number of launch plans retreived from the backend, " - "if `from-server` option is used", + help="Use this to limit number of entities to fetch", ) ) cluster_pool: str = make_click_option_field( @@ -553,32 +554,40 @@ def _run(*args, **kwargs): return _run -class DynamicLaunchPlanCommand(click.RichCommand): +class DynamicEntityLaunchCommand(click.RichCommand): """ This is a dynamic command that is created for each launch plan. This is used to execute a launch plan. It will fetch the launch plan from remote and create parameters from all the inputs of the launch plan. """ - def __init__(self, name: str, h: str, lp_name: str, **kwargs): + LP_LAUNCHER = "lp" + TASK_LAUNCHER = "task" + + def __init__(self, name: str, h: str, entity_name: str, launcher: str, **kwargs): super().__init__(name=name, help=h, **kwargs) - self._lp_name = lp_name - self._lp = None + self._entity_name = entity_name + self._launcher = launcher + self._entity = None - def _fetch_launch_plan(self, ctx: click.Context) -> FlyteLaunchPlan: - if self._lp: - return self._lp + def _fetch_entity(self, ctx: click.Context) -> typing.Union[FlyteLaunchPlan, FlyteTask]: + if self._entity: + return self._entity run_level_params: RunLevelParams = ctx.obj r = run_level_params.remote_instance() - self._lp = r.fetch_launch_plan(run_level_params.project, run_level_params.domain, self._lp_name) - return self._lp + if self._launcher == self.LP_LAUNCHER: + entity = r.fetch_launch_plan(run_level_params.project, run_level_params.domain, self._entity_name) + else: + entity = r.fetch_task(run_level_params.project, run_level_params.domain, self._entity_name) + self._entity = entity + return entity def _get_params( self, ctx: click.Context, inputs: typing.Dict[str, Variable], native_inputs: typing.Dict[str, type], - fixed: typing.Dict[str, Literal], - defaults: typing.Dict[str, Parameter], + fixed: typing.Optional[typing.Dict[str, Literal]] = None, + defaults: typing.Optional[typing.Dict[str, Parameter]] = None, ) -> typing.List["click.Parameter"]: params = [] flyte_ctx = context_manager.FlyteContextManager.current_context() @@ -586,21 +595,31 @@ def _get_params( if fixed and name in fixed: continue required = True + default_val = None if defaults and name in defaults: - required = False - params.append(to_click_option(ctx, flyte_ctx, name, var, native_inputs[name], None, required)) + if not defaults[name].required: + required = False + default_val = literal_string_repr(defaults[name].default) if defaults[name].default else None + params.append(to_click_option(ctx, flyte_ctx, name, var, native_inputs[name], default_val, required)) return params def get_params(self, ctx: click.Context) -> typing.List["click.Parameter"]: if not self.params: self.params = [] - lp = self._fetch_launch_plan(ctx) - if lp.interface: - if lp.interface.inputs: - types = TypeEngine.guess_python_types(lp.interface.inputs) - self.params = self._get_params( - ctx, lp.interface.inputs, types, lp.fixed_inputs.literals, lp.default_inputs.parameters - ) + entity = self._fetch_entity(ctx) + if entity.interface: + if entity.interface.inputs: + types = TypeEngine.guess_python_types(entity.interface.inputs) + if isinstance(entity, FlyteLaunchPlan): + self.params = self._get_params( + ctx, + entity.interface.inputs, + types, + entity.fixed_inputs.literals, + entity.default_inputs.parameters, + ) + else: + self.params = self._get_params(ctx, entity.interface.inputs, types) return super().get_params(ctx) @@ -611,40 +630,61 @@ def invoke(self, ctx: click.Context) -> typing.Any: """ run_level_params: RunLevelParams = ctx.obj r = run_level_params.remote_instance() - lp = self._fetch_launch_plan(ctx) + entity = self._fetch_entity(ctx) run_remote( r, - lp, + entity, run_level_params.project, run_level_params.domain, ctx.params, run_level_params, - type_hints=lp.python_interface.inputs if lp.python_interface else None, + type_hints=entity.python_interface.inputs if entity.python_interface else None, ) -class RemoteLaunchPlanGroup(click.RichGroup): +class RemoteEntityGroup(click.RichGroup): """ click multicommand that retrieves launchplans from a remote flyte instance and executes them. """ - COMMAND_NAME = "remote-launchplan" + LAUNCHPLAN_COMMAND = "remote-launchplan" + WORKFLOW_COMMAND = "remote-workflow" + TASK_COMMAND = "remote-task" - def __init__(self): + def __init__(self, command_name: str): super().__init__( - name="from-server", - help="Retrieve launchplans from a remote flyte instance and execute them.", + name=command_name, + help=f"Retrieve {command_name} from a remote flyte instance and execute them.", params=[ click.Option( - ["--limit"], help="Limit the number of launchplans to retrieve.", default=10, show_default=True + ["--limit", "limit"], + help=f"Limit the number of {command_name}'s to retrieve.", + default=50, + show_default=True, ) ], ) - self._lps = [] + self._command_name = command_name + self._entities = [] + + def _get_entities(self, r: FlyteRemote, project: str, domain: str, limit: int) -> typing.List[str]: + """ + Retreieves the right entities from the remote flyte instance. + """ + if self._command_name == self.LAUNCHPLAN_COMMAND: + lps = r.client.list_launch_plan_ids_paginated(project=project, domain=domain, limit=limit) + return [l.name for l in lps[0]] + elif self._command_name == self.WORKFLOW_COMMAND: + wfs = r.client.list_workflow_ids_paginated(project=project, domain=domain, limit=limit) + return [w.name for w in wfs[0]] + elif self._command_name == self.TASK_COMMAND: + tasks = r.client.list_task_ids_paginated(project=project, domain=domain, limit=limit) + return [t.name for t in tasks[0]] + return [] def list_commands(self, ctx): - if self._lps or ctx.obj is None: - return self._lps + if self._entities or ctx.obj is None: + return self._entities run_level_params: RunLevelParams = ctx.obj r = run_level_params.remote_instance() @@ -653,17 +693,28 @@ def list_commands(self, ctx): with progress: progress.start_task(task) try: - lps = r.client.list_launch_plan_ids_paginated( - project=run_level_params.project, domain=run_level_params.domain, limit=run_level_params.limit + self._entities = self._get_entities( + r, run_level_params.project, run_level_params.domain, run_level_params.limit ) - self._lps = [l.name for l in lps[0]] - return self._lps + return self._entities except FlyteSystemException as e: pretty_print_exception(e) return [] def get_command(self, ctx, name): - return DynamicLaunchPlanCommand(name=name, h="Execute a launchplan from remote.", lp_name=name) + if self._command_name in [self.LAUNCHPLAN_COMMAND, self.WORKFLOW_COMMAND]: + return DynamicEntityLaunchCommand( + name=name, + h=f"Execute a {self._command_name}.", + entity_name=name, + launcher=DynamicEntityLaunchCommand.LP_LAUNCHER, + ) + return DynamicEntityLaunchCommand( + name=name, + h=f"Execute a {self._command_name}.", + entity_name=name, + launcher=DynamicEntityLaunchCommand.TASK_LAUNCHER, + ) class WorkflowCommand(click.RichGroup): @@ -789,7 +840,11 @@ def list_commands(self, ctx, add_remote: bool = True): self._files = [str(p) for p in pathlib.Path(".").glob("*.py") if str(p) != "__init__.py"] self._files = sorted(self._files) if add_remote: - self._files = self._files + [RemoteLaunchPlanGroup.COMMAND_NAME] + self._files = self._files + [ + RemoteEntityGroup.LAUNCHPLAN_COMMAND, + RemoteEntityGroup.WORKFLOW_COMMAND, + RemoteEntityGroup.TASK_COMMAND, + ] return self._files def get_command(self, ctx, filename): @@ -800,8 +855,12 @@ def get_command(self, ctx, filename): params.update(ctx.params) params.update(ctx.obj) ctx.obj = self._run_params.from_dict(params) - if filename == RemoteLaunchPlanGroup.COMMAND_NAME: - return RemoteLaunchPlanGroup() + if filename == RemoteEntityGroup.LAUNCHPLAN_COMMAND: + return RemoteEntityGroup(RemoteEntityGroup.LAUNCHPLAN_COMMAND) + elif filename == RemoteEntityGroup.WORKFLOW_COMMAND: + return RemoteEntityGroup(RemoteEntityGroup.WORKFLOW_COMMAND) + elif filename == RemoteEntityGroup.TASK_COMMAND: + return RemoteEntityGroup(RemoteEntityGroup.TASK_COMMAND) return WorkflowCommand(filename, name=filename, help=f"Run a [workflow|task] from {filename}") diff --git a/flytekit/clis/sdk_in_container/utils.py b/flytekit/clis/sdk_in_container/utils.py index b89e6b3829..8c59b00c1c 100644 --- a/flytekit/clis/sdk_in_container/utils.py +++ b/flytekit/clis/sdk_in_container/utils.py @@ -83,7 +83,7 @@ def pretty_print_grpc_error(e: grpc.RpcError): def pretty_print_traceback(e): """ - This method will print the Traceback of a error. + This method will print the Traceback of an error. """ if e.__traceback__: stack_list = traceback.format_list(traceback.extract_tb(e.__traceback__)) @@ -115,8 +115,7 @@ def pretty_print_exception(e: Exception): if isinstance(cause, grpc.RpcError): pretty_print_grpc_error(cause) else: - click.secho(f"Underlying Exception: {cause}") - pretty_print_traceback(e) + pretty_print_traceback(cause) return if isinstance(e, grpc.RpcError): @@ -141,6 +140,8 @@ def invoke(self, ctx: click.Context) -> typing.Any: except Exception as e: if verbose > 0: click.secho("Verbose mode on") + if isinstance(e, FlyteException): + raise e.with_traceback(None) raise e pretty_print_exception(e) raise SystemExit(e) from e diff --git a/flytekit/configuration/__init__.py b/flytekit/configuration/__init__.py index b1794c73c6..aa9c2bf129 100644 --- a/flytekit/configuration/__init__.py +++ b/flytekit/configuration/__init__.py @@ -872,7 +872,7 @@ def with_serialized_context(self) -> SerializationSettings: """ Use this method to create a new SerializationSettings that has an environment variable set with the SerializedContext This is useful in transporting SerializedContext to serialized and registered tasks. - The setting will be availabe in the `env` field with the key `SERIALIZED_CONTEXT_ENV_VAR` + The setting will be available in the `env` field with the key `SERIALIZED_CONTEXT_ENV_VAR` :return: A newly constructed SerializationSettings, or self, if it already has the serializationSettings """ if self._has_serialized_context(): diff --git a/flytekit/configuration/plugin.py b/flytekit/configuration/plugin.py index 85bc811c44..051d421d04 100644 --- a/flytekit/configuration/plugin.py +++ b/flytekit/configuration/plugin.py @@ -39,6 +39,10 @@ def get_remote( def configure_pyflyte_cli(main: Group) -> Group: """Configure pyflyte's CLI.""" + @staticmethod + def secret_requires_group() -> bool: + """Return True if secrets require group entry.""" + class FlytekitPlugin: @staticmethod @@ -62,6 +66,11 @@ def configure_pyflyte_cli(main: Group) -> Group: """Configure pyflyte's CLI.""" return main + @staticmethod + def secret_requires_group() -> bool: + """Return True if secrets require group entry.""" + return True + def _get_plugin_from_entrypoint(): """Get plugin from entrypoint.""" @@ -73,7 +82,9 @@ def _get_plugin_from_entrypoint(): if len(plugins) >= 2: plugin_names = [p.name for p in plugins] - logger.info(f"Multiple plugins seen for {group}: {plugin_names}") + raise ValueError( + f"Multiple plugins installed: {plugin_names}. flytekit only supports one installed plugin at a time." + ) plugin_to_load = plugins[0] logger.info(f"Loading plugin: {plugin_to_load.name}") diff --git a/flytekit/core/array_node_map_task.py b/flytekit/core/array_node_map_task.py index 904ba51f42..e4fef9ed10 100644 --- a/flytekit/core/array_node_map_task.py +++ b/flytekit/core/array_node_map_task.py @@ -82,6 +82,7 @@ def __init__( ).hexdigest() self._name = f"{mod}.map_{f}_{h}-arraynode" + self._cmd_prefix: Optional[List[str]] = None self._concurrency: Optional[int] = concurrency self._min_successes: Optional[int] = min_successes self._min_success_ratio: Optional[float] = min_success_ratio @@ -149,6 +150,9 @@ def prepare_target(self): def get_custom(self, settings: SerializationSettings) -> Dict[str, Any]: return ArrayJob(parallelism=self._concurrency, min_success_ratio=self._min_success_ratio).to_dict() + def get_config(self, settings: SerializationSettings) -> Optional[Dict[str, str]]: + return self.python_function_task.get_config(settings) + def get_container(self, settings: SerializationSettings) -> Container: with self.prepare_target(): return self.python_function_task.get_container(settings) @@ -185,11 +189,13 @@ def get_command(self, settings: SerializationSettings) -> List[str]: *mt.loader_args(settings, self), ] - # TODO: add support for ContainerTask - # if self._cmd_prefix: - # return self._cmd_prefix + container_args + if self._cmd_prefix: + return self._cmd_prefix + container_args return container_args + def set_command_prefix(self, cmd: Optional[List[str]]): + self._cmd_prefix = cmd + def __call__(self, *args, **kwargs): """ This call method modifies the kwargs and adds kwargs from partial. diff --git a/flytekit/core/artifact.py b/flytekit/core/artifact.py index 866dd7f4ec..e32a924997 100644 --- a/flytekit/core/artifact.py +++ b/flytekit/core/artifact.py @@ -11,8 +11,7 @@ from flyteidl.artifact import artifacts_pb2 from flyteidl.core import artifact_id_pb2 as art_id from flyteidl.core.identifier_pb2 import TaskExecutionIdentifier, WorkflowExecutionIdentifier -from flyteidl.core.literals_pb2 import Literal -from flyteidl.core.types_pb2 import LiteralType +from google.protobuf.timestamp_pb2 import Timestamp from flytekit.loggers import logger from flytekit.models.literals import Literal @@ -21,10 +20,6 @@ if typing.TYPE_CHECKING: from flytekit.remote.remote import FlyteRemote -# Consider separated out into an independent field in the IDL before GA -# probably worthwhile to add a format field to the date as well -# but separating may be hard as it'll need a new element in the URI mapping. -TIME_PARTITION = "ds" TIME_PARTITION_KWARG = "time_partition" @@ -51,12 +46,10 @@ class ArtifactIDSpecification(object): having a pointer to the main artifact. """ - def __init__( - self, a: Artifact, partitions: Optional[Partitions] = None, time_partition: Optional[TimePartition] = None - ): + def __init__(self, a: Artifact): self.artifact = a - self.partitions = partitions - self.time_partition = time_partition + self.partitions: Optional[Partitions] = None + self.time_partition: Optional[TimePartition] = None # todo: add time partition arg hint def __call__(self, *args, **kwargs): @@ -72,7 +65,9 @@ def bind_partitions(self, *args, **kwargs) -> ArtifactIDSpecification: raise ValueError("Cannot bind time partition to non-time partitioned artifact") p = kwargs[TIME_PARTITION_KWARG] if isinstance(p, datetime.datetime): - self.time_partition = TimePartition(value=art_id.LabelValue(static_value=f"{p}")) + t = Timestamp() + t.FromDatetime(p) + self.time_partition = TimePartition(value=art_id.LabelValue(time_value=t)) elif isinstance(p, art_id.InputBindingData): self.time_partition = TimePartition(value=art_id.LabelValue(input_binding=p)) else: @@ -102,20 +97,29 @@ def to_partial_artifact_id(self) -> art_id.ArtifactID: # This function should only be called by transform_variable_map artifact_id = self.artifact.to_flyte_idl().artifact_id # Use the partitions from this object, but replacement is not allowed by protobuf, so generate new object - p = partitions_to_idl(self.partitions, self.time_partition) + p = partitions_to_idl(self.partitions) + tp = None + if self.artifact.time_partitioned: + if not self.time_partition: + raise ValueError( + f"Artifact {artifact_id.artifact_key} requires a time partition, but it hasn't been bound." + ) + tp = self.time_partition.to_flyte_idl() if self.artifact.partition_keys: required = len(self.artifact.partition_keys) - required += 1 if self.artifact.time_partitioned else 0 fulfilled = len(p.value) if p else 0 if required != fulfilled: raise ValueError( - f"Artifact {artifact_id.artifact_key} requires {required} partitions, but only {fulfilled} are bound." + f"Artifact {artifact_id.artifact_key} requires {required} partitions, but only {fulfilled} are " + f"bound." ) artifact_id = art_id.ArtifactID( artifact_key=artifact_id.artifact_key, partitions=p, - version=artifact_id.version, + time_partition=tp, + version=artifact_id.version, # this should almost never be set since setting it + # hardcodes the query to one version ) return artifact_id @@ -155,9 +159,7 @@ def __init__( tag: Optional[str] = None, ): if not name: - raise ValueError(f"Cannot create query without name") - if partitions and partitions.partitions and TIME_PARTITION in partitions.partitions: - raise ValueError(f"Cannot use 'ds' as a partition name, just use time partition") + raise ValueError("Cannot create query without name") # So normally, if you just do MyData.query(partitions={"region": "{{ inputs.region }}"}), it will just # use the input value to fill in the partition. But if you do @@ -203,11 +205,15 @@ def to_flyte_idl( ) return aq - p = partitions_to_idl(self.partitions, self.time_partition, bindings) + p = partitions_to_idl(self.partitions, bindings) + tp = None + if self.time_partition: + tp = self.time_partition.to_flyte_idl(bindings) i = art_id.ArtifactID( artifact_key=ak, partitions=p, + time_partition=tp, ) aq = art_id.ArtifactQuery( @@ -232,13 +238,15 @@ def __init__( other: Optional[timedelta] = None, ): if isinstance(value, str): - value = art_id.LabelValue(static_value=value) + raise ValueError(f"value to a time partition shouldn't be a str {value}") elif isinstance(value, datetime.datetime): - value = art_id.LabelValue(static_value=f"{value}") + t = Timestamp() + t.FromDatetime(value) + value = art_id.LabelValue(time_value=t) elif isinstance(value, art_id.InputBindingData): value = art_id.LabelValue(input_binding=value) # else should already be a LabelValue or None - self.value = value + self.value: art_id.LabelValue = value self.op = op self.other = other self.reference_artifact: Optional[Artifact] = None @@ -253,15 +261,11 @@ def __sub__(self, other: timedelta) -> TimePartition: tp.reference_artifact = self.reference_artifact return tp - def truncate_to_day(self): - # raise NotImplementedError("Not implemented yet") - return self - - def get_idl_partitions_for_trigger(self, bindings: typing.List[Artifact]) -> art_id.Partitions: + def get_idl_partitions_for_trigger(self, bindings: typing.List[Artifact]) -> art_id.TimePartition: if not self.reference_artifact or (self.reference_artifact and self.reference_artifact not in bindings): # basically if there's no reference artifact, or if the reference artifact isn't # in the list of triggers, then treat it like normal. - return art_id.Partitions(value={TIME_PARTITION: self.value}) + return art_id.TimePartition(value=self.value) elif self.reference_artifact in bindings: idx = bindings.index(self.reference_artifact) transform = None @@ -270,55 +274,35 @@ def get_idl_partitions_for_trigger(self, bindings: typing.List[Artifact]) -> art lv = art_id.LabelValue( triggered_binding=art_id.ArtifactBindingData( index=idx, - partition_key=TIME_PARTITION, + bind_to_time_partition=True, transform=transform, ) ) - return art_id.Partitions(value={TIME_PARTITION: lv}) + return art_id.TimePartition(value=lv) # investigate if this happens, if not, remove. else logger.warning(f"Investigate - time partition in trigger with unhandled reference artifact {self}") - return art_id.Partitions(value={TIME_PARTITION: self.value}) + raise ValueError("Time partition reference artifact not found in ") + # return art_id.Partitions(value={TIME_PARTITION: self.value}) - def to_flyte_idl(self, bindings: Optional[typing.List[Artifact]] = None) -> Optional[art_id.Partitions]: + def to_flyte_idl(self, bindings: Optional[typing.List[Artifact]] = None) -> Optional[art_id.TimePartition]: if bindings and len(bindings) > 0: return self.get_idl_partitions_for_trigger(bindings) if not self.value: # This is only for triggers - the backend needs to know of the existence of a time partition - return art_id.Partitions(value={TIME_PARTITION: art_id.LabelValue(static_value="")}) + return art_id.TimePartition() - return art_id.Partitions(value={TIME_PARTITION: self.value}) - - -def merge_idl_partitions( - p_idl: Optional[art_id.Partitions], time_p_idl: Optional[art_id.Partitions] -) -> Optional[art_id.Partitions]: - if not p_idl and not time_p_idl: - return None - p = {} - if p_idl and p_idl.value: - p.update(p_idl.value) - if time_p_idl and time_p_idl.value: - p.update(time_p_idl.value) - - return art_id.Partitions(value=p) if p else None + return art_id.TimePartition(value=self.value) def partitions_to_idl( partitions: Optional[Partitions], - time_partition: Optional[TimePartition], bindings: Optional[typing.List[Artifact]] = None, ) -> Optional[art_id.Partitions]: - partition_idl = None if partitions: - partition_idl = partitions.to_flyte_idl(bindings) - - time_p_idl = None - if time_partition: - time_p_idl = time_partition.to_flyte_idl(bindings) + return partitions.to_flyte_idl(bindings) - merged = merge_idl_partitions(partition_idl, time_p_idl) - return merged + return None class Partition(object): @@ -378,7 +362,7 @@ def get_idl_partitions_for_trigger( if not v.reference_artifact or ( v.reference_artifact and v.reference_artifact is self.reference_artifact - and not v.reference_artifact in bindings + and v.reference_artifact not in bindings ): # consider changing condition to just check for static value p[k] = art_id.LabelValue(static_value=v.value.static_value) @@ -483,10 +467,8 @@ def __init__( :param short_description: A description of the Artifact. TODO: Additional fields to come: figure out sources, and also add metadata (cards). """ - if partition_keys and TIME_PARTITION in partition_keys: - # See note on TIME_PARTITION above. If we can figure out a nice uri syntax for time partition then remove - raise ValueError("cannot use 'ds' as a partition name, just use time partitions") - + if not name: + raise ValueError("Can't instantiate an Artifact without a name.") self.project = project self.domain = domain self.name = name @@ -543,12 +525,16 @@ def time_partition(self) -> TimePartition: return self._time_partition def __str__(self): + tp_str = f" time partition={self.time_partition}\n" if self.time_partitioned else "" return ( f"Artifact: project={self.project}, domain={self.domain}, name={self.name}, version={self.version}\n" f" name={self.name}\n" f" partitions={self.partitions}\n" + f"{tp_str}" f" tags={self.tags}\n" - f" literal_type={self.literal_type}, literal={self.literal})" + f" literal_type=" + f"{self.literal_type}, " + f"literal={self.literal})" ) def __repr__(self): @@ -663,26 +649,36 @@ def artifact_id(self) -> Optional[art_id.ArtifactID]: @property def as_artifact_id(self) -> art_id.ArtifactID: + # This property is used when you want to ensure that this is a materialized artifact, all fields are known. if self.name is None or self.project is None or self.domain is None or self.version is None: raise ValueError("Cannot create artifact id without name, project, domain, version") return self.to_flyte_idl().artifact_id def embed_as_query( - self, bindings: typing.List[Artifact], partition: Optional[str] = None, expr: Optional[str] = None + self, + bindings: typing.List[Artifact], + partition: Optional[str] = None, + bind_to_time_partition: Optional[bool] = None, + expr: Optional[str] = None, ) -> art_id.ArtifactQuery: """ This should only be called in the context of a Trigger :param bindings: The list of artifacts in trigger_on :param partition: Can embed a time partition + :param bind_to_time_partition: Set to true if you want to bind to a time partition :param expr: Only valid if there's a time partition. """ # Find self in the list, raises ValueError if not there. idx = bindings.index(self) aq = art_id.ArtifactQuery( binding=art_id.ArtifactBindingData( - index=idx, partition_key=partition, transform=str(expr) if expr and partition else None + index=idx, + partition_key=partition, + bind_to_time_partition=bind_to_time_partition, + transform=str(expr) if expr and (partition or bind_to_time_partition) else None, ) ) + return aq def to_flyte_idl(self) -> artifacts_pb2.Artifact: @@ -691,7 +687,8 @@ def to_flyte_idl(self) -> artifacts_pb2.Artifact: This is here instead of translator because it's in the interface, a relatively simple proto object that's exposed to the user. """ - p = partitions_to_idl(self.partitions, self.time_partition if self.time_partitioned else None) + p = partitions_to_idl(self.partitions) + tp = self.time_partition.to_flyte_idl() if self.time_partitioned else None return artifacts_pb2.Artifact( artifact_id=art_id.ArtifactID( @@ -702,6 +699,7 @@ def to_flyte_idl(self) -> artifacts_pb2.Artifact: ), version=self.version, partitions=p, + time_partition=tp, ), spec=artifacts_pb2.ArtifactSpec(), tags=self.tags, @@ -717,9 +715,18 @@ def as_create_request(self) -> artifacts_pb2.CreateArtifactRequest: value=self.literal, type=self.literal_type, ) - partitions = partitions_to_idl(self.partitions, self.time_partition) - tag = self.tags[0] if self.tags else None - return artifacts_pb2.CreateArtifactRequest(artifact_key=ak, spec=spec, partitions=partitions, tag=tag) + partitions = partitions_to_idl(self.partitions) + + tp = None + if self._time_partition: + tv = self.time_partition.value.time_value + if not tv: + raise Exception("missing time value") + tp = self.time_partition.value.time_value + + return artifacts_pb2.CreateArtifactRequest( + artifact_key=ak, spec=spec, partitions=partitions, time_partition_value=tp + ) @classmethod def from_flyte_idl(cls, pb2: artifacts_pb2.Artifact) -> Artifact: @@ -741,17 +748,22 @@ def from_flyte_idl(cls, pb2: artifacts_pb2.Artifact) -> Artifact: if len(pb2.artifact_id.partitions.value) > 0: # static values should be the only ones set since currently we don't from_flyte_idl # anything that's not a materialized artifact. - if TIME_PARTITION in pb2.artifact_id.partitions.value: - a._time_partition = TimePartition(pb2.artifact_id.partitions.value[TIME_PARTITION].static_value) - a._time_partition.reference_artifact = a + # if TIME_PARTITION in pb2.artifact_id.partitions.value: + # a._time_partition = TimePartition(pb2.artifact_id.partitions.value[TIME_PARTITION].static_value) + # a._time_partition.reference_artifact = a a._partitions = Partitions( partitions={ k: Partition(value=v, name=k) for k, v in pb2.artifact_id.partitions.value.items() - if k != TIME_PARTITION + # if k != TIME_PARTITION } ) a.partitions.reference_artifact = a + if pb2.artifact_id.HasField("time_partition"): + ts = pb2.artifact_id.time_partition.value.time_value + dt = ts.ToDatetime() + a._time_partition = TimePartition(dt) + a._time_partition.reference_artifact = a return a diff --git a/flytekit/core/base_task.py b/flytekit/core/base_task.py index 21c7279c53..b9171f54b8 100644 --- a/flytekit/core/base_task.py +++ b/flytekit/core/base_task.py @@ -628,12 +628,8 @@ def dispatch_execute( # TODO: Logger should auto inject the current context information to indicate if the task is running within # a workflow or a subworkflow etc logger.info(f"Invoking {self.name} with inputs: {native_inputs}") - try: - with timeit("Execute user level code"): - native_outputs = self.execute(**native_inputs) - except Exception as e: - logger.exception(f"Exception when executing {e}") - raise e + with timeit("Execute user level code"): + native_outputs = self.execute(**native_inputs) if inspect.iscoroutine(native_outputs): # If native outputs is a coroutine, then this is an eager workflow. diff --git a/flytekit/core/container_task.py b/flytekit/core/container_task.py index d606675299..1b078f83a7 100644 --- a/flytekit/core/container_task.py +++ b/flytekit/core/container_task.py @@ -124,12 +124,10 @@ def _get_container(self, settings: SerializationSettings) -> _task_model.Contain args=self._args, data_loading_config=self._get_data_loading_config(), environment=env, - storage_request=self.resources.requests.storage, ephemeral_storage_request=self.resources.requests.ephemeral_storage, cpu_request=self.resources.requests.cpu, gpu_request=self.resources.requests.gpu, memory_request=self.resources.requests.mem, - storage_limit=self.resources.limits.storage, ephemeral_storage_limit=self.resources.limits.ephemeral_storage, cpu_limit=self.resources.limits.cpu, gpu_limit=self.resources.limits.gpu, diff --git a/flytekit/core/context_manager.py b/flytekit/core/context_manager.py index 833c7d8562..0b21ab24d9 100644 --- a/flytekit/core/context_manager.py +++ b/flytekit/core/context_manager.py @@ -350,7 +350,11 @@ def __getattr__(self, item: str) -> _GroupSecrets: return self._GroupSecrets(item, self) def get( - self, group: str, key: Optional[str] = None, group_version: Optional[str] = None, encode_mode: str = "r" + self, + group: Optional[str] = None, + key: Optional[str] = None, + group_version: Optional[str] = None, + encode_mode: str = "r", ) -> str: """ Retrieves a secret using the resolution order -> Env followed by file. If not found raises a ValueError @@ -361,7 +365,7 @@ def get( fpath = self.get_secrets_file(group, key, group_version) v = os.environ.get(env_var) if v is not None: - return v + return v.strip() if os.path.exists(fpath): with open(fpath, encode_mode) as f: return f.read().strip() @@ -370,7 +374,9 @@ def get( f"in Env Var:{env_var} and FilePath: {fpath}" ) - def get_secrets_env_var(self, group: str, key: Optional[str] = None, group_version: Optional[str] = None) -> str: + def get_secrets_env_var( + self, group: Optional[str] = None, key: Optional[str] = None, group_version: Optional[str] = None + ) -> str: """ Returns a string that matches the ENV Variable to look for the secrets """ @@ -378,7 +384,9 @@ def get_secrets_env_var(self, group: str, key: Optional[str] = None, group_versi l = [k.upper() for k in filter(None, (group, group_version, key))] return f"{self._env_prefix}{'_'.join(l)}" - def get_secrets_file(self, group: str, key: Optional[str] = None, group_version: Optional[str] = None) -> str: + def get_secrets_file( + self, group: Optional[str] = None, key: Optional[str] = None, group_version: Optional[str] = None + ) -> str: """ Returns a path that matches the file to look for the secrets """ @@ -388,8 +396,10 @@ def get_secrets_file(self, group: str, key: Optional[str] = None, group_version: return os.path.join(self._base_dir, *l) @staticmethod - def check_group_key(group: str): - if group is None or group == "": + def check_group_key(group: Optional[str]): + from flytekit.configuration.plugin import get_plugin + + if get_plugin().secret_requires_group() and (group is None or group == ""): raise ValueError("secrets group is a mandatory field.") @@ -662,7 +672,7 @@ def get_deck(self) -> typing.Union[str, "IPython.core.display.HTML"]: # type:ig my_task(...) ctx.get_deck() - OR if you wish to explicity display + OR if you wish to explicitly display .. code-block:: python diff --git a/flytekit/core/data_persistence.py b/flytekit/core/data_persistence.py index d1651e8548..1579ef3f6b 100644 --- a/flytekit/core/data_persistence.py +++ b/flytekit/core/data_persistence.py @@ -33,7 +33,7 @@ from flytekit.configuration import DataConfig from flytekit.core.local_fsspec import FlyteLocalFileSystem from flytekit.core.utils import timeit -from flytekit.exceptions.user import FlyteAssertion +from flytekit.exceptions.user import FlyteAssertion, FlyteValueException from flytekit.interfaces.random import random from flytekit.loggers import logger @@ -254,6 +254,8 @@ def get(self, from_path: str, to_path: str, recursive: bool = False, **kwargs): return to_path except OSError as oe: logger.debug(f"Error in getting {from_path} to {to_path} rec {recursive} {oe}") + if not file_system.exists(from_path): + raise FlyteValueException(from_path, "File not found") file_system = self.get_filesystem(get_protocol(from_path), anonymous=True) if file_system is not None: logger.debug(f"Attempting anonymous get with {file_system}") diff --git a/flytekit/core/interface.py b/flytekit/core/interface.py index 173f004cd9..b5b2e9eb0e 100644 --- a/flytekit/core/interface.py +++ b/flytekit/core/interface.py @@ -371,35 +371,28 @@ def transform_variable_map( def detect_artifact( ts: typing.Tuple[typing.Any], -) -> Tuple[Optional[art_id.ArtifactID], Optional[art_id.ArtifactTag]]: +) -> Optional[art_id.ArtifactID]: """ If the user wishes to control how Artifacts are created (i.e. naming them, etc.) this is where we pick it up and - store it in the interface. There are two fields, the ID and a tag. For this to take effect, the name field - must have been specified. + store it in the interface. """ for t in ts: - if isinstance(t, Artifact) and t.name: - tag = None - if t.tags: - tag = art_id.ArtifactTag(value=art_id.LabelValue(static_value=t.tags[0])) - - artifact_id = t.to_flyte_idl().artifact_id - - return artifact_id, tag + if isinstance(t, Artifact): + id_spec = t() + return id_spec.to_partial_artifact_id() elif isinstance(t, ArtifactIDSpecification): artifact_id = t.to_partial_artifact_id() - tag = None - if t.artifact.tags: - tag = art_id.ArtifactTag(value=art_id.LabelValue(static_value=t.artifact.tags[0])) - return artifact_id, tag + return artifact_id - return None, None + return None def transform_type(x: type, description: Optional[str] = None) -> _interface_models.Variable: - artifact_id, tag = detect_artifact(get_args(x)) + artifact_id = detect_artifact(get_args(x)) + if artifact_id: + logger.debug(f"Found artifact id spec: {artifact_id}") return _interface_models.Variable( - type=TypeEngine.to_literal_type(x), description=description, artifact_partial_id=artifact_id, artifact_tag=tag + type=TypeEngine.to_literal_type(x), description=description, artifact_partial_id=artifact_id ) diff --git a/flytekit/core/node.py b/flytekit/core/node.py index b9edddeec0..6c84877c90 100644 --- a/flytekit/core/node.py +++ b/flytekit/core/node.py @@ -210,10 +210,6 @@ def _convert_resource_overrides( if resources.gpu is not None: resource_entries.append(_resources_model.ResourceEntry(_resources_model.ResourceName.GPU, resources.gpu)) - if resources.storage is not None: - resource_entries.append( - _resources_model.ResourceEntry(_resources_model.ResourceName.STORAGE, resources.storage) - ) if resources.ephemeral_storage is not None: resource_entries.append( _resources_model.ResourceEntry( diff --git a/flytekit/core/promise.py b/flytekit/core/promise.py index 46f95f5147..f15923ab5f 100644 --- a/flytekit/core/promise.py +++ b/flytekit/core/promise.py @@ -930,7 +930,7 @@ def create_and_link_node_from_remote( :param entity: RemoteEntity :param _inputs_not_allowed: Set of all variable names that should not be provided when using this entity. Useful for Launchplans with `fixed` inputs - :param _ignorable_inputs: Set of all variable names that are optional, but if provided will be overriden. Useful + :param _ignorable_inputs: Set of all variable names that are optional, but if provided will be overridden. Useful for launchplans with `default` inputs :param kwargs: Dict[str, Any] default inputs passed from the user to this entity. Can be promises. :return: Optional[Union[Tuple[Promise], Promise, VoidPromise]] diff --git a/flytekit/core/python_auto_container.py b/flytekit/core/python_auto_container.py index 2f9d8417fd..c43e3d4d14 100644 --- a/flytekit/core/python_auto_container.py +++ b/flytekit/core/python_auto_container.py @@ -196,12 +196,10 @@ def _get_container(self, settings: SerializationSettings) -> _task_model.Contain args=self.get_command(settings=settings), data_loading_config=None, environment=env, - storage_request=self.resources.requests.storage, ephemeral_storage_request=self.resources.requests.ephemeral_storage, cpu_request=self.resources.requests.cpu, gpu_request=self.resources.requests.gpu, memory_request=self.resources.requests.mem, - storage_limit=self.resources.limits.storage, ephemeral_storage_limit=self.resources.limits.ephemeral_storage, cpu_limit=self.resources.limits.cpu, gpu_limit=self.resources.limits.gpu, diff --git a/flytekit/core/python_customized_container_task.py b/flytekit/core/python_customized_container_task.py index 07493886a2..a3d89b0979 100644 --- a/flytekit/core/python_customized_container_task.py +++ b/flytekit/core/python_customized_container_task.py @@ -165,12 +165,10 @@ def get_container(self, settings: SerializationSettings) -> _task_model.Containe args=self.get_command(settings=settings), data_loading_config=None, environment=env, - storage_request=self.resources.requests.storage, ephemeral_storage_request=self.resources.requests.ephemeral_storage, cpu_request=self.resources.requests.cpu, gpu_request=self.resources.requests.gpu, memory_request=self.resources.requests.mem, - storage_limit=self.resources.limits.storage, ephemeral_storage_limit=self.resources.limits.ephemeral_storage, cpu_limit=self.resources.limits.cpu, gpu_limit=self.resources.limits.gpu, diff --git a/flytekit/core/resources.py b/flytekit/core/resources.py index 62b880f6ed..2e8388f986 100644 --- a/flytekit/core/resources.py +++ b/flytekit/core/resources.py @@ -29,7 +29,6 @@ class Resources(object): cpu: Optional[str] = None mem: Optional[str] = None gpu: Optional[str] = None - storage: Optional[str] = None ephemeral_storage: Optional[str] = None def __post_init__(self): @@ -42,7 +41,6 @@ def _check_none_or_str(value): _check_none_or_str(self.cpu) _check_none_or_str(self.mem) _check_none_or_str(self.gpu) - _check_none_or_str(self.storage) _check_none_or_str(self.ephemeral_storage) @@ -64,8 +62,6 @@ def _convert_resources_to_resource_entries(resources: Resources) -> List[_Resour resource_entries.append(_ResourceEntry(name=_ResourceName.MEMORY, value=resources.mem)) if resources.gpu is not None: resource_entries.append(_ResourceEntry(name=_ResourceName.GPU, value=resources.gpu)) - if resources.storage is not None: - resource_entries.append(_ResourceEntry(name=_ResourceName.STORAGE, value=resources.storage)) if resources.ephemeral_storage is not None: resource_entries.append(_ResourceEntry(name=_ResourceName.EPHEMERAL_STORAGE, value=resources.ephemeral_storage)) return resource_entries diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 4bcbbeceef..4220d6e7b6 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -6,6 +6,7 @@ import datetime as _datetime import enum import inspect +import json import json as _json import mimetypes import textwrap @@ -349,20 +350,61 @@ def assert_type(self, expected_type: Type[DataClassJsonMixin], v: T): for f in dataclasses.fields(expected_type): expected_fields_dict[f.name] = f.type - for f in dataclasses.fields(type(v)): # type: ignore - original_type = f.type - expected_type = expected_fields_dict[f.name] + if isinstance(v, dict): + original_dict = v - if UnionTransformer.is_optional_type(original_type): - original_type = UnionTransformer.get_sub_type_in_optional(original_type) - if UnionTransformer.is_optional_type(expected_type): - expected_type = UnionTransformer.get_sub_type_in_optional(expected_type) + # Find the Optional keys in expected_fields_dict + optional_keys = {k for k, t in expected_fields_dict.items() if UnionTransformer.is_optional_type(t)} - val = v.__getattribute__(f.name) - if dataclasses.is_dataclass(val): - self.assert_type(expected_type, val) - elif original_type != expected_type: - raise TypeTransformerFailedError(f"Type of Val '{original_type}' is not an instance of {expected_type}") + # Remove the Optional keys from the keys of original_dict + original_key = set(original_dict.keys()) - optional_keys + expected_key = set(expected_fields_dict.keys()) - optional_keys + + # Check if original_key is missing any keys from expected_key + missing_keys = expected_key - original_key + if missing_keys: + raise TypeTransformerFailedError( + f"The original fields are missing the following keys from the dataclass fields: {list(missing_keys)}" + ) + + # Check if original_key has any extra keys that are not in expected_key + extra_keys = original_key - expected_key + if extra_keys: + raise TypeTransformerFailedError( + f"The original fields have the following extra keys that are not in dataclass fields: {list(extra_keys)}" + ) + + for k, v in original_dict.items(): + if k in expected_fields_dict: + if isinstance(v, dict): + self.assert_type(expected_fields_dict[k], v) + else: + expected_type = expected_fields_dict[k] + original_type = type(v) + if UnionTransformer.is_optional_type(expected_type): + expected_type = UnionTransformer.get_sub_type_in_optional(expected_type) + if original_type != expected_type: + raise TypeTransformerFailedError( + f"Type of Val '{original_type}' is not an instance of {expected_type}" + ) + + else: + for f in dataclasses.fields(type(v)): # type: ignore + original_type = f.type + expected_type = expected_fields_dict[f.name] + + if UnionTransformer.is_optional_type(original_type): + original_type = UnionTransformer.get_sub_type_in_optional(original_type) + if UnionTransformer.is_optional_type(expected_type): + expected_type = UnionTransformer.get_sub_type_in_optional(expected_type) + + val = v.__getattribute__(f.name) + if dataclasses.is_dataclass(val): + self.assert_type(expected_type, val) + elif original_type != expected_type: + raise TypeTransformerFailedError( + f"Type of Val '{original_type}' is not an instance of {expected_type}" + ) def get_literal_type(self, t: Type[T]) -> LiteralType: """ @@ -424,6 +466,10 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: return _type_models.LiteralType(simple=_type_models.SimpleType.STRUCT, metadata=schema, structure=ts) def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: + if isinstance(python_val, dict): + json_str = json.dumps(python_val) + return Literal(scalar=Scalar(generic=_json_format.Parse(json_str, _struct.Struct()))) + if not dataclasses.is_dataclass(python_val): raise TypeTransformerFailedError( f"{type(python_val)} is not of type @dataclass, only Dataclasses are supported for " @@ -1802,7 +1848,7 @@ def _check_and_covert_float(lv: Literal) -> float: def _check_and_convert_void(lv: Literal) -> None: if lv.scalar.none_type is None: - raise TypeTransformerFailedError(f"Cannot conver literal {lv} to None") + raise TypeTransformerFailedError(f"Cannot convert literal {lv} to None") return None diff --git a/flytekit/core/utils.py b/flytekit/core/utils.py index c24d53c4e0..17cdfb3de9 100644 --- a/flytekit/core/utils.py +++ b/flytekit/core/utils.py @@ -62,20 +62,16 @@ def _get_container_definition( command: List[str], args: Optional[List[str]] = None, data_loading_config: Optional["task_models.DataLoadingConfig"] = None, - storage_request: Optional[str] = None, ephemeral_storage_request: Optional[str] = None, cpu_request: Optional[str] = None, gpu_request: Optional[str] = None, memory_request: Optional[str] = None, - storage_limit: Optional[str] = None, ephemeral_storage_limit: Optional[str] = None, cpu_limit: Optional[str] = None, gpu_limit: Optional[str] = None, memory_limit: Optional[str] = None, environment: Optional[Dict[str, str]] = None, ) -> "task_models.Container": - storage_limit = storage_limit - storage_request = storage_request ephemeral_storage_limit = ephemeral_storage_limit ephemeral_storage_request = ephemeral_storage_request cpu_limit = cpu_limit @@ -89,10 +85,6 @@ def _get_container_definition( # TODO: Use convert_resources_to_resource_model instead of manually fixing the resources. requests = [] - if storage_request: - requests.append( - task_models.Resources.ResourceEntry(task_models.Resources.ResourceName.STORAGE, storage_request) - ) if ephemeral_storage_request: requests.append( task_models.Resources.ResourceEntry( @@ -108,8 +100,6 @@ def _get_container_definition( requests.append(task_models.Resources.ResourceEntry(task_models.Resources.ResourceName.MEMORY, memory_request)) limits = [] - if storage_limit: - limits.append(task_models.Resources.ResourceEntry(task_models.Resources.ResourceName.STORAGE, storage_limit)) if ephemeral_storage_limit: limits.append( task_models.Resources.ResourceEntry( diff --git a/flytekit/core/workflow.py b/flytekit/core/workflow.py index 2ecc7ab667..7a54ef6de5 100644 --- a/flytekit/core/workflow.py +++ b/flytekit/core/workflow.py @@ -286,7 +286,6 @@ def __call__(self, *args, **kwargs) -> Union[Tuple[Promise], Promise, VoidPromis if self.on_failure.python_interface and "err" in self.on_failure.python_interface.inputs: input_kwargs["err"] = FlyteError(failed_node_id="", message=str(exc)) self.on_failure(**input_kwargs) - exc.args = (f"Encountered error while executing workflow '{self.name}':\n {exc}", *exc.args[1:]) raise exc def execute(self, **kwargs): @@ -456,7 +455,7 @@ def execute(self, **kwargs): raise FlyteValidationException(f"Workflow not ready, wf is currently {self}") # Create a map that holds the outputs of each node. - intermediate_node_outputs: Dict[Node, Dict[str, Promise]] = {GLOBAL_START_NODE: {}} + intermediate_node_outputs: Dict[Node, Dict[str, Promise]] = {GLOBAL_START_NODE: {}} # type: ignore # Start things off with the outputs of the global input node, i.e. the inputs to the workflow. # local_execute should've already ensured that all the values in kwargs are Promise objects diff --git a/flytekit/extend/backend/base_agent.py b/flytekit/extend/backend/base_agent.py index 72bbcb698e..eb8e7562b6 100644 --- a/flytekit/extend/backend/base_agent.py +++ b/flytekit/extend/backend/base_agent.py @@ -6,7 +6,7 @@ from abc import ABC from collections import OrderedDict from functools import partial -from types import FrameType +from types import FrameType, coroutine import grpc from flyteidl.admin.agent_pb2 import ( @@ -138,7 +138,8 @@ def convert_to_flyte_state(state: str) -> State: Convert the state from the agent to the state in flyte. """ state = state.lower() - if state in ["failed", "timedout", "canceled"]: + # timedout is the state of Databricks job. https://docs.databricks.com/en/workflows/jobs/jobs-2.0-api.html#runresultstate + if state in ["failed", "timeout", "timedout", "canceled"]: return RETRYABLE_FAILURE elif state in ["done", "succeeded", "success"]: return SUCCEEDED @@ -158,15 +159,27 @@ def get_agent_secret(secret_key: str) -> str: return flytekit.current_context().secrets.get(secret_key) +def _get_grpc_context() -> grpc.ServicerContext: + from unittest.mock import MagicMock + + grpc_ctx = MagicMock(spec=grpc.ServicerContext) + return grpc_ctx + + class AsyncAgentExecutorMixin: """ This mixin class is used to run the agent task locally, and it's only used for local execution. Task should inherit from this class if the task can be run in the agent. + It can handle asynchronous tasks and synchronous tasks. + Asynchronous tasks are tasks that take a long time to complete, such as running a query. + Synchronous tasks run quickly and can return their results instantly. Sending a prompt to ChatGPT and getting a response, or retrieving some metadata from a backend system. """ - _is_canceled = None - _agent = None - _entity = None + _clean_up_task: coroutine = None + _agent: AgentBase = None + _entity: PythonTask = None + _ctx: FlyteContext = FlyteContext.current_context() + _grpc_ctx: grpc.ServicerContext = _get_grpc_context() def execute(self, **kwargs) -> typing.Any: ctx = FlyteContext.current_context() @@ -180,6 +193,13 @@ def execute(self, **kwargs) -> typing.Any: self._agent = AgentRegistry.get_agent(task_template.type) res = asyncio.run(self._create(task_template, output_prefix, kwargs)) + + # If the task is synchronous, the agent will return the output from the resource literals. + if res.HasField("resource"): + if res.resource.state != SUCCEEDED: + raise FlyteUserException(f"Failed to run the task {self._entity.name}") + return LiteralMap.from_flyte_idl(res.resource.outputs) + res = asyncio.run(self._get(resource_meta=res.resource_meta)) if res.resource.state != SUCCEEDED: @@ -198,13 +218,11 @@ async def _create( self, task_template: TaskTemplate, output_prefix: str, inputs: typing.Dict[str, typing.Any] = None ) -> CreateTaskResponse: ctx = FlyteContext.current_context() - grpc_ctx = _get_grpc_context() # Convert python inputs to literals literals = inputs or {} for k, v in inputs.items(): literals[k] = TypeEngine.to_literal(ctx, v, type(v), self._entity.interface.inputs[k].type) - literal_map = LiteralMap(literals) if isinstance(self, PythonFunctionTask): @@ -215,9 +233,9 @@ async def _create( task_template = render_task_template(task_template, output_prefix) if self._agent.asynchronous: - res = await self._agent.async_create(grpc_ctx, output_prefix, task_template, literal_map) + res = await self._agent.async_create(self._grpc_ctx, output_prefix, task_template, literal_map) else: - res = self._agent.create(grpc_ctx, output_prefix, task_template, literal_map) + res = self._agent.create(self._grpc_ctx, output_prefix, task_template, literal_map) signal.signal(signal.SIGINT, partial(self.signal_handler, res.resource_meta)) # type: ignore return res @@ -225,7 +243,6 @@ async def _create( async def _get(self, resource_meta: bytes) -> GetTaskResponse: state = RUNNING grpc_ctx = _get_grpc_context() - res = State.PENDING progress = Progress(transient=True) task = progress.add_task(f"[cyan]Running Task {self._entity.name}...", total=None) @@ -235,22 +252,24 @@ async def _get(self, resource_meta: bytes) -> GetTaskResponse: time.sleep(1) if self._agent.asynchronous: res = await self._agent.async_get(grpc_ctx, resource_meta) - if self._is_canceled: - await self._is_canceled + if self._clean_up_task: + await self._clean_up_task sys.exit(1) else: res = self._agent.get(grpc_ctx, resource_meta) state = res.resource.state - logger.info(f"Task state: {state}, State message: {res.resource.message}") + progress.print(f"Task state: {State.Name(state)}, State message: {res.resource.message}") + if hasattr(res.resource, "log_links"): + for link in res.resource.log_links: + progress.print(f"{link.name}: {link.uri}") return res def signal_handler(self, resource_meta: bytes, signum: int, frame: FrameType) -> typing.Any: - grpc_ctx = _get_grpc_context() if self._agent.asynchronous: - if self._is_canceled is None: - self._is_canceled = asyncio.create_task(self._agent.async_delete(grpc_ctx, resource_meta)) + if self._clean_up_task is None: + self._clean_up_task = asyncio.create_task(self._agent.async_delete(self._grpc_ctx, resource_meta)) else: - self._agent.delete(grpc_ctx, resource_meta) + self._agent.delete(self._grpc_ctx, resource_meta) sys.exit(1) diff --git a/flytekit/extras/accelerators.py b/flytekit/extras/accelerators.py index 3615f32bdb..6f3fac9ffd 100644 --- a/flytekit/extras/accelerators.py +++ b/flytekit/extras/accelerators.py @@ -1,3 +1,93 @@ +""" +Specifying Accelerators +========================== + +.. tags:: MachineLearning, Advanced, Hardware + +Flyte allows you to specify `gpu` resources for a given task. However, in some cases, you may want to use a different +accelerator type, such as TPU, specific variations of GPUs, or fractional GPUs. You can configure the Flyte backend to +use your preferred accelerators, and those who write workflow code can import the `flytekit.extras.accelerators` module +to specify an accelerator in the task decorator. + + +If you want to use a specific GPU device, you can pass the device name directly to the task decorator, e.g.: + +.. code-block:: + + @task( + limits=Resources(gpu="1"), + accelerator=GPUAccelerator("nvidia-tesla-v100"), + ) + def my_task() -> None: + ... + + +Base Classes +------------ +These classes can be used to create custom accelerator type constants. For example, you can create a TPU accelerator. + + + +.. currentmodule:: flytekit.extras.accelerators + +.. autosummary:: + + BaseAccelerator + GPUAccelerator + MultiInstanceGPUAccelerator + +But, often, you may want to use a well known accelerator type, and to simplify this, flytekit provides a set of +predefined accelerator constants, as described in the next section. + + +Predefined Accelerator Constants +-------------------------------- + +The `flytekit.extras.accelerators` module provides some constants for known accelerators, listed below, but this is not +a complete list. If you know the name of the accelerator, you can pass the string name to the task decorator directly. + +If using the constants, you can import them directly from the module, e.g.: + +.. code-block:: + + from flytekit.extras.accelerators import T4 + + @task( + limits=Resources(gpu="1"), + accelerator=T4, + ) + def my_task() -> None: + ... + +if you want to use a fractional GPU, you can use the ``partitioned`` method on the accelerator constant, e.g.: + +.. code-block:: + + from flytekit.extras.accelerators import A100 + + @task( + limits=Resources(gpu="1"), + accelerator=A100.partition_2g_10gb, + ) + def my_task() -> None: + ... + +.. currentmodule:: flytekit.extras.accelerators + +.. autosummary:: + + A10G + L4 + K80 + M60 + P4 + P100 + T4 + V100 + A100 + A100_80GB + +""" import abc import copy from typing import ClassVar, Generic, Optional, Type, TypeVar @@ -9,12 +99,22 @@ class BaseAccelerator(abc.ABC, Generic[T]): + """ + Base class for all accelerator types. This class is not meant to be instantiated directly. + """ + @abc.abstractmethod def to_flyte_idl(self) -> T: ... class GPUAccelerator(BaseAccelerator): + """ + Class that represents a GPU accelerator. The class can be instantiated with any valid GPU device name, but + it is recommended to use one of the pre-defined constants below, as name has to match the name of the device + configured on the cluster. + """ + def __init__(self, device: str) -> None: self._device = device @@ -22,17 +122,46 @@ def to_flyte_idl(self) -> tasks_pb2.GPUAccelerator: return tasks_pb2.GPUAccelerator(device=self._device) +#: use this constant to specify that the task should run on an +#: `NVIDIA A10 Tensor Core GPU `_ A10G = GPUAccelerator("nvidia-a10g") + +#: use this constant to specify that the task should run on an +#: `NVIDIA L4 Tensor Core GPU `_ L4 = GPUAccelerator("nvidia-l4-vws") + +#: use this constant to specify that the task should run on an +#: `NVIDIA Tesla K80 GPU `_ K80 = GPUAccelerator("nvidia-tesla-k80") + +#: use this constant to specify that the task should run on an +#: `NVIDIA Tesla M60 GPU `_ M60 = GPUAccelerator("nvidia-tesla-m60") + +#: use this constant to specify that the task should run on an +#: `NVIDIA Tesla P4 GPU `_ P4 = GPUAccelerator("nvidia-tesla-p4") + +#: use this constant to specify that the task should run on an +#: `NVIDIA Tesla P100 GPU `_ P100 = GPUAccelerator("nvidia-tesla-p100") + +#: use this constant to specify that the task should run on an +#: `NVIDIA T4 Tensor Core GPU `_ T4 = GPUAccelerator("nvidia-tesla-t4") + +#: use this constant to specify that the task should run on an +#: `NVIDIA Tesla V100 GPU `_ V100 = GPUAccelerator("nvidia-tesla-v100") class MultiInstanceGPUAccelerator(BaseAccelerator): + """ + Base class for all multi-instance GPU accelerator types. It is recommended to use one of the pre-defined constants + below, as name has to match the name of the device configured on the cluster. + For example, to specify a 10GB partition of an A100 GPU, use ``A100.partition_2g_10gb``. + """ + device: ClassVar[str] _partition_size: Optional[str] @@ -65,13 +194,44 @@ class _A100_Base(MultiInstanceGPUAccelerator): class _A100(_A100_Base): + """ + Class that represents an `NVIDIA A100 GPU `_. It is possible + to specify a partition of an A100 GPU by using the provided partitions on the class. For example, to specify a + 10GB partition, use ``A100.partition_2g_10gb``. + Refer to `Partitioned GPUs `_ + """ + partition_1g_5gb = _A100_Base.partitioned("1g.5gb") + """ + 5GB partition of an A100 GPU. + """ partition_2g_10gb = _A100_Base.partitioned("2g.10gb") + """ + 10GB partition of an A100 GPU - 2x5GB slices with 2/7th of the SM. + """ partition_3g_20gb = _A100_Base.partitioned("3g.20gb") + """ + 20GB partition of an A100 GPU - 4x5GB slices, with 3/7th fraction of SM (Streaming multiprocessor). + """ partition_4g_20gb = _A100_Base.partitioned("4g.20gb") + """ + 20GB partition of an A100 GPU - 4x5GB slices, with 4/7th fraction of SM. + """ partition_7g_40gb = _A100_Base.partitioned("7g.40gb") + """ + 40GB partition of an A100 GPU - 8x5GB slices, with 7/7th fraction of SM. + """ +#: Use this constant to specify that the task should run on an entire +#: `NVIDIA A100 GPU `_. Fractional partitions are also available. +#: +#: Use pre-defined partitions (as instance attributes). For example, to specify a 10GB partition, use +#: ``A100.partition_2g_10gb``. +#: All partitions are nested in the class as follows: +#: +#: .. autoclass:: _A100 +#: :members: A100 = _A100() @@ -80,11 +240,39 @@ class _A100_80GB_Base(MultiInstanceGPUAccelerator): class _A100_80GB(_A100_80GB_Base): + """ + Partitions of an `NVIDIA A100 80GB GPU `_. + """ + partition_1g_10gb = _A100_80GB_Base.partitioned("1g.10gb") + """ + 10GB partition of an A100 80GB GPU - 2x5GB slices with 1/7th of the SM. + """ partition_2g_20gb = _A100_80GB_Base.partitioned("2g.20gb") + """ + 2GB partition of an A100 80GB GPU - 4x5GB slices with 2/7th of the SM. + """ partition_3g_40gb = _A100_80GB_Base.partitioned("3g.40gb") + """ + 3GB partition of an A100 80GB GPU - 8x5GB slices with 3/7th of the SM. + """ partition_4g_40gb = _A100_80GB_Base.partitioned("4g.40gb") + """ + 4GB partition of an A100 80GB GPU - 8x5GB slices with 4/7th of the SM. + """ partition_7g_80gb = _A100_80GB_Base.partitioned("7g.80gb") + """ + 7GB partition of an A100 80GB GPU - 16x5GB slices with 7/7th of the SM. + """ +#: use this constant to specify that the task should run on an entire +#: `NVIDIA A100 80GB GPU `_. Fractional partitions are also available. +#: +#: Use pre-defined partitions (as instance attributes). For example, to specify a 10GB partition, use +#: ``A100.partition_2g_10gb``. +#: All available partitions are listed below: +#: +#: .. autoclass:: _A100_80GB +#: :members: A100_80GB = _A100_80GB() diff --git a/flytekit/extras/tasks/shell.py b/flytekit/extras/tasks/shell.py index 319a927743..57d7eb0109 100644 --- a/flytekit/extras/tasks/shell.py +++ b/flytekit/extras/tasks/shell.py @@ -34,7 +34,7 @@ class OutputLocation: location: typing.Union[os.PathLike, str] -def subproc_execute(command: List[str]) -> Tuple[str, str]: +def subproc_execute(command: typing.Union[List[str], str], **kwargs) -> Tuple[str, str]: """ Execute a command and capture its stdout and stderr. Useful for executing shell commands from within a python task. @@ -52,9 +52,18 @@ def subproc_execute(command: List[str]) -> Tuple[str, str]: guidance on specifying a container image in the task definition when using custom dependencies. """ + defaults = { + "stdout": subprocess.PIPE, + "stderr": subprocess.PIPE, + "text": True, + "check": True, + } + + kwargs = {**defaults, **kwargs} + try: # Execute the command and capture stdout and stderr - result = subprocess.run(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, check=True) + result = subprocess.run(command, **kwargs) # Access the stdout and stderr output return result.stdout, result.stderr diff --git a/flytekit/image_spec/image_spec.py b/flytekit/image_spec/image_spec.py index 570a2a2952..cef455acbb 100644 --- a/flytekit/image_spec/image_spec.py +++ b/flytekit/image_spec/image_spec.py @@ -29,6 +29,8 @@ class ImageSpec: env: environment variables of the image. registry: registry of the image. packages: list of python packages to install. + conda_packages: list of conda packages to install. + conda_channels: list of conda channels. requirements: path to the requirements.txt file. apt_packages: list of apt packages to install. cuda: version of cuda to install. @@ -47,6 +49,8 @@ class ImageSpec: env: Optional[typing.Dict[str, str]] = None registry: Optional[str] = None packages: Optional[List[str]] = None + conda_packages: Optional[List[str]] = None + conda_channels: Optional[List[str]] = None requirements: Optional[str] = None apt_packages: Optional[List[str]] = None cuda: Optional[str] = None @@ -193,6 +197,7 @@ def register(cls, builder_type: str, image_spec_builder: ImageSpecBuilder): cls._REGISTRY[builder_type] = image_spec_builder @classmethod + @lru_cache def build(cls, image_spec: ImageSpec): img_name = image_spec.image_name() if img_name in cls._BUILT_IMAGES or image_spec.exist(): diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py index f164ab7b25..0746e0f126 100644 --- a/flytekit/models/literals.py +++ b/flytekit/models/literals.py @@ -1,6 +1,6 @@ from datetime import datetime as _datetime from datetime import timezone as _timezone -from typing import Optional, Dict +from typing import Dict, Optional from flyteidl.core import literals_pb2 as _literals_pb2 from google.protobuf.struct_pb2 import Struct @@ -859,7 +859,7 @@ def __init__( collection: Optional[LiteralCollection] = None, map: Optional[LiteralMap] = None, hash: Optional[str] = None, - metadata: Optional[Dict[str, str]] = None, + metadata: Optional[Dict[str, str]] = None, ): """ This IDL message represents a literal value in the Flyte ecosystem. diff --git a/flytekit/models/security.py b/flytekit/models/security.py index 9af90a4b8a..a9ee7e7cb9 100644 --- a/flytekit/models/security.py +++ b/flytekit/models/security.py @@ -35,13 +35,15 @@ class MountType(Enum): Caution: May not be supported in all environments """ - group: str + group: Optional[str] = None key: Optional[str] = None group_version: Optional[str] = None mount_requirement: MountType = MountType.ANY def __post_init__(self): - if self.group is None: + from flytekit.configuration.plugin import get_plugin + + if get_plugin().secret_requires_group() and self.group is None: raise ValueError("Group is a required parameter") def to_flyte_idl(self) -> _sec.Secret: diff --git a/flytekit/models/task.py b/flytekit/models/task.py index 48a8abfde1..1da786ea6d 100644 --- a/flytekit/models/task.py +++ b/flytekit/models/task.py @@ -22,7 +22,6 @@ class ResourceName(object): CPU = _core_task.Resources.CPU GPU = _core_task.Resources.GPU MEMORY = _core_task.Resources.MEMORY - STORAGE = _core_task.Resources.STORAGE EPHEMERAL_STORAGE = _core_task.Resources.EPHEMERAL_STORAGE class ResourceEntry(_common.FlyteIdlEntity): diff --git a/flytekit/remote/backfill.py b/flytekit/remote/backfill.py index b36fc7919d..166f2e4745 100644 --- a/flytekit/remote/backfill.py +++ b/flytekit/remote/backfill.py @@ -43,7 +43,7 @@ def create_backfill_workflow( :param start_date: datetime generate a backfill starting at this datetime (exclusive) :param end_date: datetime generate a backfill ending at this datetime (inclusive) - :param for_lp: typing.Union[LaunchPlan, FlyteLaunchPlan] the backfill is generatd for this launchplan + :param for_lp: typing.Union[LaunchPlan, FlyteLaunchPlan] the backfill is generated for this launchplan :param parallel: if the backfill should be run in parallel. False (default) will run each bacfill sequentially :param per_node_timeout: timedelta Timeout to use per node :param per_node_retries: int Retries to user per node diff --git a/flytekit/remote/lazy_entity.py b/flytekit/remote/lazy_entity.py index 4755aad99d..1df2197329 100644 --- a/flytekit/remote/lazy_entity.py +++ b/flytekit/remote/lazy_entity.py @@ -10,7 +10,7 @@ class LazyEntity(RemoteEntity, typing.Generic[T]): """ Fetches the entity when the entity is called or when the entity is retrieved. - The entity is derived from RemoteEntity so that it behaves exactly like the mimiced entity. + The entity is derived from RemoteEntity so that it behaves exactly like the mimicked entity. """ def __init__(self, name: str, getter: typing.Callable[[], T], *args, **kwargs): diff --git a/flytekit/remote/remote.py b/flytekit/remote/remote.py index d6f3a094a6..46c70bd49d 100644 --- a/flytekit/remote/remote.py +++ b/flytekit/remote/remote.py @@ -1210,7 +1210,7 @@ def _resolve_identifier_kwargs( if not (ident.project and ident.domain and ident.name): raise ValueError( f"Cannot launch an execution with missing project/domain/name {ident} for entity type {type(entity)}." - f" Specify them in the execute method or when intializing FlyteRemote" + f" Specify them in the execute method or when initializing FlyteRemote" ) return ident diff --git a/flytekit/sensor/base_sensor.py b/flytekit/sensor/base_sensor.py index 60beb6aa2b..0e40055ea5 100644 --- a/flytekit/sensor/base_sensor.py +++ b/flytekit/sensor/base_sensor.py @@ -35,7 +35,7 @@ def __init__( type_hints = get_type_hints(self.poke, include_extras=True) signature = inspect.signature(self.poke) inputs = collections.OrderedDict() - for k, v in signature.parameters.items(): # type: ignore + for k, _ in signature.parameters.items(): # type: ignore annotation = type_hints.get(k, None) inputs[k] = annotation diff --git a/flytekit/tools/fast_registration.py b/flytekit/tools/fast_registration.py index 16ee24084a..0664ebbd8d 100644 --- a/flytekit/tools/fast_registration.py +++ b/flytekit/tools/fast_registration.py @@ -106,7 +106,7 @@ def download_distribution(additional_distribution: str, destination: str): """ if not os.path.isdir(destination): raise ValueError("Destination path is required to download distribution and it should be a directory") - # NOTE the os.path.join(destination, ''). This is to ensure that the given path is infact a directory and all + # NOTE the os.path.join(destination, ''). This is to ensure that the given path is in fact a directory and all # downloaded data should be copied into this directory. We do this to account for a difference in behavior in # fsspec, which requires a trailing slash in case of pre-existing directory. FlyteContextManager.current_context().file_access.get_data(additional_distribution, os.path.join(destination, "")) diff --git a/flytekit/tools/script_mode.py b/flytekit/tools/script_mode.py index e5bbde04df..5c9990d819 100644 --- a/flytekit/tools/script_mode.py +++ b/flytekit/tools/script_mode.py @@ -104,7 +104,7 @@ def copy_module_to_destination( # Takes in a TarInfo and returns the modified TarInfo: # https://docs.python.org/3/library/tarfile.html#tarinfo-objects -# intented to be passed as a filter to tarfile.add +# intended to be passed as a filter to tarfile.add # https://docs.python.org/3/library/tarfile.html#tarfile.TarFile.add def tar_strip_file_attributes(tar_info: tarfile.TarInfo) -> tarfile.TarInfo: # set time to epoch timestamp 0, aka 00:00:00 UTC on 1 January 1970 diff --git a/flytekit/tools/translator.py b/flytekit/tools/translator.py index f98fe8abaf..dc81ba55e6 100644 --- a/flytekit/tools/translator.py +++ b/flytekit/tools/translator.py @@ -774,7 +774,7 @@ def gather_dependent_entities( The ``get_serializable`` function above takes in an ``OrderedDict`` that helps keep track of dependent entities. For example, when serializing a workflow, all its tasks are also serialized. The ordered dict will also contain serialized entities that aren't as useful though, like nodes and branches. This is just a small helper function - that will pull out the serialzed tasks, workflows, and launch plans. This function is primarily used for testing. + that will pull out the serialized tasks, workflows, and launch plans. This function is primarily used for testing. :param serialized: This should be the filled in OrderedDict used in the get_serializable function above. :return: diff --git a/flytekit/trigger.py b/flytekit/trigger.py index 6623565d84..62d703f642 100644 --- a/flytekit/trigger.py +++ b/flytekit/trigger.py @@ -6,7 +6,7 @@ from flyteidl.core import identifier_pb2 as idl from flyteidl.core import interface_pb2 -from flytekit.core.artifact import TIME_PARTITION, Artifact, ArtifactQuery, Partition, TimePartition +from flytekit.core.artifact import Artifact, ArtifactQuery, Partition, TimePartition from flytekit.core.context_manager import FlyteContextManager from flytekit.core.launch_plan import LaunchPlan from flytekit.core.tracker import TrackedInstance @@ -77,7 +77,7 @@ def get_parameter_map( expr = None if v.op and v.other and isinstance(v.other, timedelta): expr = str(v.op) + isodate.duration_isoformat(v.other) - aq = v.reference_artifact.embed_as_query(self.triggers, TIME_PARTITION, expr) + aq = v.reference_artifact.embed_as_query(self.triggers, bind_to_time_partition=True, expr=expr) p = interface_pb2.Parameter(var=var, artifact_query=aq) elif isinstance(v, Partition): # The reason is that if we bind to arbitrary partitions, we'll have to start keeping track of types diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index 14337427c4..169e9d0a6d 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -123,7 +123,7 @@ def __init__( self, path: typing.Union[str, os.PathLike], downloader: typing.Optional[typing.Callable] = None, - remote_directory: typing.Optional[typing.Union[os.PathLike, typing.Literal[False]]] = None, + remote_directory: typing.Optional[typing.Union[os.PathLike, str, typing.Literal[False]]] = None, ): """ :param path: The source path that users are expected to call open() on @@ -138,7 +138,7 @@ def __init__( self._downloader = downloader or noop self._downloaded = False self._remote_directory = remote_directory - self._remote_source = None + self._remote_source: typing.Optional[str] = None def __fspath__(self): """ @@ -192,7 +192,7 @@ def downloaded(self) -> bool: return self._downloaded @property - def remote_directory(self) -> typing.Optional[typing.Union[os.PathLike, bool]]: + def remote_directory(self) -> typing.Optional[typing.Union[os.PathLike, bool, str]]: return self._remote_directory @property @@ -236,6 +236,66 @@ def new_dir(self, name: typing.Optional[str] = None) -> FlyteDirectory: def download(self) -> str: return self.__fspath__() + @classmethod + def listdir(cls, directory: FlyteDirectory) -> typing.List[typing.Union[FlyteDirectory, FlyteFile]]: + """ + This function will list all files and folders in the given directory, but without downloading the contents. + In addition, it will return a list of FlyteFile and FlyteDirectory objects that have ability to lazily download the + contents of the file/folder. For example: + + .. code-block:: python + + entity = FlyteDirectory.listdir(directory) + for e in entity: + print("s3 object:", e.remote_source) + # s3 object: s3://test-flytedir/file1.txt + # s3 object: s3://test-flytedir/file2.txt + # s3 object: s3://test-flytedir/sub_dir + + open(entity[0], "r") # This will download the file to the local disk. + open(entity[0], "r") # flytekit will read data from the local disk if you open it again. + """ + + final_path = directory.path + if directory.remote_source: + final_path = directory.remote_source + elif directory.remote_directory: + final_path = typing.cast(os.PathLike, directory.remote_directory) + + paths: typing.List[typing.Union[FlyteDirectory, FlyteFile]] = [] + file_access = FlyteContextManager.current_context().file_access + if not file_access.is_remote(final_path): + for p in os.listdir(final_path): + if os.path.isfile(os.path.join(final_path, p)): + paths.append(FlyteFile(p)) + else: + paths.append(FlyteDirectory(p)) + return paths + + def create_downloader(_remote_path: str, _local_path: str, is_multipart: bool): + return lambda: file_access.get_data(_remote_path, _local_path, is_multipart=is_multipart) + + fs = file_access.get_filesystem_for_path(final_path) + for key in fs.listdir(final_path): + remote_path = os.path.join(final_path, key["name"].split(os.sep)[-1]) + if key["type"] == "file": + local_path = file_access.get_random_local_path() + os.makedirs(pathlib.Path(local_path).parent, exist_ok=True) + downloader = create_downloader(remote_path, local_path, is_multipart=False) + + flyte_file: FlyteFile = FlyteFile(local_path, downloader=downloader) + flyte_file._remote_source = remote_path + paths.append(flyte_file) + else: + local_folder = file_access.get_random_local_directory() + downloader = create_downloader(remote_path, local_folder, is_multipart=True) + + flyte_directory: FlyteDirectory = FlyteDirectory(path=local_folder, downloader=downloader) + flyte_directory._remote_source = remote_path + paths.append(flyte_directory) + + return paths + def crawl( self, maxdepth: typing.Optional[int] = None, topdown: bool = True, **kwargs ) -> Generator[Tuple[typing.Union[str, os.PathLike[Any]], typing.Dict[Any, Any]], None, None]: diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index 4729c25e9c..6dac55e818 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -200,7 +200,7 @@ def __init__( self._downloader = downloader self._downloaded = False self._remote_path = remote_path - self._remote_source = None + self._remote_source: typing.Optional[str] = None def __fspath__(self): # This is where a delayed downloading of the file will happen diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py index ce5ab52de1..1d7af31404 100644 --- a/flytekit/types/structured/structured_dataset.py +++ b/flytekit/types/structured/structured_dataset.py @@ -177,7 +177,7 @@ def __init__(self, python_type: Type[T], protocol: Optional[str] = None, support is capable of handling. :param supported_format: Arbitrary string representing the format. If not supplied then an empty string will be used. An empty string implies that the encoder works with any format. If the format being asked - for does not exist, the transformer enginer will look for the "" endcoder instead and write a warning. + for does not exist, the transformer enginer will look for the "" encoder instead and write a warning. """ self._python_type = python_type self._protocol = protocol.replace("://", "") if protocol else None @@ -300,7 +300,7 @@ def convert_schema_type_to_structured_dataset_type( def get_supported_types(): import numpy as _np - _SUPPORTED_TYPES: typing.Dict[Type, LiteralType] = { + _SUPPORTED_TYPES: typing.Dict[Type, LiteralType] = { # type: ignore _np.int32: type_models.LiteralType(simple=type_models.SimpleType.INTEGER), _np.int64: type_models.LiteralType(simple=type_models.SimpleType.INTEGER), _np.uint32: type_models.LiteralType(simple=type_models.SimpleType.INTEGER), diff --git a/plugins/README.md b/plugins/README.md index abf47b9556..d738c5b5a4 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -23,8 +23,7 @@ All the Flytekit plugins maintained by the core team are added here. It is not n | Snowflake | ```bash pip install flytekitplugins-snowflake``` | Use Snowflake as a 'data warehouse-as-a-service' within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-snowflake.svg)](https://pypi.python.org/pypi/flytekitplugins-snowflake/) | Backend | | dbt | ```bash pip install flytekitplugins-dbt``` | Run dbt within Flyte | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-dbt.svg)](https://pypi.python.org/pypi/flytekitplugins-dbt/) | Flytekit-only | | Huggingface | ```bash pip install flytekitplugins-huggingface``` | Read & write Hugginface Datasets as Flyte StructuredDatasets | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-huggingface.svg)](https://pypi.python.org/pypi/flytekitplugins-huggingface/) | Flytekit-only | -| DuckDB | ```bash pip install flytekitplugins-duckdb``` | Run analytical workloads with ease using DuckDB. -| [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-duckdb.svg)](https://pypi.python.org/pypi/flytekitplugins-duckdb/) | Flytekit-only | +| DuckDB | ```bash pip install flytekitplugins-duckdb``` | Run analytical workloads with ease using DuckDB | [![PyPI version fury.io](https://badge.fury.io/py/flytekitplugins-duckdb.svg)](https://pypi.python.org/pypi/flytekitplugins-duckdb/) | Flytekit-only | ## Have a Plugin Idea? 💡 Please [file an issue](https://github.com/flyteorg/flyte/issues/new?assignees=&labels=untriaged%2Cplugins&template=backend-plugin-request.md&title=%5BPlugin%5D). diff --git a/plugins/flytekit-airflow/dev-requirements.in b/plugins/flytekit-airflow/dev-requirements.in index 8ee20b47d1..b3913ef810 100644 --- a/plugins/flytekit-airflow/dev-requirements.in +++ b/plugins/flytekit-airflow/dev-requirements.in @@ -1 +1,2 @@ apache-airflow-providers-apache-beam[google] +apache-airflow[google] diff --git a/plugins/flytekit-airflow/dev-requirements.txt b/plugins/flytekit-airflow/dev-requirements.txt index 744f2c092c..b7b8178724 100644 --- a/plugins/flytekit-airflow/dev-requirements.txt +++ b/plugins/flytekit-airflow/dev-requirements.txt @@ -21,8 +21,9 @@ annotated-types==0.6.0 # via pydantic anyio==4.0.0 # via httpx -apache-airflow==2.8.0 +apache-airflow[google]==2.8.0 # via + # -r dev-requirements.in # apache-airflow-providers-apache-beam # apache-airflow-providers-common-sql # apache-airflow-providers-ftp @@ -40,7 +41,9 @@ apache-airflow-providers-common-sql==1.8.0 apache-airflow-providers-ftp==3.6.0 # via apache-airflow apache-airflow-providers-google==10.11.0 - # via apache-airflow-providers-apache-beam + # via + # apache-airflow + # apache-airflow-providers-apache-beam apache-airflow-providers-http==4.6.0 # via apache-airflow apache-airflow-providers-imap==3.4.0 @@ -528,7 +531,7 @@ itsdangerous==2.1.2 # connexion # flask # flask-wtf -jinja2==3.1.2 +jinja2==3.1.3 # via # apache-airflow # flask diff --git a/plugins/flytekit-airflow/flytekitplugins/airflow/task.py b/plugins/flytekit-airflow/flytekitplugins/airflow/task.py index 04fb848cb5..b49bc061f5 100644 --- a/plugins/flytekit-airflow/flytekitplugins/airflow/task.py +++ b/plugins/flytekit-airflow/flytekitplugins/airflow/task.py @@ -146,11 +146,7 @@ def _get_airflow_instance( obj_module = importlib.import_module(name=airflow_obj.module) obj_def = getattr(obj_module, airflow_obj.name) - if ( - issubclass(obj_def, airflow_models.BaseOperator) - and not issubclass(obj_def, airflow_sensors.BaseSensorOperator) - and _is_deferrable(obj_def) - ): + if _is_deferrable(obj_def): try: return obj_def(**airflow_obj.parameters, deferrable=True) except airflow.exceptions.AirflowException as e: @@ -160,15 +156,22 @@ def _get_airflow_instance( return obj_def(**airflow_obj.parameters) -def _is_deferrable(cls: Type): +def _is_deferrable(cls: Type) -> bool: """ This function is used to check if the Airflow operator is deferrable. + If the operator is not deferrable, we run it in a container instead of the agent. """ + # Only Airflow operators are deferrable. + if not issubclass(cls, airflow_models.BaseOperator): + return False + # Airflow sensors are not deferrable. Sensor is a subclass of BaseOperator. + if issubclass(cls, airflow_sensors.BaseSensorOperator): + return False try: from airflow.providers.apache.beam.operators.beam import BeamBasePipelineOperator # Dataflow operators are not deferrable. - if not issubclass(cls, BeamBasePipelineOperator): + if issubclass(cls, BeamBasePipelineOperator): return False except ImportError: logger.debug("Failed to import BeamBasePipelineOperator") @@ -194,7 +197,7 @@ def _flyte_operator(*args, **kwargs): task_id = kwargs["task_id"] or cls.__name__ config = AirflowObj(module=cls.__module__, name=cls.__name__, parameters=kwargs) - if _is_deferrable(cls): + if not _is_deferrable(cls): # Dataflow operators are not deferrable, so we run them in a container. return AirflowContainerTask(name=task_id, task_config=config, container_image=container_image)() return AirflowTask(name=task_id, task_config=config)() diff --git a/plugins/flytekit-airflow/setup.py b/plugins/flytekit-airflow/setup.py index 91214e6dbf..682cd72c18 100644 --- a/plugins/flytekit-airflow/setup.py +++ b/plugins/flytekit-airflow/setup.py @@ -7,6 +7,7 @@ plugin_requires = [ "apache-airflow", "flytekit>=1.9.0", + "flyteidl>=1.10.6", ] __version__ = "0.0.0+develop" diff --git a/plugins/flytekit-airflow/tests/test_task.py b/plugins/flytekit-airflow/tests/test_task.py index 3d6a954b7f..f55bcef5dd 100644 --- a/plugins/flytekit-airflow/tests/test_task.py +++ b/plugins/flytekit-airflow/tests/test_task.py @@ -1,5 +1,6 @@ import jsonpickle from airflow.providers.apache.beam.operators.beam import BeamRunJavaPipelineOperator +from airflow.providers.google.cloud.operators.dataproc import DataprocCreateClusterOperator from airflow.sensors.bash import BashSensor from airflow.utils.context import Context from flytekitplugins.airflow.task import ( @@ -34,8 +35,9 @@ def test_xcom_push(): def test_is_deferrable(): - assert _is_deferrable(BeamRunJavaPipelineOperator) is True + assert _is_deferrable(BeamRunJavaPipelineOperator) is False assert _is_deferrable(BashSensor) is False + assert _is_deferrable(DataprocCreateClusterOperator) is True def test_airflow_task(): diff --git a/plugins/flytekit-envd/README.md b/plugins/flytekit-envd/README.md index 374d2334ff..3b3a168d75 100644 --- a/plugins/flytekit-envd/README.md +++ b/plugins/flytekit-envd/README.md @@ -24,3 +24,21 @@ Example # def t1() -> str: # return "hello" ``` + +This plugin also supports install packages from `conda`: + +```python +from flytekit import task, ImageSpec + +image_spec = ImageSpec( + base_image="ubuntu:20.04", + python_version="3.11", + packages=["flytekit"], + conda_packages=["pytorch", "pytorch-cuda=12.1"], + conda_channels=["pytorch", "nvidia"] +) + +@task(container_image=image_spec) +def run_pytorch(): + ... +``` diff --git a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py index 0a534a6763..45d0b4676c 100644 --- a/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py +++ b/plugins/flytekit-envd/flytekitplugins/envd/image_builder.py @@ -50,6 +50,12 @@ def build_image(self, image_spec: ImageSpec): self.execute_command(build_command) +def _create_str_from_package_list(packages): + if packages is None: + return "" + return ", ".join(f'"{name}"' for name in packages) + + def create_envd_config(image_spec: ImageSpec) -> str: base_image = DefaultImages.default_image() if image_spec.base_image is None else image_spec.base_image if image_spec.cuda: @@ -57,10 +63,12 @@ def create_envd_config(image_spec: ImageSpec) -> str: raise Exception("python_version is required when cuda and cudnn are specified") base_image = "ubuntu20.04" - packages = [] if image_spec.packages is None else image_spec.packages - apt_packages = [] if image_spec.apt_packages is None else image_spec.apt_packages + python_packages = _create_str_from_package_list(image_spec.packages) + conda_packages = _create_str_from_package_list(image_spec.conda_packages) + run_commands = _create_str_from_package_list(image_spec.commands) + conda_channels = _create_str_from_package_list(image_spec.conda_channels) + apt_packages = _create_str_from_package_list(image_spec.apt_packages) env = {"PYTHONPATH": "/root", _F_IMG_ID: image_spec.image_name()} - commands = [] if image_spec.commands is None else image_spec.commands if image_spec.env: env.update(image_spec.env) @@ -70,9 +78,9 @@ def create_envd_config(image_spec: ImageSpec) -> str: def build(): base(image="{base_image}", dev=False) - run(commands=[{', '.join(map(str, map(lambda x: f'"{x}"', commands)))}]) - install.python_packages(name=[{', '.join(map(str, map(lambda x: f'"{x}"', packages)))}]) - install.apt_packages(name=[{', '.join(map(str, map(lambda x: f'"{x}"', apt_packages)))}]) + run(commands=[{run_commands}]) + install.python_packages(name=[{python_packages}]) + install.apt_packages(name=[{apt_packages}]) runtime.environ(env={env}, extra_path=['/root']) config.pip_index(url="{pip_index}") """ @@ -80,6 +88,10 @@ def build(): cfg_path = ctx.file_access.get_random_local_path("build.envd") pathlib.Path(cfg_path).parent.mkdir(parents=True, exist_ok=True) + if conda_packages: + envd_config += " install.conda(use_mamba=True)\n" + envd_config += f" install.conda_packages(name=[{conda_packages}], channel=[{conda_channels}])\n" + if image_spec.requirements: requirement_path = f"{pathlib.Path(cfg_path).parent}{os.sep}{REQUIREMENTS_FILE_NAME}" shutil.copyfile(image_spec.requirements, requirement_path) diff --git a/plugins/flytekit-envd/tests/test_image_spec.py b/plugins/flytekit-envd/tests/test_image_spec.py index 49c8f5085a..36adebd346 100644 --- a/plugins/flytekit-envd/tests/test_image_spec.py +++ b/plugins/flytekit-envd/tests/test_image_spec.py @@ -1,4 +1,5 @@ from pathlib import Path +from textwrap import dedent from flytekitplugins.envd.image_builder import EnvdImageSpecBuilder, create_envd_config @@ -35,3 +36,37 @@ def build(): install.python(version="3.8") """ ) + + +def test_image_spec_conda(): + image_spec = ImageSpec( + base_image="ubuntu:20.04", + python_version="3.11", + packages=["flytekit"], + conda_packages=["pytorch", "cpuonly"], + conda_channels=["pytorch"], + ) + + EnvdImageSpecBuilder().build_image(image_spec) + config_path = create_envd_config(image_spec) + assert image_spec.platform == "linux/amd64" + image_name = image_spec.image_name() + contents = Path(config_path).read_text() + expected_contents = dedent( + f"""\ + # syntax=v1 + + def build(): + base(image="ubuntu:20.04", dev=False) + run(commands=[]) + install.python_packages(name=["flytekit"]) + install.apt_packages(name=[]) + runtime.environ(env={{'PYTHONPATH': '/root', '_F_IMG_ID': '{image_name}'}}, extra_path=['/root']) + config.pip_index(url="https://pypi.org/simple") + install.conda(use_mamba=True) + install.conda_packages(name=["pytorch", "cpuonly"], channel=["pytorch"]) + install.python(version="3.11") + """ + ) + + assert contents == expected_contents diff --git a/plugins/flytekit-flyin/Dockerfile b/plugins/flytekit-flyin/Dockerfile index 9d56a84826..765c0f94c0 100644 --- a/plugins/flytekit-flyin/Dockerfile +++ b/plugins/flytekit-flyin/Dockerfile @@ -27,6 +27,7 @@ RUN apt-get update \ && apt-get autoremove --yes \ && rm -rf /var/lib/{apt,dpkg,cache,log}/ \ && useradd -u 1000 flytekit \ + && chown -R flytekit:flytekit /tmp/code-server \ && chown flytekit: /root \ && chown flytekit: /home \ && : diff --git a/plugins/flytekit-flyin/flytekitplugins/flyin/utils.py b/plugins/flytekit-flyin/flytekitplugins/flyin/utils.py index 4b0a4ee2d4..3c13951a2e 100644 --- a/plugins/flytekit-flyin/flytekitplugins/flyin/utils.py +++ b/plugins/flytekit-flyin/flytekitplugins/flyin/utils.py @@ -52,6 +52,6 @@ def get_task_inputs(task_module_name, task_name, context_working_dir): task_module = load_module_from_path(task_module_name, os.path.join(context_working_dir, f"{task_module_name}.py")) task_def = getattr(task_module, task_name) native_inputs = TypeEngine.literal_map_to_kwargs( - FlyteContextManager(), idl_input_literals, task_def.python_interface.inputs + FlyteContextManager().current_context(), idl_input_literals, task_def.python_interface.inputs ) return native_inputs diff --git a/plugins/flytekit-flyin/flytekitplugins/flyin/vscode_lib/constants.py b/plugins/flytekit-flyin/flytekitplugins/flyin/vscode_lib/constants.py index faabbfcaee..52d1e8e3ff 100644 --- a/plugins/flytekit-flyin/flytekitplugins/flyin/vscode_lib/constants.py +++ b/plugins/flytekit-flyin/flytekitplugins/flyin/vscode_lib/constants.py @@ -36,3 +36,6 @@ # Context attribute name of the task function's source file path TASK_FUNCTION_SOURCE_PATH = "TASK_FUNCTION_SOURCE_PATH" + +# Subprocess constants +EXIT_CODE_SUCCESS = 0 diff --git a/plugins/flytekit-flyin/flytekitplugins/flyin/vscode_lib/decorator.py b/plugins/flytekit-flyin/flytekitplugins/flyin/vscode_lib/decorator.py index 2bde13ae5a..9a936d7e46 100644 --- a/plugins/flytekit-flyin/flytekitplugins/flyin/vscode_lib/decorator.py +++ b/plugins/flytekit-flyin/flytekitplugins/flyin/vscode_lib/decorator.py @@ -10,7 +10,7 @@ import tarfile import time from threading import Event -from typing import Callable, Optional +from typing import Callable, List, Optional import fsspec from flytekitplugins.flyin.utils import load_module_from_path @@ -23,6 +23,7 @@ from .constants import ( DOWNLOAD_DIR, EXECUTABLE_NAME, + EXIT_CODE_SUCCESS, HEARTBEAT_CHECK_SECONDS, HEARTBEAT_PATH, INTERACTIVE_DEBUGGING_FILE_NAME, @@ -42,7 +43,7 @@ def execute_command(cmd): process = subprocess.Popen(cmd, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE) logger.info(f"cmd: {cmd}") stdout, stderr = process.communicate() - if process.returncode != 0: + if process.returncode != EXIT_CODE_SUCCESS: raise RuntimeError(f"Command {cmd} failed with error: {stderr}") logger.info(f"stdout: {stdout}") logger.info(f"stderr: {stderr}") @@ -176,6 +177,27 @@ def get_code_server_info(code_server_info_dict: dict) -> str: ) +def get_installed_extensions() -> List[str]: + """ + Get the list of installed extensions. + + Returns: + List[str]: The list of installed extensions. + """ + logger = flytekit.current_context().logging + + installed_extensions = subprocess.run(["code-server", "--list-extensions"], capture_output=True, text=True) + if installed_extensions.returncode != EXIT_CODE_SUCCESS: + logger.info(f"Command code-server --list-extensions failed with error: {installed_extensions.stderr}") + return [] + + return installed_extensions.stdout.splitlines() + + +def is_extension_installed(extension: str, installed_extensions: List[str]) -> bool: + return any(installed_extension in extension for installed_extension in installed_extensions) + + def download_vscode(config: VscodeConfig): """ Download vscode server and extension from remote to local and add the directory of binary executable to $PATH. @@ -190,34 +212,34 @@ def download_vscode(config: VscodeConfig): if executable_path is not None: logger.info(f"Code server binary already exists at {executable_path}") logger.info("Skipping downloading code server...") - return + else: + logger.info("Code server is not in $PATH, start downloading code server...") + # Create DOWNLOAD_DIR if not exist + logger.info(f"DOWNLOAD_DIR: {DOWNLOAD_DIR}") + os.makedirs(DOWNLOAD_DIR, exist_ok=True) - logger.info("Code server is not in $PATH, start downloading code server...") + logger.info(f"Start downloading files to {DOWNLOAD_DIR}") + # Download remote file to local + code_server_remote_path = get_code_server_info(config.code_server_remote_paths) + code_server_tar_path = download_file(code_server_remote_path, DOWNLOAD_DIR) - # Create DOWNLOAD_DIR if not exist - logger.info(f"DOWNLOAD_DIR: {DOWNLOAD_DIR}") - os.makedirs(DOWNLOAD_DIR, exist_ok=True) + # Extract the tarball + with tarfile.open(code_server_tar_path, "r:gz") as tar: + tar.extractall(path=DOWNLOAD_DIR) - logger.info(f"Start downloading files to {DOWNLOAD_DIR}") + code_server_dir_name = get_code_server_info(config.code_server_dir_names) + code_server_bin_dir = os.path.join(DOWNLOAD_DIR, code_server_dir_name, "bin") - # Download remote file to local - code_server_remote_path = get_code_server_info(config.code_server_remote_paths) - code_server_tar_path = download_file(code_server_remote_path, DOWNLOAD_DIR) + # Add the directory of code-server binary to $PATH + os.environ["PATH"] = code_server_bin_dir + os.pathsep + os.environ["PATH"] + # If the extension already exists in the container, skip downloading + installed_extensions = get_installed_extensions() extension_paths = [] for extension in config.extension_remote_paths: - file_path = download_file(extension, DOWNLOAD_DIR) - extension_paths.append(file_path) - - # Extract the tarball - with tarfile.open(code_server_tar_path, "r:gz") as tar: - tar.extractall(path=DOWNLOAD_DIR) - - code_server_dir_name = get_code_server_info(config.code_server_dir_names) - code_server_bin_dir = os.path.join(DOWNLOAD_DIR, code_server_dir_name, "bin") - - # Add the directory of code-server binary to $PATH - os.environ["PATH"] = code_server_bin_dir + os.pathsep + os.environ["PATH"] + if not is_extension_installed(extension, installed_extensions): + file_path = download_file(extension, DOWNLOAD_DIR) + extension_paths.append(file_path) for p in extension_paths: logger.info(f"Execute extension installation command to install extension {p}") diff --git a/plugins/flytekit-flyin/tests/test_flyin_plugin.py b/plugins/flytekit-flyin/tests/test_flyin_plugin.py index 085532e8fc..23cde516ce 100644 --- a/plugins/flytekit-flyin/tests/test_flyin_plugin.py +++ b/plugins/flytekit-flyin/tests/test_flyin_plugin.py @@ -14,7 +14,12 @@ jupyter, vscode, ) -from flytekitplugins.flyin.vscode_lib.decorator import get_code_server_info +from flytekitplugins.flyin.vscode_lib.constants import EXIT_CODE_SUCCESS +from flytekitplugins.flyin.vscode_lib.decorator import ( + get_code_server_info, + get_installed_extensions, + is_extension_installed, +) from flytekit import task, workflow from flytekit.configuration import Image, ImageConfig, SerializationSettings @@ -216,6 +221,20 @@ def wf(): mock_exit.assert_called_once() +def test_is_extension_installed(): + installed_extensions = [ + "ms-python.python", + "ms-toolsai.jupyter", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.vscode-jupyter-slideshow", + ] + config = VscodeConfig() + for extension in config.extension_remote_paths: + assert is_extension_installed(extension, installed_extensions) + + def test_vscode_config(): config = VscodeConfig() assert config.code_server_remote_paths == DEFAULT_CODE_SERVER_REMOTE_PATHS @@ -345,3 +364,58 @@ def test_platform_unsupported(mock_machine, mock_code_server_info_dict): match="Automatic download is only supported on AMD64 and ARM64 architectures. If you are using a different architecture, please visit the code-server official website to manually download the appropriate version for your image.", ): get_code_server_info(mock_code_server_info_dict) + + +@mock.patch("subprocess.run") +def test_get_installed_extensions_succeeded(mock_run): + # Set up the mock process + mock_process = mock.Mock() + mock_process.returncode = EXIT_CODE_SUCCESS + mock_process.stdout = ( + "ms-python.python\n" + "ms-toolsai.jupyter\n" + "ms-toolsai.jupyter-keymap\n" + "ms-toolsai.jupyter-renderers\n" + "ms-toolsai.vscode-jupyter-cell-tags\n" + "ms-toolsai.vscode-jupyter-slideshow\n" + ) + mock_run.return_value = mock_process + + installed_extensions = get_installed_extensions() + + # Verify the correct command was called + mock_run.assert_called_once_with(["code-server", "--list-extensions"], capture_output=True, text=True) + + # Assert that the output matches the expected list of extensions + expected_extensions = [ + "ms-python.python", + "ms-toolsai.jupyter", + "ms-toolsai.jupyter-keymap", + "ms-toolsai.jupyter-renderers", + "ms-toolsai.vscode-jupyter-cell-tags", + "ms-toolsai.vscode-jupyter-slideshow", + ] + assert installed_extensions == expected_extensions + + +@mock.patch("subprocess.run") +def test_get_installed_extensions_failed(mock_run): + # Set up the mock process + mock_process = mock.Mock() + mock_process.returncode = 1 + mock_process.stdout = ( + "ms-python.python\n" + "ms-toolsai.jupyter\n" + "ms-toolsai.jupyter-keymap\n" + "ms-toolsai.jupyter-renderers\n" + "ms-toolsai.vscode-jupyter-cell-tags\n" + "ms-toolsai.vscode-jupyter-slideshow\n" + ) + mock_run.return_value = mock_process + + installed_extensions = get_installed_extensions() + + mock_run.assert_called_once_with(["code-server", "--list-extensions"], capture_output=True, text=True) + + expected_extensions = [] + assert installed_extensions == expected_extensions diff --git a/plugins/flytekit-identity-aware-proxy/README.md b/plugins/flytekit-identity-aware-proxy/README.md index c6c631707c..64477e73a8 100644 --- a/plugins/flytekit-identity-aware-proxy/README.md +++ b/plugins/flytekit-identity-aware-proxy/README.md @@ -41,7 +41,7 @@ This flytekit plugin allows users to generate ID tokens via an external command --project < fill in the gcp project id where the secret was saved > ``` - A browser window should open, asking you to login with your GCP account. Then, a succesful log in should be confirmed with *"Successfully logged into accounts.google.com"*. + A browser window should open, asking you to login with your GCP account. Then, a successful log in should be confirmed with *"Successfully logged into accounts.google.com"*. Finally, the token beginning with `eyJhbG..."` should be printed to the console. @@ -97,7 +97,7 @@ Flyte is then deployed behind the Istio ingress gateway and does not need to be ### Deployment -1. If not already done, deploy the flyte-core helm chart, [activating auth](https://docs.flyte.org/en/latest/deployment/configuration/auth_setup.html#apply-oidc-configuration). Re-use the web app client id created for IAP (see section above). Disable the default ingress in the helm values by setting `common.ingress.enabled` to `false` in the helm values file. +1. If not already done, deploy the flyte-core helm chart, [activating auth](https://docs.flyte.org/en/latest/deployment/configuration/auth_setup.html#apply-oidc-configuration). Reuse the web app client id created for IAP (see section above). Disable the default ingress in the helm values by setting `common.ingress.enabled` to `false` in the helm values file. 2. Deployment of Istio and the Istio ingress gateway ([docs](https://istio.io/latest/docs/setup/install/helm/)) @@ -288,7 +288,7 @@ Flyte is then deployed behind the Istio ingress gateway and does not need to be 7. Connect flyteadmin and flyteconsole to the istio ingress gateway: - So far, we created a GCE ingress (which creates a GCP load balancer). The load balancer is configured to forward all requests to the istio ingress gatway at the edge of the service mesh via http2 and TLS. + So far, we created a GCE ingress (which creates a GCP load balancer). The load balancer is configured to forward all requests to the istio ingress gateway at the edge of the service mesh via http2 and TLS. Next, we configure the Istio service mesh to route requests from the Istio ingress gateway to flyteadmin and flyteconsole. @@ -387,7 +387,7 @@ Flyte is then deployed behind the Istio ingress gateway and does not need to be print(remote.recent_executions()) ``` - A browser window should open and ask you to login with your Google account. You should then see confirmation that you *"Successfully logged into accounts.google.com"* (this was for the IAP), finally followd by confirmation that you *"Successfully logged into 'your flyte domain'"* (this was for Flyte itself). + A browser window should open and ask you to login with your Google account. You should then see confirmation that you *"Successfully logged into accounts.google.com"* (this was for the IAP), finally followed by confirmation that you *"Successfully logged into 'your flyte domain'"* (this was for Flyte itself). diff --git a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py index ffe21d90cd..46eb086ad0 100644 --- a/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py +++ b/plugins/flytekit-kf-pytorch/flytekitplugins/kfpytorch/task.py @@ -310,7 +310,7 @@ def _execute(self, **kwargs) -> Any: inherits from `FlyteRecoverableException`. RuntimeError: If the first exception raised in the local worker group is not and does not inherit from `FlyteRecoverableException`. - IgnoreOutputs: Raised when the task is succesfull in any worker group with index > 0. + IgnoreOutputs: Raised when the task is successful in any worker group with index > 0. """ try: from torch.distributed.launcher.api import LaunchConfig, elastic_launch diff --git a/plugins/flytekit-onnx-pytorch/dev-requirements.txt b/plugins/flytekit-onnx-pytorch/dev-requirements.txt index ef4045e458..44660a3ba9 100644 --- a/plugins/flytekit-onnx-pytorch/dev-requirements.txt +++ b/plugins/flytekit-onnx-pytorch/dev-requirements.txt @@ -11,7 +11,9 @@ charset-normalizer==3.3.2 coloredlogs==15.0.1 # via onnxruntime filelock==3.13.1 - # via torch + # via + # torch + # triton flatbuffers==23.5.26 # via onnxruntime fsspec==2023.10.0 @@ -20,7 +22,7 @@ humanfriendly==10.0 # via coloredlogs idna==3.4 # via requests -jinja2==3.1.2 +jinja2==3.1.3 # via torch markupsafe==2.1.3 # via jinja2 @@ -32,6 +34,37 @@ numpy==1.23.5 # via # onnxruntime # torchvision +nvidia-cublas-cu12==12.1.3.1 + # via + # nvidia-cudnn-cu12 + # nvidia-cusolver-cu12 + # torch +nvidia-cuda-cupti-cu12==12.1.105 + # via torch +nvidia-cuda-nvrtc-cu12==12.1.105 + # via torch +nvidia-cuda-runtime-cu12==12.1.105 + # via torch +nvidia-cudnn-cu12==8.9.2.26 + # via torch +nvidia-cufft-cu12==11.0.2.54 + # via torch +nvidia-curand-cu12==10.3.2.106 + # via torch +nvidia-cusolver-cu12==11.4.5.107 + # via torch +nvidia-cusparse-cu12==12.1.0.106 + # via + # nvidia-cusolver-cu12 + # torch +nvidia-nccl-cu12==2.18.1 + # via torch +nvidia-nvjitlink-cu12==12.3.101 + # via + # nvidia-cusolver-cu12 + # nvidia-cusparse-cu12 +nvidia-nvtx-cu12==12.1.105 + # via torch onnxruntime==1.16.1 # via -r dev-requirements.in packaging==23.2 @@ -52,6 +85,8 @@ torch==2.1.0 # via torchvision torchvision==0.16.0 # via -r dev-requirements.in +triton==2.1.0 + # via torch typing-extensions==4.8.0 # via torch urllib3==2.0.7 diff --git a/plugins/flytekit-onnx-pytorch/flytekitplugins/onnxpytorch/schema.py b/plugins/flytekit-onnx-pytorch/flytekitplugins/onnxpytorch/schema.py index 0cca0616c5..78793b84d3 100644 --- a/plugins/flytekit-onnx-pytorch/flytekitplugins/onnxpytorch/schema.py +++ b/plugins/flytekit-onnx-pytorch/flytekitplugins/onnxpytorch/schema.py @@ -33,7 +33,7 @@ class PyTorch2ONNXConfig(DataClassJsonMixin): do_constant_folding: Whether to apply constant folding for optimization. dynamic_axes: Specify axes of tensors as dynamic. keep_initializers_as_inputs: Whether to add the initializers as inputs to the graph. - custom_opsets: A dictionary of opset doman name and version. + custom_opsets: A dictionary of opset domain name and version. export_modules_as_functions: Whether to export modules as functions. """ diff --git a/plugins/flytekit-pandera/tests/test_plugin.py b/plugins/flytekit-pandera/tests/test_plugin.py index 7e73aac932..a3e7c82565 100644 --- a/plugins/flytekit-pandera/tests/test_plugin.py +++ b/plugins/flytekit-pandera/tests/test_plugin.py @@ -57,10 +57,7 @@ def wf_with_df_input(df: pandera.typing.DataFrame[InSchema]) -> pandera.typing.D with pytest.raises( pandera.errors.SchemaError, - match=( - "^Encountered error while executing workflow 'test_plugin.wf_with_df_input':\n" - " expected series 'col2' to have type float64, got object" - ), + match="expected series 'col2' to have type float64, got object", ): wf_with_df_input(df=invalid_df) @@ -75,11 +72,7 @@ def wf_invalid_output(df: pandera.typing.DataFrame[InSchema]) -> pandera.typing. with pytest.raises( TypeError, - match=( - "^Encountered error while executing workflow 'test_plugin.wf_invalid_output':\n" - " Error encountered while executing 'wf_invalid_output':\n" - " Failed to convert outputs of task" - ), + match="Error encountered while executing 'wf_invalid_output':\n" " Failed to convert outputs of task", ): wf_invalid_output(df=valid_df) diff --git a/plugins/flytekit-pydantic/flytekitplugins/pydantic/deserialization.py b/plugins/flytekit-pydantic/flytekitplugins/pydantic/deserialization.py index 77a3e44f29..a7d602b402 100644 --- a/plugins/flytekit-pydantic/flytekitplugins/pydantic/deserialization.py +++ b/plugins/flytekit-pydantic/flytekitplugins/pydantic/deserialization.py @@ -26,7 +26,7 @@ class PydanticDeserializationLiteralStore: placeholders that it is trying to deserialize. """ - literal_store: Optional[serialization.LiteralStore] = None # attachement point for the literal map + literal_store: Optional[serialization.LiteralStore] = None # attachment point for the literal map def __init__(self) -> None: raise Exception("This class should not be instantiated") diff --git a/plugins/flytekit-whylogs/flytekitplugins/whylogs/renderer.py b/plugins/flytekit-whylogs/flytekitplugins/whylogs/renderer.py index b0b3acb30f..60cf064871 100644 --- a/plugins/flytekit-whylogs/flytekitplugins/whylogs/renderer.py +++ b/plugins/flytekit-whylogs/flytekitplugins/whylogs/renderer.py @@ -1,7 +1,7 @@ from flytekit import lazy_module why = lazy_module("whylogs") -constaints = lazy_module("whylogs.core.constraints") +constraints = lazy_module("whylogs.core.constraints") pd = lazy_module("pandas") @@ -60,7 +60,7 @@ class WhylogsConstraintsRenderer: """ @staticmethod - def to_html(constraints: constaints.Constraints) -> str: + def to_html(constraints: constraints.Constraints) -> str: viz = why.viz.NotebookProfileVisualizer() report = viz.constraints_report(constraints=constraints) return report.data diff --git a/pyproject.toml b/pyproject.toml index 32dea218da..9ca9119d64 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -12,7 +12,7 @@ readme = { file = "README.md", content-type = "text/markdown" } requires-python = ">=3.8,<3.13" dependencies = [ # Please maintain an alphabetical order in the following list - "adlfs", + "adlfs>=2023.3.0,<=2023.9.2", "click>=6.6,<9.0", "cloudpickle>=2.0.0", "cookiecutter>=1.7.3", @@ -21,9 +21,9 @@ dependencies = [ "diskcache>=5.2.1", "docker>=4.0.0,<7.0.0", "docstring-parser>=0.9.0", - "flyteidl>=1.10.0", + "flyteidl>=1.10.6", "fsspec>=2023.3.0,<=2023.9.2", - "gcsfs", + "gcsfs>=2023.3.0,<=2023.9.2", "googleapis-common-protos>=1.57", "grpcio", "grpcio-status", @@ -45,7 +45,7 @@ dependencies = [ "requests>=2.18.4,<3.0.0", "rich", "rich_click", - "s3fs>=0.6.0", + "s3fs>=2023.3.0,<=2023.9.2", "statsd>=3.0.0,<4.0.0", "typing_extensions", "urllib3>=1.22,<2.0.0", @@ -138,3 +138,7 @@ ignore = [ # unused-import "F401", ] + +[tool.codespell] +ignore-words-list="ot,te,raison,fo,lits" +skip="./docs/build,./.git,*.txt" diff --git a/setup.py b/setup.py index fc1f76c84d..606849326a 100644 --- a/setup.py +++ b/setup.py @@ -1,3 +1,3 @@ from setuptools import setup -setup() \ No newline at end of file +setup() diff --git a/tests/flytekit/common/parameterizers.py b/tests/flytekit/common/parameterizers.py index 96c30b69b4..84481a37ad 100644 --- a/tests/flytekit/common/parameterizers.py +++ b/tests/flytekit/common/parameterizers.py @@ -90,7 +90,6 @@ task.Resources.ResourceEntry(task.Resources.ResourceName.CPU, "1"), task.Resources.ResourceEntry(task.Resources.ResourceName.GPU, "1"), task.Resources.ResourceEntry(task.Resources.ResourceName.MEMORY, "1G"), - task.Resources.ResourceEntry(task.Resources.ResourceName.STORAGE, "1G"), task.Resources.ResourceEntry(task.Resources.ResourceName.EPHEMERAL_STORAGE, "1G"), ] diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 3d4c3fd037..5f4ef273a3 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -91,7 +91,7 @@ flyteidl==1.5.13 # via flytekit flytekit==1.8.2 # via -r requirements.in -fonttools==4.41.1 +fonttools==4.43.0 # via matplotlib frozenlist==1.4.0 # via @@ -155,7 +155,7 @@ jeepney==0.8.0 # via # keyring # secretstorage -jinja2==3.1.2 +jinja2==3.1.3 # via cookiecutter jmespath==1.0.1 # via botocore diff --git a/tests/flytekit/integration/remote/test_remote.py b/tests/flytekit/integration/remote/test_remote.py index 4e97110bd0..f92489e9c4 100644 --- a/tests/flytekit/integration/remote/test_remote.py +++ b/tests/flytekit/integration/remote/test_remote.py @@ -95,7 +95,7 @@ def test_monitor_workflow_execution(register): execution = remote.sync_execution(execution, sync_nodes=True) if execution.node_executions: - assert execution.node_executions["start-node"].closure.phase == 3 # SUCCEEEDED + assert execution.node_executions["start-node"].closure.phase == 3 # SUCCEEDED for key in execution.node_executions: assert execution.node_executions[key].closure.phase == 3 diff --git a/tests/flytekit/unit/cli/pyflyte/test_init.py b/tests/flytekit/unit/cli/pyflyte/test_init.py index 0a66433625..97d24ca50d 100644 --- a/tests/flytekit/unit/cli/pyflyte/test_init.py +++ b/tests/flytekit/unit/cli/pyflyte/test_init.py @@ -10,7 +10,7 @@ "command", [ ["example"], - ["example", "--template", "simple-example"], + ["example", "--template", "basic-template-imagespec"], ["example", "--template", "bayesian-optimization"], ], ) diff --git a/tests/flytekit/unit/cli/pyflyte/test_plugin.py b/tests/flytekit/unit/cli/pyflyte/test_plugin.py index 6e3423a1ca..5d7b8d8bf6 100644 --- a/tests/flytekit/unit/cli/pyflyte/test_plugin.py +++ b/tests/flytekit/unit/cli/pyflyte/test_plugin.py @@ -1,6 +1,8 @@ +import re from unittest.mock import Mock, patch import click +import pytest from flytekit.configuration.plugin import FlytekitPlugin, FlytekitPluginProtocol, _get_plugin_from_entrypoint @@ -14,20 +16,19 @@ def test_get_plugin_default(entry_points): @patch("flytekit.configuration.plugin.entry_points") -def test_get_plugin_load_other_plugin(entry_points, caplog): - loaded_plugin_1 = Mock() +def test_get_plugin_errors_with_multiple_plugins(entry_points, caplog): entry_1 = Mock() entry_1.name = "entry_1" - entry_1.load.side_effect = lambda: loaded_plugin_1 entry_2 = Mock() + entry_2.name = "entry_2" entry_points.side_effect = lambda *args, **kwargs: [entry_1, entry_2] - plugin = _get_plugin_from_entrypoint() - assert plugin is loaded_plugin_1 - - assert entry_1.load.call_count == 1 - assert entry_2.load.call_count == 0 + msg = re.escape( + "Multiple plugins installed: ['entry_1', 'entry_2']. flytekit only supports one installed plugin at a time." + ) + with pytest.raises(ValueError, match=msg): + _get_plugin_from_entrypoint() class CustomPlugin(FlytekitPlugin): diff --git a/tests/flytekit/unit/cli/pyflyte/test_run.py b/tests/flytekit/unit/cli/pyflyte/test_run.py index cb8d8f6bc6..bab8b95b15 100644 --- a/tests/flytekit/unit/cli/pyflyte/test_run.py +++ b/tests/flytekit/unit/cli/pyflyte/test_run.py @@ -287,9 +287,9 @@ def test_list_default_arguments(wf_path): ) ic_result_4 = ImageConfig( - default_image=Image(name="default", fqn="flytekit", tag="tbcFqCcdAEyJqPcyYsQ15A.."), + default_image=Image(name="default", fqn="flytekit", tag="DgQMqIi61py4I4P5iOeS0Q.."), images=[ - Image(name="default", fqn="flytekit", tag="tbcFqCcdAEyJqPcyYsQ15A.."), + Image(name="default", fqn="flytekit", tag="DgQMqIi61py4I4P5iOeS0Q.."), Image(name="xyz", fqn="docker.io/xyz", tag="latest"), Image(name="abc", fqn="docker.io/abc", tag=None), ], diff --git a/tests/flytekit/unit/core/test_artifacts.py b/tests/flytekit/unit/core/test_artifacts.py index c83e5cb035..dbafa6983f 100644 --- a/tests/flytekit/unit/core/test_artifacts.py +++ b/tests/flytekit/unit/core/test_artifacts.py @@ -1,9 +1,9 @@ import datetime -import typing from collections import OrderedDict import pandas as pd import pytest +from flyteidl.core import artifact_id_pb2 as art_id from typing_extensions import Annotated from flytekit.configuration import Image, ImageConfig, SerializationSettings @@ -13,7 +13,6 @@ from flytekit.core.task import task from flytekit.core.workflow import workflow from flytekit.tools.translator import get_serializable -from flytekit.types.structured.structured_dataset import StructuredDataset default_img = Image(name="default", fqn="test", tag="tag") serialization_settings = SerializationSettings( @@ -42,10 +41,10 @@ def t1( entities = OrderedDict() t1_s = get_serializable(entities, serialization_settings, t1) - assert len(t1_s.template.interface.outputs["o0"].artifact_partial_id.partitions.value) == 3 + assert len(t1_s.template.interface.outputs["o0"].artifact_partial_id.partitions.value) == 2 p = t1_s.template.interface.outputs["o0"].artifact_partial_id.partitions.value - assert p["ds"].HasField("input_binding") - assert p["ds"].input_binding.var == "dt" + assert t1_s.template.interface.outputs["o0"].artifact_partial_id.time_partition is not None + assert t1_s.template.interface.outputs["o0"].artifact_partial_id.time_partition.value.input_binding.var == "dt" assert p["b"].HasField("input_binding") assert p["b"].input_binding.var == "b_value" assert p["a"].HasField("static_value") @@ -55,6 +54,43 @@ def t1( assert t1_s.template.interface.outputs["o0"].artifact_partial_id.artifact_key.project == "" +def test_basic_option_no_tp(): + a1_t_ab = Artifact(name="my_data", partition_keys=["a", "b"]) + assert not a1_t_ab.time_partitioned + + # trying to bind to a time partition when not so raises an error. + with pytest.raises(ValueError): + + @task + def t1x( + b_value: str, dt: datetime.datetime + ) -> Annotated[pd.DataFrame, a1_t_ab(time_partition=Inputs.dt, b=Inputs.b_value, a="manual")]: + df = pd.DataFrame({"a": [1, 2, 3], "b": [b_value, b_value, b_value]}) + return df + + @task + def t1(b_value: str, dt: datetime.datetime) -> Annotated[pd.DataFrame, a1_t_ab(b=Inputs.b_value, a="manual")]: + df = pd.DataFrame({"a": [1, 2, 3], "b": [b_value, b_value, b_value]}) + return df + + entities = OrderedDict() + t1_s = get_serializable(entities, serialization_settings, t1) + assert len(t1_s.template.interface.outputs["o0"].artifact_partial_id.partitions.value) == 2 + p = t1_s.template.interface.outputs["o0"].artifact_partial_id.partitions.value + assert t1_s.template.interface.outputs["o0"].artifact_partial_id.HasField("time_partition") is False + assert p["b"].HasField("input_binding") + + +def test_basic_option_hardcoded_tp(): + a1_t_ab = Artifact(name="my_data", time_partitioned=True) + + dt = datetime.datetime.strptime("04/05/2063", "%m/%d/%Y") + + id_spec = a1_t_ab(time_partition=dt) + assert id_spec.partitions is None + assert id_spec.time_partition.value.HasField("time_value") + + def test_basic_option_a(): a1_t_ab = Artifact(name="my_data", partition_keys=["a", "b"], time_partitioned=True) @@ -67,10 +103,23 @@ def t1( entities = OrderedDict() t1_s = get_serializable(entities, serialization_settings, t1) - assert len(t1_s.template.interface.outputs["o0"].artifact_partial_id.partitions.value) == 3 + assert len(t1_s.template.interface.outputs["o0"].artifact_partial_id.partitions.value) == 2 assert t1_s.template.interface.outputs["o0"].artifact_partial_id.version == "" assert t1_s.template.interface.outputs["o0"].artifact_partial_id.artifact_key.name == "my_data" assert t1_s.template.interface.outputs["o0"].artifact_partial_id.artifact_key.project == "" + assert t1_s.template.interface.outputs["o0"].artifact_partial_id.time_partition is not None + + +def test_basic_no_call(): + a1_t_ab = Artifact(name="my_data", partition_keys=["a", "b"], time_partitioned=True) + + # raise an error because the user hasn't () the artifact + with pytest.raises(ValueError): + + @task + def t1(b_value: str, dt: datetime.datetime) -> Annotated[pd.DataFrame, a1_t_ab]: + df = pd.DataFrame({"a": [1, 2, 3], "b": [b_value, b_value, b_value]}) + return df def test_basic_option_a2(): @@ -79,7 +128,7 @@ def test_basic_option_a2(): with pytest.raises(ValueError): @task - def t2(b_value: str) -> Annotated[pd.DataFrame, a2_ab(a=Inputs.b_value)]: + def t2x(b_value: str) -> Annotated[pd.DataFrame, a2_ab(a=Inputs.b_value)]: ... @task @@ -107,29 +156,6 @@ def t3(b_value: str) -> Annotated[pd.DataFrame, a3]: assert t3_s.template.interface.outputs["o0"].artifact_partial_id.artifact_key.name == "my_data3" -def test_controlling_aliases_when_running(): - task_alias = Artifact(name="task_artifact", tags=["latest"]) - wf_alias = Artifact(name="wf_artifact", tags=["my_v0.1.0"]) - - @task - def t1() -> Annotated[typing.Union[CustomReturn, Annotated[StructuredDataset, "avro"]], task_alias]: - return CustomReturn({"name": ["Tom", "Joseph"], "age": [20, 22]}) - - @workflow - def wf() -> Annotated[CustomReturn, wf_alias]: - u = t1() - return u - - entities = OrderedDict() - spec = get_serializable(entities, serialization_settings, t1) - tag = spec.template.interface.outputs["o0"].artifact_tag - assert tag.value.static_value == "latest" - - spec = get_serializable(entities, serialization_settings, wf) - tag = spec.template.interface.outputs["o0"].artifact_tag - assert tag.value.static_value == "my_v0.1.0" - - def test_query_basic(): aa = Artifact( name="ride_count_data", @@ -142,11 +168,10 @@ def test_query_basic(): dq_idl = data_query.to_flyte_idl() assert dq_idl.HasField("artifact_id") assert dq_idl.artifact_id.artifact_key.name == "ride_count_data" - assert len(dq_idl.artifact_id.partitions.value) == 2 - assert dq_idl.artifact_id.partitions.value["ds"].HasField("input_binding") - assert dq_idl.artifact_id.partitions.value["ds"].input_binding.var == "dt" + assert len(dq_idl.artifact_id.partitions.value) == 1 assert dq_idl.artifact_id.partitions.value["region"].HasField("input_binding") assert dq_idl.artifact_id.partitions.value["region"].input_binding.var == "blah" + assert dq_idl.artifact_id.time_partition.value.input_binding.var == "dt" def test_not_specified_behavior(): @@ -162,11 +187,12 @@ def test_not_specified_behavior(): assert wf_artifact_no_tag.partitions is None aq = wf_artifact_no_tag.query().to_flyte_idl() assert aq.artifact_id.HasField("partitions") is False + assert aq.artifact_id.HasField("time_partition") is False def test_artifact_as_promise_query(): # when artifact is partially specified, can be used as a query input - wf_artifact = Artifact(project="project1", domain="dev", name="wf_artifact", tags=["my_v0.1.0"]) + wf_artifact = Artifact(project="project1", domain="dev", name="wf_artifact") @task def t1(a: CustomReturn) -> CustomReturn: @@ -182,10 +208,9 @@ def wf(a: CustomReturn = wf_artifact.query()): lp = LaunchPlan.get_default_launch_plan(ctx, wf) entities = OrderedDict() spec = get_serializable(entities, serialization_settings, lp) - assert spec.spec.default_inputs.parameters["a"].artifact_query.artifact_tag.artifact_key.project == "project1" - assert spec.spec.default_inputs.parameters["a"].artifact_query.artifact_tag.artifact_key.domain == "dev" - assert spec.spec.default_inputs.parameters["a"].artifact_query.artifact_tag.artifact_key.name == "wf_artifact" - assert spec.spec.default_inputs.parameters["a"].artifact_query.artifact_tag.value.static_value == "my_v0.1.0" + assert spec.spec.default_inputs.parameters["a"].artifact_query.artifact_id.artifact_key.project == "project1" + assert spec.spec.default_inputs.parameters["a"].artifact_query.artifact_id.artifact_key.domain == "dev" + assert spec.spec.default_inputs.parameters["a"].artifact_query.artifact_id.artifact_key.name == "wf_artifact" def test_artifact_as_promise(): @@ -206,7 +231,6 @@ def wf2(a: CustomReturn = wf_artifact): lp = LaunchPlan.get_default_launch_plan(ctx, wf2) entities = OrderedDict() spec = get_serializable(entities, serialization_settings, lp) - x = spec.spec.default_inputs.parameters["a"] assert spec.spec.default_inputs.parameters["a"].artifact_id.artifact_key.project == "pro" assert spec.spec.default_inputs.parameters["a"].artifact_id.artifact_key.domain == "dom" assert spec.spec.default_inputs.parameters["a"].artifact_id.artifact_key.name == "key" @@ -214,3 +238,43 @@ def wf2(a: CustomReturn = wf_artifact): aq = wf_artifact.query().to_flyte_idl() assert aq.artifact_id.HasField("partitions") is True assert aq.artifact_id.partitions.value["region"].static_value == "LAX" + + +def test_partition_none(): + # confirm that we can distinguish between partitions being set to empty, and not being set + # though this is not currently used. + ak = art_id.ArtifactKey(project="p", domain="d", name="name") + no_partition = art_id.ArtifactID(artifact_key=ak, version="without_p") + assert not no_partition.HasField("partitions") + + p = art_id.Partitions() + with_partition = art_id.ArtifactID(artifact_key=ak, version="without_p", partitions=p) + assert with_partition.HasField("partitions") + + +def test_as_artf_no_partitions(): + int_artf = Artifact(name="important_int") + + @task + def greet(day_of_week: str, number: int, am: bool) -> str: + greeting = "Have a great " + day_of_week + " " + greeting += "morning" if am else "evening" + return greeting + "!" * number + + @workflow + def go_greet(day_of_week: str, number: int = int_artf.query(), am: bool = False) -> str: + return greet(day_of_week=day_of_week, number=number, am=am) + + tst_lp = LaunchPlan.create( + "morning_lp", + go_greet, + fixed_inputs={"am": True}, + default_inputs={"day_of_week": "monday"}, + ) + + entities = OrderedDict() + spec = get_serializable(entities, serialization_settings, tst_lp) + aq = spec.spec.default_inputs.parameters["number"].artifact_query + assert aq.artifact_id.artifact_key.name == "important_int" + assert not aq.artifact_id.HasField("partitions") + assert not aq.artifact_id.HasField("time_partition") diff --git a/tests/flytekit/unit/core/test_context_manager.py b/tests/flytekit/unit/core/test_context_manager.py index 2ec7eb8e19..ca22f359c9 100644 --- a/tests/flytekit/unit/core/test_context_manager.py +++ b/tests/flytekit/unit/core/test_context_manager.py @@ -1,11 +1,14 @@ import base64 import os from datetime import datetime +from pathlib import Path +from unittest.mock import Mock import mock import py import pytest +import flytekit.configuration.plugin from flytekit.configuration import ( SERIALIZED_CONTEXT_ENV_VAR, FastSerializationSettings, @@ -128,6 +131,24 @@ def test_secrets_manager_get_envvar(): assert sec.get_secrets_env_var("group") == f"{cfg.env_prefix}GROUP" +def test_secret_manager_no_group(monkeypatch): + plugin_mock = Mock() + plugin_mock.secret_requires_group.return_value = False + mock_global_plugin = {"plugin": plugin_mock} + monkeypatch.setattr(flytekit.configuration.plugin, "_GLOBAL_CONFIG", mock_global_plugin) + + sec = SecretsManager() + cfg = SecretsConfig.auto() + sec.check_group_key(None) + sec.check_group_key("") + + assert sec.get_secrets_env_var(key="ABC") == f"{cfg.env_prefix}ABC" + + default_path = Path(cfg.default_dir) + expected_path = default_path / f"{cfg.file_prefix}abc" + assert sec.get_secrets_file(key="ABC") == str(expected_path) + + def test_secrets_manager_get_file(): sec = SecretsManager() with pytest.raises(ValueError): diff --git a/tests/flytekit/unit/core/test_flyte_directory.py b/tests/flytekit/unit/core/test_flyte_directory.py index 48f778e4c7..206b58ec5d 100644 --- a/tests/flytekit/unit/core/test_flyte_directory.py +++ b/tests/flytekit/unit/core/test_flyte_directory.py @@ -5,6 +5,7 @@ import typing from unittest.mock import MagicMock +import mock import pytest import flytekit.configuration @@ -16,6 +17,7 @@ from flytekit.core.task import task from flytekit.core.type_engine import TypeEngine from flytekit.core.workflow import workflow +from flytekit.exceptions.user import FlyteAssertion from flytekit.models.core.types import BlobType from flytekit.models.literals import LiteralMap from flytekit.types.directory.types import FlyteDirectory, FlyteDirToMultipartBlobTransformer @@ -89,25 +91,24 @@ def test_transformer_to_literal_local(): TypeEngine.to_literal(ctx, 3, FlyteDirectory, lt) -# def test_transformer_to_literal_localss(): -# random_dir = context_manager.FlyteContext.current_context().file_access.get_random_local_directory() -# fs = FileAccessProvider(local_sandbox_dir=random_dir, raw_output_prefix=os.path.join(random_dir, "raw")) -# ctx = context_manager.FlyteContext.current_context() -# with context_manager.FlyteContextManager.with_context(ctx.with_file_access(fs)) as ctx: -# -# tf = FlyteDirToMultipartBlobTransformer() -# lt = tf.get_literal_type(FlyteDirectory) -# # Can't use if it's not a directory -# with pytest.raises(FlyteAssertion): -# p = "/tmp/flyte/xyz" -# path = pathlib.Path(p) -# try: -# path.unlink() -# except OSError: -# ... -# with open(p, "w") as fh: -# fh.write("hello world\n") -# tf.to_literal(ctx, FlyteDirectory(p), FlyteDirectory, lt) +def test_transformer_to_literal_local_path(): + random_dir = context_manager.FlyteContext.current_context().file_access.get_random_local_directory() + fs = FileAccessProvider(local_sandbox_dir=random_dir, raw_output_prefix=os.path.join(random_dir, "raw")) + ctx = context_manager.FlyteContext.current_context() + with context_manager.FlyteContextManager.with_context(ctx.with_file_access(fs)) as ctx: + tf = FlyteDirToMultipartBlobTransformer() + lt = tf.get_literal_type(FlyteDirectory) + # Can't use if it's not a directory + with pytest.raises(FlyteAssertion): + p = ctx.file_access.get_random_local_path() + path = pathlib.Path(p) + try: + path.unlink() + except OSError: + ... + with open(p, "w") as fh: + fh.write("hello world\n") + tf.to_literal(ctx, FlyteDirectory(p), FlyteDirectory, lt) def test_transformer_to_literal_remote(): @@ -279,3 +280,39 @@ def test_directory_guess(): fft = transformer.guess_python_type(lt) assert issubclass(fft, FlyteDirectory) assert fft.extension() == "" + + +@mock.patch("s3fs.core.S3FileSystem._lsdir") +@mock.patch("flytekit.core.data_persistence.FileAccessProvider.get_data") +def test_list_dir(mock_get_data, mock_lsdir): + remote_dir = "s3://test-flytedir" + mock_lsdir.return_value = [ + {"name": os.path.join(remote_dir, "file1.txt"), "type": "file"}, + {"name": os.path.join(remote_dir, "file2.txt"), "type": "file"}, + {"name": os.path.join(remote_dir, "subdir"), "type": "directory"}, + ] + + mock_get_data.side_effect = lambda: Exception("Should not be called") + + temp_dir = tempfile.mkdtemp(prefix="temp_example_") + file1_path = os.path.join(temp_dir, "file1.txt") + sub_dir = os.path.join(temp_dir, "subdir") + os.mkdir(sub_dir) + with open(file1_path, "w") as file1: + file1.write("Content of file1.txt") + + f = FlyteDirectory(temp_dir) + paths = FlyteDirectory.listdir(f) + assert len(paths) == 2 + + f = FlyteDirectory(path=temp_dir, remote_directory=remote_dir) + paths = FlyteDirectory.listdir(f) + assert len(paths) == 3 + + f = FlyteDirectory(path=temp_dir) + f._remote_source = remote_dir + paths = FlyteDirectory.listdir(f) + assert len(paths) == 3 + + with pytest.raises(Exception): + open(paths[0], "r") diff --git a/tests/flytekit/unit/core/test_promise.py b/tests/flytekit/unit/core/test_promise.py index 3322d4dd59..979b80b45f 100644 --- a/tests/flytekit/unit/core/test_promise.py +++ b/tests/flytekit/unit/core/test_promise.py @@ -87,7 +87,7 @@ def wf(i: int, j: int): # Even if j is not provided it will default create_and_link_node_from_remote(ctx, lp, _inputs_not_allowed={"i"}, _ignorable_inputs={"j"}) - # value of `i` cannot be overriden + # value of `i` cannot be overridden with pytest.raises( FlyteAssertion, match="ixed inputs cannot be specified. Please remove the following inputs - {'i'}" ): @@ -195,7 +195,7 @@ def my_workflow() -> (str, str, str): o3 = t2(a=f.a) return o1, o2, o3 - # Run a local execution with promises having atrribute path + # Run a local execution with promises having attribute path o1, o2, o3 = my_workflow() assert o1 == "a" assert o2 == "b" diff --git a/tests/flytekit/unit/core/test_python_function_task.py b/tests/flytekit/unit/core/test_python_function_task.py index 9ba9d4c780..498228f3fe 100644 --- a/tests/flytekit/unit/core/test_python_function_task.py +++ b/tests/flytekit/unit/core/test_python_function_task.py @@ -107,7 +107,7 @@ def foo(i: str): assert foo_metadata.cache_serialize is True assert foo_metadata.cache_version == "1.0" - # test cache, cache_serialize, and cache_version at no unecessarily set + # test cache, cache_serialize, and cache_version at no unnecessarily set @task() def bar(i: str): print(f"{i}") diff --git a/tests/flytekit/unit/core/test_resources.py b/tests/flytekit/unit/core/test_resources.py index 25a637b2d6..e11be83462 100644 --- a/tests/flytekit/unit/core/test_resources.py +++ b/tests/flytekit/unit/core/test_resources.py @@ -22,10 +22,9 @@ def test_convert_no_requests_no_limits(): ({"cpu": "2"}, _ResourceName.CPU), ({"mem": "1Gi"}, _ResourceName.MEMORY), ({"gpu": "1"}, _ResourceName.GPU), - ({"storage": "100Mb"}, _ResourceName.STORAGE), ({"ephemeral_storage": "123Mb"}, _ResourceName.EPHEMERAL_STORAGE), ), - ids=("CPU", "MEMORY", "GPU", "STORAGE", "EPHEMERAL_STORAGE"), + ids=("CPU", "MEMORY", "GPU", "EPHEMERAL_STORAGE"), ) def test_convert_requests(resource_dict: Dict[str, str], expected_resource_name: _task_models.Resources): assert len(resource_dict) == 1 @@ -48,10 +47,9 @@ def test_convert_requests(resource_dict: Dict[str, str], expected_resource_name: ({"cpu": "2"}, _ResourceName.CPU), ({"mem": "1Gi"}, _ResourceName.MEMORY), ({"gpu": "1"}, _ResourceName.GPU), - ({"storage": "100Mb"}, _ResourceName.STORAGE), ({"ephemeral_storage": "123Mb"}, _ResourceName.EPHEMERAL_STORAGE), ), - ids=("CPU", "MEMORY", "GPU", "STORAGE", "EPHEMERAL_STORAGE"), + ids=("CPU", "MEMORY", "GPU", "EPHEMERAL_STORAGE"), ) def test_convert_limits(resource_dict: Dict[str, str], expected_resource_name: _task_models.Resources): assert len(resource_dict) == 1 @@ -75,7 +73,5 @@ def test_incorrect_type_resources(): Resources(mem=1) # type: ignore with pytest.raises(AssertionError): Resources(gpu=1) # type: ignore - with pytest.raises(AssertionError): - Resources(storage=1) # type: ignore with pytest.raises(AssertionError): Resources(ephemeral_storage=1) # type: ignore diff --git a/tests/flytekit/unit/core/test_triggers.py b/tests/flytekit/unit/core/test_triggers.py index f25920d649..d00067b7d0 100644 --- a/tests/flytekit/unit/core/test_triggers.py +++ b/tests/flytekit/unit/core/test_triggers.py @@ -1,16 +1,16 @@ -from datetime import timedelta +from datetime import datetime, timedelta from flyteidl.core import artifact_id_pb2 as art_id from flyteidl.core import literals_pb2 from typing_extensions import Annotated -from flytekit.core.artifact import Artifact +from flytekit.core.artifact import Artifact, Inputs from flytekit.core.workflow import workflow from flytekit.trigger import Trigger def test_basic_11(): - # This test would translate to + # This test translates to # Trigger(trigger_on=[hourlyArtifact], # inputs={"x": hourlyArtifact}) hourlyArtifact = Artifact( @@ -21,6 +21,8 @@ def test_basic_11(): aq_idl = hourlyArtifact.embed_as_query([hourlyArtifact]) assert aq_idl.HasField("binding") assert aq_idl.binding.index == 0 + assert aq_idl.binding.partition_key == "" + assert aq_idl.binding.bind_to_time_partition is False def test_basic_1(): @@ -37,10 +39,12 @@ def test_basic_1(): aq = hourlyArtifact.query(partitions={"region": "LAX"}) aq_idl = aq.to_flyte_idl([hourlyArtifact]) - assert aq_idl.artifact_id.partitions.value["ds"].HasField("triggered_binding") - assert aq_idl.artifact_id.partitions.value["ds"].triggered_binding.index == 0 + assert aq_idl.artifact_id.time_partition.value.HasField("triggered_binding") + assert aq_idl.artifact_id.time_partition.value.triggered_binding.index == 0 + assert aq_idl.artifact_id.time_partition.value.triggered_binding.bind_to_time_partition is True assert aq_idl.artifact_id.partitions.value["some_dim"].HasField("triggered_binding") assert aq_idl.artifact_id.partitions.value["some_dim"].triggered_binding.index == 0 + assert aq_idl.artifact_id.partitions.value["some_dim"].triggered_binding.bind_to_time_partition is False assert aq_idl.artifact_id.partitions.value["region"].static_value == "LAX" @@ -50,9 +54,11 @@ def test_basic_2(): aq = dailyArtifact.query(time_partition=dailyArtifact.time_partition - timedelta(days=1)) aq_idl = aq.to_flyte_idl([dailyArtifact]) x = aq_idl.artifact_id.partitions.value - assert aq_idl.artifact_id.partitions.value["ds"].triggered_binding.index == 0 - assert aq_idl.artifact_id.partitions.value["ds"].triggered_binding.partition_key == "ds" - assert aq_idl.artifact_id.partitions.value["ds"].triggered_binding.transform is not None + assert len(x) == 0 + assert aq_idl.artifact_id.time_partition.value.triggered_binding.index == 0 + assert aq_idl.artifact_id.time_partition.value.triggered_binding.HasField("partition_key") is False + assert aq_idl.artifact_id.time_partition.value.triggered_binding.bind_to_time_partition is True + assert aq_idl.artifact_id.time_partition.value.triggered_binding.transform is not None def test_big_trigger(): @@ -77,7 +83,7 @@ def test_big_trigger(): "other_daily_upstream": hourlyArtifact.query(partitions={"region": "LAX"}), "region": "SEA", # static value that will be passed as input "other_artifact": UnrelatedArtifact.query(time_partition=dailyArtifact.time_partition), - "other_artifact_2": UnrelatedArtifact.query(time_partition=hourlyArtifact.time_partition.truncate_to_day()), + "other_artifact_2": UnrelatedArtifact.query(time_partition=hourlyArtifact.time_partition), "other_artifact_3": UnrelatedTwo.query(partitions={"rgg": hourlyArtifact.partitions.region}), }, ) @@ -91,7 +97,8 @@ def my_workflow( other_artifact: str, other_artifact_2: str, other_artifact_3: str, - ) -> Annotated[str, dailyArtifact]: + dt: datetime, + ) -> Annotated[str, dailyArtifact(time_partition=Inputs.dt)]: ... pm = t.get_parameter_map(my_workflow.python_interface.inputs, my_workflow.interface.inputs) @@ -102,17 +109,16 @@ def my_workflow( ), ) assert not pm.parameters["today_upstream"].artifact_query.binding.partition_key + assert not pm.parameters["today_upstream"].artifact_query.binding.bind_to_time_partition assert not pm.parameters["today_upstream"].artifact_query.binding.transform assert pm.parameters["yesterday_upstream"].artifact_query == art_id.ArtifactQuery( artifact_id=art_id.ArtifactID( artifact_key=art_id.ArtifactKey(project=None, domain=None, name="daily_artifact"), - partitions=art_id.Partitions( - value={ - "ds": art_id.LabelValue( - triggered_binding=art_id.ArtifactBindingData(index=0, partition_key="ds", transform="-P1D") - ), - } + time_partition=art_id.TimePartition( + value=art_id.LabelValue( + triggered_binding=art_id.ArtifactBindingData(index=0, bind_to_time_partition=True, transform="-P1D") + ), ), ), ) @@ -120,9 +126,13 @@ def my_workflow( assert pm.parameters["other_daily_upstream"].artifact_query == art_id.ArtifactQuery( artifact_id=art_id.ArtifactID( artifact_key=art_id.ArtifactKey(project=None, domain=None, name="hourly_artifact"), + time_partition=art_id.TimePartition( + value=art_id.LabelValue( + triggered_binding=art_id.ArtifactBindingData(index=1, bind_to_time_partition=True) + ) + ), partitions=art_id.Partitions( value={ - "ds": art_id.LabelValue(triggered_binding=art_id.ArtifactBindingData(index=1, partition_key="ds")), "region": art_id.LabelValue(static_value="LAX"), } ), @@ -136,10 +146,10 @@ def my_workflow( assert pm.parameters["other_artifact"].artifact_query == art_id.ArtifactQuery( artifact_id=art_id.ArtifactID( artifact_key=art_id.ArtifactKey(project=None, domain=None, name="unrelated_artifact"), - partitions=art_id.Partitions( - value={ - "ds": art_id.LabelValue(triggered_binding=art_id.ArtifactBindingData(index=0, partition_key="ds")), - } + time_partition=art_id.TimePartition( + value=art_id.LabelValue( + triggered_binding=art_id.ArtifactBindingData(index=0, bind_to_time_partition=True) + ) ), ) ) @@ -147,10 +157,10 @@ def my_workflow( assert pm.parameters["other_artifact_2"].artifact_query == art_id.ArtifactQuery( artifact_id=art_id.ArtifactID( artifact_key=art_id.ArtifactKey(project=None, domain=None, name="unrelated_artifact"), - partitions=art_id.Partitions( - value={ - "ds": art_id.LabelValue(triggered_binding=art_id.ArtifactBindingData(index=1, partition_key="ds")), - } + time_partition=art_id.TimePartition( + value=art_id.LabelValue( + triggered_binding=art_id.ArtifactBindingData(index=1, bind_to_time_partition=True) + ) ), ) ) @@ -169,8 +179,8 @@ def my_workflow( ) idl_t = t.to_flyte_idl() - assert idl_t.triggers[0].partitions.value["ds"] is not None - assert idl_t.triggers[1].partitions.value["ds"] is not None + assert idl_t.triggers[0].HasField("time_partition") + assert idl_t.triggers[1].HasField("time_partition") # Test calling it to create the LaunchPlan object which adds to the global context @t @@ -183,11 +193,14 @@ def tst_wf( other_artifact: str, other_artifact_2: str, other_artifact_3: str, - ) -> Annotated[str, dailyArtifact]: + ) -> str: ... def test_partition_only(): + # This test is different than the tests above in that we're using the value of a partition directly (as opposed to + # running a query parameterized by those partition values). The only case that we'll allow for this is the + # time partition for now. dailyArtifact = Artifact(name="daily_artifact", time_partitioned=True) t = Trigger( @@ -208,7 +221,7 @@ def tst_wf( assert pm.parameters["today_upstream"].artifact_query == art_id.ArtifactQuery( binding=art_id.ArtifactBindingData( index=0, - partition_key="ds", + bind_to_time_partition=True, transform="-P1D", ) ) diff --git a/tests/flytekit/unit/core/test_type_conversion_errors.py b/tests/flytekit/unit/core/test_type_conversion_errors.py index fbdd2c8640..e8aca2570a 100644 --- a/tests/flytekit/unit/core/test_type_conversion_errors.py +++ b/tests/flytekit/unit/core/test_type_conversion_errors.py @@ -82,11 +82,10 @@ def test_workflow_with_task_error(correct_input): with pytest.raises( TypeError, match=( - r"Encountered error while executing workflow '{}':\n" - r" Error encountered while executing 'wf_with_task_error':\n" + r"Error encountered while executing 'wf_with_task_error':\n" r" Failed to convert outputs of task '.+' at position 0:\n" r" Expected value of type \ but got .+ of type .+" - ).format(wf_with_task_error.name), + ).format(), ): wf_with_task_error(a=correct_input) @@ -96,9 +95,7 @@ def test_workflow_with_task_error(correct_input): def test_workflow_with_input_error(incorrect_input): with pytest.raises( TypeError, - match=(r"Encountered error while executing workflow '{}':\n" r" Failed argument").format( - wf_with_output_error.name - ), + match=r"Failed argument".format(), ): wf_with_output_error(a=incorrect_input) @@ -108,10 +105,7 @@ def test_workflow_with_input_error(incorrect_input): def test_workflow_with_output_error(correct_input): with pytest.raises( TypeError, - match=( - r"Encountered error while executing workflow '{}':\n" - r" Failed to convert output in position 0 of value .+, expected type \" - ).format(wf_with_output_error.name), + match=(r"Failed to convert output in position 0 of value .+, expected type \"), ): wf_with_output_error(a=correct_input) @@ -128,9 +122,6 @@ def test_workflow_with_output_error(correct_input): def test_workflow_with_multioutput_error(workflow, position, correct_input): with pytest.raises( TypeError, - match=( - r"Encountered error while executing workflow '{}':\n " - r"Failed to convert output in position {} of value .+, expected type \" - ).format(workflow.name, position), + match=(r"Failed to convert output in position {} of value .+, expected type \").format(position), ): workflow(a=correct_input, b=correct_input) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index f3fabdbf42..7eed167246 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -2,6 +2,7 @@ import datetime import json import os +import re import sys import tempfile import typing @@ -1412,6 +1413,82 @@ class Bar(DataClassJsonMixin): DataclassTransformer().assert_type(gt, pv) +@pytest.mark.skipif("pandas" not in sys.modules, reason="Pandas is not installed.") +def test_assert_dict_type(): + import pandas as pd + + @dataclass + class AnotherDataClass(DataClassJsonMixin): + z: int + + @dataclass + class Args(DataClassJsonMixin): + x: int + y: typing.Optional[str] + file: FlyteFile + dataset: StructuredDataset + another_dataclass: AnotherDataClass + + pv = tempfile.mkdtemp(prefix="flyte-") + df = pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [20, 22]}) + sd = StructuredDataset(dataframe=df, file_format="parquet") + # Test when v is a dict + vd = {"x": 3, "y": "hello", "file": FlyteFile(pv), "dataset": sd, "another_dataclass": {"z": 4}} + DataclassTransformer().assert_type(Args, vd) + + # Test when v is a dict but missing Optional keys and other keys from dataclass + md = {"x": 3, "file": FlyteFile(pv), "dataset": sd, "another_dataclass": {"z": 4}} + DataclassTransformer().assert_type(Args, md) + + # Test when v is a dict but missing non-Optional keys from dataclass + md = {"y": "hello", "file": FlyteFile(pv), "dataset": sd, "another_dataclass": {"z": 4}} + with pytest.raises( + TypeTransformerFailedError, + match=re.escape("The original fields are missing the following keys from the dataclass fields: ['x']"), + ): + DataclassTransformer().assert_type(Args, md) + + # Test when v is a dict but has extra keys that are not in dataclass + ed = {"x": 3, "y": "hello", "file": FlyteFile(pv), "dataset": sd, "another_dataclass": {"z": 4}, "z": "extra"} + with pytest.raises( + TypeTransformerFailedError, + match=re.escape("The original fields have the following extra keys that are not in dataclass fields: ['z']"), + ): + DataclassTransformer().assert_type(Args, ed) + + # Test when the type of value in the dict does not match the expected_type in the dataclass + td = {"x": "3", "y": "hello", "file": FlyteFile(pv), "dataset": sd, "another_dataclass": {"z": 4}} + with pytest.raises( + TypeTransformerFailedError, match="Type of Val '' is not an instance of " + ): + DataclassTransformer().assert_type(Args, td) + + +def test_to_literal_dict(): + @dataclass + class Args(DataClassJsonMixin): + x: int + y: typing.Optional[str] + + ctx = FlyteContext.current_context() + python_type = Args + expected = TypeEngine.to_literal_type(python_type) + + # Test when python_val is a dict + python_val = {"x": 3, "y": "hello"} + literal = DataclassTransformer().to_literal(ctx, python_val, python_type, expected) + literal_json = _json_format.MessageToJson(literal.scalar.generic) + assert json.loads(literal_json) == python_val + + # Test when python_val is not a dict and not a dataclass + python_val = "not a dict or dataclass" + with pytest.raises( + TypeTransformerFailedError, + match="not of type @dataclass, only Dataclasses are supported for user defined datatypes in Flytekit", + ): + DataclassTransformer().to_literal(ctx, python_val, python_type, expected) + + @dataclass class ArgsAssert(DataClassJSONMixin): x: int @@ -1668,7 +1745,7 @@ def test_union_of_lists(): structure=TypeStructure(tag="Typed List"), ), ] - # Tags are deliberately NOT unique beacuse they are not required to encode the deep type structure, + # Tags are deliberately NOT unique because they are not required to encode the deep type structure, # only the top-level type transformer choice # # The stored typed will be used to differentiate union variants and must produce a unique choice. @@ -1955,7 +2032,7 @@ def constant_hash(df: pd.DataFrame) -> str: pandas_df_transformer.get_literal_type(pd.DataFrame), ) assert literal_with_hash_set.hash == "h4Sh" - # Confirm tha the loaded dataframe is not affected + # Confirm that the loaded dataframe is not affected python_df = TypeEngine.to_python_value(ctx, literal_with_hash_set, pd.DataFrame) expected_df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) assert expected_df.equals(python_df) diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index 53c1e45173..bdbacf246e 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -1636,7 +1636,7 @@ def wf2(a: int, b: str) -> typing.Tuple[int, str]: with pytest.raises( ValueError, - match="Encountered error while executing workflow", + match="Error encountered while executing", ): v, s = wf1(a=10, b="hello") assert v == 11 @@ -1646,7 +1646,7 @@ def wf2(a: int, b: str) -> typing.Tuple[int, str]: with pytest.raises( ValueError, - match="Encountered error while executing workflow", + match="Error encountered while executing", ): v, s = wf2(a=10, b="hello") assert v == 11 diff --git a/tests/flytekit/unit/experimental/test_eager_workflows.py b/tests/flytekit/unit/experimental/test_eager_workflows.py index 6b01d2fc7a..d4438f91f7 100644 --- a/tests/flytekit/unit/experimental/test_eager_workflows.py +++ b/tests/flytekit/unit/experimental/test_eager_workflows.py @@ -64,7 +64,7 @@ async def eager_wf(x: int) -> int: @given(x_input=INTEGER_ST) @settings(deadline=DEADLINE, max_examples=5) def test_conditional_eager_workflow(x_input: int): - """Test eager workfow with conditional logic.""" + """Test eager workflow with conditional logic.""" @eager async def eager_wf(x: int) -> int: diff --git a/tests/flytekit/unit/extend/test_agent.py b/tests/flytekit/unit/extend/test_agent.py index 2c52b70490..0010bccb1f 100644 --- a/tests/flytekit/unit/extend/test_agent.py +++ b/tests/flytekit/unit/extend/test_agent.py @@ -90,6 +90,20 @@ async def async_delete(self, context: grpc.ServicerContext, resource_meta: bytes return DeleteTaskResponse() +class SyncDummyAgent(AgentBase): + def __init__(self): + super().__init__(task_type="sync_dummy", asynchronous=True) + + async def async_create( + self, + context: grpc.ServicerContext, + output_prefix: str, + task_template: TaskTemplate, + inputs: typing.Optional[LiteralMap] = None, + ) -> CreateTaskResponse: + return CreateTaskResponse(resource=Resource(state=SUCCEEDED, outputs=LiteralMap({}).to_flyte_idl())) + + def get_task_template(task_type: str) -> TaskTemplate: @task def simple_task(i: int): @@ -118,6 +132,7 @@ def simple_task(i: int): dummy_template = get_task_template("dummy") async_dummy_template = get_task_template("async_dummy") +sync_dummy_template = get_task_template("sync_dummy") def test_dummy_agent(): @@ -161,6 +176,16 @@ async def test_async_dummy_agent(): assert res == DeleteTaskResponse() +@pytest.mark.asyncio +async def test_sync_dummy_agent(): + AgentRegistry.register(SyncDummyAgent()) + ctx = MagicMock(spec=grpc.ServicerContext) + agent = AgentRegistry.get_agent("sync_dummy") + res = await agent.async_create(ctx, "/tmp", sync_dummy_template, task_inputs) + assert res.resource.state == SUCCEEDED + assert res.resource.outputs == LiteralMap({}).to_flyte_idl() + + @pytest.mark.asyncio async def run_agent_server(): service = AsyncAgentService() @@ -171,6 +196,9 @@ async def run_agent_server(): async_request = CreateTaskRequest( inputs=task_inputs.to_flyte_idl(), output_prefix="/tmp", template=async_dummy_template.to_flyte_idl() ) + sync_request = CreateTaskRequest( + inputs=task_inputs.to_flyte_idl(), output_prefix="/tmp", template=sync_dummy_template.to_flyte_idl() + ) fake_agent = "fake" metadata_bytes = json.dumps(asdict(Metadata(job_id=dummy_id))).encode("utf-8") @@ -188,6 +216,10 @@ async def run_agent_server(): res = await service.DeleteTask(DeleteTaskRequest(task_type="async_dummy", resource_meta=metadata_bytes), ctx) assert isinstance(res, DeleteTaskResponse) + res = await service.CreateTask(sync_request, ctx) + assert res.resource.state == SUCCEEDED + assert res.resource.outputs == LiteralMap({}).to_flyte_idl() + res = await service.GetTask(GetTaskRequest(task_type=fake_agent, resource_meta=metadata_bytes), ctx) assert res is None @@ -198,7 +230,7 @@ def test_agent_server(): def test_is_terminal_state(): assert is_terminal_state(SUCCEEDED) - assert is_terminal_state(PERMANENT_FAILURE) + assert is_terminal_state(RETRYABLE_FAILURE) assert is_terminal_state(PERMANENT_FAILURE) assert not is_terminal_state(RUNNING) diff --git a/tests/flytekit/unit/extras/tasks/test_shell.py b/tests/flytekit/unit/extras/tasks/test_shell.py index 08e5bb92af..0ca7ca66bf 100644 --- a/tests/flytekit/unit/extras/tasks/test_shell.py +++ b/tests/flytekit/unit/extras/tasks/test_shell.py @@ -11,7 +11,7 @@ import flytekit from flytekit import kwtypes from flytekit.exceptions.user import FlyteRecoverableException -from flytekit.extras.tasks.shell import OutputLocation, RawShellTask, ShellTask, get_raw_shell_task +from flytekit.extras.tasks.shell import OutputLocation, RawShellTask, ShellTask, get_raw_shell_task, subproc_execute from flytekit.types.directory import FlyteDirectory from flytekit.types.file import CSVFile, FlyteFile @@ -323,3 +323,19 @@ def test_long_run_script(): name="long-running", script=script, )() + + +def test_subproc_execute(): + cmd = ["echo", "hello"] + o, e = subproc_execute(cmd) + assert o == "hello\n" + assert e == "" + + +def test_subproc_execute_with_shell(): + with tempfile.TemporaryDirectory() as tmp: + opth = os.path.join(tmp, "test.txt") + cmd = f"echo hello > {opth}" + subproc_execute(cmd, shell=True) + cont = open(opth).read() + assert "hello" in cont diff --git a/tests/flytekit/unit/models/core/test_security.py b/tests/flytekit/unit/models/core/test_security.py index c2933f9353..a7ed006174 100644 --- a/tests/flytekit/unit/models/core/test_security.py +++ b/tests/flytekit/unit/models/core/test_security.py @@ -1,3 +1,6 @@ +from unittest.mock import Mock + +import flytekit.configuration.plugin from flytekit.models.security import Secret @@ -11,3 +14,13 @@ def test_secret(): obj2 = Secret.from_flyte_idl(obj.to_flyte_idl()) assert obj2.key is None assert obj2.group_version == "v1" + + +def test_secret_no_group(monkeypatch): + plugin_mock = Mock() + plugin_mock.secret_requires_group.return_value = False + mock_global_plugin = {"plugin": plugin_mock} + monkeypatch.setattr(flytekit.configuration.plugin, "_GLOBAL_CONFIG", mock_global_plugin) + + s = Secret(key="key") + assert s.group is None