From 06e960f314a60ea2bd279eb645aa402d994d9d47 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 1 Dec 2021 17:10:17 -0800 Subject: [PATCH 01/35] Remove flyteidl from install_requires Signed-off-by: Eduardo Apolinario --- setup.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 305e3d3700..52d9173a08 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,8 @@ ] }, install_requires=[ - "flyteidl>=0.21.4", + # TODO: put the flyteidl constraint back + # "flyteidl>=0.21.4", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=6.0.0,<7.0.0", From c9047a87b70611a0911bc1176a5c6881f19db817 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 1 Dec 2021 17:12:14 -0800 Subject: [PATCH 02/35] Expose hash in Literal Signed-off-by: Eduardo Apolinario --- flytekit/models/literals.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py index 48553005b8..01d1120124 100644 --- a/flytekit/models/literals.py +++ b/flytekit/models/literals.py @@ -726,7 +726,9 @@ def from_flyte_idl(cls, pb2_object): class Literal(_common.FlyteIdlEntity): - def __init__(self, scalar: Scalar = None, collection: LiteralCollection = None, map: LiteralMap = None): + def __init__( + self, scalar: Scalar = None, collection: LiteralCollection = None, map: LiteralMap = None, hash: str = None + ): """ This IDL message represents a literal value in the Flyte ecosystem. @@ -737,6 +739,7 @@ def __init__(self, scalar: Scalar = None, collection: LiteralCollection = None, self._scalar = scalar self._collection = collection self._map = map + self.hash = hash @property def scalar(self): @@ -778,6 +781,7 @@ def to_flyte_idl(self): scalar=self.scalar.to_flyte_idl() if self.scalar is not None else None, collection=self.collection.to_flyte_idl() if self.collection is not None else None, map=self.map.to_flyte_idl() if self.map is not None else None, + hash=self.hash, ) @classmethod @@ -794,4 +798,9 @@ def from_flyte_idl(cls, pb2_object): scalar=Scalar.from_flyte_idl(pb2_object.scalar) if pb2_object.HasField("scalar") else None, collection=collection, map=LiteralMap.from_flyte_idl(pb2_object.map) if pb2_object.HasField("map") else None, + # TODO: explain that string always have a value set in protobufs, which means + # if we want to differentiate between the case of empty string and null we would have + # to wrap that in a separate value. Instead, we're deliberately setting `hash` to + # None in case of the default value (i.e. empty string). + hash=pb2_object.hash if pb2_object.hash else None, ) From 4d21957043feb3a3fe2f3f83b6af364004c6cf6d Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 1 Dec 2021 17:13:53 -0800 Subject: [PATCH 03/35] Set hash in TypeEngine Signed-off-by: Eduardo Apolinario --- flytekit/core/type_engine.py | 39 ++++++++++++++++++++++++++++++++++-- 1 file changed, 37 insertions(+), 2 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 925f79c770..17031d2f89 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -8,7 +8,7 @@ import mimetypes import typing from abc import ABC, abstractmethod -from typing import NamedTuple, Optional, Type, cast +from typing import Callable, NamedTuple, Optional, Type, cast from dataclasses_json import DataClassJsonMixin, dataclass_json from google.protobuf import json_format as _json_format @@ -18,6 +18,7 @@ from google.protobuf.json_format import ParseDict as _ParseDict from google.protobuf.struct_pb2 import Struct from marshmallow_jsonschema import JSONSchema +from typing_extensions import Annotated, get_args, get_origin from flytekit.common.exceptions import user as user_exceptions from flytekit.common.types import primitives as _primitives @@ -34,15 +35,27 @@ DEFINITIONS = "definitions" +# TODO: expose this in flytekit.__init__ +class HashMethod(typing.Generic[T]): + def __init__(self, function: Callable[[T], str]): + self._function = function + + def calculate(self, obj: T) -> str: + return self._function(obj) + + class TypeTransformer(typing.Generic[T]): """ Base transformer type that should be implemented for every python native type that can be handled by flytekit """ - def __init__(self, name: str, t: Type[T], enable_type_assertions: bool = True): + def __init__(self, name: str, t: Type[T], enable_type_assertions: bool = True, hash_overridable: bool = False): self._t = t self._name = name self._type_assertions_enabled = enable_type_assertions + # `hash_overridable` indicates that the literals produced by this type transformer can set their hashes if needed. + # See (link to documentation where this feature is explained). + self._hash_overridable = hash_overridable @property def name(self): @@ -62,6 +75,10 @@ def type_assertions_enabled(self) -> bool: """ return self._type_assertions_enabled + @property + def hash_overridable(self) -> bool: + return self._hash_overridable + def assert_type(self, t: Type[T], v: T): if not hasattr(t, "__origin__") and not isinstance(v, t): raise TypeError(f"Type of Val '{v}' is not an instance of {t}") @@ -449,7 +466,25 @@ def to_literal(cls, ctx: FlyteContext, python_val: typing.Any, python_type: Type transformer = cls.get_transformer(python_type) if transformer.type_assertions_enabled: transformer.assert_type(python_type, python_val) + + # In case the value is an annotated type we inspect the annotations and look for hash-related annotations. + hash = None + if transformer.hash_overridable and get_origin(python_type) is Annotated: + # We are now dealing with one of two cases: + # 1. The annotated type is a `HashMethod`, which indicates that we should we should produce the hash using + # the method indicated in the annotation. + # 2. The annotated type is being used for a different purpose other than calculating hash values, in which case + # we should just continue. + for annotation in get_args(python_type)[1:]: + if not isinstance(annotation, HashMethod): + continue + hash = annotation.calculate(python_val) + break + lv = transformer.to_literal(ctx, python_val, python_type, expected) + # TODO: should we worry about empty string? + if hash is not None: + lv.hash = hash return lv @classmethod From f9273db7e7d458e1616294b2dcb22dd48bbac8b6 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 1 Dec 2021 17:15:00 -0800 Subject: [PATCH 04/35] Modify cache key calculation to take hash into account Signed-off-by: Eduardo Apolinario --- flytekit/core/local_cache.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/flytekit/core/local_cache.py b/flytekit/core/local_cache.py index 9180224207..8c908f8138 100644 --- a/flytekit/core/local_cache.py +++ b/flytekit/core/local_cache.py @@ -2,7 +2,7 @@ from diskcache import Cache -from flytekit.models.literals import LiteralMap +from flytekit.models.literals import Literal, LiteralMap # Location on the filesystem where serialized objects will be stored # TODO: read from config @@ -10,7 +10,15 @@ def _calculate_cache_key(task_name: str, cache_version: str, input_literal_map: LiteralMap) -> str: - return f"{task_name}-{cache_version}-{input_literal_map}" + # TODO: document this. + literal_map_overridden = {} + for key, literal in input_literal_map.literals.items(): + if literal.hash is not None: + literal_map_overridden[key] = Literal(hash=literal.hash) + else: + literal_map_overridden[key] = literal + + return f"{task_name}-{cache_version}-{LiteralMap(literal_map_overridden)}" class LocalTaskCache(object): From 53dff4d9f17d3764aca8fbc302ee9a6434ad1f21 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 1 Dec 2021 17:15:50 -0800 Subject: [PATCH 05/35] Opt-in PandasDataFrameTransformer Signed-off-by: Eduardo Apolinario --- flytekit/types/schema/types_pandas.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/flytekit/types/schema/types_pandas.py b/flytekit/types/schema/types_pandas.py index 41a5423c08..9ec6e548df 100644 --- a/flytekit/types/schema/types_pandas.py +++ b/flytekit/types/schema/types_pandas.py @@ -112,6 +112,9 @@ class PandasDataFrameTransformer(TypeTransformer[pandas.DataFrame]): def __init__(self): super().__init__("PandasDataFrame<->GenericSchema", pandas.DataFrame) self._parquet_engine = _PARQUETIO_ENGINES[sdk.PARQUET_ENGINE.get()] + # Pandas dataframes can have their hashes overriden to facilitate the case of caching pandas dataframes by + # value. + self._hash_overridable = True @staticmethod def _get_schema_type() -> SchemaType: From 5ae247c80f956b59cb7e88e0370c0ae08dcbaeff Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 1 Dec 2021 17:19:24 -0800 Subject: [PATCH 06/35] Add unit tests Signed-off-by: Eduardo Apolinario --- tests/flytekit/unit/core/test_local_cache.py | 101 ++++++++++++++++- tests/flytekit/unit/core/test_type_engine.py | 76 +++++++++++++ tests/flytekit/unit/core/test_type_hints.py | 110 ++++++++++++++++++- 3 files changed, 285 insertions(+), 2 deletions(-) diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index c43421bbf9..da9c4b30a5 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -5,11 +5,13 @@ import pandas from dataclasses_json import dataclass_json from pytest import fixture +from typing_extensions import Annotated -from flytekit import SQLTask, kwtypes +from flytekit import SQLTask, dynamic, kwtypes from flytekit.core.local_cache import LocalTaskCache from flytekit.core.task import TaskMetadata, task from flytekit.core.testing import task_mock +from flytekit.core.type_engine import HashMethod from flytekit.core.workflow import workflow from flytekit.types.schema import FlyteSchema @@ -250,3 +252,100 @@ def my_wf(a: int, b: str) -> (int, str): x = my_wf(a=5, b="hello") assert x == (7, "hello world") assert n_cached_task_calls == 2 + + +def test_set_integer_literal_hash_is_not_cached(): + """ + Test to confirm that the local cache is not set in the case of integers, even if we + return an annotated integer. In order to make this very explicit, we define a constant hash + function, i.e. the same value is returned by it regardless of the input. + """ + + def constant_hash_function(a: int) -> str: + return "hash" + + @task + def t0(a: int) -> Annotated[int, HashMethod(function=constant_hash_function)]: + return a + + @task(cache=True, cache_version="0.0.1") + def t1(cached_a: int) -> int: + global n_cached_task_calls + n_cached_task_calls += 1 + return cached_a + + @workflow + def wf(a: int) -> int: + annotated_a = t0(a=a) + return t1(cached_a=annotated_a) + + assert n_cached_task_calls == 0 + assert wf(a=3) == 3 + assert n_cached_task_calls == 1 + # Confirm that the value is not cached, even though we set a hash function that + # returns a constant value and that the task has only one input. + assert wf(a=2) == 2 + assert n_cached_task_calls == 2 + # Confirm that the cache is hit if we execute the workflow with the same value as previous run. + assert wf(a=2) == 2 + assert n_cached_task_calls == 2 + + +def test_pass_annotated_to_downstream_tasks(): + @task + def t0(a: int) -> Annotated[int, HashMethod(function=str)]: + return a + 1 + + @task(cache=True, cache_version="42") + def downstream_t(a: int) -> int: + global n_cached_task_calls + n_cached_task_calls += 1 + return a + 2 + + @dynamic + def t1(a: int) -> int: + v = t0(a=a) + + # We should have a cache miss in the first call to downstream_t and have a cache hit + # on the second call. + v_1 = downstream_t(a=v) + v_2 = downstream_t(a=v) + + return v_1 + v_2 + + assert n_cached_task_calls == 0 + assert t1(a=3) == (6 + 6) + assert n_cached_task_calls == 1 + + +def test_pandas_dataframe_hash(): + """ + Test that cache is hit in the case of pandas dataframes where we annotated dataframes to hash + the contents of the dataframes. + """ + + def hash_pandas_dataframe(df: pandas.DataFrame) -> str: + return str(pandas.util.hash_pandas_object(df)) + + @task + def uncached_data_reading_task() -> Annotated[pandas.DataFrame, HashMethod(hash_pandas_dataframe)]: + return pandas.DataFrame({"column_1": [1, 2, 3]}) + + @task(cache=True, cache_version="0.1") + def cached_data_processing_task(data: pandas.DataFrame) -> pandas.DataFrame: + global n_cached_task_calls + n_cached_task_calls += 1 + return data * 2 + + @workflow + def my_workflow(): + raw_data = uncached_data_reading_task() + cached_data_processing_task(data=raw_data) + + assert n_cached_task_calls == 0 + my_workflow() + assert n_cached_task_calls == 1 + + # Confirm that we see a cache hit in the case of annotated dataframes. + my_workflow() + assert n_cached_task_calls == 1 diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 6c660b7106..d60dddece4 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -5,18 +5,24 @@ from datetime import timedelta from enum import Enum +import pandas as pd import pytest from dataclasses_json import DataClassJsonMixin, dataclass_json from flyteidl.core import errors_pb2 from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct from marshmallow_jsonschema import JSONSchema +from typing_extensions import Annotated from flytekit.common.exceptions import user as user_exceptions +from flytekit.common.types.primitives import Integer from flytekit.core.context_manager import FlyteContext, FlyteContextManager +from flytekit.core.dynamic_workflow_task import dynamic +from flytekit.core.task import task from flytekit.core.type_engine import ( DataclassTransformer, DictTransformer, + HashMethod, ListTransformer, SimpleTransformer, TypeEngine, @@ -32,6 +38,7 @@ from flytekit.types.file.file import FlyteFile, FlyteFilePathTransformer from flytekit.types.pickle import FlytePickle from flytekit.types.pickle.pickle import FlytePickleTransformer +from flytekit.types.schema.types_pandas import PandasDataFrameTransformer def test_type_engine(): @@ -634,3 +641,72 @@ def test_dict_to_literal_map_with_wrong_input_type(): guessed_python_types = {"a": str} with pytest.raises(user_exceptions.FlyteTypeException): TypeEngine.dict_to_literal_map(ctx, input, guessed_python_types) + + +def test_pass_annotated_to_downstream_tasks(): + """ + Test to confirm that the loaded dataframe is not affected and can be used in @dynamic. + """ + # pandas dataframe hash function + def hash_pandas_dataframe(df: pd.DataFrame) -> str: + return str(pd.util.hash_pandas_object(df)) + + @task + def t0(a: int) -> Annotated[int, HashMethod(function=str)]: + return a + 1 + + @task + def annotated_return_task() -> Annotated[pd.DataFrame, HashMethod(hash_pandas_dataframe)]: + return pd.DataFrame({"column_1": [1, 2, 3]}) + + @task(cache=True, cache_version="42") + def downstream_t(a: int, df: pd.DataFrame) -> int: + return a + 2 + len(df) + + @dynamic + def t1(a: int) -> int: + v = t0(a=a) + df = annotated_return_task() + + # We should have a cache miss in the first call to downstream_t + v_1 = downstream_t(a=v, df=df) + v_2 = downstream_t(a=v, df=df) + + return v_1 + v_2 + + assert t1(a=3) == (6 + 6 + 6) + + +def test_literal_hash_int_not_set(): + """ + Test to confirm that annotating an integer with `HashMethod` does not force the literal to have its + hash set. + """ + ctx = FlyteContext.current_context() + lv = TypeEngine.to_literal(ctx, 42, Annotated[int, HashMethod(str)], Integer.to_flyte_literal_type()) + assert lv.scalar.primitive.integer == 42 + assert lv.hash is None + + +def test_literal_hash_to_python_value(): + """ + Test to confirm that literals can be converted to python values, regardless of the hash value set in the literal. + """ + ctx = FlyteContext.current_context() + + def constant_hash(df: pd.DataFrame) -> str: + return "h4Sh" + + df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + pandas_df_transformer = PandasDataFrameTransformer() + literal_with_hash_set = TypeEngine.to_literal( + ctx, + df, + Annotated[pd.DataFrame, HashMethod(constant_hash)], + pandas_df_transformer.get_literal_type(pd.DataFrame), + ) + assert literal_with_hash_set.hash == "h4Sh" + # Confirm tha the loaded dataframe is not affected + python_df = TypeEngine.to_python_value(ctx, literal_with_hash_set, pd.DataFrame) + expected_df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + assert expected_df.equals(python_df) diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index e9f54c829b..1406d02657 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -11,6 +11,7 @@ import pytest from dataclasses_json import dataclass_json from google.protobuf.struct_pb2 import Struct +from typing_extensions import Annotated import flytekit from flytekit import ContainerTask, Secret, SQLTask, dynamic, kwtypes, map_task @@ -24,7 +25,7 @@ from flytekit.core.resources import Resources from flytekit.core.task import TaskMetadata, task from flytekit.core.testing import patch, task_mock -from flytekit.core.type_engine import RestrictedTypeError, TypeEngine +from flytekit.core.type_engine import HashMethod, RestrictedTypeError, TypeEngine from flytekit.core.workflow import workflow from flytekit.models import literals as _literal_models from flytekit.models.core import types as _core_types @@ -1427,3 +1428,110 @@ def foo3(a: typing.Dict) -> typing.Dict: with pytest.raises(TypeError, match="Not a collection type simple: STRUCT\n but got a list \\[{'hello': 2}\\]"): foo3(a=[{"hello": 2}]) + + +def test_task_annotate_primitive_type_has_no_effect(): + @task + def plus_two( + a: int, + ) -> Annotated[int, HashMethod(str)]: # Note the use of `str` as the hash function for ints. This has no effect. + return a + 2 + + assert plus_two(a=1) == 3 + + ctx = context_manager.FlyteContextManager.current_context() + output_lm = plus_two.dispatch_execute( + ctx, + _literal_models.LiteralMap( + literals={ + "a": _literal_models.Literal( + scalar=_literal_models.Scalar(primitive=_literal_models.Primitive(integer=3)) + ) + } + ), + ) + assert output_lm.literals["o0"].scalar.primitive.integer == 5 + assert output_lm.literals["o0"].hash is None + + +def test_task_hash_return_pandas_dataframe(): + constant_value = "road-hash" + + def constant_function(df: pandas.DataFrame) -> str: + return constant_value + + @task + def t0() -> Annotated[pandas.DataFrame, HashMethod(constant_function)]: + return pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + + ctx = context_manager.FlyteContextManager.current_context() + output_lm = t0.dispatch_execute(ctx, _literal_models.LiteralMap(literals={})) + assert output_lm.literals["o0"].hash == constant_value + + # Confirm that the literal containing a hash does not have any effect on the scalar. + df = TypeEngine.to_python_value(ctx, output_lm.literals["o0"], pandas.DataFrame) + expected_df = pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + assert df.equals(expected_df) + + +def test_workflow_containing_multiple_annotated_tasks(): + def hash_function_t0(df: pandas.DataFrame) -> str: + return "hash-0" + + @task + def t0() -> Annotated[pandas.DataFrame, HashMethod(hash_function_t0)]: + return pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + + def hash_function_t1(df: pandas.DataFrame) -> str: + return "hash-1" + + @task + def t1() -> Annotated[pandas.DataFrame, HashMethod(hash_function_t1)]: + return pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}) + + @task + def t2() -> pandas.DataFrame: + return pandas.DataFrame(data={"col1": [100, 200], "col2": [300, 400]}) + + # Auxiliary task used to sum up the dataframes. It demonstrates that the use of `Annotated` does not + # have any impact in the definition and execution of cached or uncached downstream tasks + @task + def sum_dataframes(df0: pandas.DataFrame, df1: pandas.DataFrame, df2: pandas.DataFrame) -> pandas.DataFrame: + return df0 + df1 + df2 + + @workflow + def wf() -> pandas.DataFrame: + df0 = t0() + df1 = t1() + df2 = t2() + return sum_dataframes(df0=df0, df1=df1, df2=df2) + + df = wf() + + expected_df = pandas.DataFrame(data={"col1": [1 + 10 + 100, 2 + 20 + 200], "col2": [3 + 30 + 300, 4 + 40 + 400]}) + assert expected_df.equals(df) + + +def test_list_containing_multiple_annotated_pandas_dataframes(): + def hash_pandas_dataframe(df: pandas.DataFrame) -> str: + return str(pandas.util.hash_pandas_object(df)) + + @task + def produce_list_of_annotated_dataframes() -> typing.List[ + Annotated[pandas.DataFrame, HashMethod(hash_pandas_dataframe)] + ]: + return [pandas.DataFrame({"column_1": [1, 2, 3]}), pandas.DataFrame({"column_1": [4, 5, 6]})] + + @task(cache=True, cache_version="v0") + def sum_list_of_pandas_dataframes(lst: typing.List[pandas.DataFrame]) -> pandas.DataFrame: + return sum(lst) + + @workflow + def wf() -> pandas.DataFrame: + lst = produce_list_of_annotated_dataframes() + return sum_list_of_pandas_dataframes(lst=lst) + + df = wf() + + expected_df = pandas.DataFrame({"column_1": [5, 7, 9]}) + assert expected_df.equals(df) From 93057469cfa212b0b62e80eba4501fceb3b29c85 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 3 Dec 2021 15:09:43 -0800 Subject: [PATCH 07/35] Iterate using a flyteidl branch Signed-off-by: Eduardo Apolinario --- requirements.in | 1 + requirements.txt | 8 ++++---- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/requirements.in b/requirements.in index 408185fca8..e5ee7fadba 100644 --- a/requirements.in +++ b/requirements.in @@ -1,4 +1,5 @@ .[all] +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekit attrs<21 # We need to restrict constrain the versions of both jsonschema and pyyaml because of docker-compose (which is diff --git a/requirements.txt b/requirements.txt index ee632aa7d5..ebb249137b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,11 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# make requirements.txt +# pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via -r requirements.in -e file:.#egg=flytekit # via -r requirements.in ansiwrap==0.8.4 @@ -81,8 +83,6 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.8 - # via flytekit gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 From ecd8b935eaf3b1feb381450499f63da450eff996 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 3 Dec 2021 15:28:47 -0800 Subject: [PATCH 08/35] Regenerate requirements files Signed-off-by: Eduardo Apolinario --- dev-requirements.txt | 47 +++++++++++++++++-------------------- requirements.txt | 56 ++++++++++++++++++++++---------------------- 2 files changed, 49 insertions(+), 54 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index a1b1fa3827..c9d8b2a08d 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make dev-requirements.txt @@ -44,7 +44,7 @@ chardet==4.0.0 # via # -c requirements.txt # binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via # -c requirements.txt # requests @@ -67,13 +67,13 @@ cookiecutter==1.7.3 # via # -c requirements.txt # flytekit -coverage[toml]==6.1.1 +coverage[toml]==6.2 # via -r dev-requirements.in -croniter==1.0.15 +croniter==1.1.0 # via # -c requirements.txt # flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via # -c requirements.txt # paramiko @@ -90,7 +90,7 @@ deprecated==1.2.13 # via # -c requirements.txt # flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via # -c requirements.txt # flytekit @@ -112,21 +112,17 @@ dockerpty==0.4.1 # via docker-compose docopt==0.6.2 # via docker-compose -docstring-parser==0.12 +docstring-parser==0.13 # via # -c requirements.txt # flytekit -filelock==3.3.2 +filelock==3.4.0 # via virtualenv -flyteidl==0.21.8 +grpcio==1.42.0 # via # -c requirements.txt # flytekit -grpcio==1.41.1 - # via - # -c requirements.txt - # flytekit -identify==2.3.5 +identify==2.4.0 # via pre-commit idna==3.3 # via @@ -159,7 +155,7 @@ jsonschema==3.2.0 # via # -c requirements.txt # docker-compose -keyring==23.2.1 +keyring==23.4.0 # via # -c requirements.txt # flytekit @@ -167,7 +163,7 @@ markupsafe==2.0.1 # via # -c requirements.txt # jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # -c requirements.txt # dataclasses-json @@ -201,7 +197,7 @@ numpy==1.21.4 # -c requirements.txt # pandas # pyarrow -packaging==21.2 +packaging==21.3 # via # -c requirements.txt # pytest @@ -209,7 +205,7 @@ pandas==1.3.4 # via # -c requirements.txt # flytekit -paramiko==2.8.0 +paramiko==2.8.1 # via # -c requirements.txt # docker @@ -223,19 +219,18 @@ poyo==0.5.0 # via # -c requirements.txt # cookiecutter -pre-commit==2.15.0 +pre-commit==2.16.0 # via -r dev-requirements.in protobuf==3.19.1 # via # -c requirements.txt - # flyteidl # flytekit py==1.11.0 # via # -c requirements.txt # pytest # retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via # -c requirements.txt # flytekit @@ -247,7 +242,7 @@ pynacl==1.4.0 # via # -c requirements.txt # paramiko -pyparsing==2.4.7 +pyparsing==3.0.6 # via # -c requirements.txt # packaging @@ -271,7 +266,7 @@ python-dateutil==2.8.1 # croniter # flytekit # pandas -python-dotenv==0.19.1 +python-dotenv==0.19.2 # via docker-compose python-json-logger==2.0.2 # via @@ -295,7 +290,7 @@ pyyaml==5.4.1 # -c requirements.txt # docker-compose # pre-commit -regex==2021.11.9 +regex==2021.11.10 # via # -c requirements.txt # docker-image-py @@ -307,7 +302,7 @@ requests==2.26.0 # docker-compose # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via # -c requirements.txt # flytekit @@ -356,7 +351,7 @@ tomli==1.2.2 # via # -c requirements.txt # coverage -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via # -c requirements.txt # mypy diff --git a/requirements.txt b/requirements.txt index ebb249137b..91122e0a0e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# pip-compile requirements.in +# make requirements.txt # -e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl # via -r requirements.in @@ -22,13 +22,13 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -black==21.10b0 +black==21.11b1 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.20.2 +boto3==1.20.20 # via sagemaker-training -botocore==1.23.2 +botocore==1.23.20 # via # boto3 # s3transfer @@ -41,7 +41,7 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -56,9 +56,9 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via # paramiko # secretstorage @@ -72,11 +72,11 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit entrypoints==0.3 # via @@ -87,7 +87,7 @@ gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit hmsclient==0.1.1 # via flytekit @@ -99,13 +99,13 @@ inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.6 # via flytekit -ipython==7.29.0 +ipython==7.30.1 # via ipykernel ipython-genutils==0.2.0 # via # ipykernel # nbformat -jedi==0.18.0 +jedi==0.18.1 # via ipython jeepney==0.7.1 # via @@ -126,7 +126,7 @@ jsonschema==3.2.0 # via # -r requirements.in # nbformat -jupyter-client==7.0.6 +jupyter-client==7.1.0 # via # ipykernel # nbclient @@ -139,11 +139,11 @@ jupyterlab-pygments==0.1.2 # via nbconvert k8s-proto==0.0.3 # via flytekit -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -162,18 +162,18 @@ mypy-extensions==0.4.3 # typing-inspect natsort==8.0.0 # via flytekit -nbclient==0.5.5 +nbclient==0.5.9 # via # nbconvert # papermill -nbconvert==6.2.0 +nbconvert==6.3.0 # via flytekit nbformat==5.1.3 # via # nbclient # nbconvert # papermill -nest-asyncio==1.5.1 +nest-asyncio==1.5.4 # via # jupyter-client # nbclient @@ -184,7 +184,7 @@ numpy==1.21.4 # pyarrow # sagemaker-training # scipy -packaging==21.2 +packaging==21.3 # via bleach pandas==1.3.4 # via flytekit @@ -192,9 +192,9 @@ pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekit -paramiko==2.8.0 +paramiko==2.8.1 # via sagemaker-training -parso==0.8.2 +parso==0.8.3 # via jedi pathspec==0.9.0 # via black @@ -206,7 +206,7 @@ platformdirs==2.4.0 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.23 # via ipython protobuf==3.19.1 # via @@ -222,7 +222,7 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi @@ -233,7 +233,7 @@ pygments==2.10.0 # nbconvert pynacl==1.4.0 # via paramiko -pyparsing==2.4.7 +pyparsing==3.0.6 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -263,7 +263,7 @@ pyyaml==5.4.1 # papermill pyzmq==22.3.0 # via jupyter-client -regex==2021.11.9 +regex==2021.11.10 # via # black # docker-image-py @@ -273,7 +273,7 @@ requests==2.26.0 # flytekit # papermill # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit @@ -283,7 +283,7 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scipy==1.7.2 +scipy==1.7.3 # via sagemaker-training secretstorage==3.3.1 # via keyring @@ -333,7 +333,7 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via # black # typing-inspect From 44b72f6b80e4bd2c34a10933ecd98d37d8edbdb9 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 3 Dec 2021 15:36:04 -0800 Subject: [PATCH 09/35] Regenerate requirements files Signed-off-by: Eduardo Apolinario --- doc-requirements.txt | 75 +++++++++---------- requirements-spark2.txt | 60 +++++++-------- .../workflows/requirements.txt | 51 ++++++++----- 3 files changed, 97 insertions(+), 89 deletions(-) diff --git a/doc-requirements.txt b/doc-requirements.txt index 1e766d5f31..d6d6c6546e 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make doc-requirements.txt @@ -12,7 +12,7 @@ ansiwrap==0.8.4 # via papermill arrow==1.2.1 # via jinja2-time -astroid==2.8.4 +astroid==2.9.0 # via sphinx-autoapi attrs==21.2.0 # via jsonschema @@ -29,13 +29,13 @@ beautifulsoup4==4.10.0 # sphinx-material binaryornot==0.4.4 # via cookiecutter -black==21.10b0 +black==21.11b1 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.20.2 +boto3==1.20.20 # via sagemaker-training -botocore==1.23.2 +botocore==1.23.20 # via # boto3 # s3transfer @@ -48,7 +48,7 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -63,9 +63,9 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via # -r doc-requirements.in # paramiko @@ -82,11 +82,11 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit docutils==0.17.1 # via sphinx @@ -95,15 +95,13 @@ entrypoints==0.3 # jupyter-client # nbconvert # papermill -flyteidl==0.21.8 - # via flytekit furo @ git+git://github.com/flyteorg/furo@main # via -r doc-requirements.in gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.1 +grpcio==1.42.0 # via # -r doc-requirements.in # flytekit @@ -115,19 +113,17 @@ imagesize==1.3.0 # via sphinx importlib-metadata==4.8.2 # via keyring -importlib-resources==5.4.0 - # via jsonschema inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.6 # via flytekit -ipython==7.29.0 +ipython==7.30.1 # via ipykernel ipython-genutils==0.2.0 # via # ipykernel # nbformat -jedi==0.18.0 +jedi==0.18.1 # via ipython jeepney==0.7.1 # via @@ -148,7 +144,7 @@ jmespath==0.10.0 # botocore jsonschema==4.2.1 # via nbformat -jupyter-client==7.0.6 +jupyter-client==7.1.0 # via # ipykernel # nbclient @@ -161,7 +157,7 @@ jupyterlab-pygments==0.1.2 # via nbconvert k8s-proto==0.0.3 # via flytekit -keyring==23.2.1 +keyring==23.4.0 # via flytekit lazy-object-proxy==1.6.0 # via astroid @@ -169,7 +165,7 @@ lxml==4.6.4 # via sphinx-material markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -188,18 +184,18 @@ mypy-extensions==0.4.3 # typing-inspect natsort==8.0.0 # via flytekit -nbclient==0.5.5 +nbclient==0.5.9 # via # nbconvert # papermill -nbconvert==6.2.0 +nbconvert==6.3.0 # via flytekit nbformat==5.1.3 # via # nbclient # nbconvert # papermill -nest-asyncio==1.5.1 +nest-asyncio==1.5.4 # via # jupyter-client # nbclient @@ -210,7 +206,7 @@ numpy==1.21.4 # pyarrow # sagemaker-training # scipy -packaging==21.2 +packaging==21.3 # via # bleach # sphinx @@ -220,9 +216,9 @@ pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekit -paramiko==2.8.0 +paramiko==2.8.1 # via sagemaker-training -parso==0.8.2 +parso==0.8.3 # via jedi pathspec==0.9.0 # via black @@ -234,11 +230,10 @@ platformdirs==2.4.0 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.23 # via ipython protobuf==3.19.1 # via - # flyteidl # flytekit # k8s-proto # sagemaker-training @@ -250,7 +245,7 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi @@ -263,7 +258,7 @@ pygments==2.10.0 # sphinx-prompt pynacl==1.4.0 # via paramiko -pyparsing==2.4.7 +pyparsing==3.0.6 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -296,7 +291,7 @@ pyyaml==6.0 # sphinx-autoapi pyzmq==22.3.0 # via jupyter-client -regex==2021.11.9 +regex==2021.11.10 # via # black # docker-image-py @@ -307,7 +302,7 @@ requests==2.26.0 # papermill # responses # sphinx -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit @@ -317,7 +312,7 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scipy==1.7.2 +scipy==1.7.3 # via sagemaker-training secretstorage==3.3.1 # via keyring @@ -335,13 +330,13 @@ six==1.16.0 # sagemaker-training # sphinx-code-include # thrift -snowballstemmer==2.1.0 +snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via flytekit -soupsieve==2.3 +soupsieve==2.3.1 # via beautifulsoup4 -sphinx==4.2.0 +sphinx==4.3.1 # via # -r doc-requirements.in # furo @@ -361,7 +356,7 @@ sphinx-copybutton==0.4.0 # via -r doc-requirements.in sphinx-fontawesome==0.0.6 # via -r doc-requirements.in -sphinx-gallery==0.10.0 +sphinx-gallery==0.10.1 # via -r doc-requirements.in sphinx-material==0.0.35 # via -r doc-requirements.in @@ -411,7 +406,7 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via # astroid # black @@ -442,9 +437,7 @@ wrapt==1.13.3 # deprecated # flytekit zipp==3.6.0 - # via - # importlib-metadata - # importlib-resources + # via importlib-metadata zope.event==4.5.0 # via gevent zope.interface==5.4.0 diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 37a2664140..98c2f7cfac 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -1,9 +1,11 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make requirements-spark2.txt # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via -r requirements.in -e file:.#egg=flytekit # via # -r requirements-spark2.in @@ -22,13 +24,13 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -black==21.10b0 +black==21.11b1 # via papermill bleach==4.1.0 # via nbconvert -boto3==1.20.2 +boto3==1.20.20 # via sagemaker-training -botocore==1.23.2 +botocore==1.23.20 # via # boto3 # s3transfer @@ -41,7 +43,7 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -56,9 +58,9 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via # paramiko # secretstorage @@ -72,24 +74,22 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit entrypoints==0.3 # via # jupyter-client # nbconvert # papermill -flyteidl==0.21.8 - # via flytekit gevent==21.8.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit hmsclient==0.1.1 # via flytekit @@ -101,13 +101,13 @@ inotify_simple==1.2.1 # via sagemaker-training ipykernel==5.5.6 # via flytekit -ipython==7.29.0 +ipython==7.30.1 # via ipykernel ipython-genutils==0.2.0 # via # ipykernel # nbformat -jedi==0.18.0 +jedi==0.18.1 # via ipython jeepney==0.7.1 # via @@ -128,7 +128,7 @@ jsonschema==3.2.0 # via # -r requirements.in # nbformat -jupyter-client==7.0.6 +jupyter-client==7.1.0 # via # ipykernel # nbclient @@ -141,11 +141,11 @@ jupyterlab-pygments==0.1.2 # via nbconvert k8s-proto==0.0.3 # via flytekit -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -164,18 +164,18 @@ mypy-extensions==0.4.3 # typing-inspect natsort==8.0.0 # via flytekit -nbclient==0.5.5 +nbclient==0.5.9 # via # nbconvert # papermill -nbconvert==6.2.0 +nbconvert==6.3.0 # via flytekit nbformat==5.1.3 # via # nbclient # nbconvert # papermill -nest-asyncio==1.5.1 +nest-asyncio==1.5.4 # via # jupyter-client # nbclient @@ -186,7 +186,7 @@ numpy==1.21.4 # pyarrow # sagemaker-training # scipy -packaging==21.2 +packaging==21.3 # via bleach pandas==1.3.4 # via flytekit @@ -194,9 +194,9 @@ pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekit -paramiko==2.8.0 +paramiko==2.8.1 # via sagemaker-training -parso==0.8.2 +parso==0.8.3 # via jedi pathspec==0.9.0 # via black @@ -208,7 +208,7 @@ platformdirs==2.4.0 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.23 # via ipython protobuf==3.19.1 # via @@ -224,7 +224,7 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi @@ -235,7 +235,7 @@ pygments==2.10.0 # nbconvert pynacl==1.4.0 # via paramiko -pyparsing==2.4.7 +pyparsing==3.0.6 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -265,7 +265,7 @@ pyyaml==5.4.1 # papermill pyzmq==22.3.0 # via jupyter-client -regex==2021.11.9 +regex==2021.11.10 # via # black # docker-image-py @@ -275,7 +275,7 @@ requests==2.26.0 # flytekit # papermill # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit @@ -285,7 +285,7 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekit -scipy==1.7.2 +scipy==1.7.3 # via sagemaker-training secretstorage==3.3.1 # via keyring @@ -335,7 +335,7 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via # black # typing-inspect diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index a37b0c3af8..e605f635e6 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.8 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -14,7 +14,7 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -26,9 +26,9 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 +cryptography==36.0.0 # via secretstorage cycler==0.11.0 # via matplotlib @@ -38,17 +38,19 @@ decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.12 # via flytekit flytekit==0.24.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -grpcio==1.41.1 +fonttools==4.28.3 + # via matplotlib +grpcio==1.42.0 # via flytekit idna==3.3 # via requests @@ -66,13 +68,13 @@ jinja2-time==0.2.0 # via cookiecutter joblib==1.1.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -keyring==23.2.1 +keyring==23.4.0 # via flytekit kiwisolver==1.3.2 # via matplotlib markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -81,7 +83,7 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib==3.4.3 +matplotlib==3.5.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in mypy-extensions==0.4.3 # via typing-inspect @@ -93,8 +95,12 @@ numpy==1.21.4 # opencv-python # pandas # pyarrow -opencv-python==4.5.4.58 +opencv-python==4.5.4.60 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in +packaging==21.3 + # via + # matplotlib + # setuptools-scm pandas==1.3.4 # via flytekit pillow==8.4.0 @@ -107,12 +113,14 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi -pyparsing==3.0.5 - # via matplotlib +pyparsing==3.0.6 + # via + # matplotlib + # packaging python-dateutil==2.8.1 # via # arrow @@ -130,19 +138,21 @@ pytz==2018.4 # via # flytekit # pandas -regex==2021.11.9 +regex==2021.11.10 # via docker-image-py requests==2.26.0 # via # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit secretstorage==3.3.1 # via keyring +setuptools-scm==6.3.2 + # via matplotlib six==1.16.0 # via # cookiecutter @@ -156,7 +166,9 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +tomli==1.2.2 + # via setuptools-scm +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json @@ -175,3 +187,6 @@ wrapt==1.13.3 # flytekit zipp==3.6.0 # via importlib-metadata + +# The following packages are considered to be unsafe in a requirements file: +# setuptools From 22c90d397ea14ff540b15c2855d7be180bb93676 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 20 Jan 2022 17:00:25 -0800 Subject: [PATCH 10/35] Move _hash_overridable to StructureDatasetTransformerEngine Signed-off-by: Eduardo Apolinario --- flytekit/types/structured/structured_dataset.py | 14 +++++++++++--- tests/flytekit/unit/core/test_type_engine.py | 4 +++- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py index fb9ab4f79f..ba90b575f5 100644 --- a/flytekit/types/structured/structured_dataset.py +++ b/flytekit/types/structured/structured_dataset.py @@ -22,7 +22,7 @@ import pyarrow as pa from flytekit.core.context_manager import FlyteContext, FlyteContextManager -from flytekit.core.type_engine import TypeTransformer +from flytekit.core.type_engine import HashMethod, TypeTransformer from flytekit.extend import TypeEngine from flytekit.loggers import logger from flytekit.models import literals @@ -335,6 +335,9 @@ def __init__(self): super().__init__("StructuredDataset Transformer", StructuredDataset) self._type_assertions_enabled = False + # Instances of StructuredDataset can have their hashes overriden to facilitate the case of caching them. + self._hash_overridable = True + def register_handler(self, h: Handlers, default_for_type: Optional[bool] = True, override: Optional[bool] = False): """ Call this with any handler to register it with this dataframe meta-transformer @@ -534,8 +537,13 @@ def _get_dataset_type(self, t: typing.Union[Type[StructuredDataset], typing.Any] # my_cols = kwtypes(x=int, y=str) # 1. Fill in format correctly by checking for typing.annotated. For example, Annotated[pd.Dataframe, my_cols] if get_origin(t) is Annotated: - _, *hint_args = get_args(t) - if type(hint_args[0]) is collections.OrderedDict: + annotated_type, *hint_args = get_args(t) + + # TODO: Strenghten this method of detecting valid annotations. We need a way to handle multiple annotations as opposed + # to assuming that only a single annotation exists. + if type(hint_args[0]) is HashMethod: + return self._get_dataset_type(annotated_type) + elif type(hint_args[0]) is collections.OrderedDict: for k, v in hint_args[0].items(): lt = self._get_dataset_column_literal_type(v) converted_cols.append(StructuredDatasetType.DatasetColumn(name=k, literal_type=lt)) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index d000c3912f..f7d6aa128b 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -859,7 +859,9 @@ def test_literal_hash_int_not_set(): hash set. """ ctx = FlyteContext.current_context() - lv = TypeEngine.to_literal(ctx, 42, Annotated[int, HashMethod(str)], Integer.to_flyte_literal_type()) + lv = TypeEngine.to_literal( + ctx, 42, Annotated[int, HashMethod(str)], LiteralType(simple=model_types.SimpleType.INTEGER) + ) assert lv.scalar.primitive.integer == 42 assert lv.hash is None From 6ac4b44a08cbd3b7a93a2695884fc5b11ab88222 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 25 Jan 2022 17:33:41 -0800 Subject: [PATCH 11/35] Move HashMethod to flytekit.core.hash Signed-off-by: Eduardo Apolinario --- flytekit/__init__.py | 1 + flytekit/core/hash.py | 20 ++++++++++++++++++++ flytekit/core/type_engine.py | 20 ++++---------------- tests/flytekit/unit/core/test_local_cache.py | 2 +- tests/flytekit/unit/core/test_type_engine.py | 2 +- tests/flytekit/unit/core/test_type_hints.py | 3 ++- 6 files changed, 29 insertions(+), 19 deletions(-) diff --git a/flytekit/__init__.py b/flytekit/__init__.py index fd8fcc7d9c..63e9bf3687 100644 --- a/flytekit/__init__.py +++ b/flytekit/__init__.py @@ -167,6 +167,7 @@ from flytekit.core.context_manager import ExecutionParameters, FlyteContext, FlyteContextManager from flytekit.core.data_persistence import DataPersistence, DataPersistencePlugins from flytekit.core.dynamic_workflow_task import dynamic +from flytekit.core.hash import HashMethod from flytekit.core.launch_plan import LaunchPlan from flytekit.core.map_task import map_task from flytekit.core.notification import Email, PagerDuty, Slack diff --git a/flytekit/core/hash.py b/flytekit/core/hash.py index 8b2559f696..2aca94e846 100644 --- a/flytekit/core/hash.py +++ b/flytekit/core/hash.py @@ -1,3 +1,23 @@ +from typing import Callable, Generic, TypeVar + +T = TypeVar("T") + + class HashOnReferenceMixin(object): def __hash__(self): return hash(id(self)) + + +class HashMethod(Generic[T]): + """ + Flyte-specific object used to wrap the hash function for a specific type + """ + + def __init__(self, function: Callable[[T], str]): + self._function = function + + def calculate(self, obj: T) -> str: + """ + Calculate hash for `obj`. + """ + return self._function(obj) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index fa00c58421..24132c3f3e 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -8,12 +8,7 @@ import mimetypes import typing from abc import ABC, abstractmethod -from typing import Callable, NamedTuple, Optional, Type, cast - -try: - from typing import Annotated, get_args, get_origin -except ImportError: - from typing_extensions import Annotated, get_origin, get_args +from typing import NamedTuple, Optional, Type, cast from dataclasses_json import DataClassJsonMixin, dataclass_json from google.protobuf import json_format as _json_format @@ -24,8 +19,10 @@ from google.protobuf.struct_pb2 import Struct from marshmallow_enum import EnumField, LoadDumpOptions from marshmallow_jsonschema import JSONSchema +from typing_extensions import Annotated, get_args, get_origin from flytekit.core.context_manager import FlyteContext +from flytekit.core.hash import HashMethod from flytekit.core.type_helpers import load_type_from_tag from flytekit.exceptions import user as user_exceptions from flytekit.loggers import logger @@ -49,15 +46,6 @@ DEFINITIONS = "definitions" -# TODO: expose this in flytekit.__init__ -class HashMethod(typing.Generic[T]): - def __init__(self, function: Callable[[T], str]): - self._function = function - - def calculate(self, obj: T) -> str: - return self._function(obj) - - class TypeTransformer(typing.Generic[T]): """ Base transformer type that should be implemented for every python native type that can be handled by flytekit @@ -634,7 +622,7 @@ def to_literal(cls, ctx: FlyteContext, python_val: typing.Any, python_type: Type break lv = transformer.to_literal(ctx, python_val, python_type, expected) - # TODO: should we worry about empty string? + if hash is not None: lv.hash = hash return lv diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index 7431347544..922fdfcccd 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -8,10 +8,10 @@ from typing_extensions import Annotated from flytekit import SQLTask, dynamic, kwtypes +from flytekit.core.hash import HashMethod from flytekit.core.local_cache import LocalTaskCache from flytekit.core.task import TaskMetadata, task from flytekit.core.testing import task_mock -from flytekit.core.type_engine import HashMethod from flytekit.core.workflow import workflow from flytekit.types.schema import FlyteSchema diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 453c5385eb..ee47e499a8 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -21,11 +21,11 @@ from flytekit import kwtypes from flytekit.core.context_manager import FlyteContext, FlyteContextManager from flytekit.core.dynamic_workflow_task import dynamic +from flytekit.core.hash import HashMethod from flytekit.core.task import task from flytekit.core.type_engine import ( DataclassTransformer, DictTransformer, - HashMethod, ListTransformer, LiteralsResolver, SimpleTransformer, diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index 2b944cf1c1..6885e73f99 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -22,12 +22,13 @@ from flytekit.core.condition import conditional from flytekit.core.context_manager import ExecutionState, FastSerializationSettings, Image, ImageConfig from flytekit.core.data_persistence import FileAccessProvider +from flytekit.core.hash import HashMethod from flytekit.core.node import Node from flytekit.core.promise import NodeOutput, Promise, VoidPromise from flytekit.core.resources import Resources from flytekit.core.task import TaskMetadata, task from flytekit.core.testing import patch, task_mock -from flytekit.core.type_engine import HashMethod, RestrictedTypeError, TypeEngine +from flytekit.core.type_engine import RestrictedTypeError, TypeEngine from flytekit.core.workflow import workflow from flytekit.models import literals as _literal_models from flytekit.models.core import types as _core_types From 4552e788c53c5b59f086b4a39530873129c1c0c4 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 14:40:43 -0800 Subject: [PATCH 12/35] Fix `unit_test` make target Signed-off-by: Eduardo Apolinario --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index b5464e205e..adbed3a374 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,7 @@ test: lint unit_test .PHONY: unit_test unit_test: - FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit tests/flytekit_compatibility + FLYTE_SDK_USE_STRUCTURED_DATASET=FALSE pytest tests/flytekit_compatibility && FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt requirements-spark2.txt: requirements-spark2.in install-piptools From 24f67a0acc02427acaed92f6f09165bce341c7c4 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 14:42:50 -0800 Subject: [PATCH 13/35] Split `unit_test` make target in two lines Signed-off-by: Eduardo Apolinario --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index adbed3a374..404c1eb1fd 100644 --- a/Makefile +++ b/Makefile @@ -50,7 +50,8 @@ test: lint unit_test .PHONY: unit_test unit_test: - FLYTE_SDK_USE_STRUCTURED_DATASET=FALSE pytest tests/flytekit_compatibility && FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit + FLYTE_SDK_USE_STRUCTURED_DATASET=FALSE pytest tests/flytekit_compatibility && \ + FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt requirements-spark2.txt: requirements-spark2.in install-piptools From 60ecf3a4ce584b90eaf10f830715a9f8c26be4d7 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 14:46:17 -0800 Subject: [PATCH 14/35] Add assert to structured dataset compatibility test Signed-off-by: Eduardo Apolinario --- tests/flytekit_compatibility/test_structured_dataset.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/flytekit_compatibility/test_structured_dataset.py b/tests/flytekit_compatibility/test_structured_dataset.py index 20965cb802..f935305210 100644 --- a/tests/flytekit_compatibility/test_structured_dataset.py +++ b/tests/flytekit_compatibility/test_structured_dataset.py @@ -1,8 +1,12 @@ import pandas as pd +from flytekit.configuration.sdk import USE_STRUCTURED_DATASET from flytekit.core.type_engine import TypeEngine def test_pandas_is_schema_with_flag(): + # This test can only be run iff USE_STRUCTURED_DATASET is not set + assert USE_STRUCTURED_DATASET.get() is False + lt = TypeEngine.to_literal_type(pd.DataFrame) assert lt.schema is not None From 3aeedebf545d71069a79f023ebf6bd65cd85a8da Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 15:28:44 -0800 Subject: [PATCH 15/35] Remove TODO Signed-off-by: Eduardo Apolinario --- flytekit/models/literals.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py index de04dc5172..92c2c7e322 100644 --- a/flytekit/models/literals.py +++ b/flytekit/models/literals.py @@ -868,9 +868,5 @@ def from_flyte_idl(cls, pb2_object): scalar=Scalar.from_flyte_idl(pb2_object.scalar) if pb2_object.HasField("scalar") else None, collection=collection, map=LiteralMap.from_flyte_idl(pb2_object.map) if pb2_object.HasField("map") else None, - # TODO: explain that string always have a value set in protobufs, which means - # if we want to differentiate between the case of empty string and null we would have - # to wrap that in a separate value. Instead, we're deliberately setting `hash` to - # None in case of the default value (i.e. empty string). hash=pb2_object.hash if pb2_object.hash else None, ) From c2dbb540ad716e883932dd9255045c128e16c6a0 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 15:28:56 -0800 Subject: [PATCH 16/35] Regenerate plugins requirements files pointing to the right version of flyteidl. Signed-off-by: Eduardo Apolinario --- plugins/flytekit-aws-athena/requirements.in | 1 + plugins/flytekit-aws-athena/requirements.txt | 38 +++-- plugins/flytekit-aws-batch/requirements.in | 1 + plugins/flytekit-aws-batch/requirements.txt | 60 ++++--- .../flytekit-aws-sagemaker/requirements.in | 1 + .../flytekit-aws-sagemaker/requirements.txt | 43 ++--- plugins/flytekit-bigquery/requirements.in | 1 + plugins/flytekit-bigquery/requirements.txt | 35 ++-- plugins/flytekit-data-fsspec/requirements.in | 1 + plugins/flytekit-data-fsspec/requirements.txt | 44 +++-- plugins/flytekit-dolt/requirements.in | 1 + plugins/flytekit-dolt/requirements.txt | 150 ++---------------- .../requirements.in | 1 + .../requirements.txt | 87 +++++----- plugins/flytekit-hive/requirements.in | 1 + plugins/flytekit-hive/requirements.txt | 38 +++-- plugins/flytekit-k8s-pod/requirements.in | 1 + plugins/flytekit-k8s-pod/requirements.txt | 40 +++-- plugins/flytekit-kf-mpi/requirements.in | 1 + plugins/flytekit-kf-mpi/requirements.txt | 41 +++-- plugins/flytekit-kf-pytorch/requirements.in | 1 + plugins/flytekit-kf-pytorch/requirements.txt | 38 +++-- .../flytekit-kf-tensorflow/requirements.in | 1 + .../flytekit-kf-tensorflow/requirements.txt | 38 +++-- plugins/flytekit-modin/requirements.in | 1 + plugins/flytekit-pandera/requirements.in | 1 + plugins/flytekit-pandera/requirements.txt | 43 +++-- plugins/flytekit-papermill/requirements.in | 1 + plugins/flytekit-papermill/requirements.txt | 62 ++++---- plugins/flytekit-snowflake/requirements.in | 1 + plugins/flytekit-snowflake/requirements.txt | 38 +++-- plugins/flytekit-spark/requirements.in | 1 + plugins/flytekit-spark/requirements.txt | 38 +++-- plugins/flytekit-sqlalchemy/requirements.in | 1 + plugins/flytekit-sqlalchemy/requirements.txt | 38 +++-- 35 files changed, 453 insertions(+), 436 deletions(-) diff --git a/plugins/flytekit-aws-athena/requirements.in b/plugins/flytekit-aws-athena/requirements.in index fb5e57406e..cd57503478 100644 --- a/plugins/flytekit-aws-athena/requirements.in +++ b/plugins/flytekit-aws-athena/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-athena diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index a1acae494f..0fa90e3fba 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-athena # via -r requirements.in arrow==1.2.2 @@ -12,13 +16,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +48,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-athena -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -79,14 +87,12 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +107,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +126,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +139,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-aws-batch/requirements.in b/plugins/flytekit-aws-batch/requirements.in index 4c2c1fae97..8013b8f761 100644 --- a/plugins/flytekit-aws-batch/requirements.in +++ b/plugins/flytekit-aws-batch/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-awsbatch diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt index 9181490234..e0b9738666 100644 --- a/plugins/flytekit-aws-batch/requirements.txt +++ b/plugins/flytekit-aws-batch/requirements.txt @@ -1,24 +1,30 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-awsbatch # via -r requirements.in -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,8 +32,10 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -40,16 +48,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.21.23 - # via flytekit -flytekit==0.26.0 +flytekit==0.30.3 # via flytekitplugins-awsbatch -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -71,25 +81,25 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via flytekit -numpy==1.22.1 +numpy==1.22.2 # via # pandas # pyarrow -pandas==1.3.5 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter -protobuf==3.19.3 - # via - # flyteidl - # flytekit +protobuf==3.19.4 + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit -python-dateutil==2.8.1 +pycparser==2.21 + # via cffi +python-dateutil==2.8.2 # via # arrow # croniter @@ -97,7 +107,7 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -112,25 +122,27 @@ requests==2.27.1 # cookiecutter # flytekit # responses -responses==0.17.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 - # via typing-inspect +typing-extensions==4.1.1 + # via + # flytekit + # typing-inspect typing-inspect==0.7.1 # via dataclasses-json urllib3==1.26.8 diff --git a/plugins/flytekit-aws-sagemaker/requirements.in b/plugins/flytekit-aws-sagemaker/requirements.in index 1f40a6102b..6e2136aac9 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.in +++ b/plugins/flytekit-aws-sagemaker/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-awssagemaker diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index 11396f213e..48a70cf6c1 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-awssagemaker # via -r requirements.in arrow==1.2.2 @@ -12,9 +16,9 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -boto3==1.20.50 +boto3==1.21.2 # via sagemaker-training -botocore==1.23.50 +botocore==1.24.2 # via # boto3 # s3transfer @@ -27,11 +31,11 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -39,7 +43,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via @@ -57,22 +61,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-awssagemaker gevent==21.12.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring -inotify-simple==1.2.1 +inotify_simple==1.2.1 # via sagemaker-training +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -106,7 +112,7 @@ numpy==1.22.2 # pyarrow # sagemaker-training # scipy -pandas==1.4.0 +pandas==1.4.1 # via flytekit paramiko==2.9.2 # via sagemaker-training @@ -114,7 +120,6 @@ poyo==0.5.0 # via cookiecutter protobuf==3.19.4 # via - # flyteidl # flytekit # sagemaker-training psutil==5.9.0 @@ -136,7 +141,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -163,6 +168,8 @@ sagemaker-training==3.9.2 # via flytekitplugins-awssagemaker scipy==1.8.0 # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # bcrypt @@ -177,7 +184,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect @@ -199,9 +206,9 @@ wrapt==1.13.3 # flytekit zipp==3.7.0 # via importlib-metadata -zope-event==4.5.0 +zope.event==4.5.0 # via gevent -zope-interface==5.4.0 +zope.interface==5.4.0 # via gevent # The following packages are considered to be unsafe in a requirements file: diff --git a/plugins/flytekit-bigquery/requirements.in b/plugins/flytekit-bigquery/requirements.in index d01f21ac2b..c7912742b5 100644 --- a/plugins/flytekit-bigquery/requirements.in +++ b/plugins/flytekit-bigquery/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-bigquery diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt index f0bc647453..4b25a52098 100644 --- a/plugins/flytekit-bigquery/requirements.txt +++ b/plugins/flytekit-bigquery/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-bigquery # via -r requirements.in arrow==1.2.2 @@ -18,11 +22,11 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -30,7 +34,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -46,9 +50,7 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-bigquery google-api-core[grpc]==2.5.0 # via @@ -58,29 +60,29 @@ google-auth==2.6.0 # via # google-api-core # google-cloud-core -google-cloud-bigquery==2.32.0 +google-cloud-bigquery==2.33.0 # via flytekitplugins-bigquery google-cloud-core==2.2.2 # via google-cloud-bigquery google-crc32c==1.3.0 # via google-resumable-media -google-resumable-media==2.2.0 +google-resumable-media==2.2.1 # via google-cloud-bigquery googleapis-common-protos==1.54.0 # via # google-api-core # grpcio-status -grpcio==1.43.0 +grpcio==1.44.0 # via # flytekit # google-api-core # google-cloud-bigquery # grpcio-status -grpcio-status==1.43.0 +grpcio-status==1.44.0 # via google-api-core idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring jeepney==0.7.1 # via @@ -115,15 +117,14 @@ numpy==1.22.2 # pyarrow packaging==21.3 # via google-cloud-bigquery -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter -proto-plus==1.20.0 +proto-plus==1.20.2 # via google-cloud-bigquery protobuf==3.19.4 # via - # flyteidl # flytekit # google-api-core # google-cloud-bigquery @@ -153,7 +154,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -190,7 +191,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-data-fsspec/requirements.in b/plugins/flytekit-data-fsspec/requirements.in index 127b779735..4c03034cce 100644 --- a/plugins/flytekit-data-fsspec/requirements.in +++ b/plugins/flytekit-data-fsspec/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-data-fsspec diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index 4dc89fa500..344d8c1f41 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -1,26 +1,32 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-data-fsspec # via -r requirements.in arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter -botocore==1.23.24 +botocore==1.24.2 # via flytekitplugins-data-fsspec certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -28,8 +34,10 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -42,18 +50,20 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-data-fsspec fsspec==2022.1.0 # via flytekitplugins-data-fsspec -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -83,18 +93,18 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi python-dateutil==2.8.2 # via # arrow @@ -104,7 +114,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -123,6 +133,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -134,7 +146,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-dolt/requirements.in b/plugins/flytekit-dolt/requirements.in index 53d807299a..04990b3216 100644 --- a/plugins/flytekit-dolt/requirements.in +++ b/plugins/flytekit-dolt/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-dolt diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index 01dfe2cb6b..221fc36e43 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -1,160 +1,44 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # --e file:.#egg=flytekitplugins-dolt +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl # via -r requirements.in -arrow==1.2.2 - # via jinja2-time -binaryornot==0.4.4 - # via cookiecutter -certifi==2021.10.8 - # via requests -chardet==4.0.0 - # via binaryornot -charset-normalizer==2.0.11 - # via requests -checksumdir==1.2.0 - # via flytekit -click==7.1.2 +-e file:../../.#egg=flytekit # via - # cookiecutter - # flytekit -cloudpickle==2.0.0 - # via flytekit -cookiecutter==1.7.3 - # via flytekit -croniter==1.2.0 - # via flytekit -cryptography==36.0.1 - # via secretstorage + # -r requirements.in + # flytekitplugins-dolt +-e file:.#egg=flytekitplugins-dolt + # via -r requirements.in dataclasses-json==0.5.6 - # via - # dolt-integrations - # flytekit -decorator==5.1.1 - # via retry -deprecated==1.2.13 - # via flytekit -diskcache==5.4.0 - # via flytekit -docker-image-py==0.1.12 - # via flytekit -docstring-parser==0.13 - # via flytekit + # via dolt-integrations dolt-integrations==0.1.5 # via flytekitplugins-dolt doltcli==0.1.17 # via dolt-integrations -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 - # via flytekitplugins-dolt -grpcio==1.43.0 - # via flytekit -idna==3.3 - # via requests -importlib-metadata==4.10.1 - # via keyring -jinja2==3.0.3 - # via - # cookiecutter - # jinja2-time -jinja2-time==0.2.0 - # via cookiecutter -keyring==23.5.0 - # via flytekit -markupsafe==2.0.1 - # via jinja2 marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum - # marshmallow-jsonschema marshmallow-enum==1.5.1 # via dataclasses-json -marshmallow-jsonschema==0.13.0 - # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.1.0 - # via flytekit numpy==1.22.2 - # via - # pandas - # pyarrow -pandas==1.4.0 - # via - # dolt-integrations - # flytekit -poyo==0.5.0 - # via cookiecutter + # via pandas +pandas==1.4.1 + # via dolt-integrations protobuf==3.19.4 - # via - # flyteidl - # flytekit -py==1.11.0 - # via retry -pyarrow==6.0.1 - # via flytekit -pycparser==2.21 - # via cffi + # via flyteidl python-dateutil==2.8.2 - # via - # arrow - # croniter - # flytekit - # pandas -python-json-logger==2.0.2 - # via flytekit -python-slugify==5.0.2 - # via cookiecutter -pytimeparse==1.1.8 - # via flytekit + # via pandas pytz==2021.3 - # via - # flytekit - # pandas -regex==2022.1.18 - # via docker-image-py -requests==2.27.1 - # via - # cookiecutter - # flytekit - # responses -responses==0.18.0 - # via flytekit -retry==0.9.2 - # via flytekit + # via pandas six==1.16.0 - # via - # cookiecutter - # grpcio - # python-dateutil -sortedcontainers==2.4.0 - # via flytekit -statsd==3.3.0 - # via flytekit -text-unidecode==1.3 - # via python-slugify -typing-extensions==4.0.1 - # via - # flytekit - # typing-inspect + # via python-dateutil +typing-extensions==4.1.1 + # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.8 - # via - # flytekit - # requests - # responses -wheel==0.37.1 - # via flytekit -wrapt==1.13.3 - # via - # deprecated - # flytekit -zipp==3.7.0 - # via importlib-metadata diff --git a/plugins/flytekit-greatexpectations/requirements.in b/plugins/flytekit-greatexpectations/requirements.in index b92d2c6b7b..31536ee1bf 100644 --- a/plugins/flytekit-greatexpectations/requirements.in +++ b/plugins/flytekit-greatexpectations/requirements.in @@ -1,3 +1,4 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-great_expectations sqlalchemy diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index 7aa0a867dd..022622111d 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -1,17 +1,17 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-great_expectations # via -r requirements.in altair==4.2.0 # via great-expectations -appnope==0.1.2 - # via - # ipykernel - # ipython argon2-cffi==21.3.0 # via notebook argon2-cffi-bindings==21.2.0 @@ -26,21 +26,23 @@ backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter -black==21.12b0 +black==22.1.0 # via ipython bleach==4.1.0 # via nbconvert certifi==2021.10.8 # via requests cffi==1.15.0 - # via argon2-cffi-bindings + # via + # argon2-cffi-bindings + # cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # black # cookiecutter @@ -50,8 +52,10 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit debugpy==1.5.1 @@ -70,30 +74,28 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 +entrypoints==0.4 # via # altair # jupyter-client # nbconvert executing==0.8.2 # via stack-data -flyteidl==0.21.24 - # via flytekit -flytekit==0.26.1 +flytekit==0.30.3 # via flytekitplugins-great-expectations -great-expectations==0.14.3 +great-expectations==0.14.5 # via flytekitplugins-great-expectations greenlet==1.1.2 # via sqlalchemy -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via # great-expectations # keyring -ipykernel==6.7.0 +ipykernel==6.9.1 # via # ipywidgets # notebook @@ -110,6 +112,10 @@ ipywidgets==7.6.5 # via great-expectations jedi==0.18.1 # via ipython +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # altair @@ -134,7 +140,7 @@ jupyter-client==7.1.2 # ipykernel # nbclient # notebook -jupyter-core==4.9.1 +jupyter-core==4.9.2 # via # jupyter-client # nbconvert @@ -169,11 +175,11 @@ mypy-extensions==0.4.3 # via # black # typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via flytekit -nbclient==0.5.10 +nbclient==0.5.11 # via nbconvert -nbconvert==6.4.1 +nbconvert==6.4.2 # via notebook nbformat==5.1.3 # via @@ -189,7 +195,7 @@ nest-asyncio==1.5.4 # notebook notebook==6.4.8 # via widgetsnbextension -numpy==1.22.1 +numpy==1.22.2 # via # altair # great-expectations @@ -198,7 +204,7 @@ numpy==1.22.1 # scipy packaging==21.3 # via bleach -pandas==1.4.0 +pandas==1.4.1 # via # altair # flytekit @@ -213,18 +219,16 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.1 +platformdirs==2.5.0 # via black poyo==0.5.0 # via cookiecutter prometheus-client==0.13.1 # via notebook -prompt-toolkit==3.0.26 +prompt-toolkit==3.0.28 # via ipython protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit ptyprocess==0.7.0 # via # pexpect @@ -248,7 +252,7 @@ pyparsing==2.4.7 # packaging pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow # croniter @@ -258,7 +262,7 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -281,16 +285,16 @@ requests==2.27.1 # flytekit # great-expectations # responses -responses==0.17.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -ruamel-yaml==0.17.17 +ruamel.yaml==0.17.17 # via great-expectations -ruamel-yaml-clib==0.2.6 - # via ruamel-yaml -scipy==1.7.3 +scipy==1.8.0 # via great-expectations +secretstorage==3.3.1 + # via keyring send2trash==1.8.0 # via notebook six==1.16.0 @@ -298,17 +302,15 @@ six==1.16.0 # asttokens # bleach # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit sqlalchemy==1.4.31 # via # -r requirements.in # flytekitplugins-great-expectations -stack-data==0.1.4 +stack-data==0.2.0 # via ipython statsd==3.3.0 # via flytekit @@ -320,7 +322,7 @@ testpath==0.5.0 # via nbconvert text-unidecode==1.3 # via python-slugify -tomli==1.2.3 +tomli==2.0.1 # via black toolz==0.11.2 # via altair @@ -344,9 +346,10 @@ traitlets==5.1.1 # nbconvert # nbformat # notebook -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via - # black + # flytekit + # great-expectations # typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-hive/requirements.in b/plugins/flytekit-hive/requirements.in index 74c269d489..7fb543577e 100644 --- a/plugins/flytekit-hive/requirements.in +++ b/plugins/flytekit-hive/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-hive diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index 3ae057f9a0..e59b52ac4f 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-hive # via -r requirements.in arrow==1.2.2 @@ -12,13 +16,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +48,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-hive -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -79,14 +87,12 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +107,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +126,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +139,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-k8s-pod/requirements.in b/plugins/flytekit-k8s-pod/requirements.in index 987e999ec5..0e35b85b77 100644 --- a/plugins/flytekit-k8s-pod/requirements.in +++ b/plugins/flytekit-k8s-pod/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-pod diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index d60d9c46b6..3918d181db 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-pod # via -r requirements.in arrow==1.2.2 @@ -16,13 +20,15 @@ certifi==2021.10.8 # via # kubernetes # requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -30,7 +36,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -46,18 +52,20 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-pod google-auth==2.6.0 # via kubernetes -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -66,7 +74,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -kubernetes==21.7.0 +kubernetes==22.6.0 # via flytekitplugins-pod markupsafe==2.0.1 # via jinja2 @@ -89,14 +97,12 @@ numpy==1.22.2 # pyarrow oauthlib==3.2.0 # via requests-oauthlib -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -118,7 +124,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -145,6 +151,8 @@ retry==0.9.2 # via flytekit rsa==4.8 # via google-auth +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -158,7 +166,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-mpi/requirements.in b/plugins/flytekit-kf-mpi/requirements.in index dc79a19e66..7eb70eb5d8 100644 --- a/plugins/flytekit-kf-mpi/requirements.in +++ b/plugins/flytekit-kf-mpi/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-kfmpi diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt index c13f9f3b74..de6d144057 100644 --- a/plugins/flytekit-kf-mpi/requirements.txt +++ b/plugins/flytekit-kf-mpi/requirements.txt @@ -1,9 +1,14 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit + # flytekitplugins-kfmpi -e file:.#egg=flytekitplugins-kfmpi # via -r requirements.in arrow==1.2.2 @@ -12,13 +17,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,7 +33,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,18 +49,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via - # flytekit - # flytekitplugins-kfmpi -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-kfmpi -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -81,14 +88,12 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -103,7 +108,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -122,6 +127,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -133,7 +140,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-pytorch/requirements.in b/plugins/flytekit-kf-pytorch/requirements.in index 7013f7ba68..443f07f66c 100644 --- a/plugins/flytekit-kf-pytorch/requirements.in +++ b/plugins/flytekit-kf-pytorch/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-kfpytorch diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index 1696646c87..46ab5fd4a4 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-kfpytorch # via -r requirements.in arrow==1.2.2 @@ -12,13 +16,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +48,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-kfpytorch -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -79,14 +87,12 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +107,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +126,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +139,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-tensorflow/requirements.in b/plugins/flytekit-kf-tensorflow/requirements.in index 0f95a44bc5..8b57632972 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.in +++ b/plugins/flytekit-kf-tensorflow/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-kftensorflow diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index d3b6253664..dc90a674c2 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-kftensorflow # via -r requirements.in arrow==1.2.2 @@ -12,13 +16,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +48,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-kftensorflow -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -79,14 +87,12 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +107,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +126,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +139,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-modin/requirements.in b/plugins/flytekit-modin/requirements.in index 0248a83ad9..e4754638cd 100644 --- a/plugins/flytekit-modin/requirements.in +++ b/plugins/flytekit-modin/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-modin diff --git a/plugins/flytekit-pandera/requirements.in b/plugins/flytekit-pandera/requirements.in index 70e750d5e4..3ff03dd965 100644 --- a/plugins/flytekit-pandera/requirements.in +++ b/plugins/flytekit-pandera/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-pandera diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index f0b65cec17..facccfb564 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-pandera # via -r requirements.in arrow==1.2.2 @@ -12,13 +16,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +48,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-pandera -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -82,18 +90,16 @@ numpy==1.22.2 # pyarrow packaging==21.3 # via pandera -pandas==1.4.0 +pandas==1.4.1 # via # flytekit # pandera -pandera==0.8.1 +pandera==0.9.0 # via flytekitplugins-pandera poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -102,6 +108,8 @@ pyarrow==6.0.1 # pandera pycparser==2.21 # via cffi +pydantic==1.9.0 + # via pandera pyparsing==3.0.7 # via packaging python-dateutil==2.8.2 @@ -112,7 +120,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -131,6 +139,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -142,9 +152,10 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit + # pydantic # typing-inspect typing-inspect==0.7.1 # via diff --git a/plugins/flytekit-papermill/requirements.in b/plugins/flytekit-papermill/requirements.in index a9dab93776..cbc5b6102b 100644 --- a/plugins/flytekit-papermill/requirements.in +++ b/plugins/flytekit-papermill/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-papermill diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 8c3e4570e9..7449153b5a 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-papermill # via -r requirements.in ansiwrap==0.8.4 @@ -18,7 +22,7 @@ backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter -black==21.12b0 +black==22.1.0 # via ipython bleach==4.1.0 # via nbconvert @@ -28,11 +32,11 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # black # cookiecutter @@ -42,7 +46,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -64,24 +68,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 +entrypoints==0.4 # via # jupyter-client # nbconvert # papermill executing==0.8.2 # via stack-data -flyteidl==0.22.0 - # via flytekit -flytekit==0.26.1 +flytekit==0.30.3 # via flytekitplugins-papermill -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring -ipykernel==6.7.0 +ipykernel==6.9.1 # via flytekitplugins-papermill ipython==8.0.1 # via ipykernel @@ -106,7 +108,7 @@ jupyter-client==7.1.2 # via # ipykernel # nbclient -jupyter-core==4.9.1 +jupyter-core==4.9.2 # via # jupyter-client # nbconvert @@ -136,13 +138,13 @@ mypy-extensions==0.4.3 # via # black # typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via flytekit -nbclient==0.5.10 +nbclient==0.5.11 # via # nbconvert # papermill -nbconvert==6.4.1 +nbconvert==6.4.2 # via flytekitplugins-papermill nbformat==5.1.3 # via @@ -154,13 +156,13 @@ nest-asyncio==1.5.4 # ipykernel # jupyter-client # nbclient -numpy==1.22.1 +numpy==1.22.2 # via # pandas # pyarrow packaging==21.3 # via bleach -pandas==1.4.0 +pandas==1.4.1 # via flytekit pandocfilters==1.5.0 # via nbconvert @@ -174,16 +176,14 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.1 +platformdirs==2.5.0 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.26 +prompt-toolkit==3.0.28 # via ipython protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 @@ -203,7 +203,7 @@ pyparsing==3.0.7 # via packaging pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow # croniter @@ -212,7 +212,7 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -232,7 +232,7 @@ requests==2.27.1 # flytekit # papermill # responses -responses==0.17.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit @@ -243,13 +243,11 @@ six==1.16.0 # asttokens # bleach # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit -stack-data==0.1.4 +stack-data==0.2.0 # via ipython statsd==3.3.0 # via flytekit @@ -261,7 +259,7 @@ text-unidecode==1.3 # via python-slugify textwrap3==0.9.2 # via ansiwrap -tomli==1.2.3 +tomli==2.0.1 # via black tornado==6.1 # via @@ -279,9 +277,9 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via - # black + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-snowflake/requirements.in b/plugins/flytekit-snowflake/requirements.in index 2a118dd3bf..f720269c73 100644 --- a/plugins/flytekit-snowflake/requirements.in +++ b/plugins/flytekit-snowflake/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-snowflake diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index 53951100b5..e9b2b125f2 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-snowflake # via -r requirements.in arrow==1.2.2 @@ -12,13 +16,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +48,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-snowflake -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -79,14 +87,12 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +107,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +126,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +139,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-spark/requirements.in b/plugins/flytekit-spark/requirements.in index f112dbc8ef..9d2e62c77e 100644 --- a/plugins/flytekit-spark/requirements.in +++ b/plugins/flytekit-spark/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-spark diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index 75d1e32d53..2ab33fe5f3 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-spark # via -r requirements.in arrow==1.2.2 @@ -12,13 +16,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +48,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-spark -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -79,14 +87,12 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry py4j==0.10.9.3 @@ -105,7 +111,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -124,6 +130,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -135,7 +143,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-sqlalchemy/requirements.in b/plugins/flytekit-sqlalchemy/requirements.in index 21d9e97b47..e545653d01 100644 --- a/plugins/flytekit-sqlalchemy/requirements.in +++ b/plugins/flytekit-sqlalchemy/requirements.in @@ -1,2 +1,3 @@ . +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-sqlalchemy diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index 93dcc2790c..bced83587b 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -1,9 +1,13 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # +-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl + # via + # -r requirements.in + # flytekit -e file:.#egg=flytekitplugins-sqlalchemy # via -r requirements.in arrow==1.2.2 @@ -12,13 +16,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.3 # via # cookiecutter # flytekit @@ -26,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.1 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,18 +48,20 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-sqlalchemy greenlet==1.1.2 # via sqlalchemy -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -81,14 +89,12 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via - # flyteidl - # flytekit + # via flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -103,7 +109,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.0.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -122,6 +128,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -135,7 +143,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect From 0e5819999414973209b12fea28b17b3e8a38e173 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 15:32:57 -0800 Subject: [PATCH 17/35] Set hash as a property of the literal Signed-off-by: Eduardo Apolinario --- flytekit/models/literals.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py index 92c2c7e322..8d5101037e 100644 --- a/flytekit/models/literals.py +++ b/flytekit/models/literals.py @@ -809,7 +809,7 @@ def __init__( self._scalar = scalar self._collection = collection self._map = map - self.hash = hash + self._hash = hash @property def scalar(self): @@ -843,6 +843,14 @@ def value(self): """ return self.scalar or self.collection or self.map + @property + def hash(self): + """ + If not None, this value holds a hash that represents the literal for caching purposes. + :rtype: str + """ + return self._hash + def to_flyte_idl(self): """ :rtype: flyteidl.core.literals_pb2.Literal From 5054861218a9fc571a957a2ab8da9cc7754a1bf5 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 15:41:21 -0800 Subject: [PATCH 18/35] Install plugins requirements in CI. Signed-off-by: Eduardo Apolinario --- .github/workflows/pythonbuild.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 8202bf0f73..e63b869734 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -105,6 +105,7 @@ jobs: cd plugins/${{ matrix.plugin-names }} pip install -e . if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi + pip install -r requirements.txt pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit pip freeze - name: Test with coverage From 9f2d06f672a4aca18869de4796fe25287c7773b6 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 16:51:15 -0800 Subject: [PATCH 19/35] Add hash.setter Signed-off-by: Eduardo Apolinario --- flytekit/models/literals.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py index 8d5101037e..70470fda6b 100644 --- a/flytekit/models/literals.py +++ b/flytekit/models/literals.py @@ -851,6 +851,10 @@ def hash(self): """ return self._hash + @hash.setter + def hash(self, value): + self._hash = value + def to_flyte_idl(self): """ :rtype: flyteidl.core.literals_pb2.Literal From e039836502b78972fd2d04f33d480e26e435c686 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 17:07:22 -0800 Subject: [PATCH 20/35] Install flyteidl directly Signed-off-by: Eduardo Apolinario --- .github/workflows/pythonbuild.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index e63b869734..08d8a47baa 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -105,8 +105,8 @@ jobs: cd plugins/${{ matrix.plugin-names }} pip install -e . if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi - pip install -r requirements.txt pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit + pip install --no-deps -U "git+https://github.com/flyteorg/flyteidl@add-hash-to-literal#egg=flyteidl" pip freeze - name: Test with coverage run: | From 2da76f38037479a2b892aaacd2fc12f85dbb24df Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Thu, 17 Feb 2022 17:08:05 -0800 Subject: [PATCH 21/35] Revert "Regenerate plugins requirements files pointing to the right version of flyteidl." This reverts commit c2dbb540ad716e883932dd9255045c128e16c6a0. Signed-off-by: Eduardo Apolinario --- plugins/flytekit-aws-athena/requirements.in | 1 - plugins/flytekit-aws-athena/requirements.txt | 38 ++--- plugins/flytekit-aws-batch/requirements.in | 1 - plugins/flytekit-aws-batch/requirements.txt | 60 +++---- .../flytekit-aws-sagemaker/requirements.in | 1 - .../flytekit-aws-sagemaker/requirements.txt | 43 +++-- plugins/flytekit-bigquery/requirements.in | 1 - plugins/flytekit-bigquery/requirements.txt | 35 ++-- plugins/flytekit-data-fsspec/requirements.in | 1 - plugins/flytekit-data-fsspec/requirements.txt | 44 ++--- plugins/flytekit-dolt/requirements.in | 1 - plugins/flytekit-dolt/requirements.txt | 150 ++++++++++++++++-- .../requirements.in | 1 - .../requirements.txt | 87 +++++----- plugins/flytekit-hive/requirements.in | 1 - plugins/flytekit-hive/requirements.txt | 38 ++--- plugins/flytekit-k8s-pod/requirements.in | 1 - plugins/flytekit-k8s-pod/requirements.txt | 40 ++--- plugins/flytekit-kf-mpi/requirements.in | 1 - plugins/flytekit-kf-mpi/requirements.txt | 41 ++--- plugins/flytekit-kf-pytorch/requirements.in | 1 - plugins/flytekit-kf-pytorch/requirements.txt | 38 ++--- .../flytekit-kf-tensorflow/requirements.in | 1 - .../flytekit-kf-tensorflow/requirements.txt | 38 ++--- plugins/flytekit-modin/requirements.in | 1 - plugins/flytekit-pandera/requirements.in | 1 - plugins/flytekit-pandera/requirements.txt | 43 ++--- plugins/flytekit-papermill/requirements.in | 1 - plugins/flytekit-papermill/requirements.txt | 62 ++++---- plugins/flytekit-snowflake/requirements.in | 1 - plugins/flytekit-snowflake/requirements.txt | 38 ++--- plugins/flytekit-spark/requirements.in | 1 - plugins/flytekit-spark/requirements.txt | 38 ++--- plugins/flytekit-sqlalchemy/requirements.in | 1 - plugins/flytekit-sqlalchemy/requirements.txt | 38 ++--- 35 files changed, 436 insertions(+), 453 deletions(-) diff --git a/plugins/flytekit-aws-athena/requirements.in b/plugins/flytekit-aws-athena/requirements.in index cd57503478..fb5e57406e 100644 --- a/plugins/flytekit-aws-athena/requirements.in +++ b/plugins/flytekit-aws-athena/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-athena diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index 0fa90e3fba..a1acae494f 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-athena # via -r requirements.in arrow==1.2.2 @@ -16,15 +12,13 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -32,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,18 +42,16 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-athena -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -87,12 +79,14 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -107,7 +101,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -139,7 +131,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-aws-batch/requirements.in b/plugins/flytekit-aws-batch/requirements.in index 8013b8f761..4c2c1fae97 100644 --- a/plugins/flytekit-aws-batch/requirements.in +++ b/plugins/flytekit-aws-batch/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-awsbatch diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt index e0b9738666..9181490234 100644 --- a/plugins/flytekit-aws-batch/requirements.txt +++ b/plugins/flytekit-aws-batch/requirements.txt @@ -1,30 +1,24 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-awsbatch # via -r requirements.in -arrow==1.2.2 +arrow==1.2.1 # via jinja2-time binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.10 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -32,10 +26,8 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit -cryptography==36.0.1 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -48,18 +40,16 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.21.23 + # via flytekit +flytekit==0.26.0 # via flytekitplugins-awsbatch -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -81,25 +71,25 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.1.0 +natsort==8.0.2 # via flytekit -numpy==1.22.2 +numpy==1.22.1 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter -protobuf==3.19.4 - # via flytekit +protobuf==3.19.3 + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi -python-dateutil==2.8.2 +python-dateutil==2.8.1 # via # arrow # croniter @@ -107,7 +97,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -122,27 +112,25 @@ requests==2.27.1 # cookiecutter # flytekit # responses -responses==0.18.0 +responses==0.17.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter + # flytekit # grpcio # python-dateutil + # responses sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 - # via - # flytekit - # typing-inspect +typing-extensions==4.0.1 + # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json urllib3==1.26.8 diff --git a/plugins/flytekit-aws-sagemaker/requirements.in b/plugins/flytekit-aws-sagemaker/requirements.in index 6e2136aac9..1f40a6102b 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.in +++ b/plugins/flytekit-aws-sagemaker/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-awssagemaker diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index 48a70cf6c1..11396f213e 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-awssagemaker # via -r requirements.in arrow==1.2.2 @@ -16,9 +12,9 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -boto3==1.21.2 +boto3==1.20.50 # via sagemaker-training -botocore==1.24.2 +botocore==1.23.50 # via # boto3 # s3transfer @@ -31,11 +27,11 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -43,7 +39,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via @@ -61,24 +57,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-awssagemaker gevent==21.12.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -inotify_simple==1.2.1 +inotify-simple==1.2.1 # via sagemaker-training -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -112,7 +106,7 @@ numpy==1.22.2 # pyarrow # sagemaker-training # scipy -pandas==1.4.1 +pandas==1.4.0 # via flytekit paramiko==2.9.2 # via sagemaker-training @@ -120,6 +114,7 @@ poyo==0.5.0 # via cookiecutter protobuf==3.19.4 # via + # flyteidl # flytekit # sagemaker-training psutil==5.9.0 @@ -141,7 +136,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -168,8 +163,6 @@ sagemaker-training==3.9.2 # via flytekitplugins-awssagemaker scipy==1.8.0 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # bcrypt @@ -184,7 +177,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect @@ -206,9 +199,9 @@ wrapt==1.13.3 # flytekit zipp==3.7.0 # via importlib-metadata -zope.event==4.5.0 +zope-event==4.5.0 # via gevent -zope.interface==5.4.0 +zope-interface==5.4.0 # via gevent # The following packages are considered to be unsafe in a requirements file: diff --git a/plugins/flytekit-bigquery/requirements.in b/plugins/flytekit-bigquery/requirements.in index c7912742b5..d01f21ac2b 100644 --- a/plugins/flytekit-bigquery/requirements.in +++ b/plugins/flytekit-bigquery/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-bigquery diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt index 4b25a52098..f0bc647453 100644 --- a/plugins/flytekit-bigquery/requirements.txt +++ b/plugins/flytekit-bigquery/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-bigquery # via -r requirements.in arrow==1.2.2 @@ -22,11 +18,11 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -34,7 +30,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -50,7 +46,9 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-bigquery google-api-core[grpc]==2.5.0 # via @@ -60,29 +58,29 @@ google-auth==2.6.0 # via # google-api-core # google-cloud-core -google-cloud-bigquery==2.33.0 +google-cloud-bigquery==2.32.0 # via flytekitplugins-bigquery google-cloud-core==2.2.2 # via google-cloud-bigquery google-crc32c==1.3.0 # via google-resumable-media -google-resumable-media==2.2.1 +google-resumable-media==2.2.0 # via google-cloud-bigquery googleapis-common-protos==1.54.0 # via # google-api-core # grpcio-status -grpcio==1.44.0 +grpcio==1.43.0 # via # flytekit # google-api-core # google-cloud-bigquery # grpcio-status -grpcio-status==1.44.0 +grpcio-status==1.43.0 # via google-api-core idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring jeepney==0.7.1 # via @@ -117,14 +115,15 @@ numpy==1.22.2 # pyarrow packaging==21.3 # via google-cloud-bigquery -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter -proto-plus==1.20.2 +proto-plus==1.20.0 # via google-cloud-bigquery protobuf==3.19.4 # via + # flyteidl # flytekit # google-api-core # google-cloud-bigquery @@ -154,7 +153,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -191,7 +190,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-data-fsspec/requirements.in b/plugins/flytekit-data-fsspec/requirements.in index 4c03034cce..127b779735 100644 --- a/plugins/flytekit-data-fsspec/requirements.in +++ b/plugins/flytekit-data-fsspec/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-data-fsspec diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index 344d8c1f41..4dc89fa500 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -1,32 +1,26 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-data-fsspec # via -r requirements.in arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter -botocore==1.24.2 +botocore==1.23.24 # via flytekitplugins-data-fsspec certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -34,10 +28,8 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit -cryptography==36.0.1 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -50,20 +42,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-data-fsspec fsspec==2022.1.0 # via flytekitplugins-data-fsspec -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -93,18 +83,18 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.2 # via # arrow @@ -114,7 +104,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -133,8 +123,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -146,7 +134,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-dolt/requirements.in b/plugins/flytekit-dolt/requirements.in index 04990b3216..53d807299a 100644 --- a/plugins/flytekit-dolt/requirements.in +++ b/plugins/flytekit-dolt/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-dolt diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index 221fc36e43..01dfe2cb6b 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -1,44 +1,160 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via -r requirements.in --e file:../../.#egg=flytekit - # via - # -r requirements.in - # flytekitplugins-dolt -e file:.#egg=flytekitplugins-dolt # via -r requirements.in +arrow==1.2.2 + # via jinja2-time +binaryornot==0.4.4 + # via cookiecutter +certifi==2021.10.8 + # via requests +chardet==4.0.0 + # via binaryornot +charset-normalizer==2.0.11 + # via requests +checksumdir==1.2.0 + # via flytekit +click==7.1.2 + # via + # cookiecutter + # flytekit +cloudpickle==2.0.0 + # via flytekit +cookiecutter==1.7.3 + # via flytekit +croniter==1.2.0 + # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 - # via dolt-integrations + # via + # dolt-integrations + # flytekit +decorator==5.1.1 + # via retry +deprecated==1.2.13 + # via flytekit +diskcache==5.4.0 + # via flytekit +docker-image-py==0.1.12 + # via flytekit +docstring-parser==0.13 + # via flytekit dolt-integrations==0.1.5 # via flytekitplugins-dolt doltcli==0.1.17 # via dolt-integrations +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 + # via flytekitplugins-dolt +grpcio==1.43.0 + # via flytekit +idna==3.3 + # via requests +importlib-metadata==4.10.1 + # via keyring +jinja2==3.0.3 + # via + # cookiecutter + # jinja2-time +jinja2-time==0.2.0 + # via cookiecutter +keyring==23.5.0 + # via flytekit +markupsafe==2.0.1 + # via jinja2 marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum + # marshmallow-jsonschema marshmallow-enum==1.5.1 # via dataclasses-json +marshmallow-jsonschema==0.13.0 + # via flytekit mypy-extensions==0.4.3 # via typing-inspect +natsort==8.1.0 + # via flytekit numpy==1.22.2 - # via pandas -pandas==1.4.1 - # via dolt-integrations + # via + # pandas + # pyarrow +pandas==1.4.0 + # via + # dolt-integrations + # flytekit +poyo==0.5.0 + # via cookiecutter protobuf==3.19.4 - # via flyteidl + # via + # flyteidl + # flytekit +py==1.11.0 + # via retry +pyarrow==6.0.1 + # via flytekit +pycparser==2.21 + # via cffi python-dateutil==2.8.2 - # via pandas + # via + # arrow + # croniter + # flytekit + # pandas +python-json-logger==2.0.2 + # via flytekit +python-slugify==5.0.2 + # via cookiecutter +pytimeparse==1.1.8 + # via flytekit pytz==2021.3 - # via pandas + # via + # flytekit + # pandas +regex==2022.1.18 + # via docker-image-py +requests==2.27.1 + # via + # cookiecutter + # flytekit + # responses +responses==0.18.0 + # via flytekit +retry==0.9.2 + # via flytekit six==1.16.0 - # via python-dateutil -typing-extensions==4.1.1 - # via typing-inspect + # via + # cookiecutter + # grpcio + # python-dateutil +sortedcontainers==2.4.0 + # via flytekit +statsd==3.3.0 + # via flytekit +text-unidecode==1.3 + # via python-slugify +typing-extensions==4.0.1 + # via + # flytekit + # typing-inspect typing-inspect==0.7.1 # via dataclasses-json +urllib3==1.26.8 + # via + # flytekit + # requests + # responses +wheel==0.37.1 + # via flytekit +wrapt==1.13.3 + # via + # deprecated + # flytekit +zipp==3.7.0 + # via importlib-metadata diff --git a/plugins/flytekit-greatexpectations/requirements.in b/plugins/flytekit-greatexpectations/requirements.in index 31536ee1bf..b92d2c6b7b 100644 --- a/plugins/flytekit-greatexpectations/requirements.in +++ b/plugins/flytekit-greatexpectations/requirements.in @@ -1,4 +1,3 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-great_expectations sqlalchemy diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index 022622111d..7aa0a867dd 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -1,17 +1,17 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-great_expectations # via -r requirements.in altair==4.2.0 # via great-expectations +appnope==0.1.2 + # via + # ipykernel + # ipython argon2-cffi==21.3.0 # via notebook argon2-cffi-bindings==21.2.0 @@ -26,23 +26,21 @@ backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter -black==22.1.0 +black==21.12b0 # via ipython bleach==4.1.0 # via nbconvert certifi==2021.10.8 # via requests cffi==1.15.0 - # via - # argon2-cffi-bindings - # cryptography + # via argon2-cffi-bindings chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.10 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # black # cookiecutter @@ -52,10 +50,8 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit -cryptography==36.0.1 - # via secretstorage dataclasses-json==0.5.6 # via flytekit debugpy==1.5.1 @@ -74,28 +70,30 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.4 +entrypoints==0.3 # via # altair # jupyter-client # nbconvert executing==0.8.2 # via stack-data -flytekit==0.30.3 +flyteidl==0.21.24 + # via flytekit +flytekit==0.26.1 # via flytekitplugins-great-expectations -great-expectations==0.14.5 +great-expectations==0.14.3 # via flytekitplugins-great-expectations greenlet==1.1.2 # via sqlalchemy -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via # great-expectations # keyring -ipykernel==6.9.1 +ipykernel==6.7.0 # via # ipywidgets # notebook @@ -112,10 +110,6 @@ ipywidgets==7.6.5 # via great-expectations jedi==0.18.1 # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # altair @@ -140,7 +134,7 @@ jupyter-client==7.1.2 # ipykernel # nbclient # notebook -jupyter-core==4.9.2 +jupyter-core==4.9.1 # via # jupyter-client # nbconvert @@ -175,11 +169,11 @@ mypy-extensions==0.4.3 # via # black # typing-inspect -natsort==8.1.0 +natsort==8.0.2 # via flytekit -nbclient==0.5.11 +nbclient==0.5.10 # via nbconvert -nbconvert==6.4.2 +nbconvert==6.4.1 # via notebook nbformat==5.1.3 # via @@ -195,7 +189,7 @@ nest-asyncio==1.5.4 # notebook notebook==6.4.8 # via widgetsnbextension -numpy==1.22.2 +numpy==1.22.1 # via # altair # great-expectations @@ -204,7 +198,7 @@ numpy==1.22.2 # scipy packaging==21.3 # via bleach -pandas==1.4.1 +pandas==1.4.0 # via # altair # flytekit @@ -219,16 +213,18 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.5.0 +platformdirs==2.4.1 # via black poyo==0.5.0 # via cookiecutter prometheus-client==0.13.1 # via notebook -prompt-toolkit==3.0.28 +prompt-toolkit==3.0.26 # via ipython protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit ptyprocess==0.7.0 # via # pexpect @@ -252,7 +248,7 @@ pyparsing==2.4.7 # packaging pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.2 +python-dateutil==2.8.1 # via # arrow # croniter @@ -262,7 +258,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -285,16 +281,16 @@ requests==2.27.1 # flytekit # great-expectations # responses -responses==0.18.0 +responses==0.17.0 # via flytekit retry==0.9.2 # via flytekit -ruamel.yaml==0.17.17 +ruamel-yaml==0.17.17 # via great-expectations -scipy==1.8.0 +ruamel-yaml-clib==0.2.6 + # via ruamel-yaml +scipy==1.7.3 # via great-expectations -secretstorage==3.3.1 - # via keyring send2trash==1.8.0 # via notebook six==1.16.0 @@ -302,15 +298,17 @@ six==1.16.0 # asttokens # bleach # cookiecutter + # flytekit # grpcio # python-dateutil + # responses sortedcontainers==2.4.0 # via flytekit sqlalchemy==1.4.31 # via # -r requirements.in # flytekitplugins-great-expectations -stack-data==0.2.0 +stack-data==0.1.4 # via ipython statsd==3.3.0 # via flytekit @@ -322,7 +320,7 @@ testpath==0.5.0 # via nbconvert text-unidecode==1.3 # via python-slugify -tomli==2.0.1 +tomli==1.2.3 # via black toolz==0.11.2 # via altair @@ -346,10 +344,9 @@ traitlets==5.1.1 # nbconvert # nbformat # notebook -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via - # flytekit - # great-expectations + # black # typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-hive/requirements.in b/plugins/flytekit-hive/requirements.in index 7fb543577e..74c269d489 100644 --- a/plugins/flytekit-hive/requirements.in +++ b/plugins/flytekit-hive/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-hive diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index e59b52ac4f..3ae057f9a0 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-hive # via -r requirements.in arrow==1.2.2 @@ -16,15 +12,13 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -32,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,18 +42,16 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-hive -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -87,12 +79,14 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -107,7 +101,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -139,7 +131,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-k8s-pod/requirements.in b/plugins/flytekit-k8s-pod/requirements.in index 0e35b85b77..987e999ec5 100644 --- a/plugins/flytekit-k8s-pod/requirements.in +++ b/plugins/flytekit-k8s-pod/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-pod diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index 3918d181db..d60d9c46b6 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-pod # via -r requirements.in arrow==1.2.2 @@ -20,15 +16,13 @@ certifi==2021.10.8 # via # kubernetes # requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -36,7 +30,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -52,20 +46,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-pod google-auth==2.6.0 # via kubernetes -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -74,7 +66,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -kubernetes==22.6.0 +kubernetes==21.7.0 # via flytekitplugins-pod markupsafe==2.0.1 # via jinja2 @@ -97,12 +89,14 @@ numpy==1.22.2 # pyarrow oauthlib==3.2.0 # via requests-oauthlib -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -124,7 +118,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -151,8 +145,6 @@ retry==0.9.2 # via flytekit rsa==4.8 # via google-auth -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -166,7 +158,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-mpi/requirements.in b/plugins/flytekit-kf-mpi/requirements.in index 7eb70eb5d8..dc79a19e66 100644 --- a/plugins/flytekit-kf-mpi/requirements.in +++ b/plugins/flytekit-kf-mpi/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-kfmpi diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt index de6d144057..c13f9f3b74 100644 --- a/plugins/flytekit-kf-mpi/requirements.txt +++ b/plugins/flytekit-kf-mpi/requirements.txt @@ -1,14 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit - # flytekitplugins-kfmpi -e file:.#egg=flytekitplugins-kfmpi # via -r requirements.in arrow==1.2.2 @@ -17,15 +12,13 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -33,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -49,18 +42,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via + # flytekit + # flytekitplugins-kfmpi +flytekit==0.30.0 # via flytekitplugins-kfmpi -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -88,12 +81,14 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -108,7 +103,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -127,8 +122,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -140,7 +133,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-pytorch/requirements.in b/plugins/flytekit-kf-pytorch/requirements.in index 443f07f66c..7013f7ba68 100644 --- a/plugins/flytekit-kf-pytorch/requirements.in +++ b/plugins/flytekit-kf-pytorch/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-kfpytorch diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index 46ab5fd4a4..1696646c87 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-kfpytorch # via -r requirements.in arrow==1.2.2 @@ -16,15 +12,13 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -32,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,18 +42,16 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-kfpytorch -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -87,12 +79,14 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -107,7 +101,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -139,7 +131,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-tensorflow/requirements.in b/plugins/flytekit-kf-tensorflow/requirements.in index 8b57632972..0f95a44bc5 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.in +++ b/plugins/flytekit-kf-tensorflow/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-kftensorflow diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index dc90a674c2..d3b6253664 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-kftensorflow # via -r requirements.in arrow==1.2.2 @@ -16,15 +12,13 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -32,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,18 +42,16 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-kftensorflow -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -87,12 +79,14 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -107,7 +101,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -139,7 +131,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-modin/requirements.in b/plugins/flytekit-modin/requirements.in index e4754638cd..0248a83ad9 100644 --- a/plugins/flytekit-modin/requirements.in +++ b/plugins/flytekit-modin/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-modin diff --git a/plugins/flytekit-pandera/requirements.in b/plugins/flytekit-pandera/requirements.in index 3ff03dd965..70e750d5e4 100644 --- a/plugins/flytekit-pandera/requirements.in +++ b/plugins/flytekit-pandera/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-pandera diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index facccfb564..f0b65cec17 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-pandera # via -r requirements.in arrow==1.2.2 @@ -16,15 +12,13 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -32,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,18 +42,16 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-pandera -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -90,16 +82,18 @@ numpy==1.22.2 # pyarrow packaging==21.3 # via pandera -pandas==1.4.1 +pandas==1.4.0 # via # flytekit # pandera -pandera==0.9.0 +pandera==0.8.1 # via flytekitplugins-pandera poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -108,8 +102,6 @@ pyarrow==6.0.1 # pandera pycparser==2.21 # via cffi -pydantic==1.9.0 - # via pandera pyparsing==3.0.7 # via packaging python-dateutil==2.8.2 @@ -120,7 +112,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -139,8 +131,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -152,10 +142,9 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit - # pydantic # typing-inspect typing-inspect==0.7.1 # via diff --git a/plugins/flytekit-papermill/requirements.in b/plugins/flytekit-papermill/requirements.in index cbc5b6102b..a9dab93776 100644 --- a/plugins/flytekit-papermill/requirements.in +++ b/plugins/flytekit-papermill/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-papermill diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 7449153b5a..8c3e4570e9 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-papermill # via -r requirements.in ansiwrap==0.8.4 @@ -22,7 +18,7 @@ backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter -black==22.1.0 +black==21.12b0 # via ipython bleach==4.1.0 # via nbconvert @@ -32,11 +28,11 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.10 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # black # cookiecutter @@ -46,7 +42,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -68,22 +64,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.4 +entrypoints==0.3 # via # jupyter-client # nbconvert # papermill executing==0.8.2 # via stack-data -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.26.1 # via flytekitplugins-papermill -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -ipykernel==6.9.1 +ipykernel==6.7.0 # via flytekitplugins-papermill ipython==8.0.1 # via ipykernel @@ -108,7 +106,7 @@ jupyter-client==7.1.2 # via # ipykernel # nbclient -jupyter-core==4.9.2 +jupyter-core==4.9.1 # via # jupyter-client # nbconvert @@ -138,13 +136,13 @@ mypy-extensions==0.4.3 # via # black # typing-inspect -natsort==8.1.0 +natsort==8.0.2 # via flytekit -nbclient==0.5.11 +nbclient==0.5.10 # via # nbconvert # papermill -nbconvert==6.4.2 +nbconvert==6.4.1 # via flytekitplugins-papermill nbformat==5.1.3 # via @@ -156,13 +154,13 @@ nest-asyncio==1.5.4 # ipykernel # jupyter-client # nbclient -numpy==1.22.2 +numpy==1.22.1 # via # pandas # pyarrow packaging==21.3 # via bleach -pandas==1.4.1 +pandas==1.4.0 # via flytekit pandocfilters==1.5.0 # via nbconvert @@ -176,14 +174,16 @@ pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.5.0 +platformdirs==2.4.1 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.28 +prompt-toolkit==3.0.26 # via ipython protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 @@ -203,7 +203,7 @@ pyparsing==3.0.7 # via packaging pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.2 +python-dateutil==2.8.1 # via # arrow # croniter @@ -212,7 +212,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -232,7 +232,7 @@ requests==2.27.1 # flytekit # papermill # responses -responses==0.18.0 +responses==0.17.0 # via flytekit retry==0.9.2 # via flytekit @@ -243,11 +243,13 @@ six==1.16.0 # asttokens # bleach # cookiecutter + # flytekit # grpcio # python-dateutil + # responses sortedcontainers==2.4.0 # via flytekit -stack-data==0.2.0 +stack-data==0.1.4 # via ipython statsd==3.3.0 # via flytekit @@ -259,7 +261,7 @@ text-unidecode==1.3 # via python-slugify textwrap3==0.9.2 # via ansiwrap -tomli==2.0.1 +tomli==1.2.3 # via black tornado==6.1 # via @@ -277,9 +279,9 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via - # flytekit + # black # typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-snowflake/requirements.in b/plugins/flytekit-snowflake/requirements.in index f720269c73..2a118dd3bf 100644 --- a/plugins/flytekit-snowflake/requirements.in +++ b/plugins/flytekit-snowflake/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-snowflake diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index e9b2b125f2..53951100b5 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-snowflake # via -r requirements.in arrow==1.2.2 @@ -16,15 +12,13 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -32,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,18 +42,16 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-snowflake -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -87,12 +79,14 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -107,7 +101,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -139,7 +131,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-spark/requirements.in b/plugins/flytekit-spark/requirements.in index 9d2e62c77e..f112dbc8ef 100644 --- a/plugins/flytekit-spark/requirements.in +++ b/plugins/flytekit-spark/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-spark diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index 2ab33fe5f3..75d1e32d53 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-spark # via -r requirements.in arrow==1.2.2 @@ -16,15 +12,13 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -32,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,18 +42,16 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-spark -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -87,12 +79,14 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry py4j==0.10.9.3 @@ -111,7 +105,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -130,8 +124,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -143,7 +135,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-sqlalchemy/requirements.in b/plugins/flytekit-sqlalchemy/requirements.in index e545653d01..21d9e97b47 100644 --- a/plugins/flytekit-sqlalchemy/requirements.in +++ b/plugins/flytekit-sqlalchemy/requirements.in @@ -1,3 +1,2 @@ . --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekitplugins-sqlalchemy diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index bced83587b..93dcc2790c 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -1,13 +1,9 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via - # -r requirements.in - # flytekit -e file:.#egg=flytekitplugins-sqlalchemy # via -r requirements.in arrow==1.2.2 @@ -16,15 +12,13 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.3 +click==7.1.2 # via # cookiecutter # flytekit @@ -32,7 +26,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.1 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,20 +42,18 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flytekit==0.30.3 +flyteidl==0.22.0 + # via flytekit +flytekit==0.30.0 # via flytekitplugins-sqlalchemy greenlet==1.1.2 # via sqlalchemy -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.1 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -89,12 +81,14 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.1 +pandas==1.4.0 # via flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit py==1.11.0 # via retry pyarrow==6.0.1 @@ -109,7 +103,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.0.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -128,8 +122,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -143,7 +135,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via # flytekit # typing-inspect From adaa44884f76bd4c7c2225e8e309132cb9111906 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 18 Feb 2022 14:49:28 -0800 Subject: [PATCH 22/35] wip - Add support for univariate lists Signed-off-by: Eduardo Apolinario --- flytekit/core/type_engine.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index e44f3fa737..3354dad84b 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -803,7 +803,9 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: t = self.get_sub_type(python_type) lit_list = [TypeEngine.to_literal(ctx, x, t, expected.collection_type) for x in python_val] # type: ignore - return Literal(collection=LiteralCollection(literals=lit_list)) + hash = "|".join([literal.hash for literal in lit_list if literal.hash]) + print(f"hash={hash}") + return Literal(collection=LiteralCollection(literals=lit_list), hash=hash if hash != "|" else None) def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[T]: st = self.get_sub_type(expected_python_type) From 4b5f608f762a018eae1f42acc3bcf6c4cf12583a Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Fri, 18 Feb 2022 15:43:51 -0800 Subject: [PATCH 23/35] Add support for lists of annotated objects Signed-off-by: Eduardo Apolinario --- flytekit/core/type_engine.py | 9 ++++-- tests/flytekit/unit/core/test_type_engine.py | 32 ++++++++++++++++++++ tests/flytekit/unit/core/test_type_hints.py | 26 ++++++++++++++++ 3 files changed, 65 insertions(+), 2 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 3354dad84b..c50edc7dca 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -803,9 +803,14 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: t = self.get_sub_type(python_type) lit_list = [TypeEngine.to_literal(ctx, x, t, expected.collection_type) for x in python_val] # type: ignore + # In order to support caching of lists of offloaded objects we delegate the production of + # overridable hashes to each literal and concatenate them into a single string to represent + # the hash of the list. hash = "|".join([literal.hash for literal in lit_list if literal.hash]) - print(f"hash={hash}") - return Literal(collection=LiteralCollection(literals=lit_list), hash=hash if hash != "|" else None) + return Literal( + collection=LiteralCollection(literals=lit_list), + hash=hash if hash != "|" else None, + ) def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[T]: st = self.get_sub_type(expected_python_type) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 56f3ca50af..59cce41998 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -914,6 +914,38 @@ def constant_hash(df: pd.DataFrame) -> str: assert expected_df.equals(python_df) +def test_literal_hash_of_list_of_dataframes(): + """ + Test to confirm that lists of annotated dataframes override the hash of the collection + """ + ctx = FlyteContext.current_context() + + def hash_function(df: pd.DataFrame) -> str: + return str(sum(df["col1"])) + + list_transformer = ListTransformer() + type_list_of_annotated = typing.List[Annotated[pd.DataFrame, HashMethod(hash_function)]] + + df1 = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) + df2 = pd.DataFrame(data={"col1": [2, 3], "col2": [4, 5]}) + literal_collection_with_hash_set = TypeEngine.to_literal( + ctx, + [df1, df2], + type_list_of_annotated, + list_transformer.get_literal_type(type_list_of_annotated), + ) + assert literal_collection_with_hash_set.hash == "3|5" + # Confirm tha the loaded dataframes are not affected + python_dfs = TypeEngine.to_python_value(ctx, literal_collection_with_hash_set, typing.List[pd.DataFrame]) + expected_dfs = [ + pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}), + pd.DataFrame(data={"col1": [2, 3], "col2": [4, 5]}), + ] + assert len(python_dfs) == len(expected_dfs) + for i in range(len(expected_dfs)): + assert expected_dfs[i].equals(python_dfs[i]) + + def test_annotated_simple_types(): def _check_annotation(t, annotation): lt = TypeEngine.to_literal_type(t) diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index e5ed338599..293ba885fe 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -1647,6 +1647,32 @@ def t0() -> Annotated[pandas.DataFrame, HashMethod(constant_function)]: assert df.equals(expected_df) +def test_task_hash_return_list_of_pandas_dataframe(): + def hash_function(df: pandas.DataFrame) -> str: + return str(sum(df["col1"]) + sum(df["col2"])) + + @task + def t0() -> typing.List[Annotated[pandas.DataFrame, HashMethod(hash_function)]]: + return [ + pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}), + pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}), + ] + + ctx = context_manager.FlyteContextManager.current_context() + output_lm = t0.dispatch_execute(ctx, _literal_models.LiteralMap(literals={})) + assert output_lm.literals["o0"].hash == "10|100" + + # Confirm that the literal containing a hash does not have any effect on the scalar. + dfs = TypeEngine.to_python_value(ctx, output_lm.literals["o0"], typing.List[pandas.DataFrame]) + expected_dfs = [ + pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}), + pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}), + ] + assert len(dfs) == len(expected_dfs) + for i in range(len(dfs)): + assert dfs[i].equals(expected_dfs[i]) + + def test_workflow_containing_multiple_annotated_tasks(): def hash_function_t0(df: pandas.DataFrame) -> str: return "hash-0" From ecffa04e5d061e56455b97c9363f8c55903015e5 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 23 Feb 2022 08:00:22 -0800 Subject: [PATCH 24/35] Revamp generation of cache key (to cover case of literals collections and maps) Signed-off-by: Eduardo Apolinario --- flytekit/core/local_cache.py | 32 ++++++++++++++---- tests/flytekit/unit/core/test_local_cache.py | 34 ++++++++++++++++++++ 2 files changed, 59 insertions(+), 7 deletions(-) diff --git a/flytekit/core/local_cache.py b/flytekit/core/local_cache.py index 8c908f8138..4afab73955 100644 --- a/flytekit/core/local_cache.py +++ b/flytekit/core/local_cache.py @@ -1,24 +1,42 @@ +import base64 from typing import Optional +import cloudpickle from diskcache import Cache -from flytekit.models.literals import Literal, LiteralMap +from flytekit.models.literals import Literal, LiteralCollection, LiteralMap # Location on the filesystem where serialized objects will be stored # TODO: read from config CACHE_LOCATION = "~/.flyte/local-cache" +def _recursive_hash_placement(literal: Literal) -> Literal: + if literal.collection is not None: + literals = [_recursive_hash_placement(literal) for literal in literal.collection.literals] + return Literal(collection=LiteralCollection(literals=literals)) + elif literal.map is not None: + literal_map = {} + for key, literal in literal.map.literals.items(): + literal_map[key] = _recursive_hash_placement(literal) + return Literal(map=LiteralMap(literal_map)) + + # Base case + if literal.hash is not None: + return Literal(hash=literal.hash) + else: + return literal + + def _calculate_cache_key(task_name: str, cache_version: str, input_literal_map: LiteralMap) -> str: - # TODO: document this. + # Traverse the literals and replace the literal with a new literal that only contains the hash literal_map_overridden = {} for key, literal in input_literal_map.literals.items(): - if literal.hash is not None: - literal_map_overridden[key] = Literal(hash=literal.hash) - else: - literal_map_overridden[key] = literal + literal_map_overridden[key] = _recursive_hash_placement(literal) - return f"{task_name}-{cache_version}-{LiteralMap(literal_map_overridden)}" + # Pickle the literal map and use base64 encoding to generate a representation of it + b64_encoded = base64.b64encode(cloudpickle.dumps(LiteralMap(literal_map_overridden))) + return f"{task_name}-{cache_version}-{b64_encoded}" class LocalTaskCache(object): diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index 922fdfcccd..f308cc88a2 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -1,6 +1,7 @@ import datetime import typing from dataclasses import dataclass +from typing import List import pandas from dataclasses_json import dataclass_json @@ -351,6 +352,39 @@ def my_workflow(): assert n_cached_task_calls == 1 +def test_list_of_pandas_dataframe_hash(): + """ + Test that cache is hit in the case of a list of pandas dataframes where we annotated dataframes to hash + the contents of the dataframes. + """ + + def hash_pandas_dataframe(df: pandas.DataFrame) -> str: + return str(pandas.util.hash_pandas_object(df)) + + @task + def uncached_data_reading_task() -> List[Annotated[pandas.DataFrame, HashMethod(hash_pandas_dataframe)]]: + return [pandas.DataFrame({"column_1": [1, 2, 3]}), pandas.DataFrame({"column_1": [10, 20, 30]})] + + @task(cache=True, cache_version="0.1") + def cached_data_processing_task(data: List[pandas.DataFrame]) -> List[pandas.DataFrame]: + global n_cached_task_calls + n_cached_task_calls += 1 + return [df * 2 for df in data] + + @workflow + def my_workflow(): + raw_data = uncached_data_reading_task() + cached_data_processing_task(data=raw_data) + + assert n_cached_task_calls == 0 + my_workflow() + assert n_cached_task_calls == 1 + + # Confirm that we see a cache hit in the case of annotated dataframes. + my_workflow() + assert n_cached_task_calls == 1 + + """ Update SD transformer so that it can to_python_value a Schema literal - If a Schema literal is detected, copy the uri and use the new decoder to unwrap the uri From d4b0b4929eb34bcfcf1338df9a17ef2984b5b490 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 23 Feb 2022 08:01:18 -0800 Subject: [PATCH 25/35] Leave TODO for warning Signed-off-by: Eduardo Apolinario --- flytekit/core/type_engine.py | 1 + 1 file changed, 1 insertion(+) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index c50edc7dca..b00ee70556 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -881,6 +881,7 @@ def to_literal( for k, v in python_val.items(): if type(k) != str: raise ValueError("Flyte MapType expects all keys to be strings") + # TODO: log a warning for Annotated objects that contain HashMethod k_type, v_type = self.get_dict_types(python_type) lit_map[k] = TypeEngine.to_literal(ctx, v, v_type, expected.map_value_type) return Literal(map=LiteralMap(literals=lit_map)) From 4d54c595e29d5acbaba2359d3d01630f865b8265 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 23 Feb 2022 08:01:46 -0800 Subject: [PATCH 26/35] Revert "Add support for lists of annotated objects" This reverts commit 4b5f608f762a018eae1f42acc3bcf6c4cf12583a. Signed-off-by: Eduardo Apolinario --- flytekit/core/type_engine.py | 9 ++---- tests/flytekit/unit/core/test_type_engine.py | 32 -------------------- tests/flytekit/unit/core/test_type_hints.py | 26 ---------------- 3 files changed, 2 insertions(+), 65 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index b00ee70556..a931f0789b 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -803,14 +803,9 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: t = self.get_sub_type(python_type) lit_list = [TypeEngine.to_literal(ctx, x, t, expected.collection_type) for x in python_val] # type: ignore - # In order to support caching of lists of offloaded objects we delegate the production of - # overridable hashes to each literal and concatenate them into a single string to represent - # the hash of the list. hash = "|".join([literal.hash for literal in lit_list if literal.hash]) - return Literal( - collection=LiteralCollection(literals=lit_list), - hash=hash if hash != "|" else None, - ) + print(f"hash={hash}") + return Literal(collection=LiteralCollection(literals=lit_list), hash=hash if hash != "|" else None) def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[T]: st = self.get_sub_type(expected_python_type) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 59cce41998..56f3ca50af 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -914,38 +914,6 @@ def constant_hash(df: pd.DataFrame) -> str: assert expected_df.equals(python_df) -def test_literal_hash_of_list_of_dataframes(): - """ - Test to confirm that lists of annotated dataframes override the hash of the collection - """ - ctx = FlyteContext.current_context() - - def hash_function(df: pd.DataFrame) -> str: - return str(sum(df["col1"])) - - list_transformer = ListTransformer() - type_list_of_annotated = typing.List[Annotated[pd.DataFrame, HashMethod(hash_function)]] - - df1 = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}) - df2 = pd.DataFrame(data={"col1": [2, 3], "col2": [4, 5]}) - literal_collection_with_hash_set = TypeEngine.to_literal( - ctx, - [df1, df2], - type_list_of_annotated, - list_transformer.get_literal_type(type_list_of_annotated), - ) - assert literal_collection_with_hash_set.hash == "3|5" - # Confirm tha the loaded dataframes are not affected - python_dfs = TypeEngine.to_python_value(ctx, literal_collection_with_hash_set, typing.List[pd.DataFrame]) - expected_dfs = [ - pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}), - pd.DataFrame(data={"col1": [2, 3], "col2": [4, 5]}), - ] - assert len(python_dfs) == len(expected_dfs) - for i in range(len(expected_dfs)): - assert expected_dfs[i].equals(python_dfs[i]) - - def test_annotated_simple_types(): def _check_annotation(t, annotation): lt = TypeEngine.to_literal_type(t) diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index 293ba885fe..e5ed338599 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -1647,32 +1647,6 @@ def t0() -> Annotated[pandas.DataFrame, HashMethod(constant_function)]: assert df.equals(expected_df) -def test_task_hash_return_list_of_pandas_dataframe(): - def hash_function(df: pandas.DataFrame) -> str: - return str(sum(df["col1"]) + sum(df["col2"])) - - @task - def t0() -> typing.List[Annotated[pandas.DataFrame, HashMethod(hash_function)]]: - return [ - pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}), - pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}), - ] - - ctx = context_manager.FlyteContextManager.current_context() - output_lm = t0.dispatch_execute(ctx, _literal_models.LiteralMap(literals={})) - assert output_lm.literals["o0"].hash == "10|100" - - # Confirm that the literal containing a hash does not have any effect on the scalar. - dfs = TypeEngine.to_python_value(ctx, output_lm.literals["o0"], typing.List[pandas.DataFrame]) - expected_dfs = [ - pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}), - pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}), - ] - assert len(dfs) == len(expected_dfs) - for i in range(len(dfs)): - assert dfs[i].equals(expected_dfs[i]) - - def test_workflow_containing_multiple_annotated_tasks(): def hash_function_t0(df: pandas.DataFrame) -> str: return "hash-0" From 82bbc1fbe07d5a32a25f68730c39f0e4408cd361 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 23 Feb 2022 08:02:06 -0800 Subject: [PATCH 27/35] Revert "wip - Add support for univariate lists" This reverts commit adaa44884f76bd4c7c2225e8e309132cb9111906. Signed-off-by: Eduardo Apolinario --- flytekit/core/type_engine.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index a931f0789b..1a84c744a8 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -803,9 +803,7 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal: t = self.get_sub_type(python_type) lit_list = [TypeEngine.to_literal(ctx, x, t, expected.collection_type) for x in python_val] # type: ignore - hash = "|".join([literal.hash for literal in lit_list if literal.hash]) - print(f"hash={hash}") - return Literal(collection=LiteralCollection(literals=lit_list), hash=hash if hash != "|" else None) + return Literal(collection=LiteralCollection(literals=lit_list)) def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[T]: st = self.get_sub_type(expected_python_type) From a21631b85db9a7b9bd46a88e4c45de567c6ddf0e Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Mon, 28 Feb 2022 17:29:48 -0800 Subject: [PATCH 28/35] Remove docstring Signed-off-by: Eduardo Apolinario --- tests/flytekit/unit/core/test_local_cache.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index f308cc88a2..3f3e56de88 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -383,15 +383,3 @@ def my_workflow(): # Confirm that we see a cache hit in the case of annotated dataframes. my_workflow() assert n_cached_task_calls == 1 - - -""" -Update SD transformer so that it can to_python_value a Schema literal - - If a Schema literal is detected, copy the uri and use the new decoder to unwrap the uri -Update FS transformer so that it can to_python_value a StructuredDataset literal - - If a StructuredDataset literal is detected, use the uri from that instead. - -Update all plugins that can take in a FlyteSchema to also be able to take in a StructuredDataset. - -All tests should work with the presence of SD imports. -""" From 3519149d718581d076ec2201651e092678732ff7 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 1 Mar 2022 17:06:17 -0800 Subject: [PATCH 29/35] Add flyteidl>=0.23.0 Signed-off-by: Eduardo Apolinario --- setup.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 4e31ecc7fe..9a96657326 100644 --- a/setup.py +++ b/setup.py @@ -35,8 +35,7 @@ ] }, install_requires=[ - # TODO: put the flyteidl constraint back - # "flyteidl>=0.21.4", + "flyteidl>=0.23.0", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=4.0.0,<7.0.0", From 09e736d8847c4e447ae42cf0e2c5292f2c9f8fb9 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 1 Mar 2022 17:06:43 -0800 Subject: [PATCH 30/35] Remove mentions to branch flyteidl@add-hash-to-literal Signed-off-by: Eduardo Apolinario --- .github/workflows/pythonbuild.yml | 1 - dev-requirements.txt | 12 ++++++++++++ doc-requirements.txt | 12 +++++++++++- requirements-spark2.txt | 10 ++++++++-- requirements.in | 1 - requirements.txt | 10 ++++++++-- .../mock_flyte_repo/workflows/requirements.txt | 2 +- 7 files changed, 40 insertions(+), 8 deletions(-) diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index 8972e0df4b..d38b666aee 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -106,7 +106,6 @@ jobs: pip install -e . if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit - pip install --no-deps -U "git+https://github.com/flyteorg/flyteidl@add-hash-to-literal#egg=flyteidl" pip freeze - name: Test with coverage run: | diff --git a/dev-requirements.txt b/dev-requirements.txt index b34300a82a..f8bd6af672 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -116,6 +116,10 @@ docstring-parser==0.13 # flytekit filelock==3.6.0 # via virtualenv +flyteidl==0.23.0 + # via + # -c requirements.txt + # flytekit google-api-core[grpc]==2.5.0 # via # google-cloud-bigquery @@ -137,6 +141,8 @@ google-resumable-media==2.3.0 # via google-cloud-bigquery googleapis-common-protos==1.55.0 # via + # -c requirements.txt + # flyteidl # google-api-core # grpcio-status grpcio==1.44.0 @@ -250,12 +256,18 @@ proto-plus==1.20.3 protobuf==3.19.4 # via # -c requirements.txt + # flyteidl # flytekit # google-api-core # google-cloud-bigquery # googleapis-common-protos # grpcio-status # proto-plus + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via + # -c requirements.txt + # flyteidl py==1.11.0 # via # -c requirements.txt diff --git a/doc-requirements.txt b/doc-requirements.txt index 71788c0b09..07ef3d0d5b 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -63,8 +63,12 @@ docutils==0.17.1 # via # sphinx # sphinx-panels +flyteidl==0.23.0 + # via flytekit furo @ git+https://github.com/flyteorg/furo@main # via -r doc-requirements.in +googleapis-common-protos==1.55.0 + # via flyteidl grpcio==1.44.0 # via # -r doc-requirements.in @@ -121,7 +125,13 @@ pandas==1.4.1 poyo==0.5.0 # via cookiecutter protobuf==3.19.4 - # via flytekit + # via + # flyteidl + # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 90bfac547e..be50ade576 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -4,8 +4,6 @@ # # make requirements-spark2.txt # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via -r requirements.in -e file:.#egg=flytekit # via # -r requirements-spark2.in @@ -52,6 +50,10 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit +flyteidl==0.23.0 + # via flytekit +googleapis-common-protos==1.55.0 + # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 @@ -102,6 +104,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 diff --git a/requirements.in b/requirements.in index 816f9301fe..0cf01a6f13 100644 --- a/requirements.in +++ b/requirements.in @@ -1,5 +1,4 @@ .[all] --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl -e file:.#egg=flytekit attrs<21 # We need to restrict constrain the versions of both jsonschema and pyyaml because of docker-compose (which is diff --git a/requirements.txt b/requirements.txt index 54cb926700..2e51986bf4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,8 +4,6 @@ # # make requirements.txt # --e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl - # via -r requirements.in -e file:.#egg=flytekit # via -r requirements.in arrow==1.2.2 @@ -50,6 +48,10 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit +flyteidl==0.23.0 + # via flytekit +googleapis-common-protos==1.55.0 + # via flyteidl grpcio==1.44.0 # via flytekit idna==3.3 @@ -100,6 +102,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index e559b6762d..15a1f89700 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -44,7 +44,7 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.3 +flyteidl==0.23.0 # via flytekit flytekit==0.30.3 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in From 9ec103cf589b62dcf37a814cdace63f35ebeb026 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 1 Mar 2022 17:53:46 -0800 Subject: [PATCH 31/35] Bump flyteidl in plugins requirements Signed-off-by: Eduardo Apolinario --- plugins/flytekit-aws-athena/requirements.txt | 38 ++++--- plugins/flytekit-aws-batch/requirements.txt | 60 +++++++---- .../flytekit-aws-sagemaker/requirements.txt | 48 +++++---- plugins/flytekit-bigquery/requirements.txt | 38 ++++--- plugins/flytekit-data-fsspec/requirements.txt | 46 +++++--- plugins/flytekit-dolt/requirements.txt | 38 ++++--- .../requirements.txt | 102 +++++++++--------- plugins/flytekit-hive/requirements.txt | 38 ++++--- plugins/flytekit-k8s-pod/requirements.txt | 42 +++++--- plugins/flytekit-kf-mpi/requirements.txt | 38 ++++--- plugins/flytekit-kf-pytorch/requirements.txt | 38 ++++--- .../flytekit-kf-tensorflow/requirements.txt | 38 ++++--- plugins/flytekit-pandera/requirements.txt | 43 +++++--- plugins/flytekit-papermill/requirements.txt | 77 ++++++------- plugins/flytekit-snowflake/requirements.txt | 38 ++++--- plugins/flytekit-spark/requirements.txt | 38 ++++--- plugins/flytekit-sqlalchemy/requirements.txt | 38 ++++--- 17 files changed, 499 insertions(+), 299 deletions(-) diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index a1acae494f..6a2fc1f2d7 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-athena -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt index 9181490234..ee76299714 100644 --- a/plugins/flytekit-aws-batch/requirements.txt +++ b/plugins/flytekit-aws-batch/requirements.txt @@ -1,24 +1,26 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in # -e file:.#egg=flytekitplugins-awsbatch # via -r requirements.in -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,8 +28,10 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -40,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.21.23 +flyteidl==0.23.0 # via flytekit -flytekit==0.26.0 +flytekit==0.30.3 # via flytekitplugins-awsbatch -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -58,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -71,25 +81,31 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via flytekit -numpy==1.22.1 +numpy==1.22.2 # via # pandas # pyarrow -pandas==1.3.5 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter -protobuf==3.19.3 +protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit -python-dateutil==2.8.1 +pycparser==2.21 + # via cffi +python-dateutil==2.8.2 # via # arrow # croniter @@ -97,7 +113,7 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -112,25 +128,27 @@ requests==2.27.1 # cookiecutter # flytekit # responses -responses==0.17.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 - # via typing-inspect +typing-extensions==4.1.1 + # via + # flytekit + # typing-inspect typing-inspect==0.7.1 # via dataclasses-json urllib3==1.26.8 diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index 11396f213e..acb982c9a2 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,9 +12,9 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -boto3==1.20.50 +boto3==1.21.10 # via sagemaker-training -botocore==1.23.50 +botocore==1.24.10 # via # boto3 # s3transfer @@ -27,11 +27,11 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -39,7 +39,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via @@ -57,22 +57,28 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-awssagemaker gevent==21.12.0 # via sagemaker-training +googleapis-common-protos==1.55.0 + # via flyteidl greenlet==1.1.2 # via gevent -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring -inotify-simple==1.2.1 +inotify_simple==1.2.1 # via sagemaker-training +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -85,7 +91,7 @@ jmespath==0.10.0 # botocore keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -106,7 +112,7 @@ numpy==1.22.2 # pyarrow # sagemaker-training # scipy -pandas==1.4.0 +pandas==1.4.1 # via flytekit paramiko==2.9.2 # via sagemaker-training @@ -116,7 +122,11 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger # sagemaker-training +protoc-gen-swagger==0.1.0 + # via flyteidl psutil==5.9.0 # via sagemaker-training py==1.11.0 @@ -136,7 +146,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -157,12 +167,14 @@ retry==0.9.2 # via flytekit retrying==1.3.3 # via sagemaker-training -s3transfer==0.5.1 +s3transfer==0.5.2 # via boto3 sagemaker-training==3.9.2 # via flytekitplugins-awssagemaker scipy==1.8.0 # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # bcrypt @@ -177,7 +189,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect @@ -199,9 +211,9 @@ wrapt==1.13.3 # flytekit zipp==3.7.0 # via importlib-metadata -zope-event==4.5.0 +zope.event==4.5.0 # via gevent -zope-interface==5.4.0 +zope.interface==5.4.0 # via gevent # The following packages are considered to be unsafe in a requirements file: diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt index f0bc647453..b908f23e9e 100644 --- a/plugins/flytekit-bigquery/requirements.txt +++ b/plugins/flytekit-bigquery/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -18,11 +18,11 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -30,7 +30,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -46,9 +46,9 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-bigquery google-api-core[grpc]==2.5.0 # via @@ -58,29 +58,30 @@ google-auth==2.6.0 # via # google-api-core # google-cloud-core -google-cloud-bigquery==2.32.0 +google-cloud-bigquery==2.34.0 # via flytekitplugins-bigquery google-cloud-core==2.2.2 # via google-cloud-bigquery google-crc32c==1.3.0 # via google-resumable-media -google-resumable-media==2.2.0 +google-resumable-media==2.3.0 # via google-cloud-bigquery -googleapis-common-protos==1.54.0 +googleapis-common-protos==1.55.0 # via + # flyteidl # google-api-core # grpcio-status -grpcio==1.43.0 +grpcio==1.44.0 # via # flytekit # google-api-core # google-cloud-bigquery # grpcio-status -grpcio-status==1.43.0 +grpcio-status==1.44.0 # via google-api-core idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring jeepney==0.7.1 # via @@ -94,7 +95,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -115,11 +116,11 @@ numpy==1.22.2 # pyarrow packaging==21.3 # via google-cloud-bigquery -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter -proto-plus==1.20.0 +proto-plus==1.20.3 # via google-cloud-bigquery protobuf==3.19.4 # via @@ -130,6 +131,9 @@ protobuf==3.19.4 # googleapis-common-protos # grpcio-status # proto-plus + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -153,7 +157,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -190,7 +194,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index 4dc89fa500..e728ce8676 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -10,17 +10,19 @@ arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter -botocore==1.23.24 +botocore==1.24.10 # via flytekitplugins-data-fsspec certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -28,8 +30,10 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -42,18 +46,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-data-fsspec -fsspec==2022.1.0 +fsspec==2022.2.0 # via flytekitplugins-data-fsspec -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -64,7 +74,7 @@ jmespath==0.10.0 # via botocore keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -83,7 +93,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -91,10 +101,16 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi python-dateutil==2.8.2 # via # arrow @@ -104,7 +120,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -123,6 +139,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -134,7 +152,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index 01dfe2cb6b..940643fdf9 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -48,16 +50,22 @@ dolt-integrations==0.1.5 # via flytekitplugins-dolt doltcli==0.1.17 # via dolt-integrations -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-dolt -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -66,7 +74,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -85,7 +93,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via # dolt-integrations # flytekit @@ -95,6 +103,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -109,7 +121,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -128,6 +140,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -139,7 +153,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index 7aa0a867dd..7acb822eec 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -8,10 +8,6 @@ # via -r requirements.in altair==4.2.0 # via great-expectations -appnope==0.1.2 - # via - # ipykernel - # ipython argon2-cffi==21.3.0 # via notebook argon2-cffi-bindings==21.2.0 @@ -26,23 +22,22 @@ backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via ipython bleach==4.1.0 # via nbconvert certifi==2021.10.8 # via requests cffi==1.15.0 - # via argon2-cffi-bindings + # via + # argon2-cffi-bindings + # cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via - # black # cookiecutter # flytekit # great-expectations @@ -50,8 +45,10 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit debugpy==1.5.1 @@ -70,34 +67,36 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 +entrypoints==0.4 # via # altair # jupyter-client # nbconvert -executing==0.8.2 +executing==0.8.3 # via stack-data -flyteidl==0.21.24 +flyteidl==0.23.0 # via flytekit -flytekit==0.26.1 +flytekit==0.30.3 # via flytekitplugins-great-expectations -great-expectations==0.14.3 +googleapis-common-protos==1.55.0 + # via flyteidl +great-expectations==0.14.5 # via flytekitplugins-great-expectations greenlet==1.1.2 # via sqlalchemy -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via # great-expectations # keyring -ipykernel==6.7.0 +ipykernel==6.9.1 # via # ipywidgets # notebook -ipython==8.0.1 +ipython==8.1.0 # via # ipykernel # ipywidgets @@ -110,6 +109,10 @@ ipywidgets==7.6.5 # via great-expectations jedi==0.18.1 # via ipython +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # altair @@ -134,7 +137,7 @@ jupyter-client==7.1.2 # ipykernel # nbclient # notebook -jupyter-core==4.9.1 +jupyter-core==4.9.2 # via # jupyter-client # nbconvert @@ -146,7 +149,7 @@ jupyterlab-widgets==1.0.2 # via ipywidgets keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -166,14 +169,12 @@ mistune==0.8.4 # great-expectations # nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbclient==0.5.10 +nbclient==0.5.11 # via nbconvert -nbconvert==6.4.1 +nbconvert==6.4.2 # via notebook nbformat==5.1.3 # via @@ -189,7 +190,7 @@ nest-asyncio==1.5.4 # notebook notebook==6.4.8 # via widgetsnbextension -numpy==1.22.1 +numpy==1.22.2 # via # altair # great-expectations @@ -198,7 +199,7 @@ numpy==1.22.1 # scipy packaging==21.3 # via bleach -pandas==1.4.0 +pandas==1.4.1 # via # altair # flytekit @@ -207,24 +208,24 @@ pandocfilters==1.5.0 # via nbconvert parso==0.8.3 # via jedi -pathspec==0.9.0 - # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.1 - # via black poyo==0.5.0 # via cookiecutter prometheus-client==0.13.1 # via notebook -prompt-toolkit==3.0.26 +prompt-toolkit==3.0.28 # via ipython protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl ptyprocess==0.7.0 # via # pexpect @@ -248,7 +249,7 @@ pyparsing==2.4.7 # packaging pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow # croniter @@ -258,7 +259,7 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -281,16 +282,16 @@ requests==2.27.1 # flytekit # great-expectations # responses -responses==0.17.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -ruamel-yaml==0.17.17 +ruamel.yaml==0.17.17 # via great-expectations -ruamel-yaml-clib==0.2.6 - # via ruamel-yaml -scipy==1.7.3 +scipy==1.8.0 # via great-expectations +secretstorage==3.3.1 + # via keyring send2trash==1.8.0 # via notebook six==1.16.0 @@ -298,17 +299,15 @@ six==1.16.0 # asttokens # bleach # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit sqlalchemy==1.4.31 # via # -r requirements.in # flytekitplugins-great-expectations -stack-data==0.1.4 +stack-data==0.2.0 # via ipython statsd==3.3.0 # via flytekit @@ -316,12 +315,10 @@ termcolor==1.1.0 # via great-expectations terminado==0.13.1 # via notebook -testpath==0.5.0 +testpath==0.6.0 # via nbconvert text-unidecode==1.3 # via python-slugify -tomli==1.2.3 - # via black toolz==0.11.2 # via altair tornado==6.1 @@ -330,7 +327,7 @@ tornado==6.1 # jupyter-client # notebook # terminado -tqdm==4.62.3 +tqdm==4.63.0 # via great-expectations traitlets==5.1.1 # via @@ -344,9 +341,10 @@ traitlets==5.1.1 # nbconvert # nbformat # notebook -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via - # black + # flytekit + # great-expectations # typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index 3ae057f9a0..a7cf6dcbb2 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-hive -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index d60d9c46b6..7e7eea1284 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -16,13 +16,15 @@ certifi==2021.10.8 # via # kubernetes # requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -30,7 +32,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -46,18 +48,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-pod google-auth==2.6.0 # via kubernetes -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -66,9 +74,9 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -kubernetes==21.7.0 +kubernetes==23.3.0 # via flytekitplugins-pod -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -89,7 +97,7 @@ numpy==1.22.2 # pyarrow oauthlib==3.2.0 # via requests-oauthlib -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -97,6 +105,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -118,7 +130,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -145,6 +157,8 @@ retry==0.9.2 # via flytekit rsa==4.8 # via google-auth +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -158,7 +172,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect @@ -170,7 +184,7 @@ urllib3==1.26.8 # kubernetes # requests # responses -websocket-client==1.2.3 +websocket-client==1.3.1 # via kubernetes wheel==0.37.1 # via flytekit diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt index c13f9f3b74..4d5c66fcea 100644 --- a/plugins/flytekit-kf-mpi/requirements.txt +++ b/plugins/flytekit-kf-mpi/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,18 +44,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via # flytekit # flytekitplugins-kfmpi -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-kfmpi -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -62,7 +70,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -81,7 +89,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -89,6 +97,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -103,7 +115,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -122,6 +134,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -133,7 +147,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index 1696646c87..6f4496e3b4 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-kfpytorch -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index d3b6253664..2af5b7c14b 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-kftensorflow -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index f0b65cec17..62c5e3319c 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-pandera -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -82,11 +90,11 @@ numpy==1.22.2 # pyarrow packaging==21.3 # via pandera -pandas==1.4.0 +pandas==1.4.1 # via # flytekit # pandera -pandera==0.8.1 +pandera==0.9.0 # via flytekitplugins-pandera poyo==0.5.0 # via cookiecutter @@ -94,6 +102,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -102,6 +114,8 @@ pyarrow==6.0.1 # pandera pycparser==2.21 # via cffi +pydantic==1.9.0 + # via pandera pyparsing==3.0.7 # via packaging python-dateutil==2.8.2 @@ -112,7 +126,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -131,6 +145,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -142,9 +158,10 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit + # pydantic # typing-inspect typing-inspect==0.7.1 # via diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 8c3e4570e9..0a0cc6e49e 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -18,8 +18,6 @@ backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via ipython bleach==4.1.0 # via nbconvert certifi==2021.10.8 @@ -28,13 +26,12 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.10 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via - # black # cookiecutter # flytekit # papermill @@ -42,7 +39,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -64,26 +61,28 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 +entrypoints==0.4 # via # jupyter-client # nbconvert # papermill -executing==0.8.2 +executing==0.8.3 # via stack-data -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.26.1 +flytekit==0.30.3 # via flytekitplugins-papermill -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring -ipykernel==6.7.0 +ipykernel==6.9.1 # via flytekitplugins-papermill -ipython==8.0.1 +ipython==8.1.0 # via ipykernel ipython-genutils==0.2.0 # via nbformat @@ -106,7 +105,7 @@ jupyter-client==7.1.2 # via # ipykernel # nbclient -jupyter-core==4.9.1 +jupyter-core==4.9.2 # via # jupyter-client # nbconvert @@ -115,7 +114,7 @@ jupyterlab-pygments==0.1.2 # via nbconvert keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -133,16 +132,14 @@ matplotlib-inline==0.1.3 mistune==0.8.4 # via nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbclient==0.5.10 +nbclient==0.5.11 # via # nbconvert # papermill -nbconvert==6.4.1 +nbconvert==6.4.2 # via flytekitplugins-papermill nbformat==5.1.3 # via @@ -154,13 +151,13 @@ nest-asyncio==1.5.4 # ipykernel # jupyter-client # nbclient -numpy==1.22.1 +numpy==1.22.2 # via # pandas # pyarrow packaging==21.3 # via bleach -pandas==1.4.0 +pandas==1.4.1 # via flytekit pandocfilters==1.5.0 # via nbconvert @@ -168,22 +165,22 @@ papermill==2.3.4 # via flytekitplugins-papermill parso==0.8.3 # via jedi -pathspec==0.9.0 - # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython -platformdirs==2.4.1 - # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.26 +prompt-toolkit==3.0.28 # via ipython protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 @@ -203,7 +200,7 @@ pyparsing==3.0.7 # via packaging pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow # croniter @@ -212,7 +209,7 @@ python-dateutil==2.8.1 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -232,7 +229,7 @@ requests==2.27.1 # flytekit # papermill # responses -responses==0.17.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit @@ -243,31 +240,27 @@ six==1.16.0 # asttokens # bleach # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit -stack-data==0.1.4 +stack-data==0.2.0 # via ipython statsd==3.3.0 # via flytekit tenacity==8.0.1 # via papermill -testpath==0.5.0 +testpath==0.6.0 # via nbconvert text-unidecode==1.3 # via python-slugify textwrap3==0.9.2 # via ansiwrap -tomli==1.2.3 - # via black tornado==6.1 # via # ipykernel # jupyter-client -tqdm==4.62.3 +tqdm==4.63.0 # via papermill traitlets==5.1.1 # via @@ -279,9 +272,9 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via - # black + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index 53951100b5..4589cef6ef 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-snowflake -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -101,7 +113,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -120,6 +132,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -131,7 +145,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index 75d1e32d53..3b96470c9a 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,16 +44,22 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-spark -grpcio==1.43.0 +googleapis-common-protos==1.55.0 + # via flyteidl +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -60,7 +68,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -79,7 +87,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -87,6 +95,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry py4j==0.10.9.3 @@ -105,7 +117,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -124,6 +136,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -135,7 +149,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index 93dcc2790c..5d0142168c 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.9 +# This file is autogenerated by pip-compile with python 3.10 # To update, run: # # pip-compile requirements.in @@ -12,13 +12,15 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.11 +charset-normalizer==2.0.12 # via requests checksumdir==1.2.0 # via flytekit -click==7.1.2 +click==8.0.4 # via # cookiecutter # flytekit @@ -26,7 +28,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.2.0 +croniter==1.3.4 # via flytekit cryptography==36.0.1 # via secretstorage @@ -42,18 +44,24 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 +flyteidl==0.23.0 # via flytekit -flytekit==0.30.0 +flytekit==0.30.3 # via flytekitplugins-sqlalchemy +googleapis-common-protos==1.55.0 + # via flyteidl greenlet==1.1.2 # via sqlalchemy -grpcio==1.43.0 +grpcio==1.44.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.1 +importlib-metadata==4.11.2 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -62,7 +70,7 @@ jinja2-time==0.2.0 # via cookiecutter keyring==23.5.0 # via flytekit -markupsafe==2.0.1 +markupsafe==2.1.0 # via jinja2 marshmallow==3.14.1 # via @@ -81,7 +89,7 @@ numpy==1.22.2 # via # pandas # pyarrow -pandas==1.4.0 +pandas==1.4.1 # via flytekit poyo==0.5.0 # via cookiecutter @@ -89,6 +97,10 @@ protobuf==3.19.4 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry pyarrow==6.0.1 @@ -103,7 +115,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==5.0.2 +python-slugify==6.1.1 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -122,6 +134,8 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter @@ -135,7 +149,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==4.0.1 +typing-extensions==4.1.1 # via # flytekit # typing-inspect From fa4e3b254e3e6cd31bc4b0c65cb952c9e98389a8 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 1 Mar 2022 18:18:37 -0800 Subject: [PATCH 32/35] Regenerate plugins requirements again Signed-off-by: Eduardo Apolinario --- plugins/flytekit-aws-athena/requirements.txt | 2 +- plugins/flytekit-aws-batch/requirements.txt | 2 +- plugins/flytekit-aws-sagemaker/requirements.txt | 2 +- plugins/flytekit-bigquery/requirements.txt | 2 +- plugins/flytekit-data-fsspec/requirements.txt | 2 +- plugins/flytekit-dolt/requirements.txt | 2 +- plugins/flytekit-greatexpectations/requirements.txt | 2 +- plugins/flytekit-hive/requirements.txt | 2 +- plugins/flytekit-k8s-pod/requirements.txt | 2 +- plugins/flytekit-kf-mpi/requirements.txt | 2 +- plugins/flytekit-kf-pytorch/requirements.txt | 2 +- plugins/flytekit-kf-tensorflow/requirements.txt | 2 +- plugins/flytekit-pandera/requirements.txt | 2 +- plugins/flytekit-papermill/requirements.txt | 4 ++-- plugins/flytekit-snowflake/requirements.txt | 2 +- plugins/flytekit-spark/requirements.txt | 2 +- plugins/flytekit-sqlalchemy/requirements.txt | 2 +- 17 files changed, 18 insertions(+), 18 deletions(-) diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index 6a2fc1f2d7..bb210d8c13 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -121,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt index ee76299714..80a3fcf51f 100644 --- a/plugins/flytekit-aws-batch/requirements.txt +++ b/plugins/flytekit-aws-batch/requirements.txt @@ -121,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index acb982c9a2..f4b02da229 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -154,7 +154,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt index b908f23e9e..aef0a38759 100644 --- a/plugins/flytekit-bigquery/requirements.txt +++ b/plugins/flytekit-bigquery/requirements.txt @@ -165,7 +165,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index e728ce8676..b3f506d572 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -128,7 +128,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index 940643fdf9..fc9dc9f8b7 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -129,7 +129,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index 7acb822eec..31b3be4b48 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -274,7 +274,7 @@ pyzmq==22.3.0 # via # jupyter-client # notebook -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index a7cf6dcbb2..69ed1e5f3e 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -121,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index 7e7eea1284..df398913af 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -140,7 +140,7 @@ pytz==2021.3 # pandas pyyaml==6.0 # via kubernetes -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt index 4d5c66fcea..82a2317f2f 100644 --- a/plugins/flytekit-kf-mpi/requirements.txt +++ b/plugins/flytekit-kf-mpi/requirements.txt @@ -123,7 +123,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index 6f4496e3b4..0a53a62333 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -121,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index 2af5b7c14b..b782c39630 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -121,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index 62c5e3319c..7b72a0b4c0 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -134,7 +134,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 0a0cc6e49e..4e993563dc 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -1,5 +1,5 @@ # -# This file is autogenerated by pip-compile with python 3.10 +# This file is autogenerated by pip-compile with python 3.9 # To update, run: # # pip-compile requirements.in @@ -221,7 +221,7 @@ pyyaml==6.0 # via papermill pyzmq==22.3.0 # via jupyter-client -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index 4589cef6ef..2d34ec966f 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -121,7 +121,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index 3b96470c9a..6e7c6e59f0 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -125,7 +125,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index 5d0142168c..e617920549 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -123,7 +123,7 @@ pytz==2021.3 # via # flytekit # pandas -regex==2022.1.18 +regex==2022.3.2 # via docker-image-py requests==2.27.1 # via From 373b3bbdb9c106e0b5e2788625e9b09ca30b4d55 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Tue, 1 Mar 2022 19:49:20 -0800 Subject: [PATCH 33/35] Restore papermill/requirements.txt Signed-off-by: Eduardo Apolinario --- plugins/flytekit-papermill/requirements.txt | 77 +++++++++++---------- 1 file changed, 42 insertions(+), 35 deletions(-) diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index 4e993563dc..8c3e4570e9 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -18,6 +18,8 @@ backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter +black==21.12b0 + # via ipython bleach==4.1.0 # via nbconvert certifi==2021.10.8 @@ -26,12 +28,13 @@ cffi==1.15.0 # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.12 +charset-normalizer==2.0.10 # via requests checksumdir==1.2.0 # via flytekit -click==8.0.4 +click==7.1.2 # via + # black # cookiecutter # flytekit # papermill @@ -39,7 +42,7 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.3.4 +croniter==1.2.0 # via flytekit cryptography==36.0.1 # via secretstorage @@ -61,28 +64,26 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.4 +entrypoints==0.3 # via # jupyter-client # nbconvert # papermill -executing==0.8.3 +executing==0.8.2 # via stack-data -flyteidl==0.23.0 +flyteidl==0.22.0 # via flytekit -flytekit==0.30.3 +flytekit==0.26.1 # via flytekitplugins-papermill -googleapis-common-protos==1.55.0 - # via flyteidl -grpcio==1.44.0 +grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.11.2 +importlib-metadata==4.10.1 # via keyring -ipykernel==6.9.1 +ipykernel==6.7.0 # via flytekitplugins-papermill -ipython==8.1.0 +ipython==8.0.1 # via ipykernel ipython-genutils==0.2.0 # via nbformat @@ -105,7 +106,7 @@ jupyter-client==7.1.2 # via # ipykernel # nbclient -jupyter-core==4.9.2 +jupyter-core==4.9.1 # via # jupyter-client # nbconvert @@ -114,7 +115,7 @@ jupyterlab-pygments==0.1.2 # via nbconvert keyring==23.5.0 # via flytekit -markupsafe==2.1.0 +markupsafe==2.0.1 # via jinja2 marshmallow==3.14.1 # via @@ -132,14 +133,16 @@ matplotlib-inline==0.1.3 mistune==0.8.4 # via nbconvert mypy-extensions==0.4.3 - # via typing-inspect -natsort==8.1.0 + # via + # black + # typing-inspect +natsort==8.0.2 # via flytekit -nbclient==0.5.11 +nbclient==0.5.10 # via # nbconvert # papermill -nbconvert==6.4.2 +nbconvert==6.4.1 # via flytekitplugins-papermill nbformat==5.1.3 # via @@ -151,13 +154,13 @@ nest-asyncio==1.5.4 # ipykernel # jupyter-client # nbclient -numpy==1.22.2 +numpy==1.22.1 # via # pandas # pyarrow packaging==21.3 # via bleach -pandas==1.4.1 +pandas==1.4.0 # via flytekit pandocfilters==1.5.0 # via nbconvert @@ -165,22 +168,22 @@ papermill==2.3.4 # via flytekitplugins-papermill parso==0.8.3 # via jedi +pathspec==0.9.0 + # via black pexpect==4.8.0 # via ipython pickleshare==0.7.5 # via ipython +platformdirs==2.4.1 + # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.28 +prompt-toolkit==3.0.26 # via ipython protobuf==3.19.4 # via # flyteidl # flytekit - # googleapis-common-protos - # protoc-gen-swagger -protoc-gen-swagger==0.1.0 - # via flyteidl ptyprocess==0.7.0 # via pexpect pure-eval==0.2.2 @@ -200,7 +203,7 @@ pyparsing==3.0.7 # via packaging pyrsistent==0.18.1 # via jsonschema -python-dateutil==2.8.2 +python-dateutil==2.8.1 # via # arrow # croniter @@ -209,7 +212,7 @@ python-dateutil==2.8.2 # pandas python-json-logger==2.0.2 # via flytekit -python-slugify==6.1.1 +python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit @@ -221,7 +224,7 @@ pyyaml==6.0 # via papermill pyzmq==22.3.0 # via jupyter-client -regex==2022.3.2 +regex==2022.1.18 # via docker-image-py requests==2.27.1 # via @@ -229,7 +232,7 @@ requests==2.27.1 # flytekit # papermill # responses -responses==0.18.0 +responses==0.17.0 # via flytekit retry==0.9.2 # via flytekit @@ -240,27 +243,31 @@ six==1.16.0 # asttokens # bleach # cookiecutter + # flytekit # grpcio # python-dateutil + # responses sortedcontainers==2.4.0 # via flytekit -stack-data==0.2.0 +stack-data==0.1.4 # via ipython statsd==3.3.0 # via flytekit tenacity==8.0.1 # via papermill -testpath==0.6.0 +testpath==0.5.0 # via nbconvert text-unidecode==1.3 # via python-slugify textwrap3==0.9.2 # via ansiwrap +tomli==1.2.3 + # via black tornado==6.1 # via # ipykernel # jupyter-client -tqdm==4.63.0 +tqdm==4.62.3 # via papermill traitlets==5.1.1 # via @@ -272,9 +279,9 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==4.1.1 +typing-extensions==4.0.1 # via - # flytekit + # black # typing-inspect typing-inspect==0.7.1 # via dataclasses-json From 403830d4e1169fc9fd57cc18a7d586d49929ddd9 Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 2 Mar 2022 11:45:43 -0800 Subject: [PATCH 34/35] Point flytekitplugins-spark to the offloaded-objects-caching branch in papermill tests Signed-off-by: Eduardo Apolinario --- plugins/flytekit-papermill/dev-requirements.in | 2 +- plugins/flytekit-papermill/dev-requirements.txt | 14 +++++++++++++- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in index 32404e0a2d..fe323f0279 100644 --- a/plugins/flytekit-papermill/dev-requirements.in +++ b/plugins/flytekit-papermill/dev-requirements.in @@ -1,2 +1,2 @@ -git+https://github.com/flyteorg/flytekit@add-sd-make-class-methods#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark +git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark # vcs+protocol://repo_url/#egg=pkg&subdirectory=flyte diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt index ef30545338..bec6aec335 100644 --- a/plugins/flytekit-papermill/dev-requirements.txt +++ b/plugins/flytekit-papermill/dev-requirements.txt @@ -10,6 +10,8 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.10 @@ -26,6 +28,8 @@ cookiecutter==1.7.3 # via flytekit croniter==1.2.0 # via flytekit +cryptography==36.0.1 + # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.1 @@ -42,7 +46,7 @@ flyteidl==0.22.0 # via flytekit flytekit==0.30.0 # via flytekitplugins-spark -flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@add-sd-make-class-methods#subdirectory=plugins/flytekit-spark +flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#subdirectory=plugins/flytekit-spark # via -r dev-requirements.in grpcio==1.43.0 # via flytekit @@ -50,6 +54,10 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -91,6 +99,8 @@ py4j==0.10.9.3 # via pyspark pyarrow==6.0.1 # via flytekit +pycparser==2.21 + # via cffi pyspark==3.2.1 # via flytekitplugins-spark python-dateutil==2.8.1 @@ -120,6 +130,8 @@ responses==0.17.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter From 9754a0f462f495dbd77df0572719885230bd909b Mon Sep 17 00:00:00 2001 From: Eduardo Apolinario Date: Wed, 2 Mar 2022 11:53:15 -0800 Subject: [PATCH 35/35] Set flyteidl>=0.23.0 in papermill dev-requirements --- plugins/flytekit-papermill/dev-requirements.in | 1 + plugins/flytekit-papermill/dev-requirements.txt | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in index fe323f0279..7a47470868 100644 --- a/plugins/flytekit-papermill/dev-requirements.in +++ b/plugins/flytekit-papermill/dev-requirements.in @@ -1,2 +1,3 @@ +flyteidl>=0.23.0 git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark # vcs+protocol://repo_url/#egg=pkg&subdirectory=flyte diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt index bec6aec335..f777991cc3 100644 --- a/plugins/flytekit-papermill/dev-requirements.txt +++ b/plugins/flytekit-papermill/dev-requirements.txt @@ -42,12 +42,16 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.22.0 - # via flytekit +flyteidl==0.23.0 + # via + # -r dev-requirements.in + # flytekit flytekit==0.30.0 # via flytekitplugins-spark flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#subdirectory=plugins/flytekit-spark # via -r dev-requirements.in +googleapis-common-protos==1.55.0 + # via flyteidl grpcio==1.43.0 # via flytekit idna==3.3 @@ -93,6 +97,10 @@ protobuf==3.19.3 # via # flyteidl # flytekit + # googleapis-common-protos + # protoc-gen-swagger +protoc-gen-swagger==0.1.0 + # via flyteidl py==1.11.0 # via retry py4j==0.10.9.3