From 71619aa68a5bba8f4cb19a647e06a77837cc723d Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 1 Dec 2021 08:46:40 -0800 Subject: [PATCH 01/42] feat: support for annotated simple + list Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 48 +++++++++++++++++++++++++++++++++--- 1 file changed, 44 insertions(+), 4 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 939b0531df..d014addbd9 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -22,11 +22,13 @@ from flytekit.common.exceptions import user as user_exceptions from flytekit.common.types import primitives as _primitives +from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import FlyteContext from flytekit.core.type_helpers import load_type_from_tag from flytekit.loggers import logger from flytekit.models import interface as _interface_models from flytekit.models import types as _type_models +from flytekit.models.annotation import TypeAnnotation as _annotation_model from flytekit.models.core import types as _core_types from flytekit.models.literals import ( Blob, @@ -516,14 +518,22 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: TODO lets make this deterministic by using an ordered dict """ + # Step 1 if python_type in cls._REGISTRY: return cls._REGISTRY[python_type] # Step 2 if hasattr(python_type, "__origin__"): + + # Handling of annotated generics, eg: + # typing.Annotated[typing.List[int], 'foo'] + if isinstance(python_type, typing._AnnotatedAlias): + return cls.get_transformer(python_type.__origin__) + if python_type.__origin__ in cls._REGISTRY: return cls._REGISTRY[python_type.__origin__] + raise ValueError(f"Generic Type {python_type.__origin__} not supported currently in Flytekit.") # Step 3 @@ -552,7 +562,29 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: Converts a python type into a flyte specific ``LiteralType`` """ transformer = cls.get_transformer(python_type) - return transformer.get_literal_type(python_type) + res = transformer.get_literal_type(python_type) + data = None + if hasattr(python_type, "__metadata__"): + for x in python_type.__metadata__: + if not isinstance(x, FlyteAnnotation): + continue + if x.data.get("__consumed", False): + continue + data = x.data + x.data["__consumed"] = True + if data is not None: + idl_type_annotation = _annotation_model(annotations=data) + return LiteralType( + simple=res.simple, + schema=res.schema, + collection_type=res.collection_type, + map_value_type=res.map_value_type, + blob=res.blob, + enum_type=res.enum_type, + metadata=res.metadata, + annotation=idl_type_annotation, + ) + return res @classmethod def to_literal(cls, ctx: FlyteContext, python_val: typing.Any, python_type: Type, expected: LiteralType) -> Literal: @@ -680,9 +712,17 @@ def get_sub_type(t: Type[T]) -> Type[T]: """ Return the generic Type T of the List """ - if hasattr(t, "__origin__") and t.__origin__ is list: # type: ignore - if hasattr(t, "__args__"): - return t.__args__[0] # type: ignore + + if hasattr(t, "__origin__"): + + # Handle annotation on list generic, eg: + # typing.Annotated[typing.List[int, 'foo']] + if isinstance(t, typing._AnnotatedAlias) and hasattr(t.__origin__, "__origin__"): + return ListTransformer.get_sub_type(t.__origin__) + + if t.__origin__ is list and hasattr(t, "__args__"): + return t.__args__[0] + raise ValueError("Only generic univariate typing.List[T] type is supported.") def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: From ac27cc05e6b0f84a74cdefead1f733b5ae0c5b4f Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 1 Dec 2021 08:47:17 -0800 Subject: [PATCH 02/42] feat: addition of annotation att to Signed-off-by: Kenny Workman --- flytekit/models/types.py | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/flytekit/models/types.py b/flytekit/models/types.py index 719cdfd7b0..0d4003ed28 100644 --- a/flytekit/models/types.py +++ b/flytekit/models/types.py @@ -5,6 +5,7 @@ from google.protobuf import struct_pb2 as _struct from flytekit.models import common as _common +from flytekit.models.annotation import TypeAnnotation as _annotation_model from flytekit.models.core import types as _core_types @@ -108,6 +109,7 @@ def __init__( blob=None, enum_type=None, metadata=None, + annotation=None, ): """ This is a oneof message, only one of the kwargs may be set, representing one of the Flyte types. @@ -120,6 +122,8 @@ def __init__( :param flytekit.models.core.types.BlobType blob: For blob objects, this describes the type. :param flytekit.models.core.types.EnumType enum_type: For enum objects, describes an enum :param dict[Text, T] metadata: Additional data describing the type + :param flytekit.models.annotation.FlyteAnnotation annotation: Additional data + describing the type _intended to be saturated by the client_ """ self._simple = simple self._schema = schema @@ -128,6 +132,7 @@ def __init__( self._blob = blob self._enum_type = enum_type self._metadata = metadata + self._annotation = annotation @property def simple(self) -> SimpleType: @@ -166,18 +171,31 @@ def metadata(self): """ return self._metadata + @property + def annotation(self) -> _annotation_model: + """ + :rtype: flytekit.models.annotation.FlyteAnnotation + """ + return self._annotation + @metadata.setter def metadata(self, value): self._metadata = value + @annotation.setter + def annotation(self, value): + self.annotation = value + def to_flyte_idl(self): """ :rtype: flyteidl.core.types_pb2.LiteralType """ + if self.metadata is not None: metadata = _json_format.Parse(_json.dumps(self.metadata), _struct.Struct()) else: metadata = None + t = _types_pb2.LiteralType( simple=self.simple if self.simple is not None else None, schema=self.schema.to_flyte_idl() if self.schema is not None else None, @@ -186,6 +204,7 @@ def to_flyte_idl(self): blob=self.blob.to_flyte_idl() if self.blob is not None else None, enum_type=self.enum_type.to_flyte_idl() if self.enum_type else None, metadata=metadata, + annotation=self.annotation.to_flyte_idl() if self.annotation else None, ) return t @@ -209,6 +228,7 @@ def from_flyte_idl(cls, proto): blob=_core_types.BlobType.from_flyte_idl(proto.blob) if proto.HasField("blob") else None, enum_type=_core_types.EnumType.from_flyte_idl(proto.enum_type) if proto.HasField("enum_type") else None, metadata=_json_format.MessageToDict(proto.metadata) or None, + annotation=_annotation_model.from_flyte_idl(proto.annotation) if proto.HasField("annotation") else None, ) From 1417294011c425fa4e981de06913da727e67795f Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 1 Dec 2021 08:47:56 -0800 Subject: [PATCH 03/42] feat: core obj Signed-off-by: Kenny Workman --- flytekit/core/annotation.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 flytekit/core/annotation.py diff --git a/flytekit/core/annotation.py b/flytekit/core/annotation.py new file mode 100644 index 0000000000..3031da46c6 --- /dev/null +++ b/flytekit/core/annotation.py @@ -0,0 +1,27 @@ +from typing import Any, Dict + + +class FlyteAnnotation: + """A core object to add arbitrary annotations to flyte types. + + This metadata is ingested as a python dictionary and will be serialized + into fields on the flyteidl type literals. This data is not accessible at + runtime but rather can be retrieved from flyteadmin for custom presentation + of typed parameters. + + For a task definition: + + .. code-block:: python + + @task + def x(a: typing.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})]): + return + + """ + + def __init__(self, data: Dict[str, Any]): + self._data = data + + @property + def data(self): + return self._data From e089b17676d7d843de63ea46f32f129819aaa8d4 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 1 Dec 2021 08:48:45 -0800 Subject: [PATCH 04/42] feat: proto model Signed-off-by: Kenny Workman --- flytekit/models/annotation.py | 45 +++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 flytekit/models/annotation.py diff --git a/flytekit/models/annotation.py b/flytekit/models/annotation.py new file mode 100644 index 0000000000..a61affe9e3 --- /dev/null +++ b/flytekit/models/annotation.py @@ -0,0 +1,45 @@ +import json as _json +from typing import Any, Dict + +from flyteidl.core import types_pb2 as _types_pb2 +from google.protobuf import json_format as _json_format +from google.protobuf import struct_pb2 as _struct + + +class TypeAnnotation: + """Python class representation of the flyteidl TypeAnnotation message.""" + + def __init__(self, annotations: Dict[str, Any]): + self._annotations = annotations + + @property + def annotations(self) -> Dict[str, Any]: + """ + :rtype: dict[str, Any] + """ + return self._annotations + + def to_flyte_idl(self) -> _types_pb2.TypeAnnotation: + """ + :rtype: flyteidl.core.types_pb2.TypeAnnotation + """ + + if self._annotations is not None: + annotations = _json_format.Parse(_json.dumps(self.annotations), _struct.Struct()) + else: + annotations = None + + t = _types_pb2.TypeAnnotation( + annotations=annotations, + ) + + return t + + @classmethod + def from_flyte_idl(cls, proto): + """ + :param flyteidl.core.types_pb2.TypeAnnotation proto: + :rtype: TypeAnnotation + """ + + return cls(annotations=proto.annotations) From a41e54555f4da885c82e6e12a6d2570c13f0ed28 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 1 Dec 2021 08:49:10 -0800 Subject: [PATCH 05/42] feat: testing suite Signed-off-by: Kenny Workman --- .../unit/core/test_typing_annotation.py | 61 +++++++++++++++++++ 1 file changed, 61 insertions(+) create mode 100644 tests/flytekit/unit/core/test_typing_annotation.py diff --git a/tests/flytekit/unit/core/test_typing_annotation.py b/tests/flytekit/unit/core/test_typing_annotation.py new file mode 100644 index 0000000000..cfcfd6b50d --- /dev/null +++ b/tests/flytekit/unit/core/test_typing_annotation.py @@ -0,0 +1,61 @@ +import typing +from collections import OrderedDict + +from flytekit.common.translator import get_serializable +from flytekit.core import context_manager +from flytekit.core.annotation import FlyteAnnotation +from flytekit.core.context_manager import Image, ImageConfig +from flytekit.core.task import task +from flytekit.models.annotation import TypeAnnotation + +default_img = Image(name="default", fqn="test", tag="tag") +serialization_settings = context_manager.SerializationSettings( + project="project", + domain="domain", + version="version", + env=None, + image_config=ImageConfig(default_image=default_img, images=[default_img]), +) +entity_mapping = OrderedDict() + + +@task +def x(a: typing.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})], b: str): + ... + + +@task +def y0(a: typing.List[typing.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})]]): + ... + + +@task +def y1(a: typing.Annotated[typing.List[int], FlyteAnnotation({"foo": {"bar": 1}})]): + ... + + +def test_get_variable_descriptions(): + + x_tsk = get_serializable(entity_mapping, serialization_settings, x) + x_input_vars = x_tsk.template.interface.inputs + + a_ann = x_input_vars["a"].type.annotation + assert isinstance(a_ann, TypeAnnotation) + assert a_ann.annotations["foo"] == {"bar": 1} + + b_ann = x_input_vars["b"].type.annotation + assert b_ann is None + + # Annotated simple type within list generic + y0_tsk = get_serializable(entity_mapping, serialization_settings, y0) + y0_input_vars = y0_tsk.template.interface.inputs + y0_a_ann = y0_input_vars["a"].type.collection_type.annotation + assert isinstance(y0_a_ann, TypeAnnotation) + assert y0_a_ann.annotations["foo"] == {"bar": 1} + + # Annotated list generic + y1_tsk = get_serializable(entity_mapping, serialization_settings, y1) + y1_input_vars = y1_tsk.template.interface.inputs + y1_a_ann = y1_input_vars["a"].type.annotation + assert isinstance(y1_a_ann, TypeAnnotation) + assert y1_a_ann.annotations["foo"] == {"bar": 1} From eb41882df33e2f8d819da6c73c8647e876e20c8b Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 1 Dec 2021 22:11:06 -0800 Subject: [PATCH 06/42] fix: more stable typing introspection Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index d014addbd9..ee49f36d2e 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -528,8 +528,8 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: # Handling of annotated generics, eg: # typing.Annotated[typing.List[int], 'foo'] - if isinstance(python_type, typing._AnnotatedAlias): - return cls.get_transformer(python_type.__origin__) + if typing.get_origin(python_type) is typing.Annotated: + return cls.get_transformer(typing.get_args(python_type)[0]) if python_type.__origin__ in cls._REGISTRY: return cls._REGISTRY[python_type.__origin__] @@ -564,8 +564,8 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: transformer = cls.get_transformer(python_type) res = transformer.get_literal_type(python_type) data = None - if hasattr(python_type, "__metadata__"): - for x in python_type.__metadata__: + if typing.get_origin(python_type) is typing.Annotated: + for x in typing.get_args(python_type)[1:]: if not isinstance(x, FlyteAnnotation): continue if x.data.get("__consumed", False): From 83777e8bed688bc51dddf9539a192fcd79259964 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 7 Dec 2021 10:06:08 -0800 Subject: [PATCH 07/42] fix: strip legacy Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index ee49f36d2e..50ba8ef553 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -568,10 +568,7 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: for x in typing.get_args(python_type)[1:]: if not isinstance(x, FlyteAnnotation): continue - if x.data.get("__consumed", False): - continue data = x.data - x.data["__consumed"] = True if data is not None: idl_type_annotation = _annotation_model(annotations=data) return LiteralType( From 8ec23a7f7b9b08419e20dd23e0dad3516d698cbf Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 7 Dec 2021 11:35:46 -0800 Subject: [PATCH 08/42] fix: explicitly allow only one annotation Signed-off-by: Kenny Workman --- flytekit/core/annotation.py | 3 +++ flytekit/core/type_engine.py | 4 ++++ 2 files changed, 7 insertions(+) diff --git a/flytekit/core/annotation.py b/flytekit/core/annotation.py index 3031da46c6..b4a70a6469 100644 --- a/flytekit/core/annotation.py +++ b/flytekit/core/annotation.py @@ -9,6 +9,9 @@ class FlyteAnnotation: runtime but rather can be retrieved from flyteadmin for custom presentation of typed parameters. + Flytekit expects to receive a maximum of one `FlyteAnnotation` object + within each typehint. + For a task definition: .. code-block:: python diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 50ba8ef553..7d25b93cef 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -568,6 +568,10 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: for x in typing.get_args(python_type)[1:]: if not isinstance(x, FlyteAnnotation): continue + if data is not None: + raise ValueError( + f"More than one FlyteAnnotation used within {python_type} typehint. Flytekit requires a max of one." + ) data = x.data if data is not None: idl_type_annotation = _annotation_model(annotations=data) From 10ba63cc84a9940a5fdeffc5b273ae2fd34f0849 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 7 Dec 2021 11:36:57 -0800 Subject: [PATCH 09/42] feat: direct type transformer tests Signed-off-by: Kenny Workman --- tests/flytekit/unit/core/test_type_engine.py | 29 ++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 1cbc58d94d..bd64f0ef58 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -17,6 +17,7 @@ from flytekit import kwtypes from flytekit.common.exceptions import user as user_exceptions +from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import FlyteContext, FlyteContextManager from flytekit.core.type_engine import ( DataclassTransformer, @@ -29,6 +30,7 @@ dataclass_from_dict, ) from flytekit.models import types as model_types +from flytekit.models.annotation import TypeAnnotation from flytekit.models.core.types import BlobType from flytekit.models.literals import Blob, BlobMetadata, Literal, LiteralCollection, LiteralMap, Primitive, Scalar from flytekit.models.types import LiteralType, SimpleType @@ -846,3 +848,30 @@ def hello(self): lr = LiteralsResolver(lit_dict) assert lr.get("a", Foo) == foo assert hasattr(lr.get("a", Foo), "hello") is True + + +def test_annotated_simple_types(): + + t = typing.Annotated[int, FlyteAnnotation({"foo": "bar"})] + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.annotation, TypeAnnotation) + assert lt.annotation.annotations == {"foo": "bar"} + + +def test_annotated_list(): + + t = typing.Annotated[typing.List[int], FlyteAnnotation({"foo": "bar"})] + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.annotation, TypeAnnotation) + assert lt.annotation.annotations == {"foo": "bar"} + + t = typing.List[typing.Annotated[int, FlyteAnnotation({"foo": "bar"})]] + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.collection_type.annotation, TypeAnnotation) + assert lt.collection_type.annotation.annotations == {"foo": "bar"} + + +def test_multiple_annotations(): + t = typing.Annotated[int, FlyteAnnotation({"foo": "bar"}), FlyteAnnotation({"anotha": "one"})] + with pytest.raises(Exception): + TypeEngine.to_literal_type(t) From a3cc48fd5e80959d1075b822d3255f5368a8016a Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 8 Dec 2021 18:19:08 -0800 Subject: [PATCH 10/42] fix: there and back test Signed-off-by: Kenny Workman --- flytekit/models/annotation.py | 6 +++++- tests/flytekit/unit/models/test_types.py | 12 ++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/flytekit/models/annotation.py b/flytekit/models/annotation.py index a61affe9e3..e50df5d08e 100644 --- a/flytekit/models/annotation.py +++ b/flytekit/models/annotation.py @@ -24,6 +24,10 @@ def to_flyte_idl(self) -> _types_pb2.TypeAnnotation: :rtype: flyteidl.core.types_pb2.TypeAnnotation """ + import inspect + + print("\nrosa: ", self.annotations, inspect.stack()[1]) + if self._annotations is not None: annotations = _json_format.Parse(_json.dumps(self.annotations), _struct.Struct()) else: @@ -42,4 +46,4 @@ def from_flyte_idl(cls, proto): :rtype: TypeAnnotation """ - return cls(annotations=proto.annotations) + return cls(annotations=_json_format.MessageToDict(proto.annotations)) diff --git a/tests/flytekit/unit/models/test_types.py b/tests/flytekit/unit/models/test_types.py index 27b5ddf595..1de42fd565 100644 --- a/tests/flytekit/unit/models/test_types.py +++ b/tests/flytekit/unit/models/test_types.py @@ -1,7 +1,9 @@ import pytest from flyteidl.core import types_pb2 +from flytekit.core.annotation import FlyteAnnotation from flytekit.models import types as _types +from flytekit.models.annotation import TypeAnnotation from tests.flytekit.common import parameterizers @@ -81,6 +83,16 @@ def test_literal_types(): assert obj == _types.LiteralType.from_flyte_idl(obj.to_flyte_idl()) +def test_annotated_literal_types(): + obj = _types.LiteralType(simple=_types.SimpleType.INTEGER, annotation=TypeAnnotation(annotations={"foo": "bar"})) + assert obj.simple == _types.SimpleType.INTEGER + assert obj.schema is None + assert obj.collection_type is None + assert obj.map_value_type is None + assert obj.annotation.annotations == {"foo": "bar"} + assert obj == _types.LiteralType.from_flyte_idl(obj.to_flyte_idl()) + + @pytest.mark.parametrize("literal_type", parameterizers.LIST_OF_ALL_LITERAL_TYPES) def test_literal_collections(literal_type): obj = _types.LiteralType(collection_type=literal_type) From 1582dfb70890004833476205f9f15e87073f1823 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 8 Dec 2021 18:49:12 -0800 Subject: [PATCH 11/42] fix: typing_extensions for get_origin Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 7d25b93cef..173cda9737 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -19,6 +19,7 @@ from google.protobuf.struct_pb2 import Struct from marshmallow_enum import EnumField, LoadDumpOptions from marshmallow_jsonschema import JSONSchema +from typing_extensions import get_origin from flytekit.common.exceptions import user as user_exceptions from flytekit.common.types import primitives as _primitives @@ -528,7 +529,7 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: # Handling of annotated generics, eg: # typing.Annotated[typing.List[int], 'foo'] - if typing.get_origin(python_type) is typing.Annotated: + if get_origin(python_type) is typing.Annotated: return cls.get_transformer(typing.get_args(python_type)[0]) if python_type.__origin__ in cls._REGISTRY: @@ -564,7 +565,7 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: transformer = cls.get_transformer(python_type) res = transformer.get_literal_type(python_type) data = None - if typing.get_origin(python_type) is typing.Annotated: + if get_origin(python_type) is typing.Annotated: for x in typing.get_args(python_type)[1:]: if not isinstance(x, FlyteAnnotation): continue From 8c5ee52711d51fa101a6af8f992c53618bae8f55 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 8 Dec 2021 19:32:43 -0800 Subject: [PATCH 12/42] fix: more semantic list generic unwrap Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 173cda9737..3569fbe52f 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -718,9 +718,9 @@ def get_sub_type(t: Type[T]) -> Type[T]: if hasattr(t, "__origin__"): # Handle annotation on list generic, eg: - # typing.Annotated[typing.List[int, 'foo']] - if isinstance(t, typing._AnnotatedAlias) and hasattr(t.__origin__, "__origin__"): - return ListTransformer.get_sub_type(t.__origin__) + # typing.Annotated[typing.List[int], 'foo'] + if get_origin(t) is typing.Annotated: + return ListTransformer.get_sub_type(typing.get_args(t)[0]) if t.__origin__ is list and hasattr(t, "__args__"): return t.__args__[0] From a6378d4d031aa15891a95776e4b97b4dc53882d2 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Wed, 8 Dec 2021 19:32:58 -0800 Subject: [PATCH 13/42] fix: tmp requirements file with custom idl Signed-off-by: Kenny Workman --- requirements.txt | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/requirements.txt b/requirements.txt index 04cdd38a16..801006fd24 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,6 +8,10 @@ # via -r requirements.in ansiwrap==0.8.4 # via papermill +appnope==0.1.2 + # via + # ipykernel + # ipython arrow==1.2.1 # via jinja2-time attrs==20.3.0 @@ -107,10 +111,6 @@ ipython-genutils==0.2.0 # nbformat jedi==0.18.1 # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -283,8 +283,6 @@ sagemaker-training==3.9.2 # via flytekit scipy==1.7.3 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # bcrypt @@ -364,4 +362,4 @@ zope.interface==5.4.0 # The following packages are considered to be unsafe in a requirements file: # pip -# setuptools +# setuptools \ No newline at end of file From f144ffa7991f5da7a44346eea955cc13b35e8931 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Sat, 11 Dec 2021 19:21:13 -0800 Subject: [PATCH 14/42] fix: nits Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 6 ++---- flytekit/models/annotation.py | 8 +------- tests/flytekit/unit/core/test_typing_annotation.py | 1 - 3 files changed, 3 insertions(+), 12 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 3569fbe52f..202b356a22 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -29,7 +29,7 @@ from flytekit.loggers import logger from flytekit.models import interface as _interface_models from flytekit.models import types as _type_models -from flytekit.models.annotation import TypeAnnotation as _annotation_model +from flytekit.models.annotation import TypeAnnotation as TypeAnnotationModel from flytekit.models.core import types as _core_types from flytekit.models.literals import ( Blob, @@ -526,7 +526,6 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: # Step 2 if hasattr(python_type, "__origin__"): - # Handling of annotated generics, eg: # typing.Annotated[typing.List[int], 'foo'] if get_origin(python_type) is typing.Annotated: @@ -575,7 +574,7 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: ) data = x.data if data is not None: - idl_type_annotation = _annotation_model(annotations=data) + idl_type_annotation = TypeAnnotationModel(annotations=data) return LiteralType( simple=res.simple, schema=res.schema, @@ -716,7 +715,6 @@ def get_sub_type(t: Type[T]) -> Type[T]: """ if hasattr(t, "__origin__"): - # Handle annotation on list generic, eg: # typing.Annotated[typing.List[int], 'foo'] if get_origin(t) is typing.Annotated: diff --git a/flytekit/models/annotation.py b/flytekit/models/annotation.py index e50df5d08e..6647554af8 100644 --- a/flytekit/models/annotation.py +++ b/flytekit/models/annotation.py @@ -24,21 +24,15 @@ def to_flyte_idl(self) -> _types_pb2.TypeAnnotation: :rtype: flyteidl.core.types_pb2.TypeAnnotation """ - import inspect - - print("\nrosa: ", self.annotations, inspect.stack()[1]) - if self._annotations is not None: annotations = _json_format.Parse(_json.dumps(self.annotations), _struct.Struct()) else: annotations = None - t = _types_pb2.TypeAnnotation( + return types_pb2.TypeAnnotation( annotations=annotations, ) - return t - @classmethod def from_flyte_idl(cls, proto): """ diff --git a/tests/flytekit/unit/core/test_typing_annotation.py b/tests/flytekit/unit/core/test_typing_annotation.py index cfcfd6b50d..7060fab3cf 100644 --- a/tests/flytekit/unit/core/test_typing_annotation.py +++ b/tests/flytekit/unit/core/test_typing_annotation.py @@ -35,7 +35,6 @@ def y1(a: typing.Annotated[typing.List[int], FlyteAnnotation({"foo": {"bar": 1}} def test_get_variable_descriptions(): - x_tsk = get_serializable(entity_mapping, serialization_settings, x) x_input_vars = x_tsk.template.interface.inputs From 7508cc9b9bf7573167a2a1aae5a7bdacd3ccb606 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Sat, 11 Dec 2021 21:12:04 -0800 Subject: [PATCH 15/42] feat: semantic error for unsupported complex literals Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 30 ++++++++++++++++++++++++++---- 1 file changed, 26 insertions(+), 4 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 202b356a22..efc6edecb7 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -19,7 +19,7 @@ from google.protobuf.struct_pb2 import Struct from marshmallow_enum import EnumField, LoadDumpOptions from marshmallow_jsonschema import JSONSchema -from typing_extensions import get_origin +from typing_extensions import get_args, get_origin from flytekit.common.exceptions import user as user_exceptions from flytekit.common.types import primitives as _primitives @@ -233,6 +233,13 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: Extracts the Literal type definition for a Dataclass and returns a type Struct. If possible also extracts the JSONSchema for the dataclass. """ + if get_origin(t) is typing.Annotated: + raise ValueError( + f"Flytekit does not currently have support \ + for FlyteAnnotations applied to Dataclass. {t} cannot be \ + parsed." + ) + if not issubclass(t, DataClassJsonMixin): raise AssertionError( f"Dataclass {t} should be decorated with @dataclass_json to be " f"serialized correctly" @@ -765,9 +772,17 @@ def get_dict_types(t: Optional[Type[dict]]) -> typing.Tuple[Optional[type], Opti """ Return the generic Type T of the Dict """ - if hasattr(t, "__origin__") and t.__origin__ is dict: # type: ignore - if hasattr(t, "__args__"): - return t.__args__ # type: ignore + _origin = get_origin(t) + _args = get_args(t) + if _origin is not None: + if _origin is typing.Annotated: + raise ValueError( + f"Flytekit does not currently have support \ + for FlyteAnnotations applied to dicts. {t} cannot be \ + parsed." + ) + if _origin is dict and _args is not None: + return _args return None, None @staticmethod @@ -915,6 +930,13 @@ def __init__(self): super().__init__(name="DefaultEnumTransformer", t=enum.Enum) def get_literal_type(self, t: Type[T]) -> LiteralType: + if get_origin(t) is typing.Annotated: + raise ValueError( + f"Flytekit does not currently have support \ + for FlyteAnnotations applied to enums. {t} cannot be \ + parsed." + ) + values = [v.value for v in t] # type: ignore if not isinstance(values[0], str): raise AssertionError("Only EnumTypes with value of string are supported") From 8bc240b2d40cc38ca019ec17c3d5b9148d30b272 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Sat, 11 Dec 2021 21:12:14 -0800 Subject: [PATCH 16/42] fix: but Signed-off-by: Kenny Workman --- flytekit/models/annotation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/flytekit/models/annotation.py b/flytekit/models/annotation.py index 6647554af8..ced935c57e 100644 --- a/flytekit/models/annotation.py +++ b/flytekit/models/annotation.py @@ -29,7 +29,7 @@ def to_flyte_idl(self) -> _types_pb2.TypeAnnotation: else: annotations = None - return types_pb2.TypeAnnotation( + return _types_pb2.TypeAnnotation( annotations=annotations, ) From e7f6bb55284147df27c76a82df67f3c114c8b35d Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Sat, 11 Dec 2021 21:12:26 -0800 Subject: [PATCH 17/42] feat: more tests ;) Signed-off-by: Kenny Workman --- tests/flytekit/unit/core/test_type_engine.py | 76 +++++++++++++------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index bd64f0ef58..47df45bcb5 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -761,6 +761,55 @@ def test_dict_to_literal_map_with_wrong_input_type(): TypeEngine.dict_to_literal_map(ctx, input, guessed_python_types) +def test_annotated_simple_types(): + t = typing.Annotated[int, FlyteAnnotation({"foo": "bar"})] + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.annotation, TypeAnnotation) + assert lt.annotation.annotations == {"foo": "bar"} + + +def test_annotated_list(): + t = typing.Annotated[typing.List[int], FlyteAnnotation({"foo": "bar"})] + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.annotation, TypeAnnotation) + assert lt.annotation.annotations == {"foo": "bar"} + + t = typing.List[typing.Annotated[int, FlyteAnnotation({"foo": "bar"})]] + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.collection_type.annotation, TypeAnnotation) + assert lt.collection_type.annotation.annotations == {"foo": "bar"} + + +def test_type_alias(): + inner_t = typing.Annotated[int, FlyteAnnotation("foo")] + t = typing.Annotated[inner_t, FlyteAnnotation("bar")] + with pytest.raises(ValueError): + TypeEngine.to_literal_type(t) + + +def test_unsupported_complex_literals(): + + t = typing.Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})] + with pytest.raises(ValueError): + TypeEngine.to_literal_type(t) + + # Enum. + t = typing.Annotated[Color, FlyteAnnotation({"foo": "bar"})] + with pytest.raises(ValueError): + TypeEngine.to_literal_type(t) + + # Dataclass. + t = typing.Annotated[Result, FlyteAnnotation({"foo": "bar"})] + with pytest.raises(ValueError): + TypeEngine.to_literal_type(t) + + +def test_multiple_annotations(): + t = typing.Annotated[int, FlyteAnnotation({"foo": "bar"}), FlyteAnnotation({"anotha": "one"})] + with pytest.raises(Exception): + TypeEngine.to_literal_type(t) + + TestSchema = FlyteSchema[kwtypes(some_str=str)] @@ -848,30 +897,3 @@ def hello(self): lr = LiteralsResolver(lit_dict) assert lr.get("a", Foo) == foo assert hasattr(lr.get("a", Foo), "hello") is True - - -def test_annotated_simple_types(): - - t = typing.Annotated[int, FlyteAnnotation({"foo": "bar"})] - lt = TypeEngine.to_literal_type(t) - assert isinstance(lt.annotation, TypeAnnotation) - assert lt.annotation.annotations == {"foo": "bar"} - - -def test_annotated_list(): - - t = typing.Annotated[typing.List[int], FlyteAnnotation({"foo": "bar"})] - lt = TypeEngine.to_literal_type(t) - assert isinstance(lt.annotation, TypeAnnotation) - assert lt.annotation.annotations == {"foo": "bar"} - - t = typing.List[typing.Annotated[int, FlyteAnnotation({"foo": "bar"})]] - lt = TypeEngine.to_literal_type(t) - assert isinstance(lt.collection_type.annotation, TypeAnnotation) - assert lt.collection_type.annotation.annotations == {"foo": "bar"} - - -def test_multiple_annotations(): - t = typing.Annotated[int, FlyteAnnotation({"foo": "bar"}), FlyteAnnotation({"anotha": "one"})] - with pytest.raises(Exception): - TypeEngine.to_literal_type(t) From 8c2b2d8eaba3523145220281e5ae07cf500d3817 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 13 Dec 2021 16:46:34 -0800 Subject: [PATCH 18/42] fix: imports Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 5 ++--- flytekit/models/types.py | 8 ++++---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index efc6edecb7..82aeb63177 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -235,9 +235,8 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: """ if get_origin(t) is typing.Annotated: raise ValueError( - f"Flytekit does not currently have support \ - for FlyteAnnotations applied to Dataclass. {t} cannot be \ - parsed." + "Flytekit does not currently have support for FlyteAnnotations applied to Dataclass." + f"Type {t} cannot be parsed." ) if not issubclass(t, DataClassJsonMixin): diff --git a/flytekit/models/types.py b/flytekit/models/types.py index 0d4003ed28..673b1629e0 100644 --- a/flytekit/models/types.py +++ b/flytekit/models/types.py @@ -5,7 +5,7 @@ from google.protobuf import struct_pb2 as _struct from flytekit.models import common as _common -from flytekit.models.annotation import TypeAnnotation as _annotation_model +from flytekit.models.annotation import TypeAnnotation as TypeAnnotationModel from flytekit.models.core import types as _core_types @@ -172,9 +172,9 @@ def metadata(self): return self._metadata @property - def annotation(self) -> _annotation_model: + def annotation(self) -> TypeAnnotationModel: """ - :rtype: flytekit.models.annotation.FlyteAnnotation + :rtype: flytekit.models.annotation.TypeAnnotation """ return self._annotation @@ -228,7 +228,7 @@ def from_flyte_idl(cls, proto): blob=_core_types.BlobType.from_flyte_idl(proto.blob) if proto.HasField("blob") else None, enum_type=_core_types.EnumType.from_flyte_idl(proto.enum_type) if proto.HasField("enum_type") else None, metadata=_json_format.MessageToDict(proto.metadata) or None, - annotation=_annotation_model.from_flyte_idl(proto.annotation) if proto.HasField("annotation") else None, + annotation=TypeAnnotationModel.from_flyte_idl(proto.annotation) if proto.HasField("annotation") else None, ) From 60089d32ceecb9203c6be85c328f995a7211e659 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 13 Dec 2021 16:58:53 -0800 Subject: [PATCH 19/42] fix: complex annotations Signed-off-by: Kenny Workman --- tests/flytekit/unit/core/test_type_engine.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 47df45bcb5..5a8841bfda 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -762,10 +762,20 @@ def test_dict_to_literal_map_with_wrong_input_type(): def test_annotated_simple_types(): - t = typing.Annotated[int, FlyteAnnotation({"foo": "bar"})] - lt = TypeEngine.to_literal_type(t) - assert isinstance(lt.annotation, TypeAnnotation) - assert lt.annotation.annotations == {"foo": "bar"} + def _check_annotation(t, annotation): + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.annotation, TypeAnnotation) + assert lt.annotation.annotations == annotation + + _check_annotation(typing.Annotated[int, FlyteAnnotation({"foo": "bar"})], {"foo": "bar"}) + _check_annotation(typing.Annotated[int, FlyteAnnotation(["foo", "bar"])], ["foo", "bar"]) + _check_annotation( + typing.Annotated[int, FlyteAnnotation({"d": {"test": "data"}, "l": ["nested", ["list"]]})], + {"d": {"test": "data"}, "l": ["nested", ["list"]]}, + ) + _check_annotation( + typing.Annotated[int, FlyteAnnotation(InnerStruct(a=1, b="fizz", c=[1]))], InnerStruct(a=1, b="fizz", c=[1]) + ) def test_annotated_list(): @@ -788,7 +798,6 @@ def test_type_alias(): def test_unsupported_complex_literals(): - t = typing.Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})] with pytest.raises(ValueError): TypeEngine.to_literal_type(t) From fb302ad32931d145417d65034889106e8b3eddf7 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 13 Dec 2021 16:59:13 -0800 Subject: [PATCH 20/42] fix: temp requirements files for unit tests Signed-off-by: Kenny Workman --- dev-requirements.txt | 13 +------------ doc-requirements.txt | 14 +++++--------- requirements-spark2.txt | 12 +++++------- requirements.in | 1 + 4 files changed, 12 insertions(+), 28 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 8cc4f7ba3f..20a38f3e1a 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -77,7 +77,6 @@ cryptography==36.0.1 # via # -c requirements.txt # paramiko - # secretstorage dataclasses-json==0.5.6 # via # -c requirements.txt @@ -138,11 +137,6 @@ importlib-metadata==4.10.0 # keyring iniconfig==1.1.1 # via pytest -jeepney==0.7.1 - # via - # -c requirements.txt - # keyring - # secretstorage jinja2==3.0.3 # via # -c requirements.txt @@ -228,7 +222,6 @@ pre-commit==2.16.0 protobuf==3.19.1 # via # -c requirements.txt - # flyteidl # flytekit py==1.11.0 # via @@ -315,10 +308,6 @@ retry==0.9.2 # via # -c requirements.txt # flytekit -secretstorage==3.3.1 - # via - # -c requirements.txt - # keyring six==1.16.0 # via # -c requirements.txt @@ -392,4 +381,4 @@ zipp==3.6.0 # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools +# setuptools \ No newline at end of file diff --git a/doc-requirements.txt b/doc-requirements.txt index cba57b13b7..055739f0fc 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -10,6 +10,10 @@ alabaster==0.7.12 # via sphinx ansiwrap==0.8.4 # via papermill +appnope==0.1.2 + # via + # ipykernel + # ipython arrow==1.2.1 # via jinja2-time astroid==2.9.0 @@ -69,7 +73,6 @@ cryptography==36.0.1 # via # -r doc-requirements.in # paramiko - # secretstorage css-html-js-minify==2.5.5 # via sphinx-material dataclasses-json==0.5.6 @@ -127,10 +130,6 @@ ipython-genutils==0.2.0 # nbformat jedi==0.18.1 # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -236,7 +235,6 @@ prompt-toolkit==3.0.24 # via ipython protobuf==3.19.1 # via - # flyteidl # flytekit # k8s-proto # sagemaker-training @@ -315,8 +313,6 @@ sagemaker-training==3.9.2 # via flytekit scipy==1.7.3 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # bcrypt @@ -446,4 +442,4 @@ zope.interface==5.4.0 # The following packages are considered to be unsafe in a requirements file: # pip -# setuptools +# setuptools \ No newline at end of file diff --git a/requirements-spark2.txt b/requirements-spark2.txt index a48ec4b5e2..7464224875 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -10,6 +10,10 @@ # -r requirements.in ansiwrap==0.8.4 # via papermill +appnope==0.1.2 + # via + # ipykernel + # ipython arrow==1.2.1 # via jinja2-time attrs==20.3.0 @@ -109,10 +113,6 @@ ipython-genutils==0.2.0 # nbformat jedi==0.18.1 # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -285,8 +285,6 @@ sagemaker-training==3.9.2 # via flytekit scipy==1.7.3 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # bcrypt @@ -366,4 +364,4 @@ zope.interface==5.4.0 # The following packages are considered to be unsafe in a requirements file: # pip -# setuptools +# setuptools \ No newline at end of file diff --git a/requirements.in b/requirements.in index 408185fca8..9a67f5aaf3 100644 --- a/requirements.in +++ b/requirements.in @@ -7,3 +7,4 @@ attrs<21 # https://github.com/flyteorg/flyte/issues/1732. jsonschema<4 pyyaml<6 +flyteidl @ git+https://github.com/kennyworkman/flyteidl.git From df8356a6cbf5ba3ed60857d94bdebee554bc8c43 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 13 Dec 2021 17:04:27 -0800 Subject: [PATCH 21/42] fix: lint bug Signed-off-by: Kenny Workman --- .../remote/mock_flyte_repo/workflows/requirements.txt | 11 +---------- tests/flytekit/unit/models/test_types.py | 1 - 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 627e87980e..9b2f5713a0 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -10,8 +10,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.9 @@ -56,10 +54,6 @@ idna==3.3 # via requests importlib-metadata==4.10.0 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -121,7 +115,6 @@ pyparsing==3.0.6 # packaging python-dateutil==2.8.1 # via - # arrow # croniter # flytekit # matplotlib @@ -147,8 +140,6 @@ responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -180,4 +171,4 @@ wrapt==1.13.3 # deprecated # flytekit zipp==3.6.0 - # via importlib-metadata + # via importlib-metadata \ No newline at end of file diff --git a/tests/flytekit/unit/models/test_types.py b/tests/flytekit/unit/models/test_types.py index 1de42fd565..82f0b9b519 100644 --- a/tests/flytekit/unit/models/test_types.py +++ b/tests/flytekit/unit/models/test_types.py @@ -1,7 +1,6 @@ import pytest from flyteidl.core import types_pb2 -from flytekit.core.annotation import FlyteAnnotation from flytekit.models import types as _types from flytekit.models.annotation import TypeAnnotation from tests.flytekit.common import parameterizers From 084e46e6134a48c71e705ffab943f30b7c491d6a Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 13 Dec 2021 17:06:24 -0800 Subject: [PATCH 22/42] fix: tmp setup.py Signed-off-by: Kenny Workman --- setup.py | 1 - 1 file changed, 1 deletion(-) diff --git a/setup.py b/setup.py index 5bf9f83aa7..3a286ee448 100644 --- a/setup.py +++ b/setup.py @@ -64,7 +64,6 @@ ] }, install_requires=[ - "flyteidl>=0.21.4", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=4.0.0,<7.0.0", From c245cbf064fe620cca4c5957b86e41988b91f773 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 13 Dec 2021 17:16:21 -0800 Subject: [PATCH 23/42] fix: use typing_extensions Signed-off-by: Kenny Workman --- tests/flytekit/unit/core/test_type_engine.py | 26 ++++++++++--------- .../unit/core/test_typing_annotation.py | 8 +++--- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 5a8841bfda..64d4fcc0b1 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -8,6 +8,7 @@ import pandas as pd import pytest +import typing_extensions from dataclasses_json import DataClassJsonMixin, dataclass_json from flyteidl.core import errors_pb2 from google.protobuf import json_format as _json_format @@ -767,54 +768,55 @@ def _check_annotation(t, annotation): assert isinstance(lt.annotation, TypeAnnotation) assert lt.annotation.annotations == annotation - _check_annotation(typing.Annotated[int, FlyteAnnotation({"foo": "bar"})], {"foo": "bar"}) - _check_annotation(typing.Annotated[int, FlyteAnnotation(["foo", "bar"])], ["foo", "bar"]) + _check_annotation(typing_extensions.Annotated[int, FlyteAnnotation({"foo": "bar"})], {"foo": "bar"}) + _check_annotation(typing_extensions.Annotated[int, FlyteAnnotation(["foo", "bar"])], ["foo", "bar"]) _check_annotation( - typing.Annotated[int, FlyteAnnotation({"d": {"test": "data"}, "l": ["nested", ["list"]]})], + typing_extensions.Annotated[int, FlyteAnnotation({"d": {"test": "data"}, "l": ["nested", ["list"]]})], {"d": {"test": "data"}, "l": ["nested", ["list"]]}, ) _check_annotation( - typing.Annotated[int, FlyteAnnotation(InnerStruct(a=1, b="fizz", c=[1]))], InnerStruct(a=1, b="fizz", c=[1]) + typing_extensions.Annotated[int, FlyteAnnotation(InnerStruct(a=1, b="fizz", c=[1]))], + InnerStruct(a=1, b="fizz", c=[1]), ) def test_annotated_list(): - t = typing.Annotated[typing.List[int], FlyteAnnotation({"foo": "bar"})] + t = typing_extensions.Annotated[typing.List[int], FlyteAnnotation({"foo": "bar"})] lt = TypeEngine.to_literal_type(t) assert isinstance(lt.annotation, TypeAnnotation) assert lt.annotation.annotations == {"foo": "bar"} - t = typing.List[typing.Annotated[int, FlyteAnnotation({"foo": "bar"})]] + t = typing.List[typing_extensions.Annotated[int, FlyteAnnotation({"foo": "bar"})]] lt = TypeEngine.to_literal_type(t) assert isinstance(lt.collection_type.annotation, TypeAnnotation) assert lt.collection_type.annotation.annotations == {"foo": "bar"} def test_type_alias(): - inner_t = typing.Annotated[int, FlyteAnnotation("foo")] - t = typing.Annotated[inner_t, FlyteAnnotation("bar")] + inner_t = typing_extensions.Annotated[int, FlyteAnnotation("foo")] + t = typing_extensions.Annotated[inner_t, FlyteAnnotation("bar")] with pytest.raises(ValueError): TypeEngine.to_literal_type(t) def test_unsupported_complex_literals(): - t = typing.Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})] + t = typing_extensions.Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})] with pytest.raises(ValueError): TypeEngine.to_literal_type(t) # Enum. - t = typing.Annotated[Color, FlyteAnnotation({"foo": "bar"})] + t = typing_extensions.Annotated[Color, FlyteAnnotation({"foo": "bar"})] with pytest.raises(ValueError): TypeEngine.to_literal_type(t) # Dataclass. - t = typing.Annotated[Result, FlyteAnnotation({"foo": "bar"})] + t = typing_extensions.Annotated[Result, FlyteAnnotation({"foo": "bar"})] with pytest.raises(ValueError): TypeEngine.to_literal_type(t) def test_multiple_annotations(): - t = typing.Annotated[int, FlyteAnnotation({"foo": "bar"}), FlyteAnnotation({"anotha": "one"})] + t = typing_extensions.Annotated[int, FlyteAnnotation({"foo": "bar"}), FlyteAnnotation({"anotha": "one"})] with pytest.raises(Exception): TypeEngine.to_literal_type(t) diff --git a/tests/flytekit/unit/core/test_typing_annotation.py b/tests/flytekit/unit/core/test_typing_annotation.py index 7060fab3cf..9922ab67dd 100644 --- a/tests/flytekit/unit/core/test_typing_annotation.py +++ b/tests/flytekit/unit/core/test_typing_annotation.py @@ -1,6 +1,8 @@ import typing from collections import OrderedDict +import typing_extensions + from flytekit.common.translator import get_serializable from flytekit.core import context_manager from flytekit.core.annotation import FlyteAnnotation @@ -20,17 +22,17 @@ @task -def x(a: typing.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})], b: str): +def x(a: typing_extensions.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})], b: str): ... @task -def y0(a: typing.List[typing.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})]]): +def y0(a: typing.List[typing_extensions.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})]]): ... @task -def y1(a: typing.Annotated[typing.List[int], FlyteAnnotation({"foo": {"bar": 1}})]): +def y1(a: typing_extensions.Annotated[typing.List[int], FlyteAnnotation({"foo": {"bar": 1}})]): ... From 43eaa8cf121143e11d0c58c70364c014dfbd61cf Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 13 Dec 2021 17:32:38 -0800 Subject: [PATCH 24/42] fix: typing_extensions for annotated Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 82aeb63177..570e20b997 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -10,6 +10,7 @@ from abc import ABC, abstractmethod from typing import NamedTuple, Optional, Type, cast +import typing_extensions from dataclasses_json import DataClassJsonMixin, dataclass_json from google.protobuf import json_format as _json_format from google.protobuf import reflection as _proto_reflection @@ -233,7 +234,7 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: Extracts the Literal type definition for a Dataclass and returns a type Struct. If possible also extracts the JSONSchema for the dataclass. """ - if get_origin(t) is typing.Annotated: + if get_origin(t) is typing_extensions.Annotated: raise ValueError( "Flytekit does not currently have support for FlyteAnnotations applied to Dataclass." f"Type {t} cannot be parsed." @@ -533,8 +534,8 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: # Step 2 if hasattr(python_type, "__origin__"): # Handling of annotated generics, eg: - # typing.Annotated[typing.List[int], 'foo'] - if get_origin(python_type) is typing.Annotated: + # typing_extensions.Annotated[typing.List[int], 'foo'] + if get_origin(python_type) is typing_extensions.Annotated: return cls.get_transformer(typing.get_args(python_type)[0]) if python_type.__origin__ in cls._REGISTRY: @@ -570,7 +571,7 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: transformer = cls.get_transformer(python_type) res = transformer.get_literal_type(python_type) data = None - if get_origin(python_type) is typing.Annotated: + if get_origin(python_type) is typing_extensions.Annotated: for x in typing.get_args(python_type)[1:]: if not isinstance(x, FlyteAnnotation): continue @@ -722,8 +723,8 @@ def get_sub_type(t: Type[T]) -> Type[T]: if hasattr(t, "__origin__"): # Handle annotation on list generic, eg: - # typing.Annotated[typing.List[int], 'foo'] - if get_origin(t) is typing.Annotated: + # typing_extensions.Annotated[typing.List[int], 'foo'] + if get_origin(t) is typing_extensions.Annotated: return ListTransformer.get_sub_type(typing.get_args(t)[0]) if t.__origin__ is list and hasattr(t, "__args__"): @@ -774,7 +775,7 @@ def get_dict_types(t: Optional[Type[dict]]) -> typing.Tuple[Optional[type], Opti _origin = get_origin(t) _args = get_args(t) if _origin is not None: - if _origin is typing.Annotated: + if _origin is typing_extensions.Annotated: raise ValueError( f"Flytekit does not currently have support \ for FlyteAnnotations applied to dicts. {t} cannot be \ @@ -929,7 +930,7 @@ def __init__(self): super().__init__(name="DefaultEnumTransformer", t=enum.Enum) def get_literal_type(self, t: Type[T]) -> LiteralType: - if get_origin(t) is typing.Annotated: + if get_origin(t) is typing_extensions.Annotated: raise ValueError( f"Flytekit does not currently have support \ for FlyteAnnotations applied to enums. {t} cannot be \ From 1a5840f3df28cc3a0a5d4875a984055af2bc30cc Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 13 Dec 2021 18:31:22 -0800 Subject: [PATCH 25/42] fix: typing_ext Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 570e20b997..b5b1681c90 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -536,7 +536,7 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: # Handling of annotated generics, eg: # typing_extensions.Annotated[typing.List[int], 'foo'] if get_origin(python_type) is typing_extensions.Annotated: - return cls.get_transformer(typing.get_args(python_type)[0]) + return cls.get_transformer(typing_extensions.get_args(python_type)[0]) if python_type.__origin__ in cls._REGISTRY: return cls._REGISTRY[python_type.__origin__] @@ -572,7 +572,7 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: res = transformer.get_literal_type(python_type) data = None if get_origin(python_type) is typing_extensions.Annotated: - for x in typing.get_args(python_type)[1:]: + for x in typing_extensions.get_args(python_type)[1:]: if not isinstance(x, FlyteAnnotation): continue if data is not None: @@ -725,7 +725,7 @@ def get_sub_type(t: Type[T]) -> Type[T]: # Handle annotation on list generic, eg: # typing_extensions.Annotated[typing.List[int], 'foo'] if get_origin(t) is typing_extensions.Annotated: - return ListTransformer.get_sub_type(typing.get_args(t)[0]) + return ListTransformer.get_sub_type(typing_extensions.get_args(t)[0]) if t.__origin__ is list and hasattr(t, "__args__"): return t.__args__[0] From c0c30738afad5c0146bc21dc65cde15dd9dd8bb9 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 14 Dec 2021 15:32:03 -0800 Subject: [PATCH 26/42] fix: plugin tmp requirements Signed-off-by: Kenny Workman --- plugins/flytekit-aws-athena/requirements.txt | 40 ++++------ .../flytekit-aws-sagemaker/requirements.txt | 50 +++++------- plugins/flytekit-data-fsspec/requirements.txt | 42 ++++------ plugins/flytekit-dolt/requirements.txt | 40 ++++------ .../requirements.txt | 77 +++++++++---------- plugins/flytekit-hive/requirements.txt | 40 ++++------ plugins/flytekit-k8s-pod/requirements.txt | 46 ++++------- plugins/flytekit-kf-mpi/requirements.txt | 40 ++++------ plugins/flytekit-kf-pytorch/requirements.txt | 40 ++++------ .../flytekit-kf-tensorflow/requirements.txt | 40 ++++------ plugins/flytekit-modin/requirements.txt | 62 +++++++-------- plugins/flytekit-pandera/requirements.txt | 48 +++++------- plugins/flytekit-papermill/requirements.txt | 76 ++++++++---------- plugins/flytekit-snowflake/requirements.txt | 40 ++++------ plugins/flytekit-spark/requirements.txt | 40 ++++------ plugins/flytekit-sqlalchemy/requirements.txt | 42 ++++------ 16 files changed, 296 insertions(+), 467 deletions(-) diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index 4cd07971c4..f0a33ac6b5 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,47 +26,41 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-athena -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -79,13 +71,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -95,10 +87,8 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -122,12 +112,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -141,7 +129,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index b5d42c422d..4674fd6ea3 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -12,9 +12,9 @@ bcrypt==3.2.0 # via paramiko binaryornot==0.4.4 # via cookiecutter -boto3==1.20.3 +boto3==1.20.24 # via sagemaker-training -botocore==1.23.3 +botocore==1.23.24 # via # boto3 # s3transfer @@ -27,7 +27,7 @@ cffi==1.15.0 # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -39,33 +39,31 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via - # paramiko - # secretstorage +cryptography==36.0.0 + # via paramiko dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-awssagemaker -gevent==21.8.0 +gevent==21.12.0 # via sagemaker-training greenlet==1.1.2 # via gevent -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests @@ -73,10 +71,6 @@ importlib-metadata==4.8.2 # via keyring inotify_simple==1.2.1 # via sagemaker-training -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -87,11 +81,11 @@ jmespath==0.10.0 # via # boto3 # botocore -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -102,7 +96,7 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via @@ -110,9 +104,9 @@ numpy==1.21.4 # pyarrow # sagemaker-training # scipy -pandas==1.3.4 +pandas==1.3.5 # via flytekit -paramiko==2.8.0 +paramiko==2.8.1 # via sagemaker-training poyo==0.5.0 # via cookiecutter @@ -125,7 +119,7 @@ psutil==5.8.0 # via sagemaker-training py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi @@ -155,7 +149,7 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit @@ -165,10 +159,8 @@ s3transfer==0.5.0 # via boto3 sagemaker-training==3.9.2 # via flytekitplugins-awssagemaker -scipy==1.7.2 +scipy==1.7.3 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # bcrypt @@ -186,7 +178,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index d6ee7128aa..b5905fde69 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,49 +26,43 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-data-fsspec -fsspec==2021.11.0 +fsspec==2021.11.1 # via flytekitplugins-data-fsspec -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -81,13 +73,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -97,10 +89,8 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -124,12 +114,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -143,7 +131,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index f669ec8672..f1e6e1bd8e 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,10 +26,8 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via # dolt-integrations @@ -40,41 +36,37 @@ decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit dolt-integrations==0.1.5 # via flytekitplugins-dolt doltcli==0.1.15 # via dolt-integrations -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-dolt -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -85,13 +77,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via # dolt-integrations # flytekit @@ -103,10 +95,8 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -130,12 +120,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -149,7 +137,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index f5aa2f1471..f8813d43da 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -8,8 +8,14 @@ # via -r requirements.in altair==4.1.0 # via great-expectations -argon2-cffi==21.1.0 +appnope==0.1.2 + # via + # ipykernel + # ipython +argon2-cffi==21.3.0 # via notebook +argon2-cffi-bindings==21.2.0 + # via argon2-cffi arrow==1.2.1 # via jinja2-time attrs==21.2.0 @@ -23,12 +29,10 @@ bleach==4.1.0 certifi==2021.10.8 # via requests cffi==1.15.0 - # via - # argon2-cffi - # cryptography + # via argon2-cffi-bindings chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -41,10 +45,8 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit debugpy==1.5.1 @@ -57,26 +59,26 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit entrypoints==0.3 # via # altair # jupyter-client # nbconvert -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-great-expectations -great-expectations==0.13.41 +great-expectations==0.13.46 # via flytekitplugins-great-expectations greenlet==1.1.2 # via sqlalchemy -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests @@ -84,11 +86,11 @@ importlib-metadata==4.8.2 # via # great-expectations # keyring -ipykernel==6.5.0 +ipykernel==6.6.0 # via # ipywidgets # notebook -ipython==7.29.0 +ipython==7.30.1 # via # ipykernel # ipywidgets @@ -99,12 +101,8 @@ ipython-genutils==0.2.0 # notebook ipywidgets==7.6.5 # via great-expectations -jedi==0.18.0 +jedi==0.18.1 # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # altair @@ -124,7 +122,7 @@ jsonschema==4.2.1 # altair # great-expectations # nbformat -jupyter-client==7.0.6 +jupyter-client==7.1.0 # via # ipykernel # nbclient @@ -139,11 +137,11 @@ jupyterlab-pygments==0.1.2 # via nbconvert jupyterlab-widgets==1.0.2 # via ipywidgets -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -162,11 +160,11 @@ mistune==0.8.4 # nbconvert mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit -nbclient==0.5.5 +nbclient==0.5.9 # via nbconvert -nbconvert==6.2.0 +nbconvert==6.3.0 # via notebook nbformat==5.1.3 # via @@ -174,11 +172,12 @@ nbformat==5.1.3 # nbclient # nbconvert # notebook -nest-asyncio==1.5.1 +nest-asyncio==1.5.4 # via # jupyter-client # nbclient -notebook==6.4.5 + # notebook +notebook==6.4.6 # via widgetsnbextension numpy==1.21.4 # via @@ -187,16 +186,16 @@ numpy==1.21.4 # pandas # pyarrow # scipy -packaging==21.2 +packaging==21.3 # via bleach -pandas==1.3.4 +pandas==1.3.5 # via # altair # flytekit # great-expectations pandocfilters==1.5.0 # via nbconvert -parso==0.8.2 +parso==0.8.3 # via jedi pexpect==4.8.0 # via ipython @@ -206,7 +205,7 @@ poyo==0.5.0 # via cookiecutter prometheus-client==0.12.0 # via notebook -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.24 # via ipython protobuf==3.19.1 # via @@ -218,7 +217,7 @@ ptyprocess==0.7.0 # terminado py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi @@ -266,7 +265,7 @@ requests==2.26.0 # flytekit # great-expectations # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit @@ -274,10 +273,8 @@ ruamel.yaml==0.17.17 # via great-expectations ruamel.yaml.clib==0.2.6 # via ruamel.yaml -scipy==1.7.2 +scipy==1.7.3 # via great-expectations -secretstorage==3.3.1 - # via keyring send2trash==1.8.0 # via notebook six==1.16.0 @@ -290,7 +287,7 @@ six==1.16.0 # responses sortedcontainers==2.4.0 # via flytekit -sqlalchemy==1.4.26 +sqlalchemy==1.4.28 # via # -r requirements.in # flytekitplugins-great-expectations @@ -326,7 +323,7 @@ traitlets==5.1.1 # nbconvert # nbformat # notebook -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index fe120d6f80..60e7fc5ab0 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,47 +26,41 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-hive -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -79,13 +71,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -95,10 +87,8 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -122,12 +112,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -141,7 +129,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index fa01bf43ca..af5e158266 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -16,11 +16,9 @@ certifi==2021.10.8 # via # kubernetes # requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -32,51 +30,45 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-pod google-auth==2.3.3 # via kubernetes -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit -kubernetes==19.15.0 +kubernetes==20.13.0 # via flytekitplugins-pod markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -87,7 +79,7 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via @@ -95,7 +87,7 @@ numpy==1.21.4 # pyarrow oauthlib==3.1.1 # via requests-oauthlib -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -105,7 +97,7 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit pyasn1==0.4.8 # via @@ -113,8 +105,6 @@ pyasn1==0.4.8 # rsa pyasn1-modules==0.2.8 # via google-auth -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -145,14 +135,12 @@ requests==2.26.0 # responses requests-oauthlib==1.3.0 # via kubernetes -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -rsa==4.7.2 +rsa==4.8 # via google-auth -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -168,7 +156,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json @@ -178,7 +166,7 @@ urllib3==1.26.7 # kubernetes # requests # responses -websocket-client==1.2.1 +websocket-client==1.2.3 # via kubernetes wheel==0.37.0 # via flytekit diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt index c65ce6c7b3..cab4255c6e 100644 --- a/plugins/flytekit-kf-mpi/requirements.txt +++ b/plugins/flytekit-kf-mpi/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,49 +26,43 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via # flytekit # flytekitplugins-kfmpi -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-kfmpi -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -81,13 +73,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -97,10 +89,8 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -124,12 +114,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -143,7 +131,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index 2cdcd130c9..fd77987627 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,47 +26,41 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-kfpytorch -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -79,13 +71,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -95,10 +87,8 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -122,12 +112,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -141,7 +129,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index ac3a64ed65..799ebec773 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,47 +26,41 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-kftensorflow -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -79,13 +71,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -95,10 +87,8 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -122,12 +112,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -141,7 +129,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-modin/requirements.txt b/plugins/flytekit-modin/requirements.txt index ff80b3a0f6..1b82843eab 100644 --- a/plugins/flytekit-modin/requirements.txt +++ b/plugins/flytekit-modin/requirements.txt @@ -16,11 +16,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -33,31 +31,33 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 - # via flytekit -diskcache==5.2.1 + # via + # flytekit + # redis +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -filelock==3.3.2 +filelock==3.4.0 # via ray -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 - # via flytekitplugins-modin -fsspec==2021.11.0 +flytekit==0.25.0 # via flytekitplugins-modin -grpcio==1.41.1 +fsspec==2021.11.1 + # via + # flytekitplugins-modin + # modin +grpcio==1.42.0 # via # flytekit # ray @@ -65,10 +65,6 @@ idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -77,11 +73,11 @@ jinja2-time==0.2.0 # via cookiecutter jsonschema==4.2.1 # via ray -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -90,13 +86,13 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -modin==0.11.3 +modin==0.12.0 # via flytekitplugins-modin -msgpack==1.0.2 +msgpack==1.0.3 # via ray mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via @@ -104,7 +100,7 @@ numpy==1.21.4 # pandas # pyarrow # ray -packaging==21.2 +packaging==21.3 # via modin pandas==1.3.4 # via @@ -119,11 +115,9 @@ protobuf==3.19.1 # ray py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi -pyparsing==2.4.7 +pyparsing==3.0.6 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -145,9 +139,9 @@ pytz==2018.4 # pandas pyyaml==6.0 # via ray -ray==1.8.0 +ray==1.9.0 # via flytekitplugins-modin -redis==3.5.3 +redis==4.0.2 # via ray regex==2021.11.10 # via docker-image-py @@ -156,12 +150,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -175,7 +167,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index de61384e56..07555749e1 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,47 +26,41 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-pandera -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -79,20 +71,22 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pandera # pyarrow -packaging==21.2 +packaging==21.3 # via pandera -pandas==1.3.4 +pandas==1.3.5 # via # flytekit # pandera -pandera==0.7.2 +pandas-stubs==1.2.0.39 + # via pandera +pandera==0.8.0 # via flytekitplugins-pandera poyo==0.5.0 # via cookiecutter @@ -102,13 +96,11 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via # flytekit # pandera -pycparser==2.21 - # via cffi -pyparsing==2.4.7 +pyparsing==3.0.6 # via packaging python-dateutil==2.8.1 # via @@ -133,12 +125,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -152,7 +142,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index c31e0824bf..cb08ace93a 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -8,6 +8,10 @@ # via -r requirements.in ansiwrap==0.8.4 # via papermill +appnope==0.1.2 + # via + # ipykernel + # ipython arrow==1.2.1 # via jinja2-time attrs==21.2.0 @@ -16,17 +20,15 @@ backcall==0.2.0 # via ipython binaryornot==0.4.4 # via cookiecutter -black==21.10b0 +black==21.12b0 # via papermill bleach==4.1.0 # via nbconvert certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -40,10 +42,8 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit debugpy==1.5.1 @@ -56,45 +56,41 @@ defusedxml==0.7.1 # via nbconvert deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit entrypoints==0.3 # via # jupyter-client # nbconvert # papermill -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via # flytekitplugins-papermill # flytekitplugins-spark -flytekitplugins-spark==0.24.0 +flytekitplugins-spark==0.25.0 # via # -r requirements.in # flytekitplugins-papermill -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -ipykernel==6.5.0 +ipykernel==6.6.0 # via flytekitplugins-papermill -ipython==7.29.0 +ipython==7.30.1 # via ipykernel ipython-genutils==0.2.0 # via nbformat -jedi==0.18.0 +jedi==0.18.1 # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -104,7 +100,7 @@ jinja2-time==0.2.0 # via cookiecutter jsonschema==4.2.1 # via nbformat -jupyter-client==7.0.6 +jupyter-client==7.1.0 # via # ipykernel # nbclient @@ -115,11 +111,11 @@ jupyter-core==4.9.1 # nbformat jupyterlab-pygments==0.1.2 # via nbconvert -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -138,20 +134,20 @@ mypy-extensions==0.4.3 # via # black # typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit -nbclient==0.5.5 +nbclient==0.5.9 # via # nbconvert # papermill -nbconvert==6.2.0 +nbconvert==6.3.0 # via flytekitplugins-papermill nbformat==5.1.3 # via # nbclient # nbconvert # papermill -nest-asyncio==1.5.1 +nest-asyncio==1.5.4 # via # jupyter-client # nbclient @@ -159,15 +155,15 @@ numpy==1.21.4 # via # pandas # pyarrow -packaging==21.2 +packaging==21.3 # via bleach -pandas==1.3.4 +pandas==1.3.5 # via flytekit pandocfilters==1.5.0 # via nbconvert papermill==2.3.3 # via flytekitplugins-papermill -parso==0.8.2 +parso==0.8.3 # via jedi pathspec==0.9.0 # via black @@ -179,7 +175,7 @@ platformdirs==2.4.0 # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.22 +prompt-toolkit==3.0.24 # via ipython protobuf==3.19.1 # via @@ -191,16 +187,14 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi pygments==2.10.0 # via # ipython # jupyterlab-pygments # nbconvert -pyparsing==2.4.7 +pyparsing==3.0.6 # via packaging pyrsistent==0.18.0 # via jsonschema @@ -228,21 +222,17 @@ pyyaml==6.0 pyzmq==22.3.0 # via jupyter-client regex==2021.11.10 - # via - # black - # docker-image-py + # via docker-image-py requests==2.26.0 # via # cookiecutter # flytekit # papermill # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # bleach @@ -263,7 +253,7 @@ text-unidecode==1.3 # via python-slugify textwrap3==0.9.2 # via ansiwrap -tomli==1.2.2 +tomli==1.2.3 # via black tornado==6.1 # via @@ -281,7 +271,7 @@ traitlets==5.1.1 # nbclient # nbconvert # nbformat -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via # black # typing-inspect diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index 04a2dfec1d..69c4b2145e 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,47 +26,41 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-snowflake -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -79,13 +71,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -95,10 +87,8 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -122,12 +112,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -141,7 +129,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index df2baaa778..cb1eabd00f 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,47 +26,41 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-spark -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -79,13 +71,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -97,10 +89,8 @@ py==1.11.0 # via retry py4j==0.10.9.2 # via pyspark -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi pyspark==3.2.0 # via flytekitplugins-spark python-dateutil==2.8.1 @@ -126,12 +116,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -145,7 +133,7 @@ statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index bf4f5f0590..fb2864a4e9 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -12,11 +12,9 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.7 +charset-normalizer==2.0.9 # via requests checksumdir==1.2.0 # via flytekit @@ -28,49 +26,43 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.0.15 +croniter==1.1.0 # via flytekit -cryptography==35.0.0 - # via secretstorage dataclasses-json==0.5.6 # via flytekit decorator==5.1.0 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.2.1 +diskcache==5.3.0 # via flytekit docker-image-py==0.1.12 # via flytekit -docstring-parser==0.12 +docstring-parser==0.13 # via flytekit -flyteidl==0.21.8 +flyteidl==0.21.13 # via flytekit -flytekit==0.24.0 +flytekit==0.25.0 # via flytekitplugins-sqlalchemy greenlet==1.1.2 # via sqlalchemy -grpcio==1.41.1 +grpcio==1.42.0 # via flytekit idna==3.3 # via requests importlib-metadata==4.8.2 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time jinja2-time==0.2.0 # via cookiecutter -keyring==23.2.1 +keyring==23.4.0 # via flytekit markupsafe==2.0.1 # via jinja2 -marshmallow==3.14.0 +marshmallow==3.14.1 # via # dataclasses-json # marshmallow-enum @@ -81,13 +73,13 @@ marshmallow-jsonschema==0.13.0 # via flytekit mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.0 +natsort==8.0.1 # via flytekit numpy==1.21.4 # via # pandas # pyarrow -pandas==1.3.4 +pandas==1.3.5 # via flytekit poyo==0.5.0 # via cookiecutter @@ -97,10 +89,8 @@ protobuf==3.19.1 # flytekit py==1.11.0 # via retry -pyarrow==6.0.0 +pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi python-dateutil==2.8.1 # via # arrow @@ -124,12 +114,10 @@ requests==2.26.0 # cookiecutter # flytekit # responses -responses==0.15.0 +responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter @@ -139,13 +127,13 @@ six==1.16.0 # responses sortedcontainers==2.4.0 # via flytekit -sqlalchemy==1.4.26 +sqlalchemy==1.4.28 # via flytekitplugins-sqlalchemy statsd==3.3.0 # via flytekit text-unidecode==1.3 # via python-slugify -typing-extensions==3.10.0.2 +typing-extensions==4.0.1 # via typing-inspect typing-inspect==0.7.1 # via dataclasses-json From a6a6cb42a0a68704a1e540c65894d6dec06524ef Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 4 Jan 2022 19:38:21 -0800 Subject: [PATCH 27/42] fix: bump requirements Signed-off-by: Kenny Workman --- dev-requirements.txt | 11 +++++++++++ requirements-spark2.txt | 10 ++++++---- requirements.in | 1 - requirements.txt | 10 ++++++---- .../remote/mock_flyte_repo/workflows/requirements.txt | 9 +++++++++ 5 files changed, 32 insertions(+), 9 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 20a38f3e1a..f235061f06 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -77,6 +77,7 @@ cryptography==36.0.1 # via # -c requirements.txt # paramiko + # secretstorage dataclasses-json==0.5.6 # via # -c requirements.txt @@ -137,6 +138,11 @@ importlib-metadata==4.10.0 # keyring iniconfig==1.1.1 # via pytest +jeepney==0.7.1 + # via + # -c requirements.txt + # keyring + # secretstorage jinja2==3.0.3 # via # -c requirements.txt @@ -222,6 +228,7 @@ pre-commit==2.16.0 protobuf==3.19.1 # via # -c requirements.txt + # flyteidl # flytekit py==1.11.0 # via @@ -308,6 +315,10 @@ retry==0.9.2 # via # -c requirements.txt # flytekit +secretstorage==3.3.1 + # via + # -c requirements.txt + # keyring six==1.16.0 # via # -c requirements.txt diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 7464224875..52026eabe3 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -10,10 +10,6 @@ # -r requirements.in ansiwrap==0.8.4 # via papermill -appnope==0.1.2 - # via - # ipykernel - # ipython arrow==1.2.1 # via jinja2-time attrs==20.3.0 @@ -113,6 +109,10 @@ ipython-genutils==0.2.0 # nbformat jedi==0.18.1 # via ipython +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -285,6 +285,8 @@ sagemaker-training==3.9.2 # via flytekit scipy==1.7.3 # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # bcrypt diff --git a/requirements.in b/requirements.in index 9a67f5aaf3..408185fca8 100644 --- a/requirements.in +++ b/requirements.in @@ -7,4 +7,3 @@ attrs<21 # https://github.com/flyteorg/flyte/issues/1732. jsonschema<4 pyyaml<6 -flyteidl @ git+https://github.com/kennyworkman/flyteidl.git diff --git a/requirements.txt b/requirements.txt index 801006fd24..949c43b85f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -8,10 +8,6 @@ # via -r requirements.in ansiwrap==0.8.4 # via papermill -appnope==0.1.2 - # via - # ipykernel - # ipython arrow==1.2.1 # via jinja2-time attrs==20.3.0 @@ -111,6 +107,10 @@ ipython-genutils==0.2.0 # nbformat jedi==0.18.1 # via ipython +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -283,6 +283,8 @@ sagemaker-training==3.9.2 # via flytekit scipy==1.7.3 # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # bcrypt diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 9b2f5713a0..136c6eb95e 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -10,6 +10,8 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests +cffi==1.15.0 + # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.9 @@ -54,6 +56,10 @@ idna==3.3 # via requests importlib-metadata==4.10.0 # via keyring +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -115,6 +121,7 @@ pyparsing==3.0.6 # packaging python-dateutil==2.8.1 # via + # arrow # croniter # flytekit # matplotlib @@ -140,6 +147,8 @@ responses==0.16.0 # via flytekit retry==0.9.2 # via flytekit +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # cookiecutter From 9df04ea95db342a71f8c69478596e32d093dd05a Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 4 Jan 2022 19:40:23 -0800 Subject: [PATCH 28/42] fix: doc requirements Signed-off-by: Kenny Workman --- doc-requirements.txt | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/doc-requirements.txt b/doc-requirements.txt index 055739f0fc..c2056c301e 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -10,10 +10,6 @@ alabaster==0.7.12 # via sphinx ansiwrap==0.8.4 # via papermill -appnope==0.1.2 - # via - # ipykernel - # ipython arrow==1.2.1 # via jinja2-time astroid==2.9.0 @@ -73,6 +69,7 @@ cryptography==36.0.1 # via # -r doc-requirements.in # paramiko + # secretstorage css-html-js-minify==2.5.5 # via sphinx-material dataclasses-json==0.5.6 @@ -130,6 +127,10 @@ ipython-genutils==0.2.0 # nbformat jedi==0.18.1 # via ipython +jeepney==0.7.1 + # via + # keyring + # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -235,6 +236,7 @@ prompt-toolkit==3.0.24 # via ipython protobuf==3.19.1 # via + # flyteidl # flytekit # k8s-proto # sagemaker-training @@ -313,6 +315,8 @@ sagemaker-training==3.9.2 # via flytekit scipy==1.7.3 # via sagemaker-training +secretstorage==3.3.1 + # via keyring six==1.16.0 # via # bcrypt From 0ed30b8efe3397a18a3e1bdf081880237c5e6f3e Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 4 Jan 2022 20:00:52 -0800 Subject: [PATCH 29/42] fix: whitespace Signed-off-by: Kenny Workman --- dev-requirements.txt | 2 +- doc-requirements.txt | 2 +- requirements-spark2.txt | 2 +- requirements.txt | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index f235061f06..8cc4f7ba3f 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -392,4 +392,4 @@ zipp==3.6.0 # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/doc-requirements.txt b/doc-requirements.txt index c2056c301e..cba57b13b7 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -446,4 +446,4 @@ zope.interface==5.4.0 # The following packages are considered to be unsafe in a requirements file: # pip -# setuptools \ No newline at end of file +# setuptools diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 52026eabe3..a48ec4b5e2 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -366,4 +366,4 @@ zope.interface==5.4.0 # The following packages are considered to be unsafe in a requirements file: # pip -# setuptools \ No newline at end of file +# setuptools diff --git a/requirements.txt b/requirements.txt index 949c43b85f..04cdd38a16 100644 --- a/requirements.txt +++ b/requirements.txt @@ -364,4 +364,4 @@ zope.interface==5.4.0 # The following packages are considered to be unsafe in a requirements file: # pip -# setuptools \ No newline at end of file +# setuptools From 5fc72d4e2c1d34ce7b7988fc8b85cd861582fdc1 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 22:26:48 -0800 Subject: [PATCH 30/42] fix: bump flytekit Signed-off-by: Kenny Workman --- .gitignore | 2 +- dev-requirements.txt | 147 ++++++---- doc-requirements.txt | 253 ++--------------- requirements-spark2.txt | 257 ++---------------- requirements.txt | 257 ++---------------- setup.py | 1 + .../workflows/requirements.txt | 67 ++--- 7 files changed, 197 insertions(+), 787 deletions(-) diff --git a/.gitignore b/.gitignore index ec56a0f239..73c81d2a6a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ *.pyt *.pytc *.egg-info -.*.swp +.*.sw* .DS_Store venv/ .venv/ diff --git a/dev-requirements.txt b/dev-requirements.txt index 8cc4f7ba3f..bd472a45d9 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -8,7 +8,7 @@ # via # -c requirements.txt # pytest-flyte -arrow==1.2.1 +arrow==1.2.2 # via # -c requirements.txt # jinja2-time @@ -18,23 +18,20 @@ attrs==20.3.0 # jsonschema # pytest # pytest-docker -backports.entry-points-selectable==1.1.1 - # via virtualenv bcrypt==3.2.0 - # via - # -c requirements.txt - # paramiko + # via paramiko binaryornot==0.4.4 # via # -c requirements.txt # cookiecutter +cachetools==5.0.0 + # via google-auth certifi==2021.10.8 # via # -c requirements.txt # requests cffi==1.15.0 # via - # -c requirements.txt # bcrypt # cryptography # pynacl @@ -44,7 +41,7 @@ chardet==4.0.0 # via # -c requirements.txt # binaryornot -charset-normalizer==2.0.9 +charset-normalizer==2.0.11 # via # -c requirements.txt # requests @@ -67,22 +64,19 @@ cookiecutter==1.7.3 # via # -c requirements.txt # flytekit -coverage[toml]==6.2 +coverage[toml]==6.3.1 # via -r dev-requirements.in -croniter==1.1.0 +croniter==1.2.0 # via # -c requirements.txt # flytekit cryptography==36.0.1 - # via - # -c requirements.txt - # paramiko - # secretstorage + # via paramiko dataclasses-json==0.5.6 # via # -c requirements.txt # flytekit -decorator==5.1.0 +decorator==5.1.1 # via # -c requirements.txt # retry @@ -90,7 +84,7 @@ deprecated==1.2.13 # via # -c requirements.txt # flytekit -diskcache==5.3.0 +diskcache==5.4.0 # via # -c requirements.txt # flytekit @@ -116,23 +110,51 @@ docstring-parser==0.13 # via # -c requirements.txt # flytekit -filelock==3.4.0 +filelock==3.4.2 # via virtualenv -flyteidl==0.21.13 +flyteidl==0.22.0 # via # -c requirements.txt # flytekit +google-api-core[grpc]==2.5.0 + # via + # google-cloud-bigquery + # google-cloud-bigquery-storage + # google-cloud-core +google-auth==2.6.0 + # via + # google-api-core + # google-cloud-core +google-cloud-bigquery==2.32.0 + # via -r dev-requirements.in +google-cloud-bigquery-storage==2.11.0 + # via -r dev-requirements.in +google-cloud-core==2.2.2 + # via google-cloud-bigquery +google-crc32c==1.3.0 + # via google-resumable-media +google-resumable-media==2.2.0 + # via google-cloud-bigquery +googleapis-common-protos==1.54.0 + # via + # google-api-core + # grpcio-status grpcio==1.43.0 # via # -c requirements.txt # flytekit -identify==2.4.0 + # google-api-core + # google-cloud-bigquery + # grpcio-status +grpcio-status==1.43.0 + # via google-api-core +identify==2.4.8 # via pre-commit idna==3.3 # via # -c requirements.txt # requests -importlib-metadata==4.10.0 +importlib-metadata==4.10.1 # via # -c requirements.txt # keyring @@ -159,10 +181,12 @@ jsonschema==3.2.0 # via # -c requirements.txt # docker-compose -keyring==23.4.0 +keyring==23.5.0 # via # -c requirements.txt # flytekit +libcst==0.4.1 + # via google-cloud-bigquery-storage markupsafe==2.0.1 # via # -c requirements.txt @@ -183,49 +207,57 @@ marshmallow-jsonschema==0.13.0 # flytekit mock==4.0.3 # via -r dev-requirements.in -mypy==0.930 +mypy==0.931 # via -r dev-requirements.in mypy-extensions==0.4.3 # via # -c requirements.txt # mypy # typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via # -c requirements.txt # flytekit nodeenv==1.6.0 # via pre-commit -numpy==1.21.5 +numpy==1.22.2 # via # -c requirements.txt # pandas # pyarrow packaging==21.3 # via +<<<<<<< HEAD # -c requirements.txt +======= + # google-cloud-bigquery +>>>>>>> 6b5e7a8c (fix: bump flytekit) # pytest -pandas==1.3.5 +pandas==1.4.0 # via # -c requirements.txt # flytekit -paramiko==2.8.1 - # via - # -c requirements.txt - # docker -platformdirs==2.4.0 - # via - # -c requirements.txt - # virtualenv +paramiko==2.9.2 + # via docker +platformdirs==2.4.1 + # via virtualenv pluggy==1.0.0 # via pytest poyo==0.5.0 # via # -c requirements.txt # cookiecutter -pre-commit==2.16.0 +pre-commit==2.17.0 # via -r dev-requirements.in +<<<<<<< HEAD protobuf==3.19.1 +======= +proto-plus==1.20.0 + # via + # google-cloud-bigquery + # google-cloud-bigquery-storage +protobuf==3.19.4 +>>>>>>> 6b5e7a8c (fix: bump flytekit) # via # -c requirements.txt # flyteidl @@ -240,18 +272,12 @@ pyarrow==6.0.1 # -c requirements.txt # flytekit pycparser==2.21 - # via - # -c requirements.txt - # cffi -pynacl==1.4.0 - # via - # -c requirements.txt - # paramiko -pyparsing==3.0.6 - # via - # -c requirements.txt - # packaging -pyrsistent==0.18.0 + # via cffi +pynacl==1.5.0 + # via paramiko +pyparsing==3.0.7 + # via packaging +pyrsistent==0.18.1 # via # -c requirements.txt # jsonschema @@ -264,7 +290,7 @@ pytest-docker==0.10.3 # via pytest-flyte pytest-flyte @ git+git://github.com/flyteorg/pytest-flyte@main # via -r dev-requirements.in -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # -c requirements.txt # arrow @@ -295,11 +321,11 @@ pyyaml==5.4.1 # -c requirements.txt # docker-compose # pre-commit -regex==2021.11.10 +regex==2022.1.18 # via # -c requirements.txt # docker-image-py -requests==2.26.0 +requests==2.27.1 # via # -c requirements.txt # cookiecutter @@ -307,7 +333,7 @@ requests==2.26.0 # docker-compose # flytekit # responses -responses==0.16.0 +responses==0.18.0 # via # -c requirements.txt # flytekit @@ -328,9 +354,7 @@ six==1.16.0 # flytekit # grpcio # jsonschema - # pynacl # python-dateutil - # responses # virtualenv # websocket-client sortedcontainers==2.4.0 @@ -351,27 +375,36 @@ toml==0.10.2 # via # pre-commit # pytest -tomli==1.2.3 +tomli==2.0.0 # via - # -c requirements.txt # coverage # mypy typing-extensions==4.0.1 # via # -c requirements.txt +<<<<<<< HEAD +======= + # flytekit + # libcst +>>>>>>> 6b5e7a8c (fix: bump flytekit) # mypy # typing-inspect typing-inspect==0.7.1 # via # -c requirements.txt # dataclasses-json +<<<<<<< HEAD urllib3==1.26.7 +======= + # libcst +urllib3==1.26.8 +>>>>>>> 6b5e7a8c (fix: bump flytekit) # via # -c requirements.txt # flytekit # requests # responses -virtualenv==20.10.0 +virtualenv==20.13.1 # via pre-commit websocket-client==0.59.0 # via @@ -386,10 +419,10 @@ wrapt==1.13.3 # -c requirements.txt # deprecated # flytekit -zipp==3.6.0 +zipp==3.7.0 # via # -c requirements.txt # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools +# setuptools \ No newline at end of file diff --git a/doc-requirements.txt b/doc-requirements.txt index cba57b13b7..f41d812ed3 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -8,20 +8,12 @@ # via -r doc-requirements.in alabaster==0.7.12 # via sphinx -ansiwrap==0.8.4 - # via papermill -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time astroid==2.9.0 # via sphinx-autoapi -attrs==21.2.0 - # via jsonschema babel==2.9.1 # via sphinx -backcall==0.2.0 - # via ipython -bcrypt==3.2.0 - # via paramiko beautifulsoup4==4.10.0 # via # furo @@ -29,23 +21,10 @@ beautifulsoup4==4.10.0 # sphinx-material binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via papermill -bleach==4.1.0 - # via nbconvert -boto3==1.20.26 - # via sagemaker-training -botocore==1.23.26 - # via - # boto3 - # s3transfer certifi==2021.10.8 # via requests cffi==1.15.0 - # via - # bcrypt - # cryptography - # pynacl + # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.9 @@ -54,32 +33,22 @@ checksumdir==1.2.0 # via flytekit click==7.1.2 # via - # black # cookiecutter # flytekit - # hmsclient - # papermill cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.1.0 +croniter==1.2.0 # via flytekit cryptography==36.0.1 - # via - # -r doc-requirements.in - # paramiko - # secretstorage + # via -r doc-requirements.in css-html-js-minify==2.5.5 # via sphinx-material dataclasses-json==0.5.6 # via flytekit -decorator==5.1.0 - # via - # ipython - # retry -defusedxml==0.7.1 - # via nbconvert +decorator==5.1.1 + # via retry deprecated==1.2.13 # via flytekit diskcache==5.3.0 @@ -89,77 +58,34 @@ docker-image-py==0.1.12 docstring-parser==0.13 # via flytekit docutils==0.17.1 - # via sphinx -entrypoints==0.3 # via - # jupyter-client - # nbconvert - # papermill -flyteidl==0.21.13 + # sphinx + # sphinx-panels +flyteidl==0.22.0 # via flytekit furo @ git+git://github.com/flyteorg/furo@main # via -r doc-requirements.in -gevent==21.12.0 - # via sagemaker-training -greenlet==1.1.2 - # via gevent grpcio==1.43.0 # via # -r doc-requirements.in # flytekit -hmsclient==0.1.1 - # via flytekit idna==3.3 # via requests imagesize==1.3.0 # via sphinx -importlib-metadata==4.10.0 - # via keyring -inotify_simple==1.2.1 - # via sagemaker-training -ipykernel==5.5.6 - # via flytekit -ipython==7.30.1 - # via ipykernel -ipython-genutils==0.2.0 - # via - # ipykernel - # nbformat -jedi==0.18.1 - # via ipython -jeepney==0.7.1 +importlib-metadata==4.10.1 # via # keyring - # secretstorage + # sphinx jinja2==3.0.3 # via # cookiecutter # jinja2-time - # nbconvert # sphinx # sphinx-autoapi jinja2-time==0.2.0 # via cookiecutter -jmespath==0.10.0 - # via - # boto3 - # botocore -jsonschema==4.3.2 - # via nbformat -jupyter-client==7.1.0 - # via - # ipykernel - # nbclient -jupyter-core==4.9.1 - # via - # jupyter-client - # nbconvert - # nbformat -jupyterlab-pygments==0.1.2 - # via nbconvert -k8s-proto==0.0.3 - # via flytekit -keyring==23.4.0 +keyring==23.5.0 # via flytekit lazy-object-proxy==1.7.1 # via astroid @@ -176,104 +102,41 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib-inline==0.1.3 - # via ipython -mistune==0.8.4 - # via nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbclient==0.5.9 +numpy==1.22.2 # via - # nbconvert - # papermill -nbconvert==6.3.0 - # via flytekit -nbformat==5.1.3 - # via - # nbclient - # nbconvert - # papermill -nest-asyncio==1.5.4 - # via - # jupyter-client - # nbclient -numpy==1.21.5 - # via - # flytekit # pandas # pyarrow - # sagemaker-training - # scipy packaging==21.3 - # via - # bleach - # sphinx -pandas==1.3.5 - # via flytekit -pandocfilters==1.5.0 - # via nbconvert -papermill==2.3.3 + # via sphinx +pandas==1.4.0 # via flytekit -paramiko==2.8.1 - # via sagemaker-training -parso==0.8.3 - # via jedi -pathspec==0.9.0 - # via black -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -platformdirs==2.4.0 - # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.24 - # via ipython -protobuf==3.19.1 +protobuf==3.19.4 # via # flyteidl # flytekit - # k8s-proto - # sagemaker-training -psutil==5.8.0 - # via sagemaker-training -ptyprocess==0.7.0 - # via pexpect py==1.11.0 # via retry -py4j==0.10.9.2 - # via pyspark pyarrow==6.0.1 # via flytekit pycparser==2.21 # via cffi pygments==2.10.0 # via - # ipython - # jupyterlab-pygments - # nbconvert # sphinx # sphinx-prompt -pynacl==1.4.0 - # via paramiko -pyparsing==3.0.6 +pyparsing==3.0.7 # via packaging -pyrsistent==0.18.0 - # via jsonschema -pyspark==3.2.0 - # via flytekit -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow - # botocore # croniter # flytekit - # jupyter-client # pandas python-json-logger==2.0.2 # via flytekit @@ -289,55 +152,33 @@ pytz==2021.3 # flytekit # pandas pyyaml==6.0 - # via - # papermill - # sphinx-autoapi -pyzmq==22.3.0 - # via jupyter-client -regex==2021.11.10 + # via sphinx-autoapi +regex==2022.1.18 # via docker-image-py requests==2.26.0 # via # cookiecutter # flytekit - # papermill # responses # sphinx -responses==0.16.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -retrying==1.3.3 - # via sagemaker-training -s3transfer==0.5.0 - # via boto3 -sagemaker-training==3.9.2 - # via flytekit -scipy==1.7.3 - # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.16.0 # via - # bcrypt - # bleach # cookiecutter - # flytekit # grpcio # pynacl # python-dateutil - # responses - # retrying - # sagemaker-training # sphinx-code-include - # thrift snowballstemmer==2.2.0 # via sphinx sortedcontainers==2.4.0 # via flytekit soupsieve==2.3.1 # via beautifulsoup4 -sphinx==4.3.2 +sphinx==4.4.0 # via # -r doc-requirements.in # furo @@ -353,7 +194,7 @@ sphinx-autoapi==1.8.4 # via -r doc-requirements.in sphinx-code-include==1.1.1 # via -r doc-requirements.in -sphinx-copybutton==0.4.0 +sphinx-copybutton==0.5.0 # via -r doc-requirements.in sphinx-fontawesome==0.0.6 # via -r doc-requirements.in @@ -379,38 +220,12 @@ sphinxcontrib-yt==0.2.2 # via -r doc-requirements.in statsd==3.3.0 # via flytekit -tenacity==8.0.1 - # via papermill -testpath==0.5.0 - # via nbconvert text-unidecode==1.3 # via python-slugify -textwrap3==0.9.2 - # via ansiwrap -thrift==0.15.0 - # via hmsclient -tomli==1.2.3 - # via black -tornado==6.1 - # via - # ipykernel - # jupyter-client -tqdm==4.62.3 - # via papermill -traitlets==5.1.1 - # via - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat typing-extensions==4.0.1 # via # astroid - # black + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json @@ -420,16 +235,9 @@ unidecode==1.3.2 # sphinx-autoapi urllib3==1.26.7 # via - # botocore # flytekit # requests # responses -wcwidth==0.2.5 - # via prompt-toolkit -webencodings==0.5.1 - # via bleach -werkzeug==2.0.2 - # via sagemaker-training wheel==0.37.1 # via flytekit wrapt==1.13.3 @@ -437,13 +245,8 @@ wrapt==1.13.3 # astroid # deprecated # flytekit -zipp==3.6.0 +zipp==3.7.0 # via importlib-metadata -zope.event==4.5.0 - # via gevent -zope.interface==5.4.0 - # via gevent # The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools +# setuptools \ No newline at end of file diff --git a/requirements-spark2.txt b/requirements-spark2.txt index a48ec4b5e2..807fee354e 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -8,140 +8,61 @@ # via # -r requirements-spark2.in # -r requirements.in -ansiwrap==0.8.4 - # via papermill -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time attrs==20.3.0 # via # -r requirements.in # jsonschema -backcall==0.2.0 - # via ipython -bcrypt==3.2.0 - # via paramiko binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via papermill -bleach==4.1.0 - # via nbconvert -boto3==1.20.26 - # via sagemaker-training -botocore==1.23.26 - # via - # boto3 - # s3transfer certifi==2021.10.8 # via requests -cffi==1.15.0 - # via - # bcrypt - # cryptography - # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.9 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit click==7.1.2 # via - # black # cookiecutter # flytekit - # hmsclient - # papermill cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.1.0 +croniter==1.2.0 # via flytekit -cryptography==36.0.1 - # via - # paramiko - # secretstorage dataclasses-json==0.5.6 # via flytekit -decorator==5.1.0 - # via - # ipython - # retry -defusedxml==0.7.1 - # via nbconvert +decorator==5.1.1 + # via retry deprecated==1.2.13 # via flytekit -diskcache==5.3.0 +diskcache==5.4.0 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 - # via - # jupyter-client - # nbconvert - # papermill -flyteidl==0.21.13 +flyteidl==0.22.0 # via flytekit -gevent==21.12.0 - # via sagemaker-training -greenlet==1.1.2 - # via gevent grpcio==1.43.0 # via flytekit -hmsclient==0.1.1 - # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.0 +importlib-metadata==4.10.1 # via keyring -inotify_simple==1.2.1 - # via sagemaker-training -ipykernel==5.5.6 - # via flytekit -ipython==7.30.1 - # via ipykernel -ipython-genutils==0.2.0 - # via - # ipykernel - # nbformat -jedi==0.18.1 - # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time - # nbconvert jinja2-time==0.2.0 # via cookiecutter -jmespath==0.10.0 - # via - # boto3 - # botocore jsonschema==3.2.0 - # via - # -r requirements.in - # nbformat -jupyter-client==7.1.0 - # via - # ipykernel - # nbclient -jupyter-core==4.9.1 - # via - # jupyter-client - # nbconvert - # nbformat -jupyterlab-pygments==0.1.2 - # via nbconvert -k8s-proto==0.0.3 - # via flytekit -keyring==23.4.0 + # via -r requirements.in +keyring==23.5.0 # via flytekit markupsafe==2.0.1 # via jinja2 @@ -154,100 +75,33 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib-inline==0.1.3 - # via ipython -mistune==0.8.4 - # via nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbclient==0.5.9 +numpy==1.22.2 # via - # nbconvert - # papermill -nbconvert==6.3.0 - # via flytekit -nbformat==5.1.3 - # via - # nbclient - # nbconvert - # papermill -nest-asyncio==1.5.4 - # via - # jupyter-client - # nbclient -numpy==1.21.5 - # via - # flytekit # pandas # pyarrow - # sagemaker-training - # scipy -packaging==21.3 - # via bleach -pandas==1.3.5 +pandas==1.4.0 # via flytekit -pandocfilters==1.5.0 - # via nbconvert -papermill==2.3.3 - # via flytekit -paramiko==2.8.1 - # via sagemaker-training -parso==0.8.3 - # via jedi -pathspec==0.9.0 - # via black -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -platformdirs==2.4.0 - # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.24 - # via ipython -protobuf==3.19.1 +protobuf==3.19.4 # via # flyteidl # flytekit - # k8s-proto - # sagemaker-training -psutil==5.8.0 - # via sagemaker-training -ptyprocess==0.7.0 - # via pexpect py==1.11.0 # via retry -py4j==0.10.9.2 - # via pyspark pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi -pygments==2.10.0 - # via - # ipython - # jupyterlab-pygments - # nbconvert -pynacl==1.4.0 - # via paramiko -pyparsing==3.0.6 - # via packaging -pyrsistent==0.18.0 +pyrsistent==0.18.1 # via jsonschema -pyspark==3.2.0 - # via flytekit -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow - # botocore # croniter # flytekit - # jupyter-client # pandas python-json-logger==2.0.2 # via flytekit @@ -260,110 +114,49 @@ pytz==2021.3 # flytekit # pandas pyyaml==5.4.1 - # via - # -r requirements.in - # papermill -pyzmq==22.3.0 - # via jupyter-client -regex==2021.11.10 + # via -r requirements.in +regex==2022.1.18 # via docker-image-py -requests==2.26.0 +requests==2.27.1 # via # cookiecutter # flytekit - # papermill # responses -responses==0.16.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -retrying==1.3.3 - # via sagemaker-training -s3transfer==0.5.0 - # via boto3 -sagemaker-training==3.9.2 - # via flytekit -scipy==1.7.3 - # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.16.0 # via - # bcrypt - # bleach # cookiecutter - # flytekit # grpcio # jsonschema - # pynacl # python-dateutil - # responses - # retrying - # sagemaker-training - # thrift sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit -tenacity==8.0.1 - # via papermill -testpath==0.5.0 - # via nbconvert text-unidecode==1.3 # via python-slugify -textwrap3==0.9.2 - # via ansiwrap -thrift==0.15.0 - # via hmsclient -tomli==1.2.3 - # via black -tornado==6.1 - # via - # ipykernel - # jupyter-client -tqdm==4.62.3 - # via papermill -traitlets==5.1.1 - # via - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat typing-extensions==4.0.1 # via - # black + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.7 +urllib3==1.26.8 # via - # botocore # flytekit # requests # responses -wcwidth==0.2.5 - # via prompt-toolkit -webencodings==0.5.1 - # via bleach -werkzeug==2.0.2 - # via sagemaker-training wheel==0.37.1 # via flytekit wrapt==1.13.3 # via # deprecated # flytekit -zipp==3.6.0 +zipp==3.7.0 # via importlib-metadata -zope.event==4.5.0 - # via gevent -zope.interface==5.4.0 - # via gevent # The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools +# setuptools \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 04cdd38a16..3557ae4dc9 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,140 +6,61 @@ # -e file:.#egg=flytekit # via -r requirements.in -ansiwrap==0.8.4 - # via papermill -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time attrs==20.3.0 # via # -r requirements.in # jsonschema -backcall==0.2.0 - # via ipython -bcrypt==3.2.0 - # via paramiko binaryornot==0.4.4 # via cookiecutter -black==21.12b0 - # via papermill -bleach==4.1.0 - # via nbconvert -boto3==1.20.26 - # via sagemaker-training -botocore==1.23.26 - # via - # boto3 - # s3transfer certifi==2021.10.8 # via requests -cffi==1.15.0 - # via - # bcrypt - # cryptography - # pynacl chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.9 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit click==7.1.2 # via - # black # cookiecutter # flytekit - # hmsclient - # papermill cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.1.0 +croniter==1.2.0 # via flytekit -cryptography==36.0.1 - # via - # paramiko - # secretstorage dataclasses-json==0.5.6 # via flytekit -decorator==5.1.0 - # via - # ipython - # retry -defusedxml==0.7.1 - # via nbconvert +decorator==5.1.1 + # via retry deprecated==1.2.13 # via flytekit -diskcache==5.3.0 +diskcache==5.4.0 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -entrypoints==0.3 - # via - # jupyter-client - # nbconvert - # papermill -flyteidl==0.21.13 +flyteidl==0.22.0 # via flytekit -gevent==21.12.0 - # via sagemaker-training -greenlet==1.1.2 - # via gevent grpcio==1.43.0 # via flytekit -hmsclient==0.1.1 - # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.0 +importlib-metadata==4.10.1 # via keyring -inotify_simple==1.2.1 - # via sagemaker-training -ipykernel==5.5.6 - # via flytekit -ipython==7.30.1 - # via ipykernel -ipython-genutils==0.2.0 - # via - # ipykernel - # nbformat -jedi==0.18.1 - # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter # jinja2-time - # nbconvert jinja2-time==0.2.0 # via cookiecutter -jmespath==0.10.0 - # via - # boto3 - # botocore jsonschema==3.2.0 - # via - # -r requirements.in - # nbformat -jupyter-client==7.1.0 - # via - # ipykernel - # nbclient -jupyter-core==4.9.1 - # via - # jupyter-client - # nbconvert - # nbformat -jupyterlab-pygments==0.1.2 - # via nbconvert -k8s-proto==0.0.3 - # via flytekit -keyring==23.4.0 + # via -r requirements.in +keyring==23.5.0 # via flytekit markupsafe==2.0.1 # via jinja2 @@ -152,100 +73,33 @@ marshmallow-enum==1.5.1 # via dataclasses-json marshmallow-jsonschema==0.13.0 # via flytekit -matplotlib-inline==0.1.3 - # via ipython -mistune==0.8.4 - # via nbconvert mypy-extensions==0.4.3 - # via - # black - # typing-inspect -natsort==8.0.2 + # via typing-inspect +natsort==8.1.0 # via flytekit -nbclient==0.5.9 +numpy==1.22.2 # via - # nbconvert - # papermill -nbconvert==6.3.0 - # via flytekit -nbformat==5.1.3 - # via - # nbclient - # nbconvert - # papermill -nest-asyncio==1.5.4 - # via - # jupyter-client - # nbclient -numpy==1.21.5 - # via - # flytekit # pandas # pyarrow - # sagemaker-training - # scipy -packaging==21.3 - # via bleach -pandas==1.3.5 +pandas==1.4.0 # via flytekit -pandocfilters==1.5.0 - # via nbconvert -papermill==2.3.3 - # via flytekit -paramiko==2.8.1 - # via sagemaker-training -parso==0.8.3 - # via jedi -pathspec==0.9.0 - # via black -pexpect==4.8.0 - # via ipython -pickleshare==0.7.5 - # via ipython -platformdirs==2.4.0 - # via black poyo==0.5.0 # via cookiecutter -prompt-toolkit==3.0.24 - # via ipython -protobuf==3.19.1 +protobuf==3.19.4 # via # flyteidl # flytekit - # k8s-proto - # sagemaker-training -psutil==5.8.0 - # via sagemaker-training -ptyprocess==0.7.0 - # via pexpect py==1.11.0 # via retry -py4j==0.10.9.2 - # via pyspark pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi -pygments==2.10.0 - # via - # ipython - # jupyterlab-pygments - # nbconvert -pynacl==1.4.0 - # via paramiko -pyparsing==3.0.6 - # via packaging -pyrsistent==0.18.0 +pyrsistent==0.18.1 # via jsonschema -pyspark==3.2.0 - # via flytekit -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via # arrow - # botocore # croniter # flytekit - # jupyter-client # pandas python-json-logger==2.0.2 # via flytekit @@ -258,110 +112,49 @@ pytz==2021.3 # flytekit # pandas pyyaml==5.4.1 - # via - # -r requirements.in - # papermill -pyzmq==22.3.0 - # via jupyter-client -regex==2021.11.10 + # via -r requirements.in +regex==2022.1.18 # via docker-image-py -requests==2.26.0 +requests==2.27.1 # via # cookiecutter # flytekit - # papermill # responses -responses==0.16.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -retrying==1.3.3 - # via sagemaker-training -s3transfer==0.5.0 - # via boto3 -sagemaker-training==3.9.2 - # via flytekit -scipy==1.7.3 - # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.16.0 # via - # bcrypt - # bleach # cookiecutter - # flytekit # grpcio # jsonschema - # pynacl # python-dateutil - # responses - # retrying - # sagemaker-training - # thrift sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 # via flytekit -tenacity==8.0.1 - # via papermill -testpath==0.5.0 - # via nbconvert text-unidecode==1.3 # via python-slugify -textwrap3==0.9.2 - # via ansiwrap -thrift==0.15.0 - # via hmsclient -tomli==1.2.3 - # via black -tornado==6.1 - # via - # ipykernel - # jupyter-client -tqdm==4.62.3 - # via papermill -traitlets==5.1.1 - # via - # ipykernel - # ipython - # jupyter-client - # jupyter-core - # matplotlib-inline - # nbclient - # nbconvert - # nbformat typing-extensions==4.0.1 # via - # black + # flytekit # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.7 +urllib3==1.26.8 # via - # botocore # flytekit # requests # responses -wcwidth==0.2.5 - # via prompt-toolkit -webencodings==0.5.1 - # via bleach -werkzeug==2.0.2 - # via sagemaker-training wheel==0.37.1 # via flytekit wrapt==1.13.3 # via # deprecated # flytekit -zipp==3.6.0 +zipp==3.7.0 # via importlib-metadata -zope.event==4.5.0 - # via gevent -zope.interface==5.4.0 - # via gevent # The following packages are considered to be unsafe in a requirements file: -# pip -# setuptools +# setuptools \ No newline at end of file diff --git a/setup.py b/setup.py index 3a286ee448..4eac97749f 100644 --- a/setup.py +++ b/setup.py @@ -64,6 +64,7 @@ ] }, install_requires=[ + "flyteidl>=0.22.0", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=4.0.0,<7.0.0", diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index 136c6eb95e..cd8604cf99 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -4,17 +4,15 @@ # # make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt # -arrow==1.2.1 +arrow==1.2.2 # via jinja2-time binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot -charset-normalizer==2.0.9 +charset-normalizer==2.0.11 # via requests checksumdir==1.2.0 # via flytekit @@ -26,40 +24,34 @@ cloudpickle==2.0.0 # via flytekit cookiecutter==1.7.3 # via flytekit -croniter==1.1.0 +croniter==1.2.0 # via flytekit -cryptography==36.0.1 - # via secretstorage cycler==0.11.0 # via matplotlib dataclasses-json==0.5.6 # via flytekit -decorator==5.1.0 +decorator==5.1.1 # via retry deprecated==1.2.13 # via flytekit -diskcache==5.3.0 +diskcache==5.4.0 # via flytekit docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.21.13 +flyteidl==0.22.0 # via flytekit -flytekit==0.25.0 +flytekit==0.30.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -fonttools==4.28.5 +fonttools==4.29.1 # via matplotlib grpcio==1.43.0 # via flytekit idna==3.3 # via requests -importlib-metadata==4.10.0 +importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -68,7 +60,7 @@ jinja2-time==0.2.0 # via cookiecutter joblib==1.1.0 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in -keyring==23.4.0 +keyring==23.5.0 # via flytekit kiwisolver==1.3.2 # via matplotlib @@ -87,25 +79,25 @@ matplotlib==3.5.1 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in mypy-extensions==0.4.3 # via typing-inspect -natsort==8.0.2 +natsort==8.1.0 # via flytekit -numpy==1.21.5 +numpy==1.22.2 # via # matplotlib # opencv-python # pandas # pyarrow -opencv-python==4.5.4.60 +opencv-python==4.5.5.62 # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in packaging==21.3 # via matplotlib -pandas==1.3.5 +pandas==1.4.0 # via flytekit -pillow==8.4.0 +pillow==9.0.1 # via matplotlib poyo==0.5.0 # via cookiecutter -protobuf==3.19.1 +protobuf==3.19.4 # via # flyteidl # flytekit @@ -113,15 +105,12 @@ py==1.11.0 # via retry pyarrow==6.0.1 # via flytekit -pycparser==2.21 - # via cffi -pyparsing==3.0.6 +pyparsing==3.0.7 # via # matplotlib # packaging -python-dateutil==2.8.1 +python-dateutil==2.8.2 # via - # arrow # croniter # flytekit # matplotlib @@ -132,30 +121,26 @@ python-slugify==5.0.2 # via cookiecutter pytimeparse==1.1.8 # via flytekit -pytz==2018.4 +pytz==2021.3 # via # flytekit # pandas -regex==2021.11.10 +regex==2022.1.18 # via docker-image-py -requests==2.26.0 +requests==2.27.1 # via # cookiecutter # flytekit # responses -responses==0.16.0 +responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter - # flytekit # grpcio # python-dateutil - # responses sortedcontainers==2.4.0 # via flytekit statsd==3.3.0 @@ -163,10 +148,12 @@ statsd==3.3.0 text-unidecode==1.3 # via python-slugify typing-extensions==4.0.1 - # via typing-inspect + # via + # flytekit + # typing-inspect typing-inspect==0.7.1 # via dataclasses-json -urllib3==1.26.7 +urllib3==1.26.8 # via # flytekit # requests @@ -179,5 +166,5 @@ wrapt==1.13.3 # via # deprecated # flytekit -zipp==3.6.0 +zipp==3.7.0 # via importlib-metadata \ No newline at end of file From 9f62e9c0505e44110e6bc96253ec2ee31b387280 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 22:36:32 -0800 Subject: [PATCH 31/42] fix: numpy version Signed-off-by: Kenny Workman --- requirements.in | 2 ++ setup.py | 1 + 2 files changed, 3 insertions(+) diff --git a/requirements.in b/requirements.in index 408185fca8..065021a77d 100644 --- a/requirements.in +++ b/requirements.in @@ -7,3 +7,5 @@ attrs<21 # https://github.com/flyteorg/flyte/issues/1732. jsonschema<4 pyyaml<6 +# numpy dropped support for python 3.7 starting on this version. +numpy<1.22.0 diff --git a/setup.py b/setup.py index 4eac97749f..425267661f 100644 --- a/setup.py +++ b/setup.py @@ -96,6 +96,7 @@ "checksumdir>=1.2.0", "cloudpickle>=2.0.0", "cookiecutter>=1.7.3", + "numpy<=1.22.1; python_version < '3.8.0'", ], extras_require=extras_require, scripts=[ From 4dbbac9c8446e4a672fec3523f98a470fc1bb390 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 22:36:37 -0800 Subject: [PATCH 32/42] fix: lint Signed-off-by: Kenny Workman --- tests/flytekit/unit/core/test_typing_annotation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flytekit/unit/core/test_typing_annotation.py b/tests/flytekit/unit/core/test_typing_annotation.py index 9922ab67dd..b57d7a4299 100644 --- a/tests/flytekit/unit/core/test_typing_annotation.py +++ b/tests/flytekit/unit/core/test_typing_annotation.py @@ -3,12 +3,12 @@ import typing_extensions -from flytekit.common.translator import get_serializable from flytekit.core import context_manager from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import Image, ImageConfig from flytekit.core.task import task from flytekit.models.annotation import TypeAnnotation +from flytekit.common.translator import get_serializable default_img = Image(name="default", fqn="test", tag="tag") serialization_settings = context_manager.SerializationSettings( From 75c91d91c0e16694c951cbfb652b4e3d0e95ba3b Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 22:37:28 -0800 Subject: [PATCH 33/42] fix: pandas version Signed-off-by: Kenny Workman --- requirements.in | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.in b/requirements.in index 065021a77d..43bd57fdd3 100644 --- a/requirements.in +++ b/requirements.in @@ -7,5 +7,6 @@ attrs<21 # https://github.com/flyteorg/flyte/issues/1732. jsonschema<4 pyyaml<6 -# numpy dropped support for python 3.7 starting on this version. +# Both numpy and pandas dropped support for python 3.7 in their latest releases numpy<1.22.0 +pandas<1.4.0 From 183f1102e96cebbe2cb4bdb4913b2cdb8f9972bd Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 22:42:53 -0800 Subject: [PATCH 34/42] fix: bump requirements Signed-off-by: Kenny Workman --- dev-requirements.txt | 4 ++-- requirements-spark2.txt | 9 ++++++--- requirements.txt | 9 ++++++--- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index bd472a45d9..409ad5d1f6 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -220,7 +220,7 @@ natsort==8.1.0 # flytekit nodeenv==1.6.0 # via pre-commit -numpy==1.22.2 +numpy==1.21.5 # via # -c requirements.txt # pandas @@ -233,7 +233,7 @@ packaging==21.3 # google-cloud-bigquery >>>>>>> 6b5e7a8c (fix: bump flytekit) # pytest -pandas==1.4.0 +pandas==1.3.5 # via # -c requirements.txt # flytekit diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 807fee354e..87ee5b86ba 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -79,12 +79,15 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.21.5 # via + # -r requirements.in # pandas # pyarrow -pandas==1.4.0 - # via flytekit +pandas==1.3.5 + # via + # -r requirements.in + # flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 diff --git a/requirements.txt b/requirements.txt index 3557ae4dc9..10156e8bb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -77,12 +77,15 @@ mypy-extensions==0.4.3 # via typing-inspect natsort==8.1.0 # via flytekit -numpy==1.22.2 +numpy==1.21.5 # via + # -r requirements.in # pandas # pyarrow -pandas==1.4.0 - # via flytekit +pandas==1.3.5 + # via + # -r requirements.in + # flytekit poyo==0.5.0 # via cookiecutter protobuf==3.19.4 From d3f483067444ccd7a7fca726b5d88d6323983b0b Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 22:48:39 -0800 Subject: [PATCH 35/42] fix: test import Signed-off-by: Kenny Workman --- tests/flytekit/unit/core/test_typing_annotation.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/flytekit/unit/core/test_typing_annotation.py b/tests/flytekit/unit/core/test_typing_annotation.py index b57d7a4299..f999c62612 100644 --- a/tests/flytekit/unit/core/test_typing_annotation.py +++ b/tests/flytekit/unit/core/test_typing_annotation.py @@ -8,7 +8,7 @@ from flytekit.core.context_manager import Image, ImageConfig from flytekit.core.task import task from flytekit.models.annotation import TypeAnnotation -from flytekit.common.translator import get_serializable +from flytekit.tools.translator import get_serializable default_img = Image(name="default", fqn="test", tag="tag") serialization_settings = context_manager.SerializationSettings( From 854b96f4157aa5ae28618070b24b92dcb5dbdd78 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 22:58:46 -0800 Subject: [PATCH 36/42] fix: flake8 lint Signed-off-by: Kenny Workman --- flytekit/core/type_engine.py | 1 - 1 file changed, 1 deletion(-) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index b5b1681c90..08247970d6 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -20,7 +20,6 @@ from google.protobuf.struct_pb2 import Struct from marshmallow_enum import EnumField, LoadDumpOptions from marshmallow_jsonschema import JSONSchema -from typing_extensions import get_args, get_origin from flytekit.common.exceptions import user as user_exceptions from flytekit.common.types import primitives as _primitives From 2ec2d94ff58f2588afb00bc587d1200ff452edfe Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 23:23:00 -0800 Subject: [PATCH 37/42] fix: merge Signed-off-by: Kenny Workman --- dev-requirements.txt | 2 +- doc-requirements.txt | 2 +- plugins/flytekit-greatexpectations/requirements.txt | 2 +- plugins/flytekit-papermill/requirements.txt | 2 +- requirements-spark2.txt | 2 +- requirements.txt | 2 +- .../remote/mock_flyte_repo/workflows/requirements.txt | 2 +- 7 files changed, 7 insertions(+), 7 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 1af87b2074..95d809f295 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -414,4 +414,4 @@ zipp==3.7.0 # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/doc-requirements.txt b/doc-requirements.txt index cbde3f171c..8d8749c422 100644 --- a/doc-requirements.txt +++ b/doc-requirements.txt @@ -250,4 +250,4 @@ wrapt==1.13.3 zipp==3.7.0 # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index bbc775d355..f8813d43da 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -352,4 +352,4 @@ zipp==3.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index ce92c532dd..2b483ea875 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -294,4 +294,4 @@ zipp==3.6.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 8e0919b48e..ee3cffb260 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -162,4 +162,4 @@ zipp==3.7.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/requirements.txt b/requirements.txt index 10156e8bb3..53f57a2211 100644 --- a/requirements.txt +++ b/requirements.txt @@ -160,4 +160,4 @@ zipp==3.7.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index cd8604cf99..d161e9b46b 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -167,4 +167,4 @@ wrapt==1.13.3 # deprecated # flytekit zipp==3.7.0 - # via importlib-metadata \ No newline at end of file + # via importlib-metadata From 956f8e6197c77f424bbab816a4eec915e14716cf Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 23:33:47 -0800 Subject: [PATCH 38/42] fix: requirements Signed-off-by: Kenny Workman --- dev-requirements.txt | 4 ++-- requirements.txt | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 95d809f295..5dca22d856 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -243,7 +243,7 @@ proto-plus==1.19.8 # via # google-cloud-bigquery # google-cloud-bigquery-storage -protobuf==3.19.1 +protobuf==3.19.4 # via # -c requirements.txt # flyteidl @@ -414,4 +414,4 @@ zipp==3.7.0 # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools +# setuptools \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 53f57a2211..10156e8bb3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -160,4 +160,4 @@ zipp==3.7.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools +# setuptools \ No newline at end of file From 19d564258f9f07ecd0c95ae3a43645d44d6e3b24 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 23:38:44 -0800 Subject: [PATCH 39/42] fix: requirements Signed-off-by: Kenny Workman --- dev-requirements.txt | 2 +- requirements-spark2.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index 5dca22d856..dea9a03b69 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -387,7 +387,7 @@ typing-inspect==0.7.1 # via # -c requirements.txt # dataclasses-json -urllib3==1.26.7 +urllib3==1.26.8 # via # -c requirements.txt # flytekit diff --git a/requirements-spark2.txt b/requirements-spark2.txt index ee3cffb260..8e0919b48e 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -162,4 +162,4 @@ zipp==3.7.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools +# setuptools \ No newline at end of file From 755f31f85992565353b39775b57146e5f9f36c28 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Mon, 7 Feb 2022 23:39:52 -0800 Subject: [PATCH 40/42] fix: lint Signed-off-by: Kenny Workman --- dev-requirements.txt | 2 +- requirements-spark2.txt | 2 +- requirements.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/dev-requirements.txt b/dev-requirements.txt index dea9a03b69..63721ac672 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -414,4 +414,4 @@ zipp==3.7.0 # importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/requirements-spark2.txt b/requirements-spark2.txt index 8e0919b48e..ee3cffb260 100644 --- a/requirements-spark2.txt +++ b/requirements-spark2.txt @@ -162,4 +162,4 @@ zipp==3.7.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools diff --git a/requirements.txt b/requirements.txt index 10156e8bb3..53f57a2211 100644 --- a/requirements.txt +++ b/requirements.txt @@ -160,4 +160,4 @@ zipp==3.7.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools From 4d64acd1308003b3610a434aef18d702ade1e403 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 8 Feb 2022 09:57:42 -0800 Subject: [PATCH 41/42] fix: papermill req Signed-off-by: Kenny Workman --- plugins/flytekit-papermill/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index b06adde647..bec12b9b46 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -71,7 +71,7 @@ entrypoints==0.3 # papermill executing==0.8.2 # via stack-data -flyteidl==0.21.24 +flyteidl==0.22.0 # via flytekit flytekit==0.26.1 # via flytekitplugins-papermill @@ -304,4 +304,4 @@ zipp==3.7.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools +# setuptools \ No newline at end of file From 88bc4d09f580b5c0c455aaf2cf30703edb680169 Mon Sep 17 00:00:00 2001 From: Kenny Workman Date: Tue, 8 Feb 2022 10:21:21 -0800 Subject: [PATCH 42/42] fix: req Signed-off-by: Kenny Workman --- plugins/flytekit-papermill/dev-requirements.txt | 2 +- plugins/flytekit-papermill/requirements.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt index 849cef4c9f..d647708f20 100644 --- a/plugins/flytekit-papermill/dev-requirements.txt +++ b/plugins/flytekit-papermill/dev-requirements.txt @@ -38,7 +38,7 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.21.23 +flyteidl==0.22.0 # via flytekit flytekit==0.26.0 # via flytekitplugins-spark diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index bec12b9b46..8c3e4570e9 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -304,4 +304,4 @@ zipp==3.7.0 # via importlib-metadata # The following packages are considered to be unsafe in a requirements file: -# setuptools \ No newline at end of file +# setuptools