diff --git a/.gitignore b/.gitignore index ec56a0f239..73c81d2a6a 100644 --- a/.gitignore +++ b/.gitignore @@ -3,7 +3,7 @@ *.pyt *.pytc *.egg-info -.*.swp +.*.sw* .DS_Store venv/ .venv/ diff --git a/dev-requirements.txt b/dev-requirements.txt index 28c22afa15..6e17840325 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -32,7 +32,6 @@ certifi==2021.10.8 # requests cffi==1.15.0 # via - # -c requirements.txt # bcrypt # cryptography # pynacl diff --git a/flytekit/core/annotation.py b/flytekit/core/annotation.py new file mode 100644 index 0000000000..b4a70a6469 --- /dev/null +++ b/flytekit/core/annotation.py @@ -0,0 +1,30 @@ +from typing import Any, Dict + + +class FlyteAnnotation: + """A core object to add arbitrary annotations to flyte types. + + This metadata is ingested as a python dictionary and will be serialized + into fields on the flyteidl type literals. This data is not accessible at + runtime but rather can be retrieved from flyteadmin for custom presentation + of typed parameters. + + Flytekit expects to receive a maximum of one `FlyteAnnotation` object + within each typehint. + + For a task definition: + + .. code-block:: python + + @task + def x(a: typing.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})]): + return + + """ + + def __init__(self, data: Dict[str, Any]): + self._data = data + + @property + def data(self): + return self._data diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 07faf178ce..62f2daadc0 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -25,12 +25,14 @@ from marshmallow_enum import EnumField, LoadDumpOptions from marshmallow_jsonschema import JSONSchema +from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import FlyteContext from flytekit.core.type_helpers import load_type_from_tag from flytekit.exceptions import user as user_exceptions from flytekit.loggers import logger from flytekit.models import interface as _interface_models from flytekit.models import types as _type_models +from flytekit.models.annotation import TypeAnnotation as TypeAnnotationModel from flytekit.models.core import types as _core_types from flytekit.models.literals import ( Blob, @@ -235,6 +237,12 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: Extracts the Literal type definition for a Dataclass and returns a type Struct. If possible also extracts the JSONSchema for the dataclass. """ + if get_origin(t) is Annotated: + raise ValueError( + "Flytekit does not currently have support for FlyteAnnotations applied to Dataclass." + f"Type {t} cannot be parsed." + ) + if not issubclass(t, DataClassJsonMixin): raise AssertionError( f"Dataclass {t} should be decorated with @dataclass_json to be " f"serialized correctly" @@ -551,6 +559,7 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: TODO lets make this deterministic by using an ordered dict """ + # Step 1 if get_origin(python_type) is Annotated: python_type = get_args(python_type)[0] @@ -560,8 +569,14 @@ def get_transformer(cls, python_type: Type) -> TypeTransformer[T]: # Step 2 if hasattr(python_type, "__origin__"): + # Handling of annotated generics, eg: + # Annotated[typing.List[int], 'foo'] + if get_origin(python_type) is Annotated: + return cls.get_transformer(get_args(python_type)[0]) + if python_type.__origin__ in cls._REGISTRY: return cls._REGISTRY[python_type.__origin__] + raise ValueError(f"Generic Type {python_type.__origin__} not supported currently in Flytekit.") # Step 3 @@ -590,7 +605,30 @@ def to_literal_type(cls, python_type: Type) -> LiteralType: Converts a python type into a flyte specific ``LiteralType`` """ transformer = cls.get_transformer(python_type) - return transformer.get_literal_type(python_type) + res = transformer.get_literal_type(python_type) + data = None + if get_origin(python_type) is Annotated: + for x in get_args(python_type)[1:]: + if not isinstance(x, FlyteAnnotation): + continue + if data is not None: + raise ValueError( + f"More than one FlyteAnnotation used within {python_type} typehint. Flytekit requires a max of one." + ) + data = x.data + if data is not None: + idl_type_annotation = TypeAnnotationModel(annotations=data) + return LiteralType( + simple=res.simple, + schema=res.schema, + collection_type=res.collection_type, + map_value_type=res.map_value_type, + blob=res.blob, + enum_type=res.enum_type, + metadata=res.metadata, + annotation=idl_type_annotation, + ) + return res @classmethod def to_literal(cls, ctx: FlyteContext, python_val: typing.Any, python_type: Type, expected: LiteralType) -> Literal: @@ -718,9 +756,16 @@ def get_sub_type(t: Type[T]) -> Type[T]: """ Return the generic Type T of the List """ - if hasattr(t, "__origin__") and t.__origin__ is list: # type: ignore - if hasattr(t, "__args__"): - return t.__args__[0] # type: ignore + + if hasattr(t, "__origin__"): + # Handle annotation on list generic, eg: + # Annotated[typing.List[int], 'foo'] + if get_origin(t) is Annotated: + return ListTransformer.get_sub_type(get_args(t)[0]) + + if t.__origin__ is list and hasattr(t, "__args__"): + return t.__args__[0] + raise ValueError("Only generic univariate typing.List[T] type is supported.") def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]: @@ -763,9 +808,17 @@ def get_dict_types(t: Optional[Type[dict]]) -> typing.Tuple[Optional[type], Opti """ Return the generic Type T of the Dict """ - if hasattr(t, "__origin__") and t.__origin__ is dict: # type: ignore - if hasattr(t, "__args__"): - return t.__args__ # type: ignore + _origin = get_origin(t) + _args = get_args(t) + if _origin is not None: + if _origin is Annotated: + raise ValueError( + f"Flytekit does not currently have support \ + for FlyteAnnotations applied to dicts. {t} cannot be \ + parsed." + ) + if _origin is dict and _args is not None: + return _args return None, None @staticmethod @@ -913,6 +966,13 @@ def __init__(self): super().__init__(name="DefaultEnumTransformer", t=enum.Enum) def get_literal_type(self, t: Type[T]) -> LiteralType: + if get_origin(t) is Annotated: + raise ValueError( + f"Flytekit does not currently have support \ + for FlyteAnnotations applied to enums. {t} cannot be \ + parsed." + ) + values = [v.value for v in t] # type: ignore if not isinstance(values[0], str): raise AssertionError("Only EnumTypes with value of string are supported") diff --git a/flytekit/models/annotation.py b/flytekit/models/annotation.py new file mode 100644 index 0000000000..ced935c57e --- /dev/null +++ b/flytekit/models/annotation.py @@ -0,0 +1,43 @@ +import json as _json +from typing import Any, Dict + +from flyteidl.core import types_pb2 as _types_pb2 +from google.protobuf import json_format as _json_format +from google.protobuf import struct_pb2 as _struct + + +class TypeAnnotation: + """Python class representation of the flyteidl TypeAnnotation message.""" + + def __init__(self, annotations: Dict[str, Any]): + self._annotations = annotations + + @property + def annotations(self) -> Dict[str, Any]: + """ + :rtype: dict[str, Any] + """ + return self._annotations + + def to_flyte_idl(self) -> _types_pb2.TypeAnnotation: + """ + :rtype: flyteidl.core.types_pb2.TypeAnnotation + """ + + if self._annotations is not None: + annotations = _json_format.Parse(_json.dumps(self.annotations), _struct.Struct()) + else: + annotations = None + + return _types_pb2.TypeAnnotation( + annotations=annotations, + ) + + @classmethod + def from_flyte_idl(cls, proto): + """ + :param flyteidl.core.types_pb2.TypeAnnotation proto: + :rtype: TypeAnnotation + """ + + return cls(annotations=_json_format.MessageToDict(proto.annotations)) diff --git a/flytekit/models/types.py b/flytekit/models/types.py index 09ed947642..0a0a2f8ced 100644 --- a/flytekit/models/types.py +++ b/flytekit/models/types.py @@ -6,6 +6,7 @@ from google.protobuf import struct_pb2 as _struct from flytekit.models import common as _common +from flytekit.models.annotation import TypeAnnotation as TypeAnnotationModel from flytekit.models.core import types as _core_types @@ -195,6 +196,7 @@ def __init__( enum_type=None, structured_dataset_type=None, metadata=None, + annotation=None, ): """ This is a oneof message, only one of the kwargs may be set, representing one of the Flyte types. @@ -208,6 +210,8 @@ def __init__( :param flytekit.models.core.types.EnumType enum_type: For enum objects, describes an enum :param flytekit.models.core.types.StructuredDatasetType structured_dataset_type: structured dataset :param dict[Text, T] metadata: Additional data describing the type + :param flytekit.models.annotation.FlyteAnnotation annotation: Additional data + describing the type _intended to be saturated by the client_ """ self._simple = simple self._schema = schema @@ -217,6 +221,7 @@ def __init__( self._enum_type = enum_type self._structured_dataset_type = structured_dataset_type self._metadata = metadata + self._annotation = annotation @property def simple(self) -> SimpleType: @@ -259,18 +264,31 @@ def metadata(self): """ return self._metadata + @property + def annotation(self) -> TypeAnnotationModel: + """ + :rtype: flytekit.models.annotation.TypeAnnotation + """ + return self._annotation + @metadata.setter def metadata(self, value): self._metadata = value + @annotation.setter + def annotation(self, value): + self.annotation = value + def to_flyte_idl(self): """ :rtype: flyteidl.core.types_pb2.LiteralType """ + if self.metadata is not None: metadata = _json_format.Parse(_json.dumps(self.metadata), _struct.Struct()) else: metadata = None + t = _types_pb2.LiteralType( simple=self.simple if self.simple is not None else None, schema=self.schema.to_flyte_idl() if self.schema is not None else None, @@ -282,6 +300,7 @@ def to_flyte_idl(self): if self.structured_dataset_type else None, metadata=metadata, + annotation=self.annotation.to_flyte_idl() if self.annotation else None, ) return t @@ -308,6 +327,7 @@ def from_flyte_idl(cls, proto): if proto.HasField("structured_dataset_type") else None, metadata=_json_format.MessageToDict(proto.metadata) or None, + annotation=TypeAnnotationModel.from_flyte_idl(proto.annotation) if proto.HasField("annotation") else None, ) diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt index 6fe91c53c8..a1acae494f 100644 --- a/plugins/flytekit-aws-athena/requirements.txt +++ b/plugins/flytekit-aws-athena/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -54,10 +52,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt index 2624f28a98..11396f213e 100644 --- a/plugins/flytekit-aws-sagemaker/requirements.txt +++ b/plugins/flytekit-aws-sagemaker/requirements.txt @@ -73,10 +73,6 @@ importlib-metadata==4.10.1 # via keyring inotify-simple==1.2.1 # via sagemaker-training -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -167,8 +163,6 @@ sagemaker-training==3.9.2 # via flytekitplugins-awssagemaker scipy==1.8.0 # via sagemaker-training -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # bcrypt diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt index a14a5bf6cf..c644ac98e7 100644 --- a/plugins/flytekit-data-fsspec/requirements.txt +++ b/plugins/flytekit-data-fsspec/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -56,10 +54,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -128,8 +122,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt index 559d5b2085..01dfe2cb6b 100644 --- a/plugins/flytekit-dolt/requirements.txt +++ b/plugins/flytekit-dolt/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -60,10 +58,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -134,8 +128,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt index fdbb857264..624f3dc633 100644 --- a/plugins/flytekit-greatexpectations/requirements.txt +++ b/plugins/flytekit-greatexpectations/requirements.txt @@ -110,10 +110,6 @@ ipywidgets==7.6.5 # via great-expectations jedi==0.18.1 # via ipython -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # altair @@ -295,8 +291,6 @@ ruamel.yaml.clib==0.2.6 # via ruamel.yaml scipy==1.7.3 # via great-expectations -secretstorage==3.3.1 - # via keyring send2trash==1.8.0 # via notebook six==1.16.0 diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt index 269d52fceb..3ae057f9a0 100644 --- a/plugins/flytekit-hive/requirements.txt +++ b/plugins/flytekit-hive/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -54,10 +52,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt index 6574f4ab66..d60d9c46b6 100644 --- a/plugins/flytekit-k8s-pod/requirements.txt +++ b/plugins/flytekit-k8s-pod/requirements.txt @@ -16,8 +16,6 @@ certifi==2021.10.8 # via # kubernetes # requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -60,10 +58,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -151,8 +145,6 @@ retry==0.9.2 # via flytekit rsa==4.8 # via google-auth -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt index ef91243daf..c13f9f3b74 100644 --- a/plugins/flytekit-kf-mpi/requirements.txt +++ b/plugins/flytekit-kf-mpi/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -56,10 +54,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -128,8 +122,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt index 9fc8665f43..1696646c87 100644 --- a/plugins/flytekit-kf-pytorch/requirements.txt +++ b/plugins/flytekit-kf-pytorch/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -54,10 +52,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt index ae428e9215..d3b6253664 100644 --- a/plugins/flytekit-kf-tensorflow/requirements.txt +++ b/plugins/flytekit-kf-tensorflow/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -54,10 +52,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-modin/requirements.txt b/plugins/flytekit-modin/requirements.txt index adaa0b7307..4f3b01e5e4 100644 --- a/plugins/flytekit-modin/requirements.txt +++ b/plugins/flytekit-modin/requirements.txt @@ -16,8 +16,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -69,10 +67,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -166,8 +160,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt index 82df51d30e..f0b65cec17 100644 --- a/plugins/flytekit-pandera/requirements.txt +++ b/plugins/flytekit-pandera/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -54,10 +52,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -137,8 +131,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt index 849cef4c9f..d647708f20 100644 --- a/plugins/flytekit-papermill/dev-requirements.txt +++ b/plugins/flytekit-papermill/dev-requirements.txt @@ -38,7 +38,7 @@ docker-image-py==0.1.12 # via flytekit docstring-parser==0.13 # via flytekit -flyteidl==0.21.23 +flyteidl==0.22.0 # via flytekit flytekit==0.26.0 # via flytekitplugins-spark diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt index b06adde647..8c3e4570e9 100644 --- a/plugins/flytekit-papermill/requirements.txt +++ b/plugins/flytekit-papermill/requirements.txt @@ -71,7 +71,7 @@ entrypoints==0.3 # papermill executing==0.8.2 # via stack-data -flyteidl==0.21.24 +flyteidl==0.22.0 # via flytekit flytekit==0.26.1 # via flytekitplugins-papermill diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt index b563e54814..53951100b5 100644 --- a/plugins/flytekit-snowflake/requirements.txt +++ b/plugins/flytekit-snowflake/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -54,10 +52,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -126,8 +120,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt index fed298d4b4..75d1e32d53 100644 --- a/plugins/flytekit-spark/requirements.txt +++ b/plugins/flytekit-spark/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -54,10 +52,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -130,8 +124,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt index 134f8960e8..93dcc2790c 100644 --- a/plugins/flytekit-sqlalchemy/requirements.txt +++ b/plugins/flytekit-sqlalchemy/requirements.txt @@ -12,8 +12,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -56,10 +54,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -128,8 +122,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/setup.py b/setup.py index 48250dac6b..885f32b897 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ ] }, install_requires=[ - "flyteidl>=0.21.17", + "flyteidl>=0.22.0", "wheel>=0.30.0,<1.0.0", "pandas>=1.0.0,<2.0.0", "pyarrow>=4.0.0,<7.0.0", @@ -67,6 +67,7 @@ "checksumdir>=1.2.0", "cloudpickle>=2.0.0", "cookiecutter>=1.7.3", + "numpy<=1.22.1; python_version < '3.8.0'", ], extras_require=extras_require, scripts=[ diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt index dc2ddebe81..93514898b2 100644 --- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt +++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt @@ -10,8 +10,6 @@ binaryornot==0.4.4 # via cookiecutter certifi==2021.10.8 # via requests -cffi==1.15.0 - # via cryptography chardet==4.0.0 # via binaryornot charset-normalizer==2.0.11 @@ -28,8 +26,6 @@ cookiecutter==1.7.3 # via flytekit croniter==1.2.0 # via flytekit -cryptography==36.0.1 - # via secretstorage cycler==0.11.0 # via matplotlib dataclasses-json==0.5.6 @@ -56,10 +52,6 @@ idna==3.3 # via requests importlib-metadata==4.10.1 # via keyring -jeepney==0.7.1 - # via - # keyring - # secretstorage jinja2==3.0.3 # via # cookiecutter @@ -121,7 +113,6 @@ pyparsing==3.0.7 # packaging python-dateutil==2.8.2 # via - # arrow # croniter # flytekit # matplotlib @@ -147,8 +138,6 @@ responses==0.18.0 # via flytekit retry==0.9.2 # via flytekit -secretstorage==3.3.1 - # via keyring six==1.16.0 # via # cookiecutter diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index d738d12ea5..df1ae3e11f 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -9,6 +9,7 @@ import pandas as pd import pyarrow as pa import pytest +import typing_extensions from dataclasses_json import DataClassJsonMixin, dataclass_json from flyteidl.core import errors_pb2 from google.protobuf import json_format as _json_format @@ -18,6 +19,7 @@ from pandas._testing import assert_frame_equal from flytekit import kwtypes +from flytekit.core.annotation import FlyteAnnotation from flytekit.core.context_manager import FlyteContext, FlyteContextManager from flytekit.core.type_engine import ( DataclassTransformer, @@ -31,6 +33,7 @@ ) from flytekit.exceptions import user as user_exceptions from flytekit.models import types as model_types +from flytekit.models.annotation import TypeAnnotation from flytekit.models.core.types import BlobType from flytekit.models.literals import Blob, BlobMetadata, Literal, LiteralCollection, LiteralMap, Primitive, Scalar from flytekit.models.types import LiteralType, SimpleType @@ -840,6 +843,65 @@ def test_dict_to_literal_map_with_wrong_input_type(): TypeEngine.dict_to_literal_map(ctx, input, guessed_python_types) +def test_annotated_simple_types(): + def _check_annotation(t, annotation): + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.annotation, TypeAnnotation) + assert lt.annotation.annotations == annotation + + _check_annotation(typing_extensions.Annotated[int, FlyteAnnotation({"foo": "bar"})], {"foo": "bar"}) + _check_annotation(typing_extensions.Annotated[int, FlyteAnnotation(["foo", "bar"])], ["foo", "bar"]) + _check_annotation( + typing_extensions.Annotated[int, FlyteAnnotation({"d": {"test": "data"}, "l": ["nested", ["list"]]})], + {"d": {"test": "data"}, "l": ["nested", ["list"]]}, + ) + _check_annotation( + typing_extensions.Annotated[int, FlyteAnnotation(InnerStruct(a=1, b="fizz", c=[1]))], + InnerStruct(a=1, b="fizz", c=[1]), + ) + + +def test_annotated_list(): + t = typing_extensions.Annotated[typing.List[int], FlyteAnnotation({"foo": "bar"})] + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.annotation, TypeAnnotation) + assert lt.annotation.annotations == {"foo": "bar"} + + t = typing.List[typing_extensions.Annotated[int, FlyteAnnotation({"foo": "bar"})]] + lt = TypeEngine.to_literal_type(t) + assert isinstance(lt.collection_type.annotation, TypeAnnotation) + assert lt.collection_type.annotation.annotations == {"foo": "bar"} + + +def test_type_alias(): + inner_t = typing_extensions.Annotated[int, FlyteAnnotation("foo")] + t = typing_extensions.Annotated[inner_t, FlyteAnnotation("bar")] + with pytest.raises(ValueError): + TypeEngine.to_literal_type(t) + + +def test_unsupported_complex_literals(): + t = typing_extensions.Annotated[typing.Dict[int, str], FlyteAnnotation({"foo": "bar"})] + with pytest.raises(ValueError): + TypeEngine.to_literal_type(t) + + # Enum. + t = typing_extensions.Annotated[Color, FlyteAnnotation({"foo": "bar"})] + with pytest.raises(ValueError): + TypeEngine.to_literal_type(t) + + # Dataclass. + t = typing_extensions.Annotated[Result, FlyteAnnotation({"foo": "bar"})] + with pytest.raises(ValueError): + TypeEngine.to_literal_type(t) + + +def test_multiple_annotations(): + t = typing_extensions.Annotated[int, FlyteAnnotation({"foo": "bar"}), FlyteAnnotation({"anotha": "one"})] + with pytest.raises(Exception): + TypeEngine.to_literal_type(t) + + TestSchema = FlyteSchema[kwtypes(some_str=str)] diff --git a/tests/flytekit/unit/core/test_typing_annotation.py b/tests/flytekit/unit/core/test_typing_annotation.py new file mode 100644 index 0000000000..f999c62612 --- /dev/null +++ b/tests/flytekit/unit/core/test_typing_annotation.py @@ -0,0 +1,62 @@ +import typing +from collections import OrderedDict + +import typing_extensions + +from flytekit.core import context_manager +from flytekit.core.annotation import FlyteAnnotation +from flytekit.core.context_manager import Image, ImageConfig +from flytekit.core.task import task +from flytekit.models.annotation import TypeAnnotation +from flytekit.tools.translator import get_serializable + +default_img = Image(name="default", fqn="test", tag="tag") +serialization_settings = context_manager.SerializationSettings( + project="project", + domain="domain", + version="version", + env=None, + image_config=ImageConfig(default_image=default_img, images=[default_img]), +) +entity_mapping = OrderedDict() + + +@task +def x(a: typing_extensions.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})], b: str): + ... + + +@task +def y0(a: typing.List[typing_extensions.Annotated[int, FlyteAnnotation({"foo": {"bar": 1}})]]): + ... + + +@task +def y1(a: typing_extensions.Annotated[typing.List[int], FlyteAnnotation({"foo": {"bar": 1}})]): + ... + + +def test_get_variable_descriptions(): + x_tsk = get_serializable(entity_mapping, serialization_settings, x) + x_input_vars = x_tsk.template.interface.inputs + + a_ann = x_input_vars["a"].type.annotation + assert isinstance(a_ann, TypeAnnotation) + assert a_ann.annotations["foo"] == {"bar": 1} + + b_ann = x_input_vars["b"].type.annotation + assert b_ann is None + + # Annotated simple type within list generic + y0_tsk = get_serializable(entity_mapping, serialization_settings, y0) + y0_input_vars = y0_tsk.template.interface.inputs + y0_a_ann = y0_input_vars["a"].type.collection_type.annotation + assert isinstance(y0_a_ann, TypeAnnotation) + assert y0_a_ann.annotations["foo"] == {"bar": 1} + + # Annotated list generic + y1_tsk = get_serializable(entity_mapping, serialization_settings, y1) + y1_input_vars = y1_tsk.template.interface.inputs + y1_a_ann = y1_input_vars["a"].type.annotation + assert isinstance(y1_a_ann, TypeAnnotation) + assert y1_a_ann.annotations["foo"] == {"bar": 1} diff --git a/tests/flytekit/unit/models/test_types.py b/tests/flytekit/unit/models/test_types.py index 27b5ddf595..82f0b9b519 100644 --- a/tests/flytekit/unit/models/test_types.py +++ b/tests/flytekit/unit/models/test_types.py @@ -2,6 +2,7 @@ from flyteidl.core import types_pb2 from flytekit.models import types as _types +from flytekit.models.annotation import TypeAnnotation from tests.flytekit.common import parameterizers @@ -81,6 +82,16 @@ def test_literal_types(): assert obj == _types.LiteralType.from_flyte_idl(obj.to_flyte_idl()) +def test_annotated_literal_types(): + obj = _types.LiteralType(simple=_types.SimpleType.INTEGER, annotation=TypeAnnotation(annotations={"foo": "bar"})) + assert obj.simple == _types.SimpleType.INTEGER + assert obj.schema is None + assert obj.collection_type is None + assert obj.map_value_type is None + assert obj.annotation.annotations == {"foo": "bar"} + assert obj == _types.LiteralType.from_flyte_idl(obj.to_flyte_idl()) + + @pytest.mark.parametrize("literal_type", parameterizers.LIST_OF_ALL_LITERAL_TYPES) def test_literal_collections(literal_type): obj = _types.LiteralType(collection_type=literal_type)