diff --git a/flytekit/configuration/__init__.py b/flytekit/configuration/__init__.py index 71da768bea..0705a21fb9 100644 --- a/flytekit/configuration/__init__.py +++ b/flytekit/configuration/__init__.py @@ -144,7 +144,7 @@ from typing import Dict, List, Optional import yaml -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from flytekit.configuration import internal as _internal from flytekit.configuration.default_images import DefaultImages @@ -164,9 +164,8 @@ SERIALIZED_CONTEXT_ENV_VAR = "_F_SS_C" -@dataclass_json @dataclass(init=True, repr=True, eq=True, frozen=True) -class Image(object): +class Image(DataClassJsonMixin): """ Image is a structured wrapper for task container images used in object serialization. @@ -224,9 +223,8 @@ def look_up_image_info(name: str, tag: str, optional_tag: bool = False) -> Image return Image(name=name, fqn=ref["name"], tag=ref["tag"]) -@dataclass_json @dataclass(init=True, repr=True, eq=True, frozen=True) -class ImageConfig(object): +class ImageConfig(DataClassJsonMixin): """ We recommend you to use ImageConfig.auto(img_name=None) to create an ImageConfig. For example, ImageConfig.auto(img_name=""ghcr.io/flyteorg/flytecookbook:v1.0.0"") will create an ImageConfig. @@ -671,9 +669,8 @@ def for_endpoint( return c.with_params(platform=PlatformConfig.for_endpoint(endpoint, insecure), data_config=data_config) -@dataclass_json @dataclass -class EntrypointSettings(object): +class EntrypointSettings(DataClassJsonMixin): """ This object carries information about the path of the entrypoint command that will be invoked at runtime. This is where `pyflyte-execute` code can be found. This is useful for cases like pyspark execution. @@ -682,9 +679,8 @@ class EntrypointSettings(object): path: Optional[str] = None -@dataclass_json @dataclass -class FastSerializationSettings(object): +class FastSerializationSettings(DataClassJsonMixin): """ This object hold information about settings necessary to serialize an object so that it can be fast-registered. """ @@ -698,9 +694,8 @@ class FastSerializationSettings(object): # TODO: ImageConfig, python_interpreter, venv_root, fast_serialization_settings.destination_dir should be combined. -@dataclass_json -@dataclass() -class SerializationSettings(object): +@dataclass +class SerializationSettings(DataClassJsonMixin): """ These settings are provided while serializing a workflow and task, before registration. This is required to get runtime information at serialization time, as well as some defaults. diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index f1203c7fc7..1eb62f9960 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -12,7 +12,7 @@ import typing from abc import ABC, abstractmethod from functools import lru_cache -from typing import Dict, NamedTuple, Optional, Type, cast +from typing import Any, Dict, List, NamedTuple, Optional, Tuple, Type, cast from dataclasses_json import DataClassJsonMixin, dataclass_json from google.protobuf import json_format as _json_format @@ -220,8 +220,7 @@ def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: class DataclassTransformer(TypeTransformer[object]): """ - The Dataclass Transformer, provides a type transformer for arbitrary Python dataclasses, that have - @dataclass and @dataclass_json decorators. + The Dataclass Transformer provides a type transformer for dataclasses_json dataclasses. The Dataclass is converted to and from json and is transported between tasks using the proto.Structpb representation Also the type declaration will try to extract the JSON Schema for the object if possible and pass it with the @@ -233,9 +232,8 @@ class DataclassTransformer(TypeTransformer[object]): .. code-block:: python - @dataclass_json @dataclass - class Test(): + class Test(DataClassJsonMixin): a: int b: str @@ -270,9 +268,8 @@ def assert_type(self, expected_type: Type[DataClassJsonMixin], v: T): if type(v) == expected_type: return - # @dataclass_json # @dataclass - # class Foo(object): + # class Foo(DataClassJsonMixin): # a: int = 0 # # @task @@ -318,7 +315,8 @@ def get_literal_type(self, t: Type[T]) -> LiteralType: if not issubclass(t, DataClassJsonMixin): raise AssertionError( - f"Dataclass {t} should be decorated with @dataclass_json to be " f"serialized correctly" + f"Dataclass {t} should be decorated with @dataclass_json or be a subclass of DataClassJsonMixin to be " + "serialized correctly" ) schema = None try: @@ -349,7 +347,8 @@ def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], exp ) if not issubclass(type(python_val), DataClassJsonMixin): raise TypeTransformerFailedError( - f"Dataclass {python_type} should be decorated with @dataclass_json to be " f"serialized correctly" + f"Dataclass {python_type} should be decorated with @dataclass_json or be a subclass of " + "DataClassJsonMixin to be serialized correctly" ) self._serialize_flyte_type(python_val, python_type) return Literal( @@ -429,10 +428,10 @@ def _serialize_flyte_type(self, python_val: T, python_type: Type[T]) -> typing.A or issubclass(python_type, StructuredDataset) ): lv = TypeEngine.to_literal(FlyteContext.current_context(), python_val, python_type, None) - # dataclass_json package will extract the "path" from FlyteFile, FlyteDirectory, and write it to a + # dataclasses_json package will extract the "path" from FlyteFile, FlyteDirectory, and write it to a # JSON which will be stored in IDL. The path here should always be a remote path, but sometimes the # path in FlyteFile and FlyteDirectory could be a local path. Therefore, reset the python value here, - # so that dataclass_json can always get a remote path. + # so that dataclasses_json can always get a remote path. # In other words, the file transformer has special code that handles the fact that if remote_source is # set, then the real uri in the literal should be the remote source, not the path (which may be an # auto-generated random local path). To be sure we're writing the right path to the json, use the uri @@ -596,12 +595,12 @@ def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: if not dataclasses.is_dataclass(expected_python_type): raise TypeTransformerFailedError( f"{expected_python_type} is not of type @dataclass, only Dataclasses are supported for " - f"user defined datatypes in Flytekit" + "user defined datatypes in Flytekit" ) if not issubclass(expected_python_type, DataClassJsonMixin): raise TypeTransformerFailedError( - f"Dataclass {expected_python_type} should be decorated with @dataclass_json to be " - f"serialized correctly" + f"Dataclass {expected_python_type} should be decorated with @dataclass_json or be a subclass of " + "DataClassJsonMixin to be serialized correctly" ) json_str = _json_format.MessageToJson(lv.scalar.generic) dc = cast(DataClassJsonMixin, expected_python_type).from_json(json_str) @@ -1520,18 +1519,18 @@ def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: return expected_python_type(lv.scalar.primitive.string_value) # type: ignore -def convert_json_schema_to_python_class(schema: dict, schema_name) -> Type[dataclasses.dataclass()]: # type: ignore +def convert_json_schema_to_python_class(schema: Dict[str, Any], schema_name: str) -> Type[Any]: """ Generate a model class based on the provided JSON Schema :param schema: dict representing valid JSON schema :param schema_name: dataclass name of return type """ - attribute_list = [] + attribute_list: List[Tuple[str, type]] = [] for property_key, property_val in schema[schema_name]["properties"].items(): property_type = property_val["type"] # Handle list if property_val["type"] == "array": - attribute_list.append((property_key, typing.List[_get_element_type(property_val["items"])])) # type: ignore + attribute_list.append((property_key, List[_get_element_type(property_val["items"])])) # type: ignore[misc,index] # Handle dataclass and dict elif property_type == "object": if property_val.get("$ref"): @@ -1539,13 +1538,13 @@ def convert_json_schema_to_python_class(schema: dict, schema_name) -> Type[datac attribute_list.append((property_key, convert_json_schema_to_python_class(schema, name))) elif property_val.get("additionalProperties"): attribute_list.append( - (property_key, typing.Dict[str, _get_element_type(property_val["additionalProperties"])]) # type: ignore + (property_key, Dict[str, _get_element_type(property_val["additionalProperties"])]) # type: ignore[misc,index] ) else: - attribute_list.append((property_key, typing.Dict[str, _get_element_type(property_val)])) # type: ignore + attribute_list.append((property_key, Dict[str, _get_element_type(property_val)])) # type: ignore[misc,index] # Handle int, float, bool or str else: - attribute_list.append([property_key, _get_element_type(property_val)]) # type: ignore + attribute_list.append((property_key, _get_element_type(property_val))) return dataclass_json(dataclasses.make_dataclass(schema_name, attribute_list)) diff --git a/flytekit/extras/pytorch/checkpoint.py b/flytekit/extras/pytorch/checkpoint.py index c7561f13f4..9e5841a1d5 100644 --- a/flytekit/extras/pytorch/checkpoint.py +++ b/flytekit/extras/pytorch/checkpoint.py @@ -4,7 +4,7 @@ from typing import Any, Callable, Dict, NamedTuple, Optional, Type, Union import torch -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from typing_extensions import Protocol from flytekit.core.context_manager import FlyteContext @@ -20,9 +20,8 @@ class IsDataclass(Protocol): __post_init__: Optional[Callable] -@dataclass_json @dataclass -class PyTorchCheckpoint: +class PyTorchCheckpoint(DataClassJsonMixin): """ This class is helpful to save a checkpoint. """ diff --git a/flytekit/extras/tensorflow/record.py b/flytekit/extras/tensorflow/record.py index 17e7c37ddd..0258d379ac 100644 --- a/flytekit/extras/tensorflow/record.py +++ b/flytekit/extras/tensorflow/record.py @@ -3,7 +3,7 @@ from typing import Optional, Tuple, Type, Union import tensorflow as tf -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from tensorflow.python.data.ops.readers import TFRecordDatasetV2 from typing_extensions import Annotated, get_args, get_origin @@ -16,9 +16,8 @@ from flytekit.types.file import TFRecordFile -@dataclass_json @dataclass -class TFRecordDatasetConfig: +class TFRecordDatasetConfig(DataClassJsonMixin): """ TFRecordDatasetConfig can be used while creating tf.data.TFRecordDataset comprising record of one or more TFRecord files. diff --git a/flytekit/types/directory/types.py b/flytekit/types/directory/types.py index f4f23eb72f..6d863b7c66 100644 --- a/flytekit/types/directory/types.py +++ b/flytekit/types/directory/types.py @@ -10,7 +10,7 @@ from uuid import UUID import fsspec -from dataclasses_json import config, dataclass_json +from dataclasses_json import DataClassJsonMixin, config from fsspec.utils import get_protocol from marshmallow import fields @@ -30,9 +30,8 @@ def noop(): ... -@dataclass_json @dataclass -class FlyteDirectory(os.PathLike, typing.Generic[T]): +class FlyteDirectory(DataClassJsonMixin, os.PathLike, typing.Generic[T]): path: PathType = field(default=None, metadata=config(mm_field=fields.String())) # type: ignore """ .. warning:: diff --git a/flytekit/types/file/file.py b/flytekit/types/file/file.py index f33b2854ac..d949379705 100644 --- a/flytekit/types/file/file.py +++ b/flytekit/types/file/file.py @@ -6,7 +6,7 @@ from contextlib import contextmanager from dataclasses import dataclass, field -from dataclasses_json import config, dataclass_json +from dataclasses_json import DataClassJsonMixin, config from marshmallow import fields from flytekit.core.context_manager import FlyteContext, FlyteContextManager @@ -25,9 +25,8 @@ def noop(): T = typing.TypeVar("T") -@dataclass_json @dataclass -class FlyteFile(os.PathLike, typing.Generic[T]): +class FlyteFile(DataClassJsonMixin, os.PathLike, typing.Generic[T]): path: typing.Union[str, os.PathLike] = field( default=None, metadata=config(mm_field=fields.String()) ) # type: ignore diff --git a/flytekit/types/schema/types.py b/flytekit/types/schema/types.py index 7ac98d27c6..dc7ca816ba 100644 --- a/flytekit/types/schema/types.py +++ b/flytekit/types/schema/types.py @@ -11,7 +11,7 @@ import numpy as _np import pandas -from dataclasses_json import config, dataclass_json +from dataclasses_json import DataClassJsonMixin, config from marshmallow import fields from flytekit.core.context_manager import FlyteContext, FlyteContextManager @@ -179,9 +179,8 @@ def get_handler(cls, t: Type) -> SchemaHandler: return cls._SCHEMA_HANDLERS[t] -@dataclass_json @dataclass -class FlyteSchema(object): +class FlyteSchema(DataClassJsonMixin): remote_path: typing.Optional[str] = field(default=None, metadata=config(mm_field=fields.String())) """ This is the main schema class that users should use. diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py index 5a4ef43d1a..a88de49974 100644 --- a/flytekit/types/structured/structured_dataset.py +++ b/flytekit/types/structured/structured_dataset.py @@ -8,7 +8,7 @@ from typing import Dict, Generator, Optional, Type, Union import _datetime -from dataclasses_json import config, dataclass_json +from dataclasses_json import DataClassJsonMixin, config from fsspec.utils import get_protocol from marshmallow import fields from typing_extensions import Annotated, TypeAlias, get_args, get_origin @@ -43,9 +43,8 @@ GENERIC_PROTOCOL: str = "generic protocol" -@dataclass_json @dataclass -class StructuredDataset(object): +class StructuredDataset(DataClassJsonMixin): """ This is the user facing StructuredDataset class. Please don't confuse it with the literals.StructuredDataset class (that is just a model, a Python class representation of the protobuf). diff --git a/plugins/flytekit-aws-batch/flytekitplugins/awsbatch/task.py b/plugins/flytekit-aws-batch/flytekitplugins/awsbatch/task.py index 0e67b2e50b..e0326f112b 100644 --- a/plugins/flytekit-aws-batch/flytekitplugins/awsbatch/task.py +++ b/plugins/flytekit-aws-batch/flytekitplugins/awsbatch/task.py @@ -1,7 +1,7 @@ from dataclasses import dataclass from typing import Any, Callable, Dict, List, Optional -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from google.protobuf import json_format from google.protobuf.struct_pb2 import Struct @@ -10,9 +10,8 @@ from flytekit.extend import TaskPlugins -@dataclass_json @dataclass -class AWSBatchConfig(object): +class AWSBatchConfig(DataClassJsonMixin): """ Use this to configure SubmitJobInput for a AWS batch job. Task's marked with this will automatically execute natively onto AWS batch service. @@ -27,7 +26,7 @@ class AWSBatchConfig(object): def to_dict(self): s = Struct() - s.update(self.to_dict()) + s.update(super().to_dict()) return json_format.MessageToDict(s) diff --git a/plugins/flytekit-dbt/flytekitplugins/dbt/schema.py b/plugins/flytekit-dbt/flytekitplugins/dbt/schema.py index 3634118b38..6163e440b1 100644 --- a/plugins/flytekit-dbt/flytekitplugins/dbt/schema.py +++ b/plugins/flytekit-dbt/flytekitplugins/dbt/schema.py @@ -2,12 +2,11 @@ from dataclasses import dataclass from typing import List, Optional -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin -@dataclass_json @dataclass -class BaseDBTInput: +class BaseDBTInput(DataClassJsonMixin): """ Base class for DBT Task Input. @@ -76,9 +75,8 @@ def to_args(self) -> List[str]: return args -@dataclass_json @dataclass -class BaseDBTOutput: +class BaseDBTOutput(DataClassJsonMixin): """ Base class for output of DBT task. @@ -94,7 +92,6 @@ class BaseDBTOutput: exit_code: int -@dataclass_json @dataclass class DBTRunInput(BaseDBTInput): """ @@ -131,7 +128,6 @@ def to_args(self) -> List[str]: return args -@dataclass_json @dataclass class DBTRunOutput(BaseDBTOutput): """ @@ -149,7 +145,6 @@ class DBTRunOutput(BaseDBTOutput): raw_manifest: str -@dataclass_json @dataclass class DBTTestInput(BaseDBTInput): """ @@ -187,7 +182,6 @@ def to_args(self) -> List[str]: return args -@dataclass_json @dataclass class DBTTestOutput(BaseDBTOutput): """ @@ -205,7 +199,6 @@ class DBTTestOutput(BaseDBTOutput): raw_manifest: str -@dataclass_json @dataclass class DBTFreshnessInput(BaseDBTInput): """ @@ -243,7 +236,6 @@ def to_args(self) -> List[str]: return args -@dataclass_json @dataclass class DBTFreshnessOutput(BaseDBTOutput): """ diff --git a/plugins/flytekit-dolt/flytekitplugins/dolt/schema.py b/plugins/flytekit-dolt/flytekitplugins/dolt/schema.py index 8f6867b47f..b5832557ba 100644 --- a/plugins/flytekit-dolt/flytekitplugins/dolt/schema.py +++ b/plugins/flytekit-dolt/flytekitplugins/dolt/schema.py @@ -6,7 +6,7 @@ import dolt_integrations.core as dolt_int import doltcli as dolt import pandas -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from google.protobuf.json_format import MessageToDict from google.protobuf.struct_pb2 import Struct @@ -17,9 +17,8 @@ from flytekit.models.types import LiteralType -@dataclass_json @dataclass -class DoltConfig: +class DoltConfig(DataClassJsonMixin): db_path: str tablename: typing.Optional[str] = None sql: typing.Optional[str] = None @@ -29,9 +28,8 @@ class DoltConfig: remote_conf: typing.Optional[dolt_int.Remote] = None -@dataclass_json @dataclass -class DoltTable: +class DoltTable(DataClassJsonMixin): config: DoltConfig data: typing.Optional[pandas.DataFrame] = None diff --git a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py index 39d2758417..c57af980a0 100644 --- a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py +++ b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/schema.py @@ -5,7 +5,7 @@ from typing import Any, Dict, List, Optional, Tuple, Type, Union import great_expectations as ge -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from great_expectations.checkpoint import SimpleCheckpoint from great_expectations.core.run_identifier import RunIdentifier from great_expectations.core.util import convert_to_json_serializable @@ -23,9 +23,8 @@ from .task import BatchRequestConfig -@dataclass_json @dataclass -class GreatExpectationsFlyteConfig(object): +class GreatExpectationsFlyteConfig(DataClassJsonMixin): """ Use this configuration to configure GreatExpectations Plugin. diff --git a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/task.py b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/task.py index 185ec20daa..8fe53e1e95 100644 --- a/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/task.py +++ b/plugins/flytekit-greatexpectations/flytekitplugins/great_expectations/task.py @@ -5,7 +5,7 @@ from typing import Any, Dict, List, Optional, Type, Union import great_expectations as ge -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from great_expectations.checkpoint import SimpleCheckpoint from great_expectations.core.run_identifier import RunIdentifier from great_expectations.core.util import convert_to_json_serializable @@ -19,9 +19,8 @@ from flytekit.types.schema import FlyteSchema -@dataclass_json @dataclass -class BatchRequestConfig(object): +class BatchRequestConfig(DataClassJsonMixin): """ Use this configuration to configure Batch Request. A BatchRequest can either be a simple BatchRequest or a RuntimeBatchRequest. diff --git a/plugins/flytekit-kf-pytorch/tests/test_elastic_task.py b/plugins/flytekit-kf-pytorch/tests/test_elastic_task.py index 0d134a5e18..d8b163eff5 100644 --- a/plugins/flytekit-kf-pytorch/tests/test_elastic_task.py +++ b/plugins/flytekit-kf-pytorch/tests/test_elastic_task.py @@ -6,16 +6,15 @@ import pytest import torch import torch.distributed as dist -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from flytekitplugins.kfpytorch.task import Elastic import flytekit from flytekit import task, workflow -@dataclass_json @dataclass -class Config: +class Config(DataClassJsonMixin): lr: float = 1e-5 bs: int = 64 name: str = "foo" diff --git a/plugins/flytekit-onnx-pytorch/flytekitplugins/onnxpytorch/schema.py b/plugins/flytekit-onnx-pytorch/flytekitplugins/onnxpytorch/schema.py index a0676fdd90..1dcbc066d2 100644 --- a/plugins/flytekit-onnx-pytorch/flytekitplugins/onnxpytorch/schema.py +++ b/plugins/flytekit-onnx-pytorch/flytekitplugins/onnxpytorch/schema.py @@ -4,7 +4,7 @@ from typing import Dict, List, Optional, Tuple, Type, Union import torch -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from torch.onnx import OperatorExportTypes, TrainingMode from typing_extensions import Annotated, get_args, get_origin @@ -16,9 +16,8 @@ from flytekit.types.file import ONNXFile -@dataclass_json @dataclass -class PyTorch2ONNXConfig: +class PyTorch2ONNXConfig(DataClassJsonMixin): """ PyTorch2ONNXConfig is the config used during the pytorch to ONNX conversion. @@ -53,9 +52,8 @@ class PyTorch2ONNXConfig: export_modules_as_functions: Union[bool, set[Type]] = False -@dataclass_json @dataclass -class PyTorch2ONNX: +class PyTorch2ONNX(DataClassJsonMixin): model: Union[torch.nn.Module, torch.jit.ScriptModule, torch.jit.ScriptFunction] = field(default=None) diff --git a/plugins/flytekit-onnx-scikitlearn/flytekitplugins/onnxscikitlearn/schema.py b/plugins/flytekit-onnx-scikitlearn/flytekitplugins/onnxscikitlearn/schema.py index 3305396e20..979e9bdcab 100644 --- a/plugins/flytekit-onnx-scikitlearn/flytekitplugins/onnxscikitlearn/schema.py +++ b/plugins/flytekit-onnx-scikitlearn/flytekitplugins/onnxscikitlearn/schema.py @@ -5,7 +5,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Type, Union import skl2onnx.common.data_types -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from skl2onnx import convert_sklearn from sklearn.base import BaseEstimator from typing_extensions import Annotated, get_args, get_origin @@ -18,9 +18,8 @@ from flytekit.types.file import ONNXFile -@dataclass_json @dataclass -class ScikitLearn2ONNXConfig: +class ScikitLearn2ONNXConfig(DataClassJsonMixin): """ ScikitLearn2ONNXConfig is the config used during the scikitlearn to ONNX conversion. @@ -71,9 +70,8 @@ def __post_init__(self): raise ValueError("All types in final_types must be in skl2onnx.common.data_types") -@dataclass_json @dataclass -class ScikitLearn2ONNX: +class ScikitLearn2ONNX(DataClassJsonMixin): model: BaseEstimator = field(default=None) diff --git a/plugins/flytekit-onnx-tensorflow/flytekitplugins/onnxtensorflow/schema.py b/plugins/flytekit-onnx-tensorflow/flytekitplugins/onnxtensorflow/schema.py index af8095e0fb..28ed2c5c62 100644 --- a/plugins/flytekit-onnx-tensorflow/flytekitplugins/onnxtensorflow/schema.py +++ b/plugins/flytekit-onnx-tensorflow/flytekitplugins/onnxtensorflow/schema.py @@ -4,7 +4,7 @@ import numpy as np import tensorflow as tf import tf2onnx -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from typing_extensions import Annotated, get_args, get_origin from flytekit import FlyteContext @@ -15,9 +15,8 @@ from flytekit.types.file import ONNXFile -@dataclass_json @dataclass -class TensorFlow2ONNXConfig: +class TensorFlow2ONNXConfig(DataClassJsonMixin): """ TensorFlow2ONNXConfig is the config used during the tensorflow to ONNX conversion. @@ -46,9 +45,8 @@ class TensorFlow2ONNXConfig: large_model: bool = False -@dataclass_json @dataclass -class TensorFlow2ONNX: +class TensorFlow2ONNX(DataClassJsonMixin): model: tf.keras.Model = field(default=None) diff --git a/plugins/flytekit-papermill/tests/testdata/datatype.py b/plugins/flytekit-papermill/tests/testdata/datatype.py index 8e07ef052b..86e4aa0aa0 100644 --- a/plugins/flytekit-papermill/tests/testdata/datatype.py +++ b/plugins/flytekit-papermill/tests/testdata/datatype.py @@ -1,9 +1,8 @@ from dataclasses import dataclass -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin -@dataclass_json @dataclass -class X: +class X(DataClassJsonMixin): x: int diff --git a/tests/flytekit/unit/cli/pyflyte/default_arguments/dataclass_wf.py b/tests/flytekit/unit/cli/pyflyte/default_arguments/dataclass_wf.py index d9ba207cf2..a88cfb93ea 100644 --- a/tests/flytekit/unit/cli/pyflyte/default_arguments/dataclass_wf.py +++ b/tests/flytekit/unit/cli/pyflyte/default_arguments/dataclass_wf.py @@ -1,13 +1,12 @@ from dataclasses import dataclass -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from flytekit import task, workflow -@dataclass_json @dataclass -class DataclassA: +class DataclassA(DataClassJsonMixin): a: str b: int diff --git a/tests/flytekit/unit/cli/pyflyte/workflow.py b/tests/flytekit/unit/cli/pyflyte/workflow.py index 311f141a22..bcb22bda25 100644 --- a/tests/flytekit/unit/cli/pyflyte/workflow.py +++ b/tests/flytekit/unit/cli/pyflyte/workflow.py @@ -4,7 +4,7 @@ from dataclasses import dataclass import pandas as pd -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from typing_extensions import Annotated from flytekit import kwtypes, task, workflow @@ -28,9 +28,8 @@ def show_sd(in_sd: StructuredDataset): print(df) -@dataclass_json @dataclass -class MyDataclass(object): +class MyDataclass(DataClassJsonMixin): i: int a: typing.List[str] diff --git a/tests/flytekit/unit/core/test_complex_nesting.py b/tests/flytekit/unit/core/test_complex_nesting.py index c8c643cc67..7534197f98 100644 --- a/tests/flytekit/unit/core/test_complex_nesting.py +++ b/tests/flytekit/unit/core/test_complex_nesting.py @@ -4,7 +4,7 @@ from typing import List import pytest -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from flytekit.configuration import Image, ImageConfig, SerializationSettings from flytekit.core.context_manager import ExecutionState, FlyteContextManager @@ -14,31 +14,27 @@ from flytekit.types.file import FlyteFile -@dataclass_json @dataclass -class MyProxyConfiguration: +class MyProxyConfiguration(DataClassJsonMixin): # File and directory paths kept as 'str' so Flyte doesn't manage these static resources splat_data_dir: str apriori_file: str -@dataclass_json @dataclass -class MyProxyParameters: +class MyProxyParameters(DataClassJsonMixin): id: str job_i_step: int -@dataclass_json @dataclass -class MyAprioriConfiguration: +class MyAprioriConfiguration(DataClassJsonMixin): static_data_dir: FlyteDirectory external_data_dir: FlyteDirectory -@dataclass_json @dataclass -class MyInput: +class MyInput(DataClassJsonMixin): main_product: FlyteFile apriori_config: MyAprioriConfiguration proxy_config: MyProxyConfiguration diff --git a/tests/flytekit/unit/core/test_dataclass.py b/tests/flytekit/unit/core/test_dataclass.py index db49d2312c..34350ca40b 100644 --- a/tests/flytekit/unit/core/test_dataclass.py +++ b/tests/flytekit/unit/core/test_dataclass.py @@ -1,16 +1,15 @@ from dataclasses import dataclass from typing import List -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from flytekit.core.task import task from flytekit.core.workflow import workflow def test_dataclass(): - @dataclass_json @dataclass - class AppParams(object): + class AppParams(DataClassJsonMixin): snapshotDate: str region: str preprocess: bool diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py index 2ee5e34674..1569a258f4 100644 --- a/tests/flytekit/unit/core/test_local_cache.py +++ b/tests/flytekit/unit/core/test_local_cache.py @@ -6,7 +6,7 @@ import pandas import pandas as pd import pytest -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from pytest import fixture from typing_extensions import Annotated @@ -164,9 +164,8 @@ def my_wf() -> FlyteSchema: def test_wf_custom_types(): - @dataclass_json @dataclass - class MyCustomType(object): + class MyCustomType(DataClassJsonMixin): x: int y: str diff --git a/tests/flytekit/unit/core/test_promise.py b/tests/flytekit/unit/core/test_promise.py index c1eb15912b..6a487b464a 100644 --- a/tests/flytekit/unit/core/test_promise.py +++ b/tests/flytekit/unit/core/test_promise.py @@ -2,7 +2,7 @@ from dataclasses import dataclass import pytest -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from typing_extensions import Annotated from flytekit import LaunchPlan, task, workflow @@ -92,9 +92,8 @@ def wf(i: int, j: int): create_and_link_node_from_remote(ctx, lp, _inputs_not_allowed={"i"}, _ignorable_inputs={"j"}, j=15) -@dataclass_json @dataclass -class MyDataclass(object): +class MyDataclass(DataClassJsonMixin): i: int a: typing.List[str] diff --git a/tests/flytekit/unit/core/test_type_delayed.py b/tests/flytekit/unit/core/test_type_delayed.py index 3e6824788d..a47a0b88f8 100644 --- a/tests/flytekit/unit/core/test_type_delayed.py +++ b/tests/flytekit/unit/core/test_type_delayed.py @@ -3,7 +3,7 @@ import typing from dataclasses import dataclass -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from typing_extensions import Annotated # type: ignore from flytekit.core import context_manager @@ -11,9 +11,8 @@ from flytekit.core.type_engine import TypeEngine -@dataclass_json @dataclass -class Foo(object): +class Foo(DataClassJsonMixin): x: int y: str z: typing.Dict[str, str] diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 411ffa85dc..56d337af13 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -13,7 +13,7 @@ import pyarrow as pa import pytest import typing_extensions -from dataclasses_json import DataClassJsonMixin, dataclass_json +from dataclasses_json import DataClassJsonMixin from flyteidl.core import errors_pb2 from google.protobuf import json_format as _json_format from google.protobuf import struct_pb2 as _struct @@ -149,15 +149,13 @@ def test_list_of_dict_getting_python_value(): def test_list_of_single_dataclass(): - @dataclass_json - @dataclass() - class Bar(object): + @dataclass + class Bar(DataClassJsonMixin): v: typing.Optional[typing.List[int]] w: typing.Optional[typing.List[float]] - @dataclass_json - @dataclass() - class Foo(object): + @dataclass + class Foo(DataClassJsonMixin): a: typing.Optional[typing.List[str]] b: Bar @@ -219,18 +217,16 @@ def __class_getitem__(cls, item: Type[T]): def test_list_of_dataclass_getting_python_value(): - @dataclass_json - @dataclass() - class Bar(object): + @dataclass + class Bar(DataClassJsonMixin): v: typing.Union[int, None] w: typing.Optional[str] x: float y: str z: typing.Dict[str, bool] - @dataclass_json - @dataclass() - class Foo(object): + @dataclass + class Foo(DataClassJsonMixin): u: typing.Optional[int] v: typing.Optional[int] w: int @@ -380,9 +376,8 @@ def recursive_assert(lit: LiteralType, expected: LiteralType, expected_depth: in def test_convert_json_schema_to_python_class(): - @dataclass_json @dataclass - class Foo(object): + class Foo(DataClassJsonMixin): x: int y: str @@ -492,40 +487,35 @@ def test_zero_floats(): assert TypeEngine.to_python_value(ctx, l1, float) == 0 -@dataclass_json @dataclass -class InnerStruct(object): +class InnerStruct(DataClassJsonMixin): a: int b: typing.Optional[str] c: typing.List[int] -@dataclass_json @dataclass -class TestStruct(object): +class TestStruct(DataClassJsonMixin): s: InnerStruct m: typing.Dict[str, str] -@dataclass_json @dataclass -class TestStructB(object): +class TestStructB(DataClassJsonMixin): s: InnerStruct m: typing.Dict[int, str] n: typing.Optional[typing.List[typing.List[int]]] = None o: typing.Optional[typing.Dict[int, typing.Dict[int, int]]] = None -@dataclass_json @dataclass -class TestStructC(object): +class TestStructC(DataClassJsonMixin): s: InnerStruct m: typing.Dict[str, int] -@dataclass_json @dataclass -class TestStructD(object): +class TestStructD(DataClassJsonMixin): s: InnerStruct m: typing.Dict[str, typing.List[int]] @@ -535,9 +525,8 @@ def __init__(self): self._a = "Hello" -@dataclass_json @dataclass -class UnsupportedNestedStruct(object): +class UnsupportedNestedStruct(DataClassJsonMixin): a: int s: UnsupportedSchemaType @@ -623,14 +612,12 @@ def test_dataclass_int_preserving(): def test_optional_flytefile_in_dataclass(mock_upload_dir): mock_upload_dir.return_value = True - @dataclass_json @dataclass - class A(object): + class A(DataClassJsonMixin): a: int - @dataclass_json @dataclass - class TestFileStruct(object): + class TestFileStruct(DataClassJsonMixin): a: FlyteFile b: typing.Optional[FlyteFile] b_prime: typing.Optional[FlyteFile] @@ -704,18 +691,16 @@ class TestFileStruct(object): def test_flyte_file_in_dataclass(): - @dataclass_json @dataclass - class TestInnerFileStruct(object): + class TestInnerFileStruct(DataClassJsonMixin): a: JPEGImageFile b: typing.List[FlyteFile] c: typing.Dict[str, FlyteFile] d: typing.List[FlyteFile] e: typing.Dict[str, FlyteFile] - @dataclass_json @dataclass - class TestFileStruct(object): + class TestFileStruct(DataClassJsonMixin): a: FlyteFile b: TestInnerFileStruct @@ -749,18 +734,16 @@ class TestFileStruct(object): def test_flyte_directory_in_dataclass(): - @dataclass_json @dataclass - class TestInnerFileStruct(object): + class TestInnerFileStruct(DataClassJsonMixin): a: TensorboardLogs b: typing.List[FlyteDirectory] c: typing.Dict[str, FlyteDirectory] d: typing.List[FlyteDirectory] e: typing.Dict[str, FlyteDirectory] - @dataclass_json @dataclass - class TestFileStruct(object): + class TestFileStruct(DataClassJsonMixin): a: FlyteDirectory b: TestInnerFileStruct @@ -800,16 +783,14 @@ def test_structured_dataset_in_dataclass(): df = pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [20, 22]}) People = Annotated[StructuredDataset, "parquet", kwtypes(Name=str, Age=int)] - @dataclass_json @dataclass - class InnerDatasetStruct(object): + class InnerDatasetStruct(DataClassJsonMixin): a: StructuredDataset b: typing.List[Annotated[StructuredDataset, "parquet"]] c: typing.Dict[str, Annotated[StructuredDataset, kwtypes(Name=str, Age=int)]] - @dataclass_json @dataclass - class DatasetStruct(object): + class DatasetStruct(DataClassJsonMixin): a: People b: InnerDatasetStruct @@ -955,15 +936,13 @@ def test_union_type(): def test_assert_dataclass_type(): - @dataclass_json @dataclass - class Args(object): + class Args(DataClassJsonMixin): x: int y: typing.Optional[str] - @dataclass_json @dataclass - class Schema(object): + class Schema(DataClassJsonMixin): x: typing.Optional[Args] = None pt = Schema @@ -973,9 +952,8 @@ class Schema(object): DataclassTransformer().assert_type(gt, pv) DataclassTransformer().assert_type(Schema, pv) - @dataclass_json @dataclass - class Bar(object): + class Bar(DataClassJsonMixin): x: int pv = Bar(x=3) @@ -1280,9 +1258,8 @@ def __init__(self, number: int): def test_enum_in_dataclass(): - @dataclass_json @dataclass - class Datum(object): + class Datum(DataClassJsonMixin): x: int y: Color @@ -1537,16 +1514,14 @@ def test_multiple_annotations(): TestSchema = FlyteSchema[kwtypes(some_str=str)] # type: ignore -@dataclass_json @dataclass -class InnerResult: +class InnerResult(DataClassJsonMixin): number: int schema: TestSchema # type: ignore -@dataclass_json @dataclass -class Result: +class Result(DataClassJsonMixin): result: InnerResult schema: TestSchema # type: ignore @@ -1566,9 +1541,8 @@ def test_schema_in_dataclass(): def test_guess_of_dataclass(): - @dataclass_json - @dataclass() - class Foo(object): + @dataclass + class Foo(DataClassJsonMixin): x: int y: str z: typing.Dict[str, int] diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index 875c56a4b7..5622e5c886 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -13,7 +13,7 @@ import pandas import pandas as pd import pytest -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from google.protobuf.struct_pb2 import Struct from pandas._testing import assert_frame_equal from typing_extensions import Annotated, get_origin @@ -387,15 +387,13 @@ def test_user_demo_test(mock_sql): def test_flyte_file_in_dataclass(): - @dataclass_json @dataclass - class InnerFileStruct(object): + class InnerFileStruct(DataClassJsonMixin): a: FlyteFile b: PNGImageFile - @dataclass_json @dataclass - class FileStruct(object): + class FileStruct(DataClassJsonMixin): a: FlyteFile b: InnerFileStruct @@ -437,15 +435,13 @@ def wf(path: str) -> (os.PathLike, FlyteFile): def test_flyte_directory_in_dataclass(): - @dataclass_json @dataclass - class InnerFileStruct(object): + class InnerFileStruct(DataClassJsonMixin): a: FlyteDirectory b: TensorboardLogs - @dataclass_json @dataclass - class FileStruct(object): + class FileStruct(DataClassJsonMixin): a: FlyteDirectory b: InnerFileStruct @@ -470,14 +466,12 @@ def wf(path: str) -> os.PathLike: def test_structured_dataset_in_dataclass(): df = pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [20, 22]}) - @dataclass_json @dataclass - class InnerDatasetStruct(object): + class InnerDatasetStruct(DataClassJsonMixin): a: StructuredDataset - @dataclass_json @dataclass - class DatasetStruct(object): + class DatasetStruct(DataClassJsonMixin): a: StructuredDataset b: InnerDatasetStruct @@ -1087,9 +1081,8 @@ def t1(a: int) -> MyCustomType: def test_wf_custom_types(): - @dataclass_json @dataclass - class MyCustomType(object): + class MyCustomType(DataClassJsonMixin): x: int y: str @@ -1137,9 +1130,8 @@ def wf(a: int) -> typing.Dict[str, Foo]: def test_dataclass_more(): - @dataclass_json @dataclass - class Datum(object): + class Datum(DataClassJsonMixin): x: int y: str z: typing.Dict[int, str] @@ -1166,9 +1158,8 @@ class Color(Enum): GREEN = "green" BLUE = "blue" - @dataclass_json @dataclass - class Datum(object): + class Datum(DataClassJsonMixin): x: int y: Color @@ -1186,15 +1177,13 @@ def wf(x: int) -> Datum: def test_flyte_schema_dataclass(): TestSchema = FlyteSchema[kwtypes(some_str=str)] - @dataclass_json @dataclass - class InnerResult: + class InnerResult(DataClassJsonMixin): number: int schema: TestSchema - @dataclass_json @dataclass - class Result: + class Result(DataClassJsonMixin): result: InnerResult schema: TestSchema @@ -1514,16 +1503,14 @@ def t2() -> dict: def test_guess_dict4(): - @dataclass_json @dataclass - class Foo(object): + class Foo(DataClassJsonMixin): x: int y: str z: typing.Dict[str, str] - @dataclass_json @dataclass - class Bar(object): + class Bar(DataClassJsonMixin): x: int y: dict z: Foo diff --git a/tests/flytekit/unit/extras/pytorch/test_checkpoint.py b/tests/flytekit/unit/extras/pytorch/test_checkpoint.py index 49ad083285..10721bf237 100644 --- a/tests/flytekit/unit/extras/pytorch/test_checkpoint.py +++ b/tests/flytekit/unit/extras/pytorch/test_checkpoint.py @@ -5,16 +5,15 @@ import torch.nn as nn import torch.nn.functional as F import torch.optim as optim -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin from flytekit import task, workflow from flytekit.core.type_engine import TypeTransformerFailedError from flytekit.extras.pytorch import PyTorchCheckpoint -@dataclass_json @dataclass -class Hyperparameters: +class Hyperparameters(DataClassJsonMixin): epochs: int loss: float diff --git a/tests/flytekit/unit/extras/tasks/test_shell.py b/tests/flytekit/unit/extras/tasks/test_shell.py index 580cec4394..e70515ec73 100644 --- a/tests/flytekit/unit/extras/tasks/test_shell.py +++ b/tests/flytekit/unit/extras/tasks/test_shell.py @@ -5,7 +5,7 @@ from dataclasses import dataclass import pytest -from dataclasses_json import dataclass_json +from dataclasses_json import DataClassJsonMixin import flytekit from flytekit import kwtypes @@ -215,9 +215,8 @@ def test_reuse_variables_for_both_inputs_and_outputs(): def test_can_use_complex_types_for_inputs_to_f_string_template(): - @dataclass_json @dataclass - class InputArgs: + class InputArgs(DataClassJsonMixin): in_file: CSVFile t = ShellTask(