diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml index d77197717f..7d5604be60 100644 --- a/.github/workflows/pythonbuild.yml +++ b/.github/workflows/pythonbuild.yml @@ -148,7 +148,7 @@ jobs: python-version: 3.8 - name: Install dependencies run: | - python -m pip install --upgrade pip==21.2.4 setuptools wheel + python -m pip install --upgrade pip setuptools wheel pip install -r doc-requirements.txt - name: Build the documentation run: make -C docs html diff --git a/dataset_example.py b/dataset_example.py index ecbaabe4f8..4e034fc226 100644 --- a/dataset_example.py +++ b/dataset_example.py @@ -10,8 +10,6 @@ import pandas as pd import pyarrow as pa import pyarrow.parquet as pq -import pyspark.sql.dataframe -from pyspark.sql import SparkSession from flytekit import FlyteContext, kwtypes, task, workflow from flytekit.models import literals @@ -27,8 +25,8 @@ ) from flytekit.types.structured.utils import get_filesystem -PANDAS_PATH = "/tmp/pandas" -NUMPY_PATH = "/tmp/numpy" +PANDAS_PATH = "s3://flyte-batch/my-s3-bucket/test-data/pandas" +NUMPY_PATH = "s3://flyte-batch/my-s3-bucket/test-data/numpy" BQ_PATH = "bq://photo-313016:flyte.new_table3" # https://github.com/flyteorg/flyte/issues/523 @@ -60,7 +58,7 @@ def t2(dataframe: pd.DataFrame) -> Annotated[pd.DataFrame, arrow_schema]: @task def t3(dataset: StructuredDataset[my_cols]) -> StructuredDataset[my_cols]: # s3 (parquet) -> pandas -> s3 (parquet) - print("Pandas dataframe") + print("Pandas dataframe:") print(dataset.open(pd.DataFrame).all()) # In the example, we download dataset when we open it. # Here we won't upload anything, since we're returning just the input object. @@ -148,6 +146,8 @@ def decode( FLYTE_DATASET_TRANSFORMER.register_handler(NumpyEncodingHandlers(np.ndarray, "/", "parquet")) FLYTE_DATASET_TRANSFORMER.register_handler(NumpyDecodingHandlers(np.ndarray, "/", "parquet")) +FLYTE_DATASET_TRANSFORMER.register_handler(NumpyEncodingHandlers(np.ndarray, "s3://", "parquet")) +FLYTE_DATASET_TRANSFORMER.register_handler(NumpyDecodingHandlers(np.ndarray, "s3://", "parquet")) @task @@ -163,17 +163,6 @@ def t10(dataset: StructuredDataset[my_cols]) -> np.ndarray: return np_array -@task -def t11(dataframe: pyspark.sql.dataframe.DataFrame) -> StructuredDataset[my_cols]: - return StructuredDataset(dataframe) - - -@task -def t12(dataset: StructuredDataset[my_cols]) -> pyspark.sql.dataframe.DataFrame: - spark_df = dataset.open(pyspark.sql.dataframe.DataFrame).all() - return spark_df - - @task def generate_pandas() -> pd.DataFrame: return pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [20, 22]}) @@ -189,24 +178,11 @@ def generate_arrow() -> pa.Table: return pa.Table.from_pandas(pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [20, 22]})) -@task -def generate_spark_dataframe() -> pyspark.sql.dataframe.DataFrame: - data = [ - {"Category": "A", "ID": 1, "Value": 121.44, "Truth": True}, - {"Category": "B", "ID": 2, "Value": 300.01, "Truth": False}, - {"Category": "C", "ID": 3, "Value": 10.99, "Truth": None}, - {"Category": "E", "ID": 4, "Value": 33.87, "Truth": True}, - ] - spark = SparkSession.builder.getOrCreate() - return spark.createDataFrame(data) - - @workflow() def wf(): df = generate_pandas() np_array = generate_numpy() arrow_df = generate_arrow() - spark_df = generate_spark_dataframe() t1(dataframe=df) t1a(dataframe=df) t2(dataframe=df) @@ -220,8 +196,6 @@ def wf(): t8a(dataframe=arrow_df) t9(dataframe=np_array) t10(dataset=StructuredDataset(uri=NUMPY_PATH)) - dataset = t11(dataframe=spark_df) - t12(dataset=dataset) if __name__ == "__main__": diff --git a/dev-requirements.in b/dev-requirements.in index 9743d0fc20..c8eb2c0601 100644 --- a/dev-requirements.in +++ b/dev-requirements.in @@ -1,6 +1,6 @@ -c requirements.txt -git+git://github.com/flyteorg/pytest-flyte@main#egg=pytest-flyte +git+https://github.com/flyteorg/pytest-flyte@main#egg=pytest-flyte coverage[toml] joblib mock diff --git a/dev-requirements.txt b/dev-requirements.txt index 28c974c3ba..08d48d1988 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -2,7 +2,7 @@ # This file is autogenerated by pip-compile with python 3.9 # To update, run: # -# make dev-requirements.txt +# pip-compile dev-requirements.in # -e file:.#egg=flytekit # via @@ -77,7 +77,6 @@ cryptography==36.0.1 # via # -c requirements.txt # paramiko - # secretstorage dataclasses-json==0.5.6 # via # -c requirements.txt @@ -138,11 +137,6 @@ importlib-metadata==4.10.0 # keyring iniconfig==1.1.1 # via pytest -jeepney==0.7.1 - # via - # -c requirements.txt - # keyring - # secretstorage jinja2==3.0.3 # via # -c requirements.txt @@ -262,7 +256,7 @@ pytest==6.2.5 # pytest-flyte pytest-docker==0.10.3 # via pytest-flyte -pytest-flyte @ git+git://github.com/flyteorg/pytest-flyte@main +pytest-flyte @ git+https://github.com/flyteorg/pytest-flyte@main # via -r dev-requirements.in python-dateutil==2.8.1 # via @@ -315,10 +309,6 @@ retry==0.9.2 # via # -c requirements.txt # flytekit -secretstorage==3.3.1 - # via - # -c requirements.txt - # keyring six==1.16.0 # via # -c requirements.txt diff --git a/flytekit/core/interface.py b/flytekit/core/interface.py index 216e337282..263ac05fb7 100644 --- a/flytekit/core/interface.py +++ b/flytekit/core/interface.py @@ -275,7 +275,11 @@ def transform_function_to_interface(fn: Callable, docstring: Optional[Docstring] For now the fancy object, maybe in the future a dumb object. """ - type_hints = typing.get_type_hints(fn) + try: + # include_extras can only be used in python >= 3.9 + type_hints = typing.get_type_hints(fn, include_extras=True) + except TypeError: + type_hints = typing.get_type_hints(fn) signature = inspect.signature(fn) return_annotation = type_hints.get("return", None) diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py index 9ef9057eba..9cca58b4bc 100644 --- a/flytekit/core/type_engine.py +++ b/flytekit/core/type_engine.py @@ -42,8 +42,9 @@ Primitive, Scalar, Schema, + StructuredDatasetMetadata, ) -from flytekit.models.types import LiteralType, SimpleType +from flytekit.models.types import LiteralType, SimpleType, StructuredDatasetType T = typing.TypeVar("T") DEFINITIONS = "definitions" @@ -280,6 +281,7 @@ def _serialize_flyte_type(self, python_val: T, python_type: Type[T]): from flytekit.types.directory.types import FlyteDirectory from flytekit.types.file import FlyteFile from flytekit.types.schema.types import FlyteSchema + from flytekit.types.structured.structured_dataset import StructuredDataset for f in dataclasses.fields(python_type): v = python_val.__getattribute__(f.name) @@ -288,6 +290,7 @@ def _serialize_flyte_type(self, python_val: T, python_type: Type[T]): issubclass(field_type, FlyteSchema) or issubclass(field_type, FlyteFile) or issubclass(field_type, FlyteDirectory) + or issubclass(field_type, StructuredDataset) ): lv = TypeEngine.to_literal(FlyteContext.current_context(), v, field_type, None) # dataclass_json package will extract the "path" from FlyteFile, FlyteDirectory, and write it to a @@ -300,6 +303,13 @@ def _serialize_flyte_type(self, python_val: T, python_type: Type[T]): # as determined by the transformer. if issubclass(field_type, FlyteFile) or issubclass(field_type, FlyteDirectory): python_val.__setattr__(f.name, field_type(path=lv.scalar.blob.uri)) + elif issubclass(field_type, StructuredDataset): + python_val.__setattr__( + f.name, + field_type( + uri=lv.scalar.structured_dataset.uri, + ), + ) elif dataclasses.is_dataclass(field_type): self._serialize_flyte_type(v, field_type) @@ -308,6 +318,7 @@ def _deserialize_flyte_type(self, python_val: T, expected_python_type: Type) -> from flytekit.types.directory.types import FlyteDirectory, FlyteDirToMultipartBlobTransformer from flytekit.types.file.file import FlyteFile, FlyteFilePathTransformer from flytekit.types.schema.types import FlyteSchema, FlyteSchemaTransformer + from flytekit.types.structured.structured_dataset import StructuredDataset, StructuredDatasetTransformerEngine if not dataclasses.is_dataclass(expected_python_type): return python_val @@ -353,6 +364,21 @@ def _deserialize_flyte_type(self, python_val: T, expected_python_type: Type) -> ), expected_python_type, ) + elif issubclass(expected_python_type, StructuredDataset): + return StructuredDatasetTransformerEngine().to_python_value( + FlyteContext.current_context(), + Literal( + scalar=Scalar( + structured_dataset=StructuredDataset( + metadata=StructuredDatasetMetadata( + structured_dataset_type=StructuredDatasetType(format=python_val.file_format) + ), + uri=python_val.uri, + ) + ) + ), + expected_python_type, + ) else: for f in dataclasses.fields(expected_python_type): value = python_val.__getattribute__(f.name) diff --git a/flytekit/models/types.py b/flytekit/models/types.py index 062dc9da8f..99cb39ed14 100644 --- a/flytekit/models/types.py +++ b/flytekit/models/types.py @@ -165,7 +165,7 @@ def external_schema_bytes(self) -> bytes: def to_flyte_idl(self) -> _types_pb2.StructuredDatasetType: return _types_pb2.StructuredDatasetType( columns=[c.to_flyte_idl() for c in self.columns] if self.columns else None, - format=self._format, + format=self.format, external_schema_type=self.external_schema_type if self.external_schema_type else None, external_schema_bytes=self.external_schema_bytes if self.external_schema_bytes else None, ) diff --git a/flytekit/types/structured/basic_dfs.py b/flytekit/types/structured/basic_dfs.py index 6ec7970ae1..aa6640534c 100644 --- a/flytekit/types/structured/basic_dfs.py +++ b/flytekit/types/structured/basic_dfs.py @@ -61,11 +61,7 @@ def decode( path = flyte_value.uri local_dir = ctx.file_access.get_random_local_directory() ctx.file_access.get_data(path, local_dir, is_multipart=True) - frames = [pandas.read_parquet(os.path.join(local_dir, f)) for f in os.listdir(local_dir)] - if len(frames) == 1: - return frames[0] - elif len(frames) > 1: - return pandas.concat(frames, copy=True) + return pd.read_parquet(local_dir) class ArrowToParquetEncodingHandler(StructuredDatasetEncoder): diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py index 6c4b9dea1d..2b4d651b2e 100644 --- a/flytekit/types/structured/structured_dataset.py +++ b/flytekit/types/structured/structured_dataset.py @@ -1,12 +1,17 @@ from __future__ import annotations import collections +import os import re import types import typing from abc import ABC, abstractmethod +from dataclasses import dataclass, field from typing import Dict, Generator, Optional, Type, Union +from dataclasses_json import config, dataclass_json +from marshmallow import fields + try: from typing import Annotated, get_args, get_origin except ImportError: @@ -37,7 +42,11 @@ PARQUET = "parquet" +@dataclass_json +@dataclass class StructuredDataset(object): + uri: typing.Optional[os.PathLike] = field(default=None, metadata=config(mm_field=fields.String())) + file_format: typing.Optional[str] = field(default=PARQUET, metadata=config(mm_field=fields.String())) """ This is the user facing StructuredDataset class. Please don't confuse it with the literals.StructuredDataset class (that is just a model, a Python class representation of the protobuf). @@ -93,9 +102,12 @@ def __init__( dataframe: typing.Optional[typing.Any] = None, uri: Optional[str] = None, metadata: typing.Optional[literals.StructuredDatasetMetadata] = None, + **kwargs, ): self._dataframe = dataframe - self._uri = uri + # Make these fields public, so that the dataclass transformer can set a value for it + # https://github.com/flyteorg/flytekit/blob/bcc8541bd6227b532f8462563fe8aac902242b21/flytekit/core/type_engine.py#L298 + self.uri = uri # This is a special attribute that indicates if the data was either downloaded or uploaded self._metadata = metadata # This is not for users to set, the transformer will set this. @@ -107,18 +119,6 @@ def __init__( def dataframe(self) -> Type[typing.Any]: return self._dataframe - @property - def uri(self) -> Optional[str]: - return self._uri - - @uri.setter - def uri(self, uri: str): - self._uri = uri - - @property - def file_format(self) -> str: - return self.FILE_FORMAT - @property def metadata(self) -> Optional[StructuredDatasetMetadata]: return self._metadata @@ -192,6 +192,11 @@ def encode( :param ctx: :param structured_dataset: This is a StructuredDataset wrapper object. See more info above. + :param structured_dataset_type: This the StructuredDatasetType, as found in the LiteralType of the interface + of the task that invoked this encoding call. It is passed along to encoders so that authors of encoders + can include it in the returned literals.StructuredDataset. See the IDL for more information on why this + literal in particular carries the type information along with it. If the encoder doesn't supply it, it will + also be filled in after the encoder runs by the transformer engine. :return: This function should return a StructuredDataset literal object. Do not confuse this with the StructuredDataset wrapper class used as input to this function - that is the user facing Python class. This function needs to return the IDL StructuredDataset. @@ -306,11 +311,9 @@ def _finder(self, handler_map, df_type: Type, protocol: str, format: str): raise ValueError(f"Failed to find a handler for {df_type}, protocol {protocol}, fmt {format}") def get_encoder(self, df_type: Type, protocol: str, format: str): - protocol.replace("://", "") return self._finder(self.ENCODERS, df_type, protocol, format) def get_decoder(self, df_type: Type, protocol: str, format: str): - protocol.replace("://", "") return self._finder(self.DECODERS, df_type, protocol, format) def _handler_finder(self, h: Handlers) -> Dict[str, Handlers]: @@ -365,8 +368,11 @@ def to_literal( ) -> Literal: # Make a copy in case we need to hand off to encoders, since we can't be sure of mutations. # Check first to see if it's even an SD type. For backwards compatibility, we may be getting a + if get_origin(python_type) is Annotated: + python_type = get_args(python_type)[0] sdt = StructuredDatasetType(format=self.DEFAULT_FORMATS.get(python_type, None)) - if expected.structured_dataset_type: + + if expected and expected.structured_dataset_type: sdt = StructuredDatasetType( columns=expected.structured_dataset_type.columns, format=expected.structured_dataset_type.format, @@ -374,9 +380,6 @@ def to_literal( external_schema_bytes=expected.structured_dataset_type.external_schema_bytes, ) - if get_origin(python_type) is Annotated: - python_type = get_args(python_type)[0] - # If the type signature has the StructuredDataset class, it will, or at least should, also be a # StructuredDataset instance. if issubclass(python_type, StructuredDataset): @@ -431,7 +434,7 @@ def to_literal( # Otherwise assume it's a dataframe instance. Wrap it with some defaults fmt = self.DEFAULT_FORMATS[python_type] protocol = self.DEFAULT_PROTOCOLS[python_type] - meta = StructuredDatasetMetadata(structured_dataset_type=expected.structured_dataset_type) + meta = StructuredDatasetMetadata(structured_dataset_type=expected.structured_dataset_type if expected else None) sd = StructuredDataset(dataframe=python_val, metadata=meta) return self.encode(ctx, sd, python_type, protocol, fmt, sdt) @@ -463,16 +466,21 @@ def encode( def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> T: # The literal that we get in might be an old FlyteSchema. # We'll continue to support this for the time being. + if get_origin(expected_python_type) is Annotated: + expected_python_type = get_args(expected_python_type)[0] if lv.scalar.schema is not None: + sd = StructuredDataset() + sd_literal = literals.StructuredDataset( + uri=lv.scalar.schema.uri, + metadata=literals.StructuredDatasetMetadata( + # Dataframe will always be serialized to parquet file by FlyteSchema transformer + structured_dataset_type=StructuredDatasetType(format=PARQUET) + ), + ) + sd._literal_sd = sd_literal if issubclass(expected_python_type, StructuredDataset): - raise ValueError("We do not support FlyteSchema -> StructuredDataset transformations") + return sd else: - sd_literal = literals.StructuredDataset( - uri=lv.scalar.schema.uri, - metadata=literals.StructuredDatasetMetadata( - structured_dataset_type=StructuredDatasetType(format=self.DEFAULT_FORMATS[expected_python_type]) - ), - ) return self.open_as(ctx, sd_literal, df_type=expected_python_type) # Either a StructuredDataset type or some dataframe type. diff --git a/tests/flytekit/unit/core/hint_handling/__init__.py b/tests/flytekit/unit/core/hint_handling/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/flytekit/unit/core/hint_handling/a.py b/tests/flytekit/unit/core/hint_handling/a.py new file mode 100644 index 0000000000..bfa5674930 --- /dev/null +++ b/tests/flytekit/unit/core/hint_handling/a.py @@ -0,0 +1,19 @@ +try: + from typing import Annotated +except ImportError: + from typing_extensions import Annotated + + +class AA(object): + ... + + +my_aa = Annotated[AA, "some annotation"] + + +def t1(in1: int) -> AA: + return AA() + + +def t2(in1: int) -> my_aa: + return my_aa() diff --git a/tests/flytekit/unit/core/hint_handling/b.py b/tests/flytekit/unit/core/hint_handling/b.py new file mode 100644 index 0000000000..3db472b069 --- /dev/null +++ b/tests/flytekit/unit/core/hint_handling/b.py @@ -0,0 +1,22 @@ +from __future__ import annotations + +""" +There's no difference betweeen this file and a.py except that we have postponed evaluation +of annotations turned on here. +""" +try: + from typing import Annotated +except ImportError: + from typing_extensions import Annotated + +from .a import AA + +my_aa = Annotated[AA, "some annotation"] + + +def t1(in1: int) -> AA: + return AA() + + +def t2(in1: int) -> my_aa: + return my_aa() diff --git a/tests/flytekit/unit/core/hint_handling/test_hint_handling.py b/tests/flytekit/unit/core/hint_handling/test_hint_handling.py new file mode 100644 index 0000000000..e3c6b942d2 --- /dev/null +++ b/tests/flytekit/unit/core/hint_handling/test_hint_handling.py @@ -0,0 +1,35 @@ +try: + from typing import get_args, get_origin, get_type_hints +except ImportError: + from typing_extensions import get_origin, get_args, get_type_hints + +import inspect + +from .a import t1 as a_t1 +from .a import t2 as a_t2 +from .b import t1 as b_t1 +from .b import t2 as b_t2 + + +def printer(fn): + print(f"In {fn.__module__} FN: {fn.__name__}") + try: + type_hints = get_type_hints(fn, include_extras=True) + except TypeError: + type_hints = get_type_hints(fn) + print(f" Type hints {type_hints} return type hint") + signature = inspect.signature(fn) + print(f" Inspect signature: {signature}") + print(f" Inspect signature return_annotation: {signature.return_annotation}") + print("") + return_type = type_hints.get("return", None) + print(f" Origin: {get_origin(return_type)}") + print(f" Args: {get_args(return_type)}") + print("===============================\n") + + +def test_hinting(): + printer(a_t1) + printer(a_t2) + printer(b_t1) + printer(b_t2) diff --git a/tests/flytekit/unit/core/test_interface.py b/tests/flytekit/unit/core/test_interface.py index 8e55ee1bb4..2317bab1c2 100644 --- a/tests/flytekit/unit/core/test_interface.py +++ b/tests/flytekit/unit/core/test_interface.py @@ -15,6 +15,11 @@ from flytekit.types.file import FlyteFile from flytekit.types.pickle import FlytePickle +try: + from typing import Annotated +except ImportError: + from typing_extensions import Annotated + def test_extract_only(): def x() -> typing.NamedTuple("NT1", x_str=str, y_int=int): @@ -184,6 +189,16 @@ def z(a: int = 7, b: str = "eleven") -> typing.Tuple[int, str]: assert not params.parameters["b"].required assert params.parameters["b"].default.scalar.primitive.string_value == "eleven" + def z(a: Annotated[int, "some annotation"]) -> Annotated[int, "some annotation"]: + return a + + our_interface = transform_function_to_interface(z) + params = transform_inputs_to_parameters(ctx, our_interface) + assert params.parameters["a"].required + assert params.parameters["a"].default is None + assert our_interface.inputs == {"a": Annotated[int, "some annotation"]} + assert our_interface.outputs == {"o0": Annotated[int, "some annotation"]} + def test_parameters_with_docstring(): ctx = context_manager.FlyteContext.current_context() diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py index 2f6a7a3ad1..2de16e6c9b 100644 --- a/tests/flytekit/unit/core/test_type_engine.py +++ b/tests/flytekit/unit/core/test_type_engine.py @@ -41,6 +41,7 @@ from flytekit.types.pickle import FlytePickle from flytekit.types.pickle.pickle import FlytePickleTransformer from flytekit.types.schema import FlyteSchema +from flytekit.types.structured.structured_dataset import StructuredDataset def test_type_engine(): @@ -586,6 +587,35 @@ class TestFileStruct(object): assert o.b.c["hello"].path == ot.b.c["hello"].path +def test_structured_dataset_in_dataclass(): + df = pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [20, 22]}) + + @dataclass_json + @dataclass + class InnerDatasetStruct(object): + a: StructuredDataset + + @dataclass_json + @dataclass + class DatasetStruct(object): + a: StructuredDataset + b: InnerDatasetStruct + + sd = StructuredDataset(dataframe=df, file_format="parquet") + o = DatasetStruct(a=sd, b=InnerDatasetStruct(a=sd)) + + ctx = FlyteContext.current_context() + tf = DataclassTransformer() + lt = tf.get_literal_type(DatasetStruct) + lv = tf.to_literal(ctx, o, DatasetStruct, lt) + ot = tf.to_python_value(ctx, lv=lv, expected_python_type=DatasetStruct) + + assert_frame_equal(df, ot.a.open(pd.DataFrame).all()) + assert_frame_equal(df, ot.b.a.open(pd.DataFrame).all()) + assert "parquet" == ot.a.file_format + assert "parquet" == ot.b.a.file_format + + # Enums should have string values class Color(Enum): RED = "red" diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py index 3f7b4c901c..54c1c77b8c 100644 --- a/tests/flytekit/unit/core/test_type_hints.py +++ b/tests/flytekit/unit/core/test_type_hints.py @@ -9,9 +9,11 @@ from enum import Enum import pandas +import pandas as pd import pytest from dataclasses_json import dataclass_json from google.protobuf.struct_pb2 import Struct +from pandas._testing import assert_frame_equal import flytekit from flytekit import ContainerTask, Secret, SQLTask, dynamic, kwtypes, map_task @@ -35,6 +37,7 @@ from flytekit.types.directory import FlyteDirectory, TensorboardLogs from flytekit.types.file import FlyteFile, PNGImageFile from flytekit.types.schema import FlyteSchema, SchemaOpenMode +from flytekit.types.structured.structured_dataset import StructuredDataset serialization_settings = context_manager.SerializationSettings( project="proj", @@ -431,6 +434,36 @@ def wf(path: str) -> os.PathLike: assert "/tmp/flyte/" in wf(path="s3://somewhere").path +def test_structured_dataset_in_dataclass(): + df = pd.DataFrame({"Name": ["Tom", "Joseph"], "Age": [20, 22]}) + + @dataclass_json + @dataclass + class InnerDatasetStruct(object): + a: StructuredDataset + + @dataclass_json + @dataclass + class DatasetStruct(object): + a: StructuredDataset + b: InnerDatasetStruct + + @task + def t1(path: str) -> DatasetStruct: + sd = StructuredDataset(dataframe=df, uri=path) + return DatasetStruct(a=sd, b=InnerDatasetStruct(a=sd)) + + @workflow + def wf(path: str) -> DatasetStruct: + return t1(path=path) + + res = wf(path="/tmp/somewhere") + assert "parquet" == res.a.file_format + assert "parquet" == res.b.a.file_format + assert_frame_equal(df, res.a.open(pd.DataFrame).all()) + assert_frame_equal(df, res.b.a.open(pd.DataFrame).all()) + + def test_wf1_with_map(): @task def t1(a: int) -> int: