From 06e960f314a60ea2bd279eb645aa402d994d9d47 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 1 Dec 2021 17:10:17 -0800
Subject: [PATCH 01/35] Remove flyteidl from install_requires

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 setup.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/setup.py b/setup.py
index 305e3d3700..52d9173a08 100644
--- a/setup.py
+++ b/setup.py
@@ -64,7 +64,8 @@
         ]
     },
     install_requires=[
-        "flyteidl>=0.21.4",
+        # TODO: put the flyteidl constraint back
+        # "flyteidl>=0.21.4",
         "wheel>=0.30.0,<1.0.0",
         "pandas>=1.0.0,<2.0.0",
         "pyarrow>=6.0.0,<7.0.0",

From c9047a87b70611a0911bc1176a5c6881f19db817 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 1 Dec 2021 17:12:14 -0800
Subject: [PATCH 02/35] Expose hash in Literal

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/models/literals.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py
index 48553005b8..01d1120124 100644
--- a/flytekit/models/literals.py
+++ b/flytekit/models/literals.py
@@ -726,7 +726,9 @@ def from_flyte_idl(cls, pb2_object):
 
 
 class Literal(_common.FlyteIdlEntity):
-    def __init__(self, scalar: Scalar = None, collection: LiteralCollection = None, map: LiteralMap = None):
+    def __init__(
+        self, scalar: Scalar = None, collection: LiteralCollection = None, map: LiteralMap = None, hash: str = None
+    ):
         """
         This IDL message represents a literal value in the Flyte ecosystem.
 
@@ -737,6 +739,7 @@ def __init__(self, scalar: Scalar = None, collection: LiteralCollection = None,
         self._scalar = scalar
         self._collection = collection
         self._map = map
+        self.hash = hash
 
     @property
     def scalar(self):
@@ -778,6 +781,7 @@ def to_flyte_idl(self):
             scalar=self.scalar.to_flyte_idl() if self.scalar is not None else None,
             collection=self.collection.to_flyte_idl() if self.collection is not None else None,
             map=self.map.to_flyte_idl() if self.map is not None else None,
+            hash=self.hash,
         )
 
     @classmethod
@@ -794,4 +798,9 @@ def from_flyte_idl(cls, pb2_object):
             scalar=Scalar.from_flyte_idl(pb2_object.scalar) if pb2_object.HasField("scalar") else None,
             collection=collection,
             map=LiteralMap.from_flyte_idl(pb2_object.map) if pb2_object.HasField("map") else None,
+            # TODO: explain that string always have a value set in protobufs, which means
+            # if we want to differentiate between the case of empty string and null we would have
+            # to wrap that in a separate value. Instead, we're deliberately setting `hash` to
+            # None in case of the default value (i.e. empty string).
+            hash=pb2_object.hash if pb2_object.hash else None,
         )

From 4d21957043feb3a3fe2f3f83b6af364004c6cf6d Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 1 Dec 2021 17:13:53 -0800
Subject: [PATCH 03/35] Set hash in TypeEngine

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/core/type_engine.py | 39 ++++++++++++++++++++++++++++++++++--
 1 file changed, 37 insertions(+), 2 deletions(-)

diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py
index 925f79c770..17031d2f89 100644
--- a/flytekit/core/type_engine.py
+++ b/flytekit/core/type_engine.py
@@ -8,7 +8,7 @@
 import mimetypes
 import typing
 from abc import ABC, abstractmethod
-from typing import NamedTuple, Optional, Type, cast
+from typing import Callable, NamedTuple, Optional, Type, cast
 
 from dataclasses_json import DataClassJsonMixin, dataclass_json
 from google.protobuf import json_format as _json_format
@@ -18,6 +18,7 @@
 from google.protobuf.json_format import ParseDict as _ParseDict
 from google.protobuf.struct_pb2 import Struct
 from marshmallow_jsonschema import JSONSchema
+from typing_extensions import Annotated, get_args, get_origin
 
 from flytekit.common.exceptions import user as user_exceptions
 from flytekit.common.types import primitives as _primitives
@@ -34,15 +35,27 @@
 DEFINITIONS = "definitions"
 
 
+# TODO: expose this in flytekit.__init__
+class HashMethod(typing.Generic[T]):
+    def __init__(self, function: Callable[[T], str]):
+        self._function = function
+
+    def calculate(self, obj: T) -> str:
+        return self._function(obj)
+
+
 class TypeTransformer(typing.Generic[T]):
     """
     Base transformer type that should be implemented for every python native type that can be handled by flytekit
     """
 
-    def __init__(self, name: str, t: Type[T], enable_type_assertions: bool = True):
+    def __init__(self, name: str, t: Type[T], enable_type_assertions: bool = True, hash_overridable: bool = False):
         self._t = t
         self._name = name
         self._type_assertions_enabled = enable_type_assertions
+        # `hash_overridable` indicates that the literals produced by this type transformer can set their hashes if needed.
+        # See (link to documentation where this feature is explained).
+        self._hash_overridable = hash_overridable
 
     @property
     def name(self):
@@ -62,6 +75,10 @@ def type_assertions_enabled(self) -> bool:
         """
         return self._type_assertions_enabled
 
+    @property
+    def hash_overridable(self) -> bool:
+        return self._hash_overridable
+
     def assert_type(self, t: Type[T], v: T):
         if not hasattr(t, "__origin__") and not isinstance(v, t):
             raise TypeError(f"Type of Val '{v}' is not an instance of {t}")
@@ -449,7 +466,25 @@ def to_literal(cls, ctx: FlyteContext, python_val: typing.Any, python_type: Type
         transformer = cls.get_transformer(python_type)
         if transformer.type_assertions_enabled:
             transformer.assert_type(python_type, python_val)
+
+        # In case the value is an annotated type we inspect the annotations and look for hash-related annotations.
+        hash = None
+        if transformer.hash_overridable and get_origin(python_type) is Annotated:
+            # We are now dealing with one of two cases:
+            # 1. The annotated type is a `HashMethod`, which indicates that we should we should produce the hash using
+            #    the method indicated in the annotation.
+            # 2. The annotated type is being used for a different purpose other than calculating hash values, in which case
+            #    we should just continue.
+            for annotation in get_args(python_type)[1:]:
+                if not isinstance(annotation, HashMethod):
+                    continue
+                hash = annotation.calculate(python_val)
+                break
+
         lv = transformer.to_literal(ctx, python_val, python_type, expected)
+        # TODO: should we worry about empty string?
+        if hash is not None:
+            lv.hash = hash
         return lv
 
     @classmethod

From f9273db7e7d458e1616294b2dcb22dd48bbac8b6 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 1 Dec 2021 17:15:00 -0800
Subject: [PATCH 04/35] Modify cache key calculation to take hash into account

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/core/local_cache.py | 12 ++++++++++--
 1 file changed, 10 insertions(+), 2 deletions(-)

diff --git a/flytekit/core/local_cache.py b/flytekit/core/local_cache.py
index 9180224207..8c908f8138 100644
--- a/flytekit/core/local_cache.py
+++ b/flytekit/core/local_cache.py
@@ -2,7 +2,7 @@
 
 from diskcache import Cache
 
-from flytekit.models.literals import LiteralMap
+from flytekit.models.literals import Literal, LiteralMap
 
 # Location on the filesystem where serialized objects will be stored
 # TODO: read from config
@@ -10,7 +10,15 @@
 
 
 def _calculate_cache_key(task_name: str, cache_version: str, input_literal_map: LiteralMap) -> str:
-    return f"{task_name}-{cache_version}-{input_literal_map}"
+    # TODO: document this.
+    literal_map_overridden = {}
+    for key, literal in input_literal_map.literals.items():
+        if literal.hash is not None:
+            literal_map_overridden[key] = Literal(hash=literal.hash)
+        else:
+            literal_map_overridden[key] = literal
+
+    return f"{task_name}-{cache_version}-{LiteralMap(literal_map_overridden)}"
 
 
 class LocalTaskCache(object):

From 53dff4d9f17d3764aca8fbc302ee9a6434ad1f21 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 1 Dec 2021 17:15:50 -0800
Subject: [PATCH 05/35] Opt-in PandasDataFrameTransformer

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/types/schema/types_pandas.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/flytekit/types/schema/types_pandas.py b/flytekit/types/schema/types_pandas.py
index 41a5423c08..9ec6e548df 100644
--- a/flytekit/types/schema/types_pandas.py
+++ b/flytekit/types/schema/types_pandas.py
@@ -112,6 +112,9 @@ class PandasDataFrameTransformer(TypeTransformer[pandas.DataFrame]):
     def __init__(self):
         super().__init__("PandasDataFrame<->GenericSchema", pandas.DataFrame)
         self._parquet_engine = _PARQUETIO_ENGINES[sdk.PARQUET_ENGINE.get()]
+        # Pandas dataframes can have their hashes overriden to facilitate the case of caching pandas dataframes by
+        # value.
+        self._hash_overridable = True
 
     @staticmethod
     def _get_schema_type() -> SchemaType:

From 5ae247c80f956b59cb7e88e0370c0ae08dcbaeff Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 1 Dec 2021 17:19:24 -0800
Subject: [PATCH 06/35] Add unit tests

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 tests/flytekit/unit/core/test_local_cache.py | 101 ++++++++++++++++-
 tests/flytekit/unit/core/test_type_engine.py |  76 +++++++++++++
 tests/flytekit/unit/core/test_type_hints.py  | 110 ++++++++++++++++++-
 3 files changed, 285 insertions(+), 2 deletions(-)

diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py
index c43421bbf9..da9c4b30a5 100644
--- a/tests/flytekit/unit/core/test_local_cache.py
+++ b/tests/flytekit/unit/core/test_local_cache.py
@@ -5,11 +5,13 @@
 import pandas
 from dataclasses_json import dataclass_json
 from pytest import fixture
+from typing_extensions import Annotated
 
-from flytekit import SQLTask, kwtypes
+from flytekit import SQLTask, dynamic, kwtypes
 from flytekit.core.local_cache import LocalTaskCache
 from flytekit.core.task import TaskMetadata, task
 from flytekit.core.testing import task_mock
+from flytekit.core.type_engine import HashMethod
 from flytekit.core.workflow import workflow
 from flytekit.types.schema import FlyteSchema
 
@@ -250,3 +252,100 @@ def my_wf(a: int, b: str) -> (int, str):
     x = my_wf(a=5, b="hello")
     assert x == (7, "hello world")
     assert n_cached_task_calls == 2
+
+
+def test_set_integer_literal_hash_is_not_cached():
+    """
+    Test to confirm that the local cache is not set in the case of integers, even if we
+    return an annotated integer. In order to make this very explicit, we define a constant hash
+    function, i.e. the same value is returned by it regardless of the input.
+    """
+
+    def constant_hash_function(a: int) -> str:
+        return "hash"
+
+    @task
+    def t0(a: int) -> Annotated[int, HashMethod(function=constant_hash_function)]:
+        return a
+
+    @task(cache=True, cache_version="0.0.1")
+    def t1(cached_a: int) -> int:
+        global n_cached_task_calls
+        n_cached_task_calls += 1
+        return cached_a
+
+    @workflow
+    def wf(a: int) -> int:
+        annotated_a = t0(a=a)
+        return t1(cached_a=annotated_a)
+
+    assert n_cached_task_calls == 0
+    assert wf(a=3) == 3
+    assert n_cached_task_calls == 1
+    # Confirm that the value is not cached, even though we set a hash function that
+    # returns a constant value and that the task has only one input.
+    assert wf(a=2) == 2
+    assert n_cached_task_calls == 2
+    # Confirm that the cache is hit if we execute the workflow with the same value as previous run.
+    assert wf(a=2) == 2
+    assert n_cached_task_calls == 2
+
+
+def test_pass_annotated_to_downstream_tasks():
+    @task
+    def t0(a: int) -> Annotated[int, HashMethod(function=str)]:
+        return a + 1
+
+    @task(cache=True, cache_version="42")
+    def downstream_t(a: int) -> int:
+        global n_cached_task_calls
+        n_cached_task_calls += 1
+        return a + 2
+
+    @dynamic
+    def t1(a: int) -> int:
+        v = t0(a=a)
+
+        # We should have a cache miss in the first call to downstream_t and have a cache hit
+        # on the second call.
+        v_1 = downstream_t(a=v)
+        v_2 = downstream_t(a=v)
+
+        return v_1 + v_2
+
+    assert n_cached_task_calls == 0
+    assert t1(a=3) == (6 + 6)
+    assert n_cached_task_calls == 1
+
+
+def test_pandas_dataframe_hash():
+    """
+    Test that cache is hit in the case of pandas dataframes where we annotated dataframes to hash
+    the contents of the dataframes.
+    """
+
+    def hash_pandas_dataframe(df: pandas.DataFrame) -> str:
+        return str(pandas.util.hash_pandas_object(df))
+
+    @task
+    def uncached_data_reading_task() -> Annotated[pandas.DataFrame, HashMethod(hash_pandas_dataframe)]:
+        return pandas.DataFrame({"column_1": [1, 2, 3]})
+
+    @task(cache=True, cache_version="0.1")
+    def cached_data_processing_task(data: pandas.DataFrame) -> pandas.DataFrame:
+        global n_cached_task_calls
+        n_cached_task_calls += 1
+        return data * 2
+
+    @workflow
+    def my_workflow():
+        raw_data = uncached_data_reading_task()
+        cached_data_processing_task(data=raw_data)
+
+    assert n_cached_task_calls == 0
+    my_workflow()
+    assert n_cached_task_calls == 1
+
+    # Confirm that we see a cache hit in the case of annotated dataframes.
+    my_workflow()
+    assert n_cached_task_calls == 1
diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py
index 6c660b7106..d60dddece4 100644
--- a/tests/flytekit/unit/core/test_type_engine.py
+++ b/tests/flytekit/unit/core/test_type_engine.py
@@ -5,18 +5,24 @@
 from datetime import timedelta
 from enum import Enum
 
+import pandas as pd
 import pytest
 from dataclasses_json import DataClassJsonMixin, dataclass_json
 from flyteidl.core import errors_pb2
 from google.protobuf import json_format as _json_format
 from google.protobuf import struct_pb2 as _struct
 from marshmallow_jsonschema import JSONSchema
+from typing_extensions import Annotated
 
 from flytekit.common.exceptions import user as user_exceptions
+from flytekit.common.types.primitives import Integer
 from flytekit.core.context_manager import FlyteContext, FlyteContextManager
+from flytekit.core.dynamic_workflow_task import dynamic
+from flytekit.core.task import task
 from flytekit.core.type_engine import (
     DataclassTransformer,
     DictTransformer,
+    HashMethod,
     ListTransformer,
     SimpleTransformer,
     TypeEngine,
@@ -32,6 +38,7 @@
 from flytekit.types.file.file import FlyteFile, FlyteFilePathTransformer
 from flytekit.types.pickle import FlytePickle
 from flytekit.types.pickle.pickle import FlytePickleTransformer
+from flytekit.types.schema.types_pandas import PandasDataFrameTransformer
 
 
 def test_type_engine():
@@ -634,3 +641,72 @@ def test_dict_to_literal_map_with_wrong_input_type():
     guessed_python_types = {"a": str}
     with pytest.raises(user_exceptions.FlyteTypeException):
         TypeEngine.dict_to_literal_map(ctx, input, guessed_python_types)
+
+
+def test_pass_annotated_to_downstream_tasks():
+    """
+    Test to confirm that the loaded dataframe is not affected and can be used in @dynamic.
+    """
+    # pandas dataframe hash function
+    def hash_pandas_dataframe(df: pd.DataFrame) -> str:
+        return str(pd.util.hash_pandas_object(df))
+
+    @task
+    def t0(a: int) -> Annotated[int, HashMethod(function=str)]:
+        return a + 1
+
+    @task
+    def annotated_return_task() -> Annotated[pd.DataFrame, HashMethod(hash_pandas_dataframe)]:
+        return pd.DataFrame({"column_1": [1, 2, 3]})
+
+    @task(cache=True, cache_version="42")
+    def downstream_t(a: int, df: pd.DataFrame) -> int:
+        return a + 2 + len(df)
+
+    @dynamic
+    def t1(a: int) -> int:
+        v = t0(a=a)
+        df = annotated_return_task()
+
+        # We should have a cache miss in the first call to downstream_t
+        v_1 = downstream_t(a=v, df=df)
+        v_2 = downstream_t(a=v, df=df)
+
+        return v_1 + v_2
+
+    assert t1(a=3) == (6 + 6 + 6)
+
+
+def test_literal_hash_int_not_set():
+    """
+    Test to confirm that annotating an integer with `HashMethod` does not force the literal to have its
+    hash set.
+    """
+    ctx = FlyteContext.current_context()
+    lv = TypeEngine.to_literal(ctx, 42, Annotated[int, HashMethod(str)], Integer.to_flyte_literal_type())
+    assert lv.scalar.primitive.integer == 42
+    assert lv.hash is None
+
+
+def test_literal_hash_to_python_value():
+    """
+    Test to confirm that literals can be converted to python values, regardless of the hash value set in the literal.
+    """
+    ctx = FlyteContext.current_context()
+
+    def constant_hash(df: pd.DataFrame) -> str:
+        return "h4Sh"
+
+    df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
+    pandas_df_transformer = PandasDataFrameTransformer()
+    literal_with_hash_set = TypeEngine.to_literal(
+        ctx,
+        df,
+        Annotated[pd.DataFrame, HashMethod(constant_hash)],
+        pandas_df_transformer.get_literal_type(pd.DataFrame),
+    )
+    assert literal_with_hash_set.hash == "h4Sh"
+    # Confirm tha the loaded dataframe is not affected
+    python_df = TypeEngine.to_python_value(ctx, literal_with_hash_set, pd.DataFrame)
+    expected_df = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
+    assert expected_df.equals(python_df)
diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py
index e9f54c829b..1406d02657 100644
--- a/tests/flytekit/unit/core/test_type_hints.py
+++ b/tests/flytekit/unit/core/test_type_hints.py
@@ -11,6 +11,7 @@
 import pytest
 from dataclasses_json import dataclass_json
 from google.protobuf.struct_pb2 import Struct
+from typing_extensions import Annotated
 
 import flytekit
 from flytekit import ContainerTask, Secret, SQLTask, dynamic, kwtypes, map_task
@@ -24,7 +25,7 @@
 from flytekit.core.resources import Resources
 from flytekit.core.task import TaskMetadata, task
 from flytekit.core.testing import patch, task_mock
-from flytekit.core.type_engine import RestrictedTypeError, TypeEngine
+from flytekit.core.type_engine import HashMethod, RestrictedTypeError, TypeEngine
 from flytekit.core.workflow import workflow
 from flytekit.models import literals as _literal_models
 from flytekit.models.core import types as _core_types
@@ -1427,3 +1428,110 @@ def foo3(a: typing.Dict) -> typing.Dict:
 
     with pytest.raises(TypeError, match="Not a collection type simple: STRUCT\n but got a list \\[{'hello': 2}\\]"):
         foo3(a=[{"hello": 2}])
+
+
+def test_task_annotate_primitive_type_has_no_effect():
+    @task
+    def plus_two(
+        a: int,
+    ) -> Annotated[int, HashMethod(str)]:  # Note the use of `str` as the hash function for ints. This has no effect.
+        return a + 2
+
+    assert plus_two(a=1) == 3
+
+    ctx = context_manager.FlyteContextManager.current_context()
+    output_lm = plus_two.dispatch_execute(
+        ctx,
+        _literal_models.LiteralMap(
+            literals={
+                "a": _literal_models.Literal(
+                    scalar=_literal_models.Scalar(primitive=_literal_models.Primitive(integer=3))
+                )
+            }
+        ),
+    )
+    assert output_lm.literals["o0"].scalar.primitive.integer == 5
+    assert output_lm.literals["o0"].hash is None
+
+
+def test_task_hash_return_pandas_dataframe():
+    constant_value = "road-hash"
+
+    def constant_function(df: pandas.DataFrame) -> str:
+        return constant_value
+
+    @task
+    def t0() -> Annotated[pandas.DataFrame, HashMethod(constant_function)]:
+        return pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
+
+    ctx = context_manager.FlyteContextManager.current_context()
+    output_lm = t0.dispatch_execute(ctx, _literal_models.LiteralMap(literals={}))
+    assert output_lm.literals["o0"].hash == constant_value
+
+    # Confirm that the literal containing a hash does not have any effect on the scalar.
+    df = TypeEngine.to_python_value(ctx, output_lm.literals["o0"], pandas.DataFrame)
+    expected_df = pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
+    assert df.equals(expected_df)
+
+
+def test_workflow_containing_multiple_annotated_tasks():
+    def hash_function_t0(df: pandas.DataFrame) -> str:
+        return "hash-0"
+
+    @task
+    def t0() -> Annotated[pandas.DataFrame, HashMethod(hash_function_t0)]:
+        return pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
+
+    def hash_function_t1(df: pandas.DataFrame) -> str:
+        return "hash-1"
+
+    @task
+    def t1() -> Annotated[pandas.DataFrame, HashMethod(hash_function_t1)]:
+        return pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]})
+
+    @task
+    def t2() -> pandas.DataFrame:
+        return pandas.DataFrame(data={"col1": [100, 200], "col2": [300, 400]})
+
+    # Auxiliary task used to sum up the dataframes. It demonstrates that the use of `Annotated` does not
+    # have any impact in the definition and execution of cached or uncached downstream tasks
+    @task
+    def sum_dataframes(df0: pandas.DataFrame, df1: pandas.DataFrame, df2: pandas.DataFrame) -> pandas.DataFrame:
+        return df0 + df1 + df2
+
+    @workflow
+    def wf() -> pandas.DataFrame:
+        df0 = t0()
+        df1 = t1()
+        df2 = t2()
+        return sum_dataframes(df0=df0, df1=df1, df2=df2)
+
+    df = wf()
+
+    expected_df = pandas.DataFrame(data={"col1": [1 + 10 + 100, 2 + 20 + 200], "col2": [3 + 30 + 300, 4 + 40 + 400]})
+    assert expected_df.equals(df)
+
+
+def test_list_containing_multiple_annotated_pandas_dataframes():
+    def hash_pandas_dataframe(df: pandas.DataFrame) -> str:
+        return str(pandas.util.hash_pandas_object(df))
+
+    @task
+    def produce_list_of_annotated_dataframes() -> typing.List[
+        Annotated[pandas.DataFrame, HashMethod(hash_pandas_dataframe)]
+    ]:
+        return [pandas.DataFrame({"column_1": [1, 2, 3]}), pandas.DataFrame({"column_1": [4, 5, 6]})]
+
+    @task(cache=True, cache_version="v0")
+    def sum_list_of_pandas_dataframes(lst: typing.List[pandas.DataFrame]) -> pandas.DataFrame:
+        return sum(lst)
+
+    @workflow
+    def wf() -> pandas.DataFrame:
+        lst = produce_list_of_annotated_dataframes()
+        return sum_list_of_pandas_dataframes(lst=lst)
+
+    df = wf()
+
+    expected_df = pandas.DataFrame({"column_1": [5, 7, 9]})
+    assert expected_df.equals(df)

From 93057469cfa212b0b62e80eba4501fceb3b29c85 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Fri, 3 Dec 2021 15:09:43 -0800
Subject: [PATCH 07/35] Iterate using a flyteidl branch

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 requirements.in  | 1 +
 requirements.txt | 8 ++++----
 2 files changed, 5 insertions(+), 4 deletions(-)

diff --git a/requirements.in b/requirements.in
index 408185fca8..e5ee7fadba 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,4 +1,5 @@
 .[all]
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekit
 attrs<21
 # We need to restrict constrain the versions of both jsonschema and pyyaml because of docker-compose (which is
diff --git a/requirements.txt b/requirements.txt
index ee632aa7d5..ebb249137b 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -1,9 +1,11 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
-#    make requirements.txt
+#    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via -r requirements.in
 -e file:.#egg=flytekit
     # via -r requirements.in
 ansiwrap==0.8.4
@@ -81,8 +83,6 @@ entrypoints==0.3
     #   jupyter-client
     #   nbconvert
     #   papermill
-flyteidl==0.21.8
-    # via flytekit
 gevent==21.8.0
     # via sagemaker-training
 greenlet==1.1.2

From ecd8b935eaf3b1feb381450499f63da450eff996 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Fri, 3 Dec 2021 15:28:47 -0800
Subject: [PATCH 08/35] Regenerate requirements files

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 dev-requirements.txt | 47 +++++++++++++++++--------------------
 requirements.txt     | 56 ++++++++++++++++++++++----------------------
 2 files changed, 49 insertions(+), 54 deletions(-)

diff --git a/dev-requirements.txt b/dev-requirements.txt
index a1b1fa3827..c9d8b2a08d 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    make dev-requirements.txt
@@ -44,7 +44,7 @@ chardet==4.0.0
     # via
     #   -c requirements.txt
     #   binaryornot
-charset-normalizer==2.0.7
+charset-normalizer==2.0.9
     # via
     #   -c requirements.txt
     #   requests
@@ -67,13 +67,13 @@ cookiecutter==1.7.3
     # via
     #   -c requirements.txt
     #   flytekit
-coverage[toml]==6.1.1
+coverage[toml]==6.2
     # via -r dev-requirements.in
-croniter==1.0.15
+croniter==1.1.0
     # via
     #   -c requirements.txt
     #   flytekit
-cryptography==35.0.0
+cryptography==36.0.0
     # via
     #   -c requirements.txt
     #   paramiko
@@ -90,7 +90,7 @@ deprecated==1.2.13
     # via
     #   -c requirements.txt
     #   flytekit
-diskcache==5.2.1
+diskcache==5.3.0
     # via
     #   -c requirements.txt
     #   flytekit
@@ -112,21 +112,17 @@ dockerpty==0.4.1
     # via docker-compose
 docopt==0.6.2
     # via docker-compose
-docstring-parser==0.12
+docstring-parser==0.13
     # via
     #   -c requirements.txt
     #   flytekit
-filelock==3.3.2
+filelock==3.4.0
     # via virtualenv
-flyteidl==0.21.8
+grpcio==1.42.0
     # via
     #   -c requirements.txt
     #   flytekit
-grpcio==1.41.1
-    # via
-    #   -c requirements.txt
-    #   flytekit
-identify==2.3.5
+identify==2.4.0
     # via pre-commit
 idna==3.3
     # via
@@ -159,7 +155,7 @@ jsonschema==3.2.0
     # via
     #   -c requirements.txt
     #   docker-compose
-keyring==23.2.1
+keyring==23.4.0
     # via
     #   -c requirements.txt
     #   flytekit
@@ -167,7 +163,7 @@ markupsafe==2.0.1
     # via
     #   -c requirements.txt
     #   jinja2
-marshmallow==3.14.0
+marshmallow==3.14.1
     # via
     #   -c requirements.txt
     #   dataclasses-json
@@ -201,7 +197,7 @@ numpy==1.21.4
     #   -c requirements.txt
     #   pandas
     #   pyarrow
-packaging==21.2
+packaging==21.3
     # via
     #   -c requirements.txt
     #   pytest
@@ -209,7 +205,7 @@ pandas==1.3.4
     # via
     #   -c requirements.txt
     #   flytekit
-paramiko==2.8.0
+paramiko==2.8.1
     # via
     #   -c requirements.txt
     #   docker
@@ -223,19 +219,18 @@ poyo==0.5.0
     # via
     #   -c requirements.txt
     #   cookiecutter
-pre-commit==2.15.0
+pre-commit==2.16.0
     # via -r dev-requirements.in
 protobuf==3.19.1
     # via
     #   -c requirements.txt
-    #   flyteidl
     #   flytekit
 py==1.11.0
     # via
     #   -c requirements.txt
     #   pytest
     #   retry
-pyarrow==6.0.0
+pyarrow==6.0.1
     # via
     #   -c requirements.txt
     #   flytekit
@@ -247,7 +242,7 @@ pynacl==1.4.0
     # via
     #   -c requirements.txt
     #   paramiko
-pyparsing==2.4.7
+pyparsing==3.0.6
     # via
     #   -c requirements.txt
     #   packaging
@@ -271,7 +266,7 @@ python-dateutil==2.8.1
     #   croniter
     #   flytekit
     #   pandas
-python-dotenv==0.19.1
+python-dotenv==0.19.2
     # via docker-compose
 python-json-logger==2.0.2
     # via
@@ -295,7 +290,7 @@ pyyaml==5.4.1
     #   -c requirements.txt
     #   docker-compose
     #   pre-commit
-regex==2021.11.9
+regex==2021.11.10
     # via
     #   -c requirements.txt
     #   docker-image-py
@@ -307,7 +302,7 @@ requests==2.26.0
     #   docker-compose
     #   flytekit
     #   responses
-responses==0.15.0
+responses==0.16.0
     # via
     #   -c requirements.txt
     #   flytekit
@@ -356,7 +351,7 @@ tomli==1.2.2
     # via
     #   -c requirements.txt
     #   coverage
-typing-extensions==3.10.0.2
+typing-extensions==4.0.1
     # via
     #   -c requirements.txt
     #   mypy
diff --git a/requirements.txt b/requirements.txt
index ebb249137b..91122e0a0e 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -2,7 +2,7 @@
 # This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
-#    pip-compile requirements.in
+#    make requirements.txt
 #
 -e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
     # via -r requirements.in
@@ -22,13 +22,13 @@ bcrypt==3.2.0
     # via paramiko
 binaryornot==0.4.4
     # via cookiecutter
-black==21.10b0
+black==21.11b1
     # via papermill
 bleach==4.1.0
     # via nbconvert
-boto3==1.20.2
+boto3==1.20.20
     # via sagemaker-training
-botocore==1.23.2
+botocore==1.23.20
     # via
     #   boto3
     #   s3transfer
@@ -41,7 +41,7 @@ cffi==1.15.0
     #   pynacl
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.7
+charset-normalizer==2.0.9
     # via requests
 checksumdir==1.2.0
     # via flytekit
@@ -56,9 +56,9 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.0.15
+croniter==1.1.0
     # via flytekit
-cryptography==35.0.0
+cryptography==36.0.0
     # via
     #   paramiko
     #   secretstorage
@@ -72,11 +72,11 @@ defusedxml==0.7.1
     # via nbconvert
 deprecated==1.2.13
     # via flytekit
-diskcache==5.2.1
+diskcache==5.3.0
     # via flytekit
 docker-image-py==0.1.12
     # via flytekit
-docstring-parser==0.12
+docstring-parser==0.13
     # via flytekit
 entrypoints==0.3
     # via
@@ -87,7 +87,7 @@ gevent==21.8.0
     # via sagemaker-training
 greenlet==1.1.2
     # via gevent
-grpcio==1.41.1
+grpcio==1.42.0
     # via flytekit
 hmsclient==0.1.1
     # via flytekit
@@ -99,13 +99,13 @@ inotify_simple==1.2.1
     # via sagemaker-training
 ipykernel==5.5.6
     # via flytekit
-ipython==7.29.0
+ipython==7.30.1
     # via ipykernel
 ipython-genutils==0.2.0
     # via
     #   ipykernel
     #   nbformat
-jedi==0.18.0
+jedi==0.18.1
     # via ipython
 jeepney==0.7.1
     # via
@@ -126,7 +126,7 @@ jsonschema==3.2.0
     # via
     #   -r requirements.in
     #   nbformat
-jupyter-client==7.0.6
+jupyter-client==7.1.0
     # via
     #   ipykernel
     #   nbclient
@@ -139,11 +139,11 @@ jupyterlab-pygments==0.1.2
     # via nbconvert
 k8s-proto==0.0.3
     # via flytekit
-keyring==23.2.1
+keyring==23.4.0
     # via flytekit
 markupsafe==2.0.1
     # via jinja2
-marshmallow==3.14.0
+marshmallow==3.14.1
     # via
     #   dataclasses-json
     #   marshmallow-enum
@@ -162,18 +162,18 @@ mypy-extensions==0.4.3
     #   typing-inspect
 natsort==8.0.0
     # via flytekit
-nbclient==0.5.5
+nbclient==0.5.9
     # via
     #   nbconvert
     #   papermill
-nbconvert==6.2.0
+nbconvert==6.3.0
     # via flytekit
 nbformat==5.1.3
     # via
     #   nbclient
     #   nbconvert
     #   papermill
-nest-asyncio==1.5.1
+nest-asyncio==1.5.4
     # via
     #   jupyter-client
     #   nbclient
@@ -184,7 +184,7 @@ numpy==1.21.4
     #   pyarrow
     #   sagemaker-training
     #   scipy
-packaging==21.2
+packaging==21.3
     # via bleach
 pandas==1.3.4
     # via flytekit
@@ -192,9 +192,9 @@ pandocfilters==1.5.0
     # via nbconvert
 papermill==2.3.3
     # via flytekit
-paramiko==2.8.0
+paramiko==2.8.1
     # via sagemaker-training
-parso==0.8.2
+parso==0.8.3
     # via jedi
 pathspec==0.9.0
     # via black
@@ -206,7 +206,7 @@ platformdirs==2.4.0
     # via black
 poyo==0.5.0
     # via cookiecutter
-prompt-toolkit==3.0.22
+prompt-toolkit==3.0.23
     # via ipython
 protobuf==3.19.1
     # via
@@ -222,7 +222,7 @@ py==1.11.0
     # via retry
 py4j==0.10.9.2
     # via pyspark
-pyarrow==6.0.0
+pyarrow==6.0.1
     # via flytekit
 pycparser==2.21
     # via cffi
@@ -233,7 +233,7 @@ pygments==2.10.0
     #   nbconvert
 pynacl==1.4.0
     # via paramiko
-pyparsing==2.4.7
+pyparsing==3.0.6
     # via packaging
 pyrsistent==0.18.0
     # via jsonschema
@@ -263,7 +263,7 @@ pyyaml==5.4.1
     #   papermill
 pyzmq==22.3.0
     # via jupyter-client
-regex==2021.11.9
+regex==2021.11.10
     # via
     #   black
     #   docker-image-py
@@ -273,7 +273,7 @@ requests==2.26.0
     #   flytekit
     #   papermill
     #   responses
-responses==0.15.0
+responses==0.16.0
     # via flytekit
 retry==0.9.2
     # via flytekit
@@ -283,7 +283,7 @@ s3transfer==0.5.0
     # via boto3
 sagemaker-training==3.9.2
     # via flytekit
-scipy==1.7.2
+scipy==1.7.3
     # via sagemaker-training
 secretstorage==3.3.1
     # via keyring
@@ -333,7 +333,7 @@ traitlets==5.1.1
     #   nbclient
     #   nbconvert
     #   nbformat
-typing-extensions==3.10.0.2
+typing-extensions==4.0.1
     # via
     #   black
     #   typing-inspect

From 44b72f6b80e4bd2c34a10933ecd98d37d8edbdb9 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Fri, 3 Dec 2021 15:36:04 -0800
Subject: [PATCH 09/35] Regenerate requirements files

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 doc-requirements.txt                          | 75 +++++++++----------
 requirements-spark2.txt                       | 60 +++++++--------
 .../workflows/requirements.txt                | 51 ++++++++-----
 3 files changed, 97 insertions(+), 89 deletions(-)

diff --git a/doc-requirements.txt b/doc-requirements.txt
index 1e766d5f31..d6d6c6546e 100644
--- a/doc-requirements.txt
+++ b/doc-requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    make doc-requirements.txt
@@ -12,7 +12,7 @@ ansiwrap==0.8.4
     # via papermill
 arrow==1.2.1
     # via jinja2-time
-astroid==2.8.4
+astroid==2.9.0
     # via sphinx-autoapi
 attrs==21.2.0
     # via jsonschema
@@ -29,13 +29,13 @@ beautifulsoup4==4.10.0
     #   sphinx-material
 binaryornot==0.4.4
     # via cookiecutter
-black==21.10b0
+black==21.11b1
     # via papermill
 bleach==4.1.0
     # via nbconvert
-boto3==1.20.2
+boto3==1.20.20
     # via sagemaker-training
-botocore==1.23.2
+botocore==1.23.20
     # via
     #   boto3
     #   s3transfer
@@ -48,7 +48,7 @@ cffi==1.15.0
     #   pynacl
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.7
+charset-normalizer==2.0.9
     # via requests
 checksumdir==1.2.0
     # via flytekit
@@ -63,9 +63,9 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.0.15
+croniter==1.1.0
     # via flytekit
-cryptography==35.0.0
+cryptography==36.0.0
     # via
     #   -r doc-requirements.in
     #   paramiko
@@ -82,11 +82,11 @@ defusedxml==0.7.1
     # via nbconvert
 deprecated==1.2.13
     # via flytekit
-diskcache==5.2.1
+diskcache==5.3.0
     # via flytekit
 docker-image-py==0.1.12
     # via flytekit
-docstring-parser==0.12
+docstring-parser==0.13
     # via flytekit
 docutils==0.17.1
     # via sphinx
@@ -95,15 +95,13 @@ entrypoints==0.3
     #   jupyter-client
     #   nbconvert
     #   papermill
-flyteidl==0.21.8
-    # via flytekit
 furo @ git+git://github.com/flyteorg/furo@main
     # via -r doc-requirements.in
 gevent==21.8.0
     # via sagemaker-training
 greenlet==1.1.2
     # via gevent
-grpcio==1.41.1
+grpcio==1.42.0
     # via
     #   -r doc-requirements.in
     #   flytekit
@@ -115,19 +113,17 @@ imagesize==1.3.0
     # via sphinx
 importlib-metadata==4.8.2
     # via keyring
-importlib-resources==5.4.0
-    # via jsonschema
 inotify_simple==1.2.1
     # via sagemaker-training
 ipykernel==5.5.6
     # via flytekit
-ipython==7.29.0
+ipython==7.30.1
     # via ipykernel
 ipython-genutils==0.2.0
     # via
     #   ipykernel
     #   nbformat
-jedi==0.18.0
+jedi==0.18.1
     # via ipython
 jeepney==0.7.1
     # via
@@ -148,7 +144,7 @@ jmespath==0.10.0
     #   botocore
 jsonschema==4.2.1
     # via nbformat
-jupyter-client==7.0.6
+jupyter-client==7.1.0
     # via
     #   ipykernel
     #   nbclient
@@ -161,7 +157,7 @@ jupyterlab-pygments==0.1.2
     # via nbconvert
 k8s-proto==0.0.3
     # via flytekit
-keyring==23.2.1
+keyring==23.4.0
     # via flytekit
 lazy-object-proxy==1.6.0
     # via astroid
@@ -169,7 +165,7 @@ lxml==4.6.4
     # via sphinx-material
 markupsafe==2.0.1
     # via jinja2
-marshmallow==3.14.0
+marshmallow==3.14.1
     # via
     #   dataclasses-json
     #   marshmallow-enum
@@ -188,18 +184,18 @@ mypy-extensions==0.4.3
     #   typing-inspect
 natsort==8.0.0
     # via flytekit
-nbclient==0.5.5
+nbclient==0.5.9
     # via
     #   nbconvert
     #   papermill
-nbconvert==6.2.0
+nbconvert==6.3.0
     # via flytekit
 nbformat==5.1.3
     # via
     #   nbclient
     #   nbconvert
     #   papermill
-nest-asyncio==1.5.1
+nest-asyncio==1.5.4
     # via
     #   jupyter-client
     #   nbclient
@@ -210,7 +206,7 @@ numpy==1.21.4
     #   pyarrow
     #   sagemaker-training
     #   scipy
-packaging==21.2
+packaging==21.3
     # via
     #   bleach
     #   sphinx
@@ -220,9 +216,9 @@ pandocfilters==1.5.0
     # via nbconvert
 papermill==2.3.3
     # via flytekit
-paramiko==2.8.0
+paramiko==2.8.1
     # via sagemaker-training
-parso==0.8.2
+parso==0.8.3
     # via jedi
 pathspec==0.9.0
     # via black
@@ -234,11 +230,10 @@ platformdirs==2.4.0
     # via black
 poyo==0.5.0
     # via cookiecutter
-prompt-toolkit==3.0.22
+prompt-toolkit==3.0.23
     # via ipython
 protobuf==3.19.1
     # via
-    #   flyteidl
     #   flytekit
     #   k8s-proto
     #   sagemaker-training
@@ -250,7 +245,7 @@ py==1.11.0
     # via retry
 py4j==0.10.9.2
     # via pyspark
-pyarrow==6.0.0
+pyarrow==6.0.1
     # via flytekit
 pycparser==2.21
     # via cffi
@@ -263,7 +258,7 @@ pygments==2.10.0
     #   sphinx-prompt
 pynacl==1.4.0
     # via paramiko
-pyparsing==2.4.7
+pyparsing==3.0.6
     # via packaging
 pyrsistent==0.18.0
     # via jsonschema
@@ -296,7 +291,7 @@ pyyaml==6.0
     #   sphinx-autoapi
 pyzmq==22.3.0
     # via jupyter-client
-regex==2021.11.9
+regex==2021.11.10
     # via
     #   black
     #   docker-image-py
@@ -307,7 +302,7 @@ requests==2.26.0
     #   papermill
     #   responses
     #   sphinx
-responses==0.15.0
+responses==0.16.0
     # via flytekit
 retry==0.9.2
     # via flytekit
@@ -317,7 +312,7 @@ s3transfer==0.5.0
     # via boto3
 sagemaker-training==3.9.2
     # via flytekit
-scipy==1.7.2
+scipy==1.7.3
     # via sagemaker-training
 secretstorage==3.3.1
     # via keyring
@@ -335,13 +330,13 @@ six==1.16.0
     #   sagemaker-training
     #   sphinx-code-include
     #   thrift
-snowballstemmer==2.1.0
+snowballstemmer==2.2.0
     # via sphinx
 sortedcontainers==2.4.0
     # via flytekit
-soupsieve==2.3
+soupsieve==2.3.1
     # via beautifulsoup4
-sphinx==4.2.0
+sphinx==4.3.1
     # via
     #   -r doc-requirements.in
     #   furo
@@ -361,7 +356,7 @@ sphinx-copybutton==0.4.0
     # via -r doc-requirements.in
 sphinx-fontawesome==0.0.6
     # via -r doc-requirements.in
-sphinx-gallery==0.10.0
+sphinx-gallery==0.10.1
     # via -r doc-requirements.in
 sphinx-material==0.0.35
     # via -r doc-requirements.in
@@ -411,7 +406,7 @@ traitlets==5.1.1
     #   nbclient
     #   nbconvert
     #   nbformat
-typing-extensions==3.10.0.2
+typing-extensions==4.0.1
     # via
     #   astroid
     #   black
@@ -442,9 +437,7 @@ wrapt==1.13.3
     #   deprecated
     #   flytekit
 zipp==3.6.0
-    # via
-    #   importlib-metadata
-    #   importlib-resources
+    # via importlib-metadata
 zope.event==4.5.0
     # via gevent
 zope.interface==5.4.0
diff --git a/requirements-spark2.txt b/requirements-spark2.txt
index 37a2664140..98c2f7cfac 100644
--- a/requirements-spark2.txt
+++ b/requirements-spark2.txt
@@ -1,9 +1,11 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    make requirements-spark2.txt
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via -r requirements.in
 -e file:.#egg=flytekit
     # via
     #   -r requirements-spark2.in
@@ -22,13 +24,13 @@ bcrypt==3.2.0
     # via paramiko
 binaryornot==0.4.4
     # via cookiecutter
-black==21.10b0
+black==21.11b1
     # via papermill
 bleach==4.1.0
     # via nbconvert
-boto3==1.20.2
+boto3==1.20.20
     # via sagemaker-training
-botocore==1.23.2
+botocore==1.23.20
     # via
     #   boto3
     #   s3transfer
@@ -41,7 +43,7 @@ cffi==1.15.0
     #   pynacl
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.7
+charset-normalizer==2.0.9
     # via requests
 checksumdir==1.2.0
     # via flytekit
@@ -56,9 +58,9 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.0.15
+croniter==1.1.0
     # via flytekit
-cryptography==35.0.0
+cryptography==36.0.0
     # via
     #   paramiko
     #   secretstorage
@@ -72,24 +74,22 @@ defusedxml==0.7.1
     # via nbconvert
 deprecated==1.2.13
     # via flytekit
-diskcache==5.2.1
+diskcache==5.3.0
     # via flytekit
 docker-image-py==0.1.12
     # via flytekit
-docstring-parser==0.12
+docstring-parser==0.13
     # via flytekit
 entrypoints==0.3
     # via
     #   jupyter-client
     #   nbconvert
     #   papermill
-flyteidl==0.21.8
-    # via flytekit
 gevent==21.8.0
     # via sagemaker-training
 greenlet==1.1.2
     # via gevent
-grpcio==1.41.1
+grpcio==1.42.0
     # via flytekit
 hmsclient==0.1.1
     # via flytekit
@@ -101,13 +101,13 @@ inotify_simple==1.2.1
     # via sagemaker-training
 ipykernel==5.5.6
     # via flytekit
-ipython==7.29.0
+ipython==7.30.1
     # via ipykernel
 ipython-genutils==0.2.0
     # via
     #   ipykernel
     #   nbformat
-jedi==0.18.0
+jedi==0.18.1
     # via ipython
 jeepney==0.7.1
     # via
@@ -128,7 +128,7 @@ jsonschema==3.2.0
     # via
     #   -r requirements.in
     #   nbformat
-jupyter-client==7.0.6
+jupyter-client==7.1.0
     # via
     #   ipykernel
     #   nbclient
@@ -141,11 +141,11 @@ jupyterlab-pygments==0.1.2
     # via nbconvert
 k8s-proto==0.0.3
     # via flytekit
-keyring==23.2.1
+keyring==23.4.0
     # via flytekit
 markupsafe==2.0.1
     # via jinja2
-marshmallow==3.14.0
+marshmallow==3.14.1
     # via
     #   dataclasses-json
     #   marshmallow-enum
@@ -164,18 +164,18 @@ mypy-extensions==0.4.3
     #   typing-inspect
 natsort==8.0.0
     # via flytekit
-nbclient==0.5.5
+nbclient==0.5.9
     # via
     #   nbconvert
     #   papermill
-nbconvert==6.2.0
+nbconvert==6.3.0
     # via flytekit
 nbformat==5.1.3
     # via
     #   nbclient
     #   nbconvert
     #   papermill
-nest-asyncio==1.5.1
+nest-asyncio==1.5.4
     # via
     #   jupyter-client
     #   nbclient
@@ -186,7 +186,7 @@ numpy==1.21.4
     #   pyarrow
     #   sagemaker-training
     #   scipy
-packaging==21.2
+packaging==21.3
     # via bleach
 pandas==1.3.4
     # via flytekit
@@ -194,9 +194,9 @@ pandocfilters==1.5.0
     # via nbconvert
 papermill==2.3.3
     # via flytekit
-paramiko==2.8.0
+paramiko==2.8.1
     # via sagemaker-training
-parso==0.8.2
+parso==0.8.3
     # via jedi
 pathspec==0.9.0
     # via black
@@ -208,7 +208,7 @@ platformdirs==2.4.0
     # via black
 poyo==0.5.0
     # via cookiecutter
-prompt-toolkit==3.0.22
+prompt-toolkit==3.0.23
     # via ipython
 protobuf==3.19.1
     # via
@@ -224,7 +224,7 @@ py==1.11.0
     # via retry
 py4j==0.10.9.2
     # via pyspark
-pyarrow==6.0.0
+pyarrow==6.0.1
     # via flytekit
 pycparser==2.21
     # via cffi
@@ -235,7 +235,7 @@ pygments==2.10.0
     #   nbconvert
 pynacl==1.4.0
     # via paramiko
-pyparsing==2.4.7
+pyparsing==3.0.6
     # via packaging
 pyrsistent==0.18.0
     # via jsonschema
@@ -265,7 +265,7 @@ pyyaml==5.4.1
     #   papermill
 pyzmq==22.3.0
     # via jupyter-client
-regex==2021.11.9
+regex==2021.11.10
     # via
     #   black
     #   docker-image-py
@@ -275,7 +275,7 @@ requests==2.26.0
     #   flytekit
     #   papermill
     #   responses
-responses==0.15.0
+responses==0.16.0
     # via flytekit
 retry==0.9.2
     # via flytekit
@@ -285,7 +285,7 @@ s3transfer==0.5.0
     # via boto3
 sagemaker-training==3.9.2
     # via flytekit
-scipy==1.7.2
+scipy==1.7.3
     # via sagemaker-training
 secretstorage==3.3.1
     # via keyring
@@ -335,7 +335,7 @@ traitlets==5.1.1
     #   nbclient
     #   nbconvert
     #   nbformat
-typing-extensions==3.10.0.2
+typing-extensions==4.0.1
     # via
     #   black
     #   typing-inspect
diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
index a37b0c3af8..e605f635e6 100644
--- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
+++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.8
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    make tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
@@ -14,7 +14,7 @@ cffi==1.15.0
     # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.7
+charset-normalizer==2.0.9
     # via requests
 checksumdir==1.2.0
     # via flytekit
@@ -26,9 +26,9 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.0.15
+croniter==1.1.0
     # via flytekit
-cryptography==35.0.0
+cryptography==36.0.0
     # via secretstorage
 cycler==0.11.0
     # via matplotlib
@@ -38,17 +38,19 @@ decorator==5.1.0
     # via retry
 deprecated==1.2.13
     # via flytekit
-diskcache==5.2.1
+diskcache==5.3.0
     # via flytekit
 docker-image-py==0.1.12
     # via flytekit
-docstring-parser==0.12
+docstring-parser==0.13
     # via flytekit
-flyteidl==0.21.8
+flyteidl==0.21.12
     # via flytekit
 flytekit==0.24.0
     # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in
-grpcio==1.41.1
+fonttools==4.28.3
+    # via matplotlib
+grpcio==1.42.0
     # via flytekit
 idna==3.3
     # via requests
@@ -66,13 +68,13 @@ jinja2-time==0.2.0
     # via cookiecutter
 joblib==1.1.0
     # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in
-keyring==23.2.1
+keyring==23.4.0
     # via flytekit
 kiwisolver==1.3.2
     # via matplotlib
 markupsafe==2.0.1
     # via jinja2
-marshmallow==3.14.0
+marshmallow==3.14.1
     # via
     #   dataclasses-json
     #   marshmallow-enum
@@ -81,7 +83,7 @@ marshmallow-enum==1.5.1
     # via dataclasses-json
 marshmallow-jsonschema==0.13.0
     # via flytekit
-matplotlib==3.4.3
+matplotlib==3.5.0
     # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in
 mypy-extensions==0.4.3
     # via typing-inspect
@@ -93,8 +95,12 @@ numpy==1.21.4
     #   opencv-python
     #   pandas
     #   pyarrow
-opencv-python==4.5.4.58
+opencv-python==4.5.4.60
     # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in
+packaging==21.3
+    # via
+    #   matplotlib
+    #   setuptools-scm
 pandas==1.3.4
     # via flytekit
 pillow==8.4.0
@@ -107,12 +113,14 @@ protobuf==3.19.1
     #   flytekit
 py==1.11.0
     # via retry
-pyarrow==6.0.0
+pyarrow==6.0.1
     # via flytekit
 pycparser==2.21
     # via cffi
-pyparsing==3.0.5
-    # via matplotlib
+pyparsing==3.0.6
+    # via
+    #   matplotlib
+    #   packaging
 python-dateutil==2.8.1
     # via
     #   arrow
@@ -130,19 +138,21 @@ pytz==2018.4
     # via
     #   flytekit
     #   pandas
-regex==2021.11.9
+regex==2021.11.10
     # via docker-image-py
 requests==2.26.0
     # via
     #   cookiecutter
     #   flytekit
     #   responses
-responses==0.15.0
+responses==0.16.0
     # via flytekit
 retry==0.9.2
     # via flytekit
 secretstorage==3.3.1
     # via keyring
+setuptools-scm==6.3.2
+    # via matplotlib
 six==1.16.0
     # via
     #   cookiecutter
@@ -156,7 +166,9 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==3.10.0.2
+tomli==1.2.2
+    # via setuptools-scm
+typing-extensions==4.0.1
     # via typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
@@ -175,3 +187,6 @@ wrapt==1.13.3
     #   flytekit
 zipp==3.6.0
     # via importlib-metadata
+
+# The following packages are considered to be unsafe in a requirements file:
+# setuptools

From 22c90d397ea14ff540b15c2855d7be180bb93676 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 20 Jan 2022 17:00:25 -0800
Subject: [PATCH 10/35] Move _hash_overridable to
 StructureDatasetTransformerEngine

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/types/structured/structured_dataset.py | 14 +++++++++++---
 tests/flytekit/unit/core/test_type_engine.py    |  4 +++-
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/flytekit/types/structured/structured_dataset.py b/flytekit/types/structured/structured_dataset.py
index fb9ab4f79f..ba90b575f5 100644
--- a/flytekit/types/structured/structured_dataset.py
+++ b/flytekit/types/structured/structured_dataset.py
@@ -22,7 +22,7 @@
 import pyarrow as pa
 
 from flytekit.core.context_manager import FlyteContext, FlyteContextManager
-from flytekit.core.type_engine import TypeTransformer
+from flytekit.core.type_engine import HashMethod, TypeTransformer
 from flytekit.extend import TypeEngine
 from flytekit.loggers import logger
 from flytekit.models import literals
@@ -335,6 +335,9 @@ def __init__(self):
         super().__init__("StructuredDataset Transformer", StructuredDataset)
         self._type_assertions_enabled = False
 
+        # Instances of StructuredDataset can have their hashes overriden to facilitate the case of caching them.
+        self._hash_overridable = True
+
     def register_handler(self, h: Handlers, default_for_type: Optional[bool] = True, override: Optional[bool] = False):
         """
         Call this with any handler to register it with this dataframe meta-transformer
@@ -534,8 +537,13 @@ def _get_dataset_type(self, t: typing.Union[Type[StructuredDataset], typing.Any]
         # my_cols = kwtypes(x=int, y=str)
         # 1. Fill in format correctly by checking for typing.annotated. For example, Annotated[pd.Dataframe, my_cols]
         if get_origin(t) is Annotated:
-            _, *hint_args = get_args(t)
-            if type(hint_args[0]) is collections.OrderedDict:
+            annotated_type, *hint_args = get_args(t)
+
+            # TODO: Strenghten this method of detecting valid annotations. We need a way to handle multiple annotations as opposed
+            # to assuming that only a single annotation exists.
+            if type(hint_args[0]) is HashMethod:
+                return self._get_dataset_type(annotated_type)
+            elif type(hint_args[0]) is collections.OrderedDict:
                 for k, v in hint_args[0].items():
                     lt = self._get_dataset_column_literal_type(v)
                     converted_cols.append(StructuredDatasetType.DatasetColumn(name=k, literal_type=lt))
diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py
index d000c3912f..f7d6aa128b 100644
--- a/tests/flytekit/unit/core/test_type_engine.py
+++ b/tests/flytekit/unit/core/test_type_engine.py
@@ -859,7 +859,9 @@ def test_literal_hash_int_not_set():
     hash set.
     """
     ctx = FlyteContext.current_context()
-    lv = TypeEngine.to_literal(ctx, 42, Annotated[int, HashMethod(str)], Integer.to_flyte_literal_type())
+    lv = TypeEngine.to_literal(
+        ctx, 42, Annotated[int, HashMethod(str)], LiteralType(simple=model_types.SimpleType.INTEGER)
+    )
     assert lv.scalar.primitive.integer == 42
     assert lv.hash is None
 

From 6ac4b44a08cbd3b7a93a2695884fc5b11ab88222 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Tue, 25 Jan 2022 17:33:41 -0800
Subject: [PATCH 11/35] Move HashMethod to flytekit.core.hash

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/__init__.py                         |  1 +
 flytekit/core/hash.py                        | 20 ++++++++++++++++++++
 flytekit/core/type_engine.py                 | 20 ++++----------------
 tests/flytekit/unit/core/test_local_cache.py |  2 +-
 tests/flytekit/unit/core/test_type_engine.py |  2 +-
 tests/flytekit/unit/core/test_type_hints.py  |  3 ++-
 6 files changed, 29 insertions(+), 19 deletions(-)

diff --git a/flytekit/__init__.py b/flytekit/__init__.py
index fd8fcc7d9c..63e9bf3687 100644
--- a/flytekit/__init__.py
+++ b/flytekit/__init__.py
@@ -167,6 +167,7 @@
 from flytekit.core.context_manager import ExecutionParameters, FlyteContext, FlyteContextManager
 from flytekit.core.data_persistence import DataPersistence, DataPersistencePlugins
 from flytekit.core.dynamic_workflow_task import dynamic
+from flytekit.core.hash import HashMethod
 from flytekit.core.launch_plan import LaunchPlan
 from flytekit.core.map_task import map_task
 from flytekit.core.notification import Email, PagerDuty, Slack
diff --git a/flytekit/core/hash.py b/flytekit/core/hash.py
index 8b2559f696..2aca94e846 100644
--- a/flytekit/core/hash.py
+++ b/flytekit/core/hash.py
@@ -1,3 +1,23 @@
+from typing import Callable, Generic, TypeVar
+
+T = TypeVar("T")
+
+
 class HashOnReferenceMixin(object):
     def __hash__(self):
         return hash(id(self))
+
+
+class HashMethod(Generic[T]):
+    """
+    Flyte-specific object used to wrap the hash function for a specific type
+    """
+
+    def __init__(self, function: Callable[[T], str]):
+        self._function = function
+
+    def calculate(self, obj: T) -> str:
+        """
+        Calculate hash for `obj`.
+        """
+        return self._function(obj)
diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py
index fa00c58421..24132c3f3e 100644
--- a/flytekit/core/type_engine.py
+++ b/flytekit/core/type_engine.py
@@ -8,12 +8,7 @@
 import mimetypes
 import typing
 from abc import ABC, abstractmethod
-from typing import Callable, NamedTuple, Optional, Type, cast
-
-try:
-    from typing import Annotated, get_args, get_origin
-except ImportError:
-    from typing_extensions import Annotated, get_origin, get_args
+from typing import NamedTuple, Optional, Type, cast
 
 from dataclasses_json import DataClassJsonMixin, dataclass_json
 from google.protobuf import json_format as _json_format
@@ -24,8 +19,10 @@
 from google.protobuf.struct_pb2 import Struct
 from marshmallow_enum import EnumField, LoadDumpOptions
 from marshmallow_jsonschema import JSONSchema
+from typing_extensions import Annotated, get_args, get_origin
 
 from flytekit.core.context_manager import FlyteContext
+from flytekit.core.hash import HashMethod
 from flytekit.core.type_helpers import load_type_from_tag
 from flytekit.exceptions import user as user_exceptions
 from flytekit.loggers import logger
@@ -49,15 +46,6 @@
 DEFINITIONS = "definitions"
 
 
-# TODO: expose this in flytekit.__init__
-class HashMethod(typing.Generic[T]):
-    def __init__(self, function: Callable[[T], str]):
-        self._function = function
-
-    def calculate(self, obj: T) -> str:
-        return self._function(obj)
-
-
 class TypeTransformer(typing.Generic[T]):
     """
     Base transformer type that should be implemented for every python native type that can be handled by flytekit
@@ -634,7 +622,7 @@ def to_literal(cls, ctx: FlyteContext, python_val: typing.Any, python_type: Type
                 break
 
         lv = transformer.to_literal(ctx, python_val, python_type, expected)
-        # TODO: should we worry about empty string?
+
         if hash is not None:
             lv.hash = hash
         return lv
diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py
index 7431347544..922fdfcccd 100644
--- a/tests/flytekit/unit/core/test_local_cache.py
+++ b/tests/flytekit/unit/core/test_local_cache.py
@@ -8,10 +8,10 @@
 from typing_extensions import Annotated
 
 from flytekit import SQLTask, dynamic, kwtypes
+from flytekit.core.hash import HashMethod
 from flytekit.core.local_cache import LocalTaskCache
 from flytekit.core.task import TaskMetadata, task
 from flytekit.core.testing import task_mock
-from flytekit.core.type_engine import HashMethod
 from flytekit.core.workflow import workflow
 from flytekit.types.schema import FlyteSchema
 
diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py
index 453c5385eb..ee47e499a8 100644
--- a/tests/flytekit/unit/core/test_type_engine.py
+++ b/tests/flytekit/unit/core/test_type_engine.py
@@ -21,11 +21,11 @@
 from flytekit import kwtypes
 from flytekit.core.context_manager import FlyteContext, FlyteContextManager
 from flytekit.core.dynamic_workflow_task import dynamic
+from flytekit.core.hash import HashMethod
 from flytekit.core.task import task
 from flytekit.core.type_engine import (
     DataclassTransformer,
     DictTransformer,
-    HashMethod,
     ListTransformer,
     LiteralsResolver,
     SimpleTransformer,
diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py
index 2b944cf1c1..6885e73f99 100644
--- a/tests/flytekit/unit/core/test_type_hints.py
+++ b/tests/flytekit/unit/core/test_type_hints.py
@@ -22,12 +22,13 @@
 from flytekit.core.condition import conditional
 from flytekit.core.context_manager import ExecutionState, FastSerializationSettings, Image, ImageConfig
 from flytekit.core.data_persistence import FileAccessProvider
+from flytekit.core.hash import HashMethod
 from flytekit.core.node import Node
 from flytekit.core.promise import NodeOutput, Promise, VoidPromise
 from flytekit.core.resources import Resources
 from flytekit.core.task import TaskMetadata, task
 from flytekit.core.testing import patch, task_mock
-from flytekit.core.type_engine import HashMethod, RestrictedTypeError, TypeEngine
+from flytekit.core.type_engine import RestrictedTypeError, TypeEngine
 from flytekit.core.workflow import workflow
 from flytekit.models import literals as _literal_models
 from flytekit.models.core import types as _core_types

From 4552e788c53c5b59f086b4a39530873129c1c0c4 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 14:40:43 -0800
Subject: [PATCH 12/35] Fix `unit_test` make target

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 Makefile | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index b5464e205e..adbed3a374 100644
--- a/Makefile
+++ b/Makefile
@@ -50,7 +50,7 @@ test: lint unit_test
 
 .PHONY: unit_test
 unit_test:
-	FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit tests/flytekit_compatibility
+	FLYTE_SDK_USE_STRUCTURED_DATASET=FALSE pytest tests/flytekit_compatibility && FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit
 
 requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt
 requirements-spark2.txt: requirements-spark2.in install-piptools

From 24f67a0acc02427acaed92f6f09165bce341c7c4 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 14:42:50 -0800
Subject: [PATCH 13/35] Split `unit_test` make target in two lines

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 Makefile | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/Makefile b/Makefile
index adbed3a374..404c1eb1fd 100644
--- a/Makefile
+++ b/Makefile
@@ -50,7 +50,8 @@ test: lint unit_test
 
 .PHONY: unit_test
 unit_test:
-	FLYTE_SDK_USE_STRUCTURED_DATASET=FALSE pytest tests/flytekit_compatibility && FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit
+	FLYTE_SDK_USE_STRUCTURED_DATASET=FALSE pytest tests/flytekit_compatibility && \
+		FLYTE_SDK_USE_STRUCTURED_DATASET=TRUE pytest tests/flytekit/unit
 
 requirements-spark2.txt: export CUSTOM_COMPILE_COMMAND := make requirements-spark2.txt
 requirements-spark2.txt: requirements-spark2.in install-piptools

From 60ecf3a4ce584b90eaf10f830715a9f8c26be4d7 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 14:46:17 -0800
Subject: [PATCH 14/35] Add assert to structured dataset compatibility test

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 tests/flytekit_compatibility/test_structured_dataset.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/tests/flytekit_compatibility/test_structured_dataset.py b/tests/flytekit_compatibility/test_structured_dataset.py
index 20965cb802..f935305210 100644
--- a/tests/flytekit_compatibility/test_structured_dataset.py
+++ b/tests/flytekit_compatibility/test_structured_dataset.py
@@ -1,8 +1,12 @@
 import pandas as pd
 
+from flytekit.configuration.sdk import USE_STRUCTURED_DATASET
 from flytekit.core.type_engine import TypeEngine
 
 
 def test_pandas_is_schema_with_flag():
+    # This test can only be run iff USE_STRUCTURED_DATASET is not set
+    assert USE_STRUCTURED_DATASET.get() is False
+
     lt = TypeEngine.to_literal_type(pd.DataFrame)
     assert lt.schema is not None

From 3aeedebf545d71069a79f023ebf6bd65cd85a8da Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 15:28:44 -0800
Subject: [PATCH 15/35] Remove TODO

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/models/literals.py | 4 ----
 1 file changed, 4 deletions(-)

diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py
index de04dc5172..92c2c7e322 100644
--- a/flytekit/models/literals.py
+++ b/flytekit/models/literals.py
@@ -868,9 +868,5 @@ def from_flyte_idl(cls, pb2_object):
             scalar=Scalar.from_flyte_idl(pb2_object.scalar) if pb2_object.HasField("scalar") else None,
             collection=collection,
             map=LiteralMap.from_flyte_idl(pb2_object.map) if pb2_object.HasField("map") else None,
-            # TODO: explain that string always have a value set in protobufs, which means
-            # if we want to differentiate between the case of empty string and null we would have
-            # to wrap that in a separate value. Instead, we're deliberately setting `hash` to
-            # None in case of the default value (i.e. empty string).
             hash=pb2_object.hash if pb2_object.hash else None,
         )

From c2dbb540ad716e883932dd9255045c128e16c6a0 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 15:28:56 -0800
Subject: [PATCH 16/35] Regenerate plugins requirements files pointing to the
 right version of flyteidl.

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 plugins/flytekit-aws-athena/requirements.in   |   1 +
 plugins/flytekit-aws-athena/requirements.txt  |  38 +++--
 plugins/flytekit-aws-batch/requirements.in    |   1 +
 plugins/flytekit-aws-batch/requirements.txt   |  60 ++++---
 .../flytekit-aws-sagemaker/requirements.in    |   1 +
 .../flytekit-aws-sagemaker/requirements.txt   |  43 ++---
 plugins/flytekit-bigquery/requirements.in     |   1 +
 plugins/flytekit-bigquery/requirements.txt    |  35 ++--
 plugins/flytekit-data-fsspec/requirements.in  |   1 +
 plugins/flytekit-data-fsspec/requirements.txt |  44 +++--
 plugins/flytekit-dolt/requirements.in         |   1 +
 plugins/flytekit-dolt/requirements.txt        | 150 ++----------------
 .../requirements.in                           |   1 +
 .../requirements.txt                          |  87 +++++-----
 plugins/flytekit-hive/requirements.in         |   1 +
 plugins/flytekit-hive/requirements.txt        |  38 +++--
 plugins/flytekit-k8s-pod/requirements.in      |   1 +
 plugins/flytekit-k8s-pod/requirements.txt     |  40 +++--
 plugins/flytekit-kf-mpi/requirements.in       |   1 +
 plugins/flytekit-kf-mpi/requirements.txt      |  41 +++--
 plugins/flytekit-kf-pytorch/requirements.in   |   1 +
 plugins/flytekit-kf-pytorch/requirements.txt  |  38 +++--
 .../flytekit-kf-tensorflow/requirements.in    |   1 +
 .../flytekit-kf-tensorflow/requirements.txt   |  38 +++--
 plugins/flytekit-modin/requirements.in        |   1 +
 plugins/flytekit-pandera/requirements.in      |   1 +
 plugins/flytekit-pandera/requirements.txt     |  43 +++--
 plugins/flytekit-papermill/requirements.in    |   1 +
 plugins/flytekit-papermill/requirements.txt   |  62 ++++----
 plugins/flytekit-snowflake/requirements.in    |   1 +
 plugins/flytekit-snowflake/requirements.txt   |  38 +++--
 plugins/flytekit-spark/requirements.in        |   1 +
 plugins/flytekit-spark/requirements.txt       |  38 +++--
 plugins/flytekit-sqlalchemy/requirements.in   |   1 +
 plugins/flytekit-sqlalchemy/requirements.txt  |  38 +++--
 35 files changed, 453 insertions(+), 436 deletions(-)

diff --git a/plugins/flytekit-aws-athena/requirements.in b/plugins/flytekit-aws-athena/requirements.in
index fb5e57406e..cd57503478 100644
--- a/plugins/flytekit-aws-athena/requirements.in
+++ b/plugins/flytekit-aws-athena/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-athena
diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt
index a1acae494f..0fa90e3fba 100644
--- a/plugins/flytekit-aws-athena/requirements.txt
+++ b/plugins/flytekit-aws-athena/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-athena
     # via -r requirements.in
 arrow==1.2.2
@@ -12,13 +16,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +32,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +48,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-athena
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -79,14 +87,12 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +107,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +126,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +139,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-aws-batch/requirements.in b/plugins/flytekit-aws-batch/requirements.in
index 4c2c1fae97..8013b8f761 100644
--- a/plugins/flytekit-aws-batch/requirements.in
+++ b/plugins/flytekit-aws-batch/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-awsbatch
diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt
index 9181490234..e0b9738666 100644
--- a/plugins/flytekit-aws-batch/requirements.txt
+++ b/plugins/flytekit-aws-batch/requirements.txt
@@ -1,24 +1,30 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-awsbatch
     # via -r requirements.in
-arrow==1.2.1
+arrow==1.2.2
     # via jinja2-time
 binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.10
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,8 +32,10 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
+cryptography==36.0.1
+    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 decorator==5.1.1
@@ -40,16 +48,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.21.23
-    # via flytekit
-flytekit==0.26.0
+flytekit==0.30.3
     # via flytekitplugins-awsbatch
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -71,25 +81,25 @@ marshmallow-jsonschema==0.13.0
     # via flytekit
 mypy-extensions==0.4.3
     # via typing-inspect
-natsort==8.0.2
+natsort==8.1.0
     # via flytekit
-numpy==1.22.1
+numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.3.5
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
-protobuf==3.19.3
-    # via
-    #   flyteidl
-    #   flytekit
+protobuf==3.19.4
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
     # via flytekit
-python-dateutil==2.8.1
+pycparser==2.21
+    # via cffi
+python-dateutil==2.8.2
     # via
     #   arrow
     #   croniter
@@ -97,7 +107,7 @@ python-dateutil==2.8.1
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -112,25 +122,27 @@ requests==2.27.1
     #   cookiecutter
     #   flytekit
     #   responses
-responses==0.17.0
+responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
-    #   flytekit
     #   grpcio
     #   python-dateutil
-    #   responses
 sortedcontainers==2.4.0
     # via flytekit
 statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
-    # via typing-inspect
+typing-extensions==4.1.1
+    # via
+    #   flytekit
+    #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
 urllib3==1.26.8
diff --git a/plugins/flytekit-aws-sagemaker/requirements.in b/plugins/flytekit-aws-sagemaker/requirements.in
index 1f40a6102b..6e2136aac9 100644
--- a/plugins/flytekit-aws-sagemaker/requirements.in
+++ b/plugins/flytekit-aws-sagemaker/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-awssagemaker
diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt
index 11396f213e..48a70cf6c1 100644
--- a/plugins/flytekit-aws-sagemaker/requirements.txt
+++ b/plugins/flytekit-aws-sagemaker/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-awssagemaker
     # via -r requirements.in
 arrow==1.2.2
@@ -12,9 +16,9 @@ bcrypt==3.2.0
     # via paramiko
 binaryornot==0.4.4
     # via cookiecutter
-boto3==1.20.50
+boto3==1.21.2
     # via sagemaker-training
-botocore==1.23.50
+botocore==1.24.2
     # via
     #   boto3
     #   s3transfer
@@ -27,11 +31,11 @@ cffi==1.15.0
     #   pynacl
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -39,7 +43,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via
@@ -57,22 +61,24 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-awssagemaker
 gevent==21.12.0
     # via sagemaker-training
 greenlet==1.1.2
     # via gevent
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
-inotify-simple==1.2.1
+inotify_simple==1.2.1
     # via sagemaker-training
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -106,7 +112,7 @@ numpy==1.22.2
     #   pyarrow
     #   sagemaker-training
     #   scipy
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 paramiko==2.9.2
     # via sagemaker-training
@@ -114,7 +120,6 @@ poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
     # via
-    #   flyteidl
     #   flytekit
     #   sagemaker-training
 psutil==5.9.0
@@ -136,7 +141,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -163,6 +168,8 @@ sagemaker-training==3.9.2
     # via flytekitplugins-awssagemaker
 scipy==1.8.0
     # via sagemaker-training
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   bcrypt
@@ -177,7 +184,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
@@ -199,9 +206,9 @@ wrapt==1.13.3
     #   flytekit
 zipp==3.7.0
     # via importlib-metadata
-zope-event==4.5.0
+zope.event==4.5.0
     # via gevent
-zope-interface==5.4.0
+zope.interface==5.4.0
     # via gevent
 
 # The following packages are considered to be unsafe in a requirements file:
diff --git a/plugins/flytekit-bigquery/requirements.in b/plugins/flytekit-bigquery/requirements.in
index d01f21ac2b..c7912742b5 100644
--- a/plugins/flytekit-bigquery/requirements.in
+++ b/plugins/flytekit-bigquery/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-bigquery
diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt
index f0bc647453..4b25a52098 100644
--- a/plugins/flytekit-bigquery/requirements.txt
+++ b/plugins/flytekit-bigquery/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-bigquery
     # via -r requirements.in
 arrow==1.2.2
@@ -18,11 +22,11 @@ cffi==1.15.0
     # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -30,7 +34,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -46,9 +50,7 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-bigquery
 google-api-core[grpc]==2.5.0
     # via
@@ -58,29 +60,29 @@ google-auth==2.6.0
     # via
     #   google-api-core
     #   google-cloud-core
-google-cloud-bigquery==2.32.0
+google-cloud-bigquery==2.33.0
     # via flytekitplugins-bigquery
 google-cloud-core==2.2.2
     # via google-cloud-bigquery
 google-crc32c==1.3.0
     # via google-resumable-media
-google-resumable-media==2.2.0
+google-resumable-media==2.2.1
     # via google-cloud-bigquery
 googleapis-common-protos==1.54.0
     # via
     #   google-api-core
     #   grpcio-status
-grpcio==1.43.0
+grpcio==1.44.0
     # via
     #   flytekit
     #   google-api-core
     #   google-cloud-bigquery
     #   grpcio-status
-grpcio-status==1.43.0
+grpcio-status==1.44.0
     # via google-api-core
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
 jeepney==0.7.1
     # via
@@ -115,15 +117,14 @@ numpy==1.22.2
     #   pyarrow
 packaging==21.3
     # via google-cloud-bigquery
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
-proto-plus==1.20.0
+proto-plus==1.20.2
     # via google-cloud-bigquery
 protobuf==3.19.4
     # via
-    #   flyteidl
     #   flytekit
     #   google-api-core
     #   google-cloud-bigquery
@@ -153,7 +154,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -190,7 +191,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-data-fsspec/requirements.in b/plugins/flytekit-data-fsspec/requirements.in
index 127b779735..4c03034cce 100644
--- a/plugins/flytekit-data-fsspec/requirements.in
+++ b/plugins/flytekit-data-fsspec/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-data-fsspec
diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt
index 4dc89fa500..344d8c1f41 100644
--- a/plugins/flytekit-data-fsspec/requirements.txt
+++ b/plugins/flytekit-data-fsspec/requirements.txt
@@ -1,26 +1,32 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-data-fsspec
     # via -r requirements.in
 arrow==1.2.2
     # via jinja2-time
 binaryornot==0.4.4
     # via cookiecutter
-botocore==1.23.24
+botocore==1.24.2
     # via flytekitplugins-data-fsspec
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -28,8 +34,10 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
+cryptography==36.0.1
+    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 decorator==5.1.1
@@ -42,18 +50,20 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-data-fsspec
 fsspec==2022.1.0
     # via flytekitplugins-data-fsspec
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -83,18 +93,18 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
     # via flytekit
+pycparser==2.21
+    # via cffi
 python-dateutil==2.8.2
     # via
     #   arrow
@@ -104,7 +114,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -123,6 +133,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -134,7 +146,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-dolt/requirements.in b/plugins/flytekit-dolt/requirements.in
index 53d807299a..04990b3216 100644
--- a/plugins/flytekit-dolt/requirements.in
+++ b/plugins/flytekit-dolt/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-dolt
diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt
index 01dfe2cb6b..221fc36e43 100644
--- a/plugins/flytekit-dolt/requirements.txt
+++ b/plugins/flytekit-dolt/requirements.txt
@@ -1,160 +1,44 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e file:.#egg=flytekitplugins-dolt
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
     # via -r requirements.in
-arrow==1.2.2
-    # via jinja2-time
-binaryornot==0.4.4
-    # via cookiecutter
-certifi==2021.10.8
-    # via requests
-chardet==4.0.0
-    # via binaryornot
-charset-normalizer==2.0.11
-    # via requests
-checksumdir==1.2.0
-    # via flytekit
-click==7.1.2
+-e file:../../.#egg=flytekit
     # via
-    #   cookiecutter
-    #   flytekit
-cloudpickle==2.0.0
-    # via flytekit
-cookiecutter==1.7.3
-    # via flytekit
-croniter==1.2.0
-    # via flytekit
-cryptography==36.0.1
-    # via secretstorage
+    #   -r requirements.in
+    #   flytekitplugins-dolt
+-e file:.#egg=flytekitplugins-dolt
+    # via -r requirements.in
 dataclasses-json==0.5.6
-    # via
-    #   dolt-integrations
-    #   flytekit
-decorator==5.1.1
-    # via retry
-deprecated==1.2.13
-    # via flytekit
-diskcache==5.4.0
-    # via flytekit
-docker-image-py==0.1.12
-    # via flytekit
-docstring-parser==0.13
-    # via flytekit
+    # via dolt-integrations
 dolt-integrations==0.1.5
     # via flytekitplugins-dolt
 doltcli==0.1.17
     # via dolt-integrations
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
-    # via flytekitplugins-dolt
-grpcio==1.43.0
-    # via flytekit
-idna==3.3
-    # via requests
-importlib-metadata==4.10.1
-    # via keyring
-jinja2==3.0.3
-    # via
-    #   cookiecutter
-    #   jinja2-time
-jinja2-time==0.2.0
-    # via cookiecutter
-keyring==23.5.0
-    # via flytekit
-markupsafe==2.0.1
-    # via jinja2
 marshmallow==3.14.1
     # via
     #   dataclasses-json
     #   marshmallow-enum
-    #   marshmallow-jsonschema
 marshmallow-enum==1.5.1
     # via dataclasses-json
-marshmallow-jsonschema==0.13.0
-    # via flytekit
 mypy-extensions==0.4.3
     # via typing-inspect
-natsort==8.1.0
-    # via flytekit
 numpy==1.22.2
-    # via
-    #   pandas
-    #   pyarrow
-pandas==1.4.0
-    # via
-    #   dolt-integrations
-    #   flytekit
-poyo==0.5.0
-    # via cookiecutter
+    # via pandas
+pandas==1.4.1
+    # via dolt-integrations
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
-py==1.11.0
-    # via retry
-pyarrow==6.0.1
-    # via flytekit
-pycparser==2.21
-    # via cffi
+    # via flyteidl
 python-dateutil==2.8.2
-    # via
-    #   arrow
-    #   croniter
-    #   flytekit
-    #   pandas
-python-json-logger==2.0.2
-    # via flytekit
-python-slugify==5.0.2
-    # via cookiecutter
-pytimeparse==1.1.8
-    # via flytekit
+    # via pandas
 pytz==2021.3
-    # via
-    #   flytekit
-    #   pandas
-regex==2022.1.18
-    # via docker-image-py
-requests==2.27.1
-    # via
-    #   cookiecutter
-    #   flytekit
-    #   responses
-responses==0.18.0
-    # via flytekit
-retry==0.9.2
-    # via flytekit
+    # via pandas
 six==1.16.0
-    # via
-    #   cookiecutter
-    #   grpcio
-    #   python-dateutil
-sortedcontainers==2.4.0
-    # via flytekit
-statsd==3.3.0
-    # via flytekit
-text-unidecode==1.3
-    # via python-slugify
-typing-extensions==4.0.1
-    # via
-    #   flytekit
-    #   typing-inspect
+    # via python-dateutil
+typing-extensions==4.1.1
+    # via typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
-urllib3==1.26.8
-    # via
-    #   flytekit
-    #   requests
-    #   responses
-wheel==0.37.1
-    # via flytekit
-wrapt==1.13.3
-    # via
-    #   deprecated
-    #   flytekit
-zipp==3.7.0
-    # via importlib-metadata
diff --git a/plugins/flytekit-greatexpectations/requirements.in b/plugins/flytekit-greatexpectations/requirements.in
index b92d2c6b7b..31536ee1bf 100644
--- a/plugins/flytekit-greatexpectations/requirements.in
+++ b/plugins/flytekit-greatexpectations/requirements.in
@@ -1,3 +1,4 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-great_expectations
 sqlalchemy
diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt
index 7aa0a867dd..022622111d 100644
--- a/plugins/flytekit-greatexpectations/requirements.txt
+++ b/plugins/flytekit-greatexpectations/requirements.txt
@@ -1,17 +1,17 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-great_expectations
     # via -r requirements.in
 altair==4.2.0
     # via great-expectations
-appnope==0.1.2
-    # via
-    #   ipykernel
-    #   ipython
 argon2-cffi==21.3.0
     # via notebook
 argon2-cffi-bindings==21.2.0
@@ -26,21 +26,23 @@ backcall==0.2.0
     # via ipython
 binaryornot==0.4.4
     # via cookiecutter
-black==21.12b0
+black==22.1.0
     # via ipython
 bleach==4.1.0
     # via nbconvert
 certifi==2021.10.8
     # via requests
 cffi==1.15.0
-    # via argon2-cffi-bindings
+    # via
+    #   argon2-cffi-bindings
+    #   cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.10
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   black
     #   cookiecutter
@@ -50,8 +52,10 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
+cryptography==36.0.1
+    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 debugpy==1.5.1
@@ -70,30 +74,28 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-entrypoints==0.3
+entrypoints==0.4
     # via
     #   altair
     #   jupyter-client
     #   nbconvert
 executing==0.8.2
     # via stack-data
-flyteidl==0.21.24
-    # via flytekit
-flytekit==0.26.1
+flytekit==0.30.3
     # via flytekitplugins-great-expectations
-great-expectations==0.14.3
+great-expectations==0.14.5
     # via flytekitplugins-great-expectations
 greenlet==1.1.2
     # via sqlalchemy
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via
     #   great-expectations
     #   keyring
-ipykernel==6.7.0
+ipykernel==6.9.1
     # via
     #   ipywidgets
     #   notebook
@@ -110,6 +112,10 @@ ipywidgets==7.6.5
     # via great-expectations
 jedi==0.18.1
     # via ipython
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   altair
@@ -134,7 +140,7 @@ jupyter-client==7.1.2
     #   ipykernel
     #   nbclient
     #   notebook
-jupyter-core==4.9.1
+jupyter-core==4.9.2
     # via
     #   jupyter-client
     #   nbconvert
@@ -169,11 +175,11 @@ mypy-extensions==0.4.3
     # via
     #   black
     #   typing-inspect
-natsort==8.0.2
+natsort==8.1.0
     # via flytekit
-nbclient==0.5.10
+nbclient==0.5.11
     # via nbconvert
-nbconvert==6.4.1
+nbconvert==6.4.2
     # via notebook
 nbformat==5.1.3
     # via
@@ -189,7 +195,7 @@ nest-asyncio==1.5.4
     #   notebook
 notebook==6.4.8
     # via widgetsnbextension
-numpy==1.22.1
+numpy==1.22.2
     # via
     #   altair
     #   great-expectations
@@ -198,7 +204,7 @@ numpy==1.22.1
     #   scipy
 packaging==21.3
     # via bleach
-pandas==1.4.0
+pandas==1.4.1
     # via
     #   altair
     #   flytekit
@@ -213,18 +219,16 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.4.1
+platformdirs==2.5.0
     # via black
 poyo==0.5.0
     # via cookiecutter
 prometheus-client==0.13.1
     # via notebook
-prompt-toolkit==3.0.26
+prompt-toolkit==3.0.28
     # via ipython
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 ptyprocess==0.7.0
     # via
     #   pexpect
@@ -248,7 +252,7 @@ pyparsing==2.4.7
     #   packaging
 pyrsistent==0.18.1
     # via jsonschema
-python-dateutil==2.8.1
+python-dateutil==2.8.2
     # via
     #   arrow
     #   croniter
@@ -258,7 +262,7 @@ python-dateutil==2.8.1
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -281,16 +285,16 @@ requests==2.27.1
     #   flytekit
     #   great-expectations
     #   responses
-responses==0.17.0
+responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-ruamel-yaml==0.17.17
+ruamel.yaml==0.17.17
     # via great-expectations
-ruamel-yaml-clib==0.2.6
-    # via ruamel-yaml
-scipy==1.7.3
+scipy==1.8.0
     # via great-expectations
+secretstorage==3.3.1
+    # via keyring
 send2trash==1.8.0
     # via notebook
 six==1.16.0
@@ -298,17 +302,15 @@ six==1.16.0
     #   asttokens
     #   bleach
     #   cookiecutter
-    #   flytekit
     #   grpcio
     #   python-dateutil
-    #   responses
 sortedcontainers==2.4.0
     # via flytekit
 sqlalchemy==1.4.31
     # via
     #   -r requirements.in
     #   flytekitplugins-great-expectations
-stack-data==0.1.4
+stack-data==0.2.0
     # via ipython
 statsd==3.3.0
     # via flytekit
@@ -320,7 +322,7 @@ testpath==0.5.0
     # via nbconvert
 text-unidecode==1.3
     # via python-slugify
-tomli==1.2.3
+tomli==2.0.1
     # via black
 toolz==0.11.2
     # via altair
@@ -344,9 +346,10 @@ traitlets==5.1.1
     #   nbconvert
     #   nbformat
     #   notebook
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
-    #   black
+    #   flytekit
+    #   great-expectations
     #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
diff --git a/plugins/flytekit-hive/requirements.in b/plugins/flytekit-hive/requirements.in
index 74c269d489..7fb543577e 100644
--- a/plugins/flytekit-hive/requirements.in
+++ b/plugins/flytekit-hive/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-hive
diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt
index 3ae057f9a0..e59b52ac4f 100644
--- a/plugins/flytekit-hive/requirements.txt
+++ b/plugins/flytekit-hive/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-hive
     # via -r requirements.in
 arrow==1.2.2
@@ -12,13 +16,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +32,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +48,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-hive
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -79,14 +87,12 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +107,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +126,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +139,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-k8s-pod/requirements.in b/plugins/flytekit-k8s-pod/requirements.in
index 987e999ec5..0e35b85b77 100644
--- a/plugins/flytekit-k8s-pod/requirements.in
+++ b/plugins/flytekit-k8s-pod/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-pod
diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt
index d60d9c46b6..3918d181db 100644
--- a/plugins/flytekit-k8s-pod/requirements.txt
+++ b/plugins/flytekit-k8s-pod/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-pod
     # via -r requirements.in
 arrow==1.2.2
@@ -16,13 +20,15 @@ certifi==2021.10.8
     # via
     #   kubernetes
     #   requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -30,7 +36,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -46,18 +52,20 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-pod
 google-auth==2.6.0
     # via kubernetes
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -66,7 +74,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-kubernetes==21.7.0
+kubernetes==22.6.0
     # via flytekitplugins-pod
 markupsafe==2.0.1
     # via jinja2
@@ -89,14 +97,12 @@ numpy==1.22.2
     #   pyarrow
 oauthlib==3.2.0
     # via requests-oauthlib
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -118,7 +124,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -145,6 +151,8 @@ retry==0.9.2
     # via flytekit
 rsa==4.8
     # via google-auth
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -158,7 +166,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-kf-mpi/requirements.in b/plugins/flytekit-kf-mpi/requirements.in
index dc79a19e66..7eb70eb5d8 100644
--- a/plugins/flytekit-kf-mpi/requirements.in
+++ b/plugins/flytekit-kf-mpi/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-kfmpi
diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt
index c13f9f3b74..de6d144057 100644
--- a/plugins/flytekit-kf-mpi/requirements.txt
+++ b/plugins/flytekit-kf-mpi/requirements.txt
@@ -1,9 +1,14 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
+    #   flytekitplugins-kfmpi
 -e file:.#egg=flytekitplugins-kfmpi
     # via -r requirements.in
 arrow==1.2.2
@@ -12,13 +17,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +33,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,18 +49,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via
-    #   flytekit
-    #   flytekitplugins-kfmpi
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-kfmpi
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -81,14 +88,12 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -103,7 +108,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -122,6 +127,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -133,7 +140,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-kf-pytorch/requirements.in b/plugins/flytekit-kf-pytorch/requirements.in
index 7013f7ba68..443f07f66c 100644
--- a/plugins/flytekit-kf-pytorch/requirements.in
+++ b/plugins/flytekit-kf-pytorch/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-kfpytorch
diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt
index 1696646c87..46ab5fd4a4 100644
--- a/plugins/flytekit-kf-pytorch/requirements.txt
+++ b/plugins/flytekit-kf-pytorch/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-kfpytorch
     # via -r requirements.in
 arrow==1.2.2
@@ -12,13 +16,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +32,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +48,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-kfpytorch
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -79,14 +87,12 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +107,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +126,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +139,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-kf-tensorflow/requirements.in b/plugins/flytekit-kf-tensorflow/requirements.in
index 0f95a44bc5..8b57632972 100644
--- a/plugins/flytekit-kf-tensorflow/requirements.in
+++ b/plugins/flytekit-kf-tensorflow/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-kftensorflow
diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt
index d3b6253664..dc90a674c2 100644
--- a/plugins/flytekit-kf-tensorflow/requirements.txt
+++ b/plugins/flytekit-kf-tensorflow/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-kftensorflow
     # via -r requirements.in
 arrow==1.2.2
@@ -12,13 +16,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +32,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +48,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-kftensorflow
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -79,14 +87,12 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +107,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +126,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +139,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-modin/requirements.in b/plugins/flytekit-modin/requirements.in
index 0248a83ad9..e4754638cd 100644
--- a/plugins/flytekit-modin/requirements.in
+++ b/plugins/flytekit-modin/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-modin
diff --git a/plugins/flytekit-pandera/requirements.in b/plugins/flytekit-pandera/requirements.in
index 70e750d5e4..3ff03dd965 100644
--- a/plugins/flytekit-pandera/requirements.in
+++ b/plugins/flytekit-pandera/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-pandera
diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt
index f0b65cec17..facccfb564 100644
--- a/plugins/flytekit-pandera/requirements.txt
+++ b/plugins/flytekit-pandera/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-pandera
     # via -r requirements.in
 arrow==1.2.2
@@ -12,13 +16,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +32,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +48,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-pandera
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -82,18 +90,16 @@ numpy==1.22.2
     #   pyarrow
 packaging==21.3
     # via pandera
-pandas==1.4.0
+pandas==1.4.1
     # via
     #   flytekit
     #   pandera
-pandera==0.8.1
+pandera==0.9.0
     # via flytekitplugins-pandera
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -102,6 +108,8 @@ pyarrow==6.0.1
     #   pandera
 pycparser==2.21
     # via cffi
+pydantic==1.9.0
+    # via pandera
 pyparsing==3.0.7
     # via packaging
 python-dateutil==2.8.2
@@ -112,7 +120,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -131,6 +139,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -142,9 +152,10 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
+    #   pydantic
     #   typing-inspect
 typing-inspect==0.7.1
     # via
diff --git a/plugins/flytekit-papermill/requirements.in b/plugins/flytekit-papermill/requirements.in
index a9dab93776..cbc5b6102b 100644
--- a/plugins/flytekit-papermill/requirements.in
+++ b/plugins/flytekit-papermill/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-papermill
diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt
index 8c3e4570e9..7449153b5a 100644
--- a/plugins/flytekit-papermill/requirements.txt
+++ b/plugins/flytekit-papermill/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-papermill
     # via -r requirements.in
 ansiwrap==0.8.4
@@ -18,7 +22,7 @@ backcall==0.2.0
     # via ipython
 binaryornot==0.4.4
     # via cookiecutter
-black==21.12b0
+black==22.1.0
     # via ipython
 bleach==4.1.0
     # via nbconvert
@@ -28,11 +32,11 @@ cffi==1.15.0
     # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.10
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   black
     #   cookiecutter
@@ -42,7 +46,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -64,24 +68,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-entrypoints==0.3
+entrypoints==0.4
     # via
     #   jupyter-client
     #   nbconvert
     #   papermill
 executing==0.8.2
     # via stack-data
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.26.1
+flytekit==0.30.3
     # via flytekitplugins-papermill
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
-ipykernel==6.7.0
+ipykernel==6.9.1
     # via flytekitplugins-papermill
 ipython==8.0.1
     # via ipykernel
@@ -106,7 +108,7 @@ jupyter-client==7.1.2
     # via
     #   ipykernel
     #   nbclient
-jupyter-core==4.9.1
+jupyter-core==4.9.2
     # via
     #   jupyter-client
     #   nbconvert
@@ -136,13 +138,13 @@ mypy-extensions==0.4.3
     # via
     #   black
     #   typing-inspect
-natsort==8.0.2
+natsort==8.1.0
     # via flytekit
-nbclient==0.5.10
+nbclient==0.5.11
     # via
     #   nbconvert
     #   papermill
-nbconvert==6.4.1
+nbconvert==6.4.2
     # via flytekitplugins-papermill
 nbformat==5.1.3
     # via
@@ -154,13 +156,13 @@ nest-asyncio==1.5.4
     #   ipykernel
     #   jupyter-client
     #   nbclient
-numpy==1.22.1
+numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
 packaging==21.3
     # via bleach
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 pandocfilters==1.5.0
     # via nbconvert
@@ -174,16 +176,14 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.4.1
+platformdirs==2.5.0
     # via black
 poyo==0.5.0
     # via cookiecutter
-prompt-toolkit==3.0.26
+prompt-toolkit==3.0.28
     # via ipython
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
@@ -203,7 +203,7 @@ pyparsing==3.0.7
     # via packaging
 pyrsistent==0.18.1
     # via jsonschema
-python-dateutil==2.8.1
+python-dateutil==2.8.2
     # via
     #   arrow
     #   croniter
@@ -212,7 +212,7 @@ python-dateutil==2.8.1
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -232,7 +232,7 @@ requests==2.27.1
     #   flytekit
     #   papermill
     #   responses
-responses==0.17.0
+responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
@@ -243,13 +243,11 @@ six==1.16.0
     #   asttokens
     #   bleach
     #   cookiecutter
-    #   flytekit
     #   grpcio
     #   python-dateutil
-    #   responses
 sortedcontainers==2.4.0
     # via flytekit
-stack-data==0.1.4
+stack-data==0.2.0
     # via ipython
 statsd==3.3.0
     # via flytekit
@@ -261,7 +259,7 @@ text-unidecode==1.3
     # via python-slugify
 textwrap3==0.9.2
     # via ansiwrap
-tomli==1.2.3
+tomli==2.0.1
     # via black
 tornado==6.1
     # via
@@ -279,9 +277,9 @@ traitlets==5.1.1
     #   nbclient
     #   nbconvert
     #   nbformat
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
-    #   black
+    #   flytekit
     #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
diff --git a/plugins/flytekit-snowflake/requirements.in b/plugins/flytekit-snowflake/requirements.in
index 2a118dd3bf..f720269c73 100644
--- a/plugins/flytekit-snowflake/requirements.in
+++ b/plugins/flytekit-snowflake/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-snowflake
diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt
index 53951100b5..e9b2b125f2 100644
--- a/plugins/flytekit-snowflake/requirements.txt
+++ b/plugins/flytekit-snowflake/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-snowflake
     # via -r requirements.in
 arrow==1.2.2
@@ -12,13 +16,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +32,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +48,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-snowflake
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -79,14 +87,12 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +107,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +126,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +139,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-spark/requirements.in b/plugins/flytekit-spark/requirements.in
index f112dbc8ef..9d2e62c77e 100644
--- a/plugins/flytekit-spark/requirements.in
+++ b/plugins/flytekit-spark/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-spark
diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt
index 75d1e32d53..2ab33fe5f3 100644
--- a/plugins/flytekit-spark/requirements.txt
+++ b/plugins/flytekit-spark/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-spark
     # via -r requirements.in
 arrow==1.2.2
@@ -12,13 +16,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +32,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +48,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-spark
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -79,14 +87,12 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 py4j==0.10.9.3
@@ -105,7 +111,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -124,6 +130,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -135,7 +143,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-sqlalchemy/requirements.in b/plugins/flytekit-sqlalchemy/requirements.in
index 21d9e97b47..e545653d01 100644
--- a/plugins/flytekit-sqlalchemy/requirements.in
+++ b/plugins/flytekit-sqlalchemy/requirements.in
@@ -1,2 +1,3 @@
 .
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-sqlalchemy
diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt
index 93dcc2790c..bced83587b 100644
--- a/plugins/flytekit-sqlalchemy/requirements.txt
+++ b/plugins/flytekit-sqlalchemy/requirements.txt
@@ -1,9 +1,13 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
+-e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
+    # via
+    #   -r requirements.in
+    #   flytekit
 -e file:.#egg=flytekitplugins-sqlalchemy
     # via -r requirements.in
 arrow==1.2.2
@@ -12,13 +16,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.3
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +32,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.1
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,18 +48,20 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-sqlalchemy
 greenlet==1.1.2
     # via sqlalchemy
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -81,14 +89,12 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via
-    #   flyteidl
-    #   flytekit
+    # via flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -103,7 +109,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.0.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -122,6 +128,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -135,7 +143,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect

From 0e5819999414973209b12fea28b17b3e8a38e173 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 15:32:57 -0800
Subject: [PATCH 17/35] Set hash as a property of the literal

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/models/literals.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py
index 92c2c7e322..8d5101037e 100644
--- a/flytekit/models/literals.py
+++ b/flytekit/models/literals.py
@@ -809,7 +809,7 @@ def __init__(
         self._scalar = scalar
         self._collection = collection
         self._map = map
-        self.hash = hash
+        self._hash = hash
 
     @property
     def scalar(self):
@@ -843,6 +843,14 @@ def value(self):
         """
         return self.scalar or self.collection or self.map
 
+    @property
+    def hash(self):
+        """
+        If not None, this value holds a hash that represents the literal for caching purposes.
+        :rtype: str
+        """
+        return self._hash
+
     def to_flyte_idl(self):
         """
         :rtype: flyteidl.core.literals_pb2.Literal

From 5054861218a9fc571a957a2ab8da9cc7754a1bf5 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 15:41:21 -0800
Subject: [PATCH 18/35] Install plugins requirements in CI.

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 .github/workflows/pythonbuild.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
index 8202bf0f73..e63b869734 100644
--- a/.github/workflows/pythonbuild.yml
+++ b/.github/workflows/pythonbuild.yml
@@ -105,6 +105,7 @@ jobs:
         cd plugins/${{ matrix.plugin-names }}
         pip install -e .
         if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
+        pip install -r requirements.txt
         pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
         pip freeze
     - name: Test with coverage

From 9f2d06f672a4aca18869de4796fe25287c7773b6 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 16:51:15 -0800
Subject: [PATCH 19/35] Add hash.setter

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/models/literals.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/flytekit/models/literals.py b/flytekit/models/literals.py
index 8d5101037e..70470fda6b 100644
--- a/flytekit/models/literals.py
+++ b/flytekit/models/literals.py
@@ -851,6 +851,10 @@ def hash(self):
         """
         return self._hash
 
+    @hash.setter
+    def hash(self, value):
+        self._hash = value
+
     def to_flyte_idl(self):
         """
         :rtype: flyteidl.core.literals_pb2.Literal

From e039836502b78972fd2d04f33d480e26e435c686 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 17:07:22 -0800
Subject: [PATCH 20/35] Install flyteidl directly

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 .github/workflows/pythonbuild.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
index e63b869734..08d8a47baa 100644
--- a/.github/workflows/pythonbuild.yml
+++ b/.github/workflows/pythonbuild.yml
@@ -105,8 +105,8 @@ jobs:
         cd plugins/${{ matrix.plugin-names }}
         pip install -e .
         if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
-        pip install -r requirements.txt
         pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
+        pip install --no-deps -U "git+https://github.com/flyteorg/flyteidl@add-hash-to-literal#egg=flyteidl"
         pip freeze
     - name: Test with coverage
       run: |

From 2da76f38037479a2b892aaacd2fc12f85dbb24df Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Thu, 17 Feb 2022 17:08:05 -0800
Subject: [PATCH 21/35] Revert "Regenerate plugins requirements files pointing
 to the right version of flyteidl."

This reverts commit c2dbb540ad716e883932dd9255045c128e16c6a0.

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 plugins/flytekit-aws-athena/requirements.in   |   1 -
 plugins/flytekit-aws-athena/requirements.txt  |  38 ++---
 plugins/flytekit-aws-batch/requirements.in    |   1 -
 plugins/flytekit-aws-batch/requirements.txt   |  60 +++----
 .../flytekit-aws-sagemaker/requirements.in    |   1 -
 .../flytekit-aws-sagemaker/requirements.txt   |  43 +++--
 plugins/flytekit-bigquery/requirements.in     |   1 -
 plugins/flytekit-bigquery/requirements.txt    |  35 ++--
 plugins/flytekit-data-fsspec/requirements.in  |   1 -
 plugins/flytekit-data-fsspec/requirements.txt |  44 ++---
 plugins/flytekit-dolt/requirements.in         |   1 -
 plugins/flytekit-dolt/requirements.txt        | 150 ++++++++++++++++--
 .../requirements.in                           |   1 -
 .../requirements.txt                          |  87 +++++-----
 plugins/flytekit-hive/requirements.in         |   1 -
 plugins/flytekit-hive/requirements.txt        |  38 ++---
 plugins/flytekit-k8s-pod/requirements.in      |   1 -
 plugins/flytekit-k8s-pod/requirements.txt     |  40 ++---
 plugins/flytekit-kf-mpi/requirements.in       |   1 -
 plugins/flytekit-kf-mpi/requirements.txt      |  41 ++---
 plugins/flytekit-kf-pytorch/requirements.in   |   1 -
 plugins/flytekit-kf-pytorch/requirements.txt  |  38 ++---
 .../flytekit-kf-tensorflow/requirements.in    |   1 -
 .../flytekit-kf-tensorflow/requirements.txt   |  38 ++---
 plugins/flytekit-modin/requirements.in        |   1 -
 plugins/flytekit-pandera/requirements.in      |   1 -
 plugins/flytekit-pandera/requirements.txt     |  43 ++---
 plugins/flytekit-papermill/requirements.in    |   1 -
 plugins/flytekit-papermill/requirements.txt   |  62 ++++----
 plugins/flytekit-snowflake/requirements.in    |   1 -
 plugins/flytekit-snowflake/requirements.txt   |  38 ++---
 plugins/flytekit-spark/requirements.in        |   1 -
 plugins/flytekit-spark/requirements.txt       |  38 ++---
 plugins/flytekit-sqlalchemy/requirements.in   |   1 -
 plugins/flytekit-sqlalchemy/requirements.txt  |  38 ++---
 35 files changed, 436 insertions(+), 453 deletions(-)

diff --git a/plugins/flytekit-aws-athena/requirements.in b/plugins/flytekit-aws-athena/requirements.in
index cd57503478..fb5e57406e 100644
--- a/plugins/flytekit-aws-athena/requirements.in
+++ b/plugins/flytekit-aws-athena/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-athena
diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt
index 0fa90e3fba..a1acae494f 100644
--- a/plugins/flytekit-aws-athena/requirements.txt
+++ b/plugins/flytekit-aws-athena/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-athena
     # via -r requirements.in
 arrow==1.2.2
@@ -16,15 +12,13 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -32,7 +26,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -48,18 +42,16 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-athena
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -87,12 +79,14 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -107,7 +101,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -126,8 +120,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -139,7 +131,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-aws-batch/requirements.in b/plugins/flytekit-aws-batch/requirements.in
index 8013b8f761..4c2c1fae97 100644
--- a/plugins/flytekit-aws-batch/requirements.in
+++ b/plugins/flytekit-aws-batch/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-awsbatch
diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt
index e0b9738666..9181490234 100644
--- a/plugins/flytekit-aws-batch/requirements.txt
+++ b/plugins/flytekit-aws-batch/requirements.txt
@@ -1,30 +1,24 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-awsbatch
     # via -r requirements.in
-arrow==1.2.2
+arrow==1.2.1
     # via jinja2-time
 binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.10
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -32,10 +26,8 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
-cryptography==36.0.1
-    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 decorator==5.1.1
@@ -48,18 +40,16 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.21.23
+    # via flytekit
+flytekit==0.26.0
     # via flytekitplugins-awsbatch
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -81,25 +71,25 @@ marshmallow-jsonschema==0.13.0
     # via flytekit
 mypy-extensions==0.4.3
     # via typing-inspect
-natsort==8.1.0
+natsort==8.0.2
     # via flytekit
-numpy==1.22.2
+numpy==1.22.1
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.3.5
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
-protobuf==3.19.4
-    # via flytekit
+protobuf==3.19.3
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
     # via flytekit
-pycparser==2.21
-    # via cffi
-python-dateutil==2.8.2
+python-dateutil==2.8.1
     # via
     #   arrow
     #   croniter
@@ -107,7 +97,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -122,27 +112,25 @@ requests==2.27.1
     #   cookiecutter
     #   flytekit
     #   responses
-responses==0.18.0
+responses==0.17.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
+    #   flytekit
     #   grpcio
     #   python-dateutil
+    #   responses
 sortedcontainers==2.4.0
     # via flytekit
 statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
-    # via
-    #   flytekit
-    #   typing-inspect
+typing-extensions==4.0.1
+    # via typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
 urllib3==1.26.8
diff --git a/plugins/flytekit-aws-sagemaker/requirements.in b/plugins/flytekit-aws-sagemaker/requirements.in
index 6e2136aac9..1f40a6102b 100644
--- a/plugins/flytekit-aws-sagemaker/requirements.in
+++ b/plugins/flytekit-aws-sagemaker/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-awssagemaker
diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt
index 48a70cf6c1..11396f213e 100644
--- a/plugins/flytekit-aws-sagemaker/requirements.txt
+++ b/plugins/flytekit-aws-sagemaker/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-awssagemaker
     # via -r requirements.in
 arrow==1.2.2
@@ -16,9 +12,9 @@ bcrypt==3.2.0
     # via paramiko
 binaryornot==0.4.4
     # via cookiecutter
-boto3==1.21.2
+boto3==1.20.50
     # via sagemaker-training
-botocore==1.24.2
+botocore==1.23.50
     # via
     #   boto3
     #   s3transfer
@@ -31,11 +27,11 @@ cffi==1.15.0
     #   pynacl
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -43,7 +39,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via
@@ -61,24 +57,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-awssagemaker
 gevent==21.12.0
     # via sagemaker-training
 greenlet==1.1.2
     # via gevent
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-inotify_simple==1.2.1
+inotify-simple==1.2.1
     # via sagemaker-training
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -112,7 +106,7 @@ numpy==1.22.2
     #   pyarrow
     #   sagemaker-training
     #   scipy
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 paramiko==2.9.2
     # via sagemaker-training
@@ -120,6 +114,7 @@ poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
     # via
+    #   flyteidl
     #   flytekit
     #   sagemaker-training
 psutil==5.9.0
@@ -141,7 +136,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -168,8 +163,6 @@ sagemaker-training==3.9.2
     # via flytekitplugins-awssagemaker
 scipy==1.8.0
     # via sagemaker-training
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   bcrypt
@@ -184,7 +177,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
@@ -206,9 +199,9 @@ wrapt==1.13.3
     #   flytekit
 zipp==3.7.0
     # via importlib-metadata
-zope.event==4.5.0
+zope-event==4.5.0
     # via gevent
-zope.interface==5.4.0
+zope-interface==5.4.0
     # via gevent
 
 # The following packages are considered to be unsafe in a requirements file:
diff --git a/plugins/flytekit-bigquery/requirements.in b/plugins/flytekit-bigquery/requirements.in
index c7912742b5..d01f21ac2b 100644
--- a/plugins/flytekit-bigquery/requirements.in
+++ b/plugins/flytekit-bigquery/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-bigquery
diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt
index 4b25a52098..f0bc647453 100644
--- a/plugins/flytekit-bigquery/requirements.txt
+++ b/plugins/flytekit-bigquery/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-bigquery
     # via -r requirements.in
 arrow==1.2.2
@@ -22,11 +18,11 @@ cffi==1.15.0
     # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -34,7 +30,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -50,7 +46,9 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-bigquery
 google-api-core[grpc]==2.5.0
     # via
@@ -60,29 +58,29 @@ google-auth==2.6.0
     # via
     #   google-api-core
     #   google-cloud-core
-google-cloud-bigquery==2.33.0
+google-cloud-bigquery==2.32.0
     # via flytekitplugins-bigquery
 google-cloud-core==2.2.2
     # via google-cloud-bigquery
 google-crc32c==1.3.0
     # via google-resumable-media
-google-resumable-media==2.2.1
+google-resumable-media==2.2.0
     # via google-cloud-bigquery
 googleapis-common-protos==1.54.0
     # via
     #   google-api-core
     #   grpcio-status
-grpcio==1.44.0
+grpcio==1.43.0
     # via
     #   flytekit
     #   google-api-core
     #   google-cloud-bigquery
     #   grpcio-status
-grpcio-status==1.44.0
+grpcio-status==1.43.0
     # via google-api-core
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
 jeepney==0.7.1
     # via
@@ -117,14 +115,15 @@ numpy==1.22.2
     #   pyarrow
 packaging==21.3
     # via google-cloud-bigquery
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
-proto-plus==1.20.2
+proto-plus==1.20.0
     # via google-cloud-bigquery
 protobuf==3.19.4
     # via
+    #   flyteidl
     #   flytekit
     #   google-api-core
     #   google-cloud-bigquery
@@ -154,7 +153,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -191,7 +190,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-data-fsspec/requirements.in b/plugins/flytekit-data-fsspec/requirements.in
index 4c03034cce..127b779735 100644
--- a/plugins/flytekit-data-fsspec/requirements.in
+++ b/plugins/flytekit-data-fsspec/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-data-fsspec
diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt
index 344d8c1f41..4dc89fa500 100644
--- a/plugins/flytekit-data-fsspec/requirements.txt
+++ b/plugins/flytekit-data-fsspec/requirements.txt
@@ -1,32 +1,26 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-data-fsspec
     # via -r requirements.in
 arrow==1.2.2
     # via jinja2-time
 binaryornot==0.4.4
     # via cookiecutter
-botocore==1.24.2
+botocore==1.23.24
     # via flytekitplugins-data-fsspec
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -34,10 +28,8 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
-cryptography==36.0.1
-    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 decorator==5.1.1
@@ -50,20 +42,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-data-fsspec
 fsspec==2022.1.0
     # via flytekitplugins-data-fsspec
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -93,18 +83,18 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
     # via flytekit
-pycparser==2.21
-    # via cffi
 python-dateutil==2.8.2
     # via
     #   arrow
@@ -114,7 +104,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -133,8 +123,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -146,7 +134,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-dolt/requirements.in b/plugins/flytekit-dolt/requirements.in
index 04990b3216..53d807299a 100644
--- a/plugins/flytekit-dolt/requirements.in
+++ b/plugins/flytekit-dolt/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-dolt
diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt
index 221fc36e43..01dfe2cb6b 100644
--- a/plugins/flytekit-dolt/requirements.txt
+++ b/plugins/flytekit-dolt/requirements.txt
@@ -1,44 +1,160 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via -r requirements.in
--e file:../../.#egg=flytekit
-    # via
-    #   -r requirements.in
-    #   flytekitplugins-dolt
 -e file:.#egg=flytekitplugins-dolt
     # via -r requirements.in
+arrow==1.2.2
+    # via jinja2-time
+binaryornot==0.4.4
+    # via cookiecutter
+certifi==2021.10.8
+    # via requests
+chardet==4.0.0
+    # via binaryornot
+charset-normalizer==2.0.11
+    # via requests
+checksumdir==1.2.0
+    # via flytekit
+click==7.1.2
+    # via
+    #   cookiecutter
+    #   flytekit
+cloudpickle==2.0.0
+    # via flytekit
+cookiecutter==1.7.3
+    # via flytekit
+croniter==1.2.0
+    # via flytekit
+cryptography==36.0.1
+    # via secretstorage
 dataclasses-json==0.5.6
-    # via dolt-integrations
+    # via
+    #   dolt-integrations
+    #   flytekit
+decorator==5.1.1
+    # via retry
+deprecated==1.2.13
+    # via flytekit
+diskcache==5.4.0
+    # via flytekit
+docker-image-py==0.1.12
+    # via flytekit
+docstring-parser==0.13
+    # via flytekit
 dolt-integrations==0.1.5
     # via flytekitplugins-dolt
 doltcli==0.1.17
     # via dolt-integrations
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
+    # via flytekitplugins-dolt
+grpcio==1.43.0
+    # via flytekit
+idna==3.3
+    # via requests
+importlib-metadata==4.10.1
+    # via keyring
+jinja2==3.0.3
+    # via
+    #   cookiecutter
+    #   jinja2-time
+jinja2-time==0.2.0
+    # via cookiecutter
+keyring==23.5.0
+    # via flytekit
+markupsafe==2.0.1
+    # via jinja2
 marshmallow==3.14.1
     # via
     #   dataclasses-json
     #   marshmallow-enum
+    #   marshmallow-jsonschema
 marshmallow-enum==1.5.1
     # via dataclasses-json
+marshmallow-jsonschema==0.13.0
+    # via flytekit
 mypy-extensions==0.4.3
     # via typing-inspect
+natsort==8.1.0
+    # via flytekit
 numpy==1.22.2
-    # via pandas
-pandas==1.4.1
-    # via dolt-integrations
+    # via
+    #   pandas
+    #   pyarrow
+pandas==1.4.0
+    # via
+    #   dolt-integrations
+    #   flytekit
+poyo==0.5.0
+    # via cookiecutter
 protobuf==3.19.4
-    # via flyteidl
+    # via
+    #   flyteidl
+    #   flytekit
+py==1.11.0
+    # via retry
+pyarrow==6.0.1
+    # via flytekit
+pycparser==2.21
+    # via cffi
 python-dateutil==2.8.2
-    # via pandas
+    # via
+    #   arrow
+    #   croniter
+    #   flytekit
+    #   pandas
+python-json-logger==2.0.2
+    # via flytekit
+python-slugify==5.0.2
+    # via cookiecutter
+pytimeparse==1.1.8
+    # via flytekit
 pytz==2021.3
-    # via pandas
+    # via
+    #   flytekit
+    #   pandas
+regex==2022.1.18
+    # via docker-image-py
+requests==2.27.1
+    # via
+    #   cookiecutter
+    #   flytekit
+    #   responses
+responses==0.18.0
+    # via flytekit
+retry==0.9.2
+    # via flytekit
 six==1.16.0
-    # via python-dateutil
-typing-extensions==4.1.1
-    # via typing-inspect
+    # via
+    #   cookiecutter
+    #   grpcio
+    #   python-dateutil
+sortedcontainers==2.4.0
+    # via flytekit
+statsd==3.3.0
+    # via flytekit
+text-unidecode==1.3
+    # via python-slugify
+typing-extensions==4.0.1
+    # via
+    #   flytekit
+    #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
+urllib3==1.26.8
+    # via
+    #   flytekit
+    #   requests
+    #   responses
+wheel==0.37.1
+    # via flytekit
+wrapt==1.13.3
+    # via
+    #   deprecated
+    #   flytekit
+zipp==3.7.0
+    # via importlib-metadata
diff --git a/plugins/flytekit-greatexpectations/requirements.in b/plugins/flytekit-greatexpectations/requirements.in
index 31536ee1bf..b92d2c6b7b 100644
--- a/plugins/flytekit-greatexpectations/requirements.in
+++ b/plugins/flytekit-greatexpectations/requirements.in
@@ -1,4 +1,3 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-great_expectations
 sqlalchemy
diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt
index 022622111d..7aa0a867dd 100644
--- a/plugins/flytekit-greatexpectations/requirements.txt
+++ b/plugins/flytekit-greatexpectations/requirements.txt
@@ -1,17 +1,17 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-great_expectations
     # via -r requirements.in
 altair==4.2.0
     # via great-expectations
+appnope==0.1.2
+    # via
+    #   ipykernel
+    #   ipython
 argon2-cffi==21.3.0
     # via notebook
 argon2-cffi-bindings==21.2.0
@@ -26,23 +26,21 @@ backcall==0.2.0
     # via ipython
 binaryornot==0.4.4
     # via cookiecutter
-black==22.1.0
+black==21.12b0
     # via ipython
 bleach==4.1.0
     # via nbconvert
 certifi==2021.10.8
     # via requests
 cffi==1.15.0
-    # via
-    #   argon2-cffi-bindings
-    #   cryptography
+    # via argon2-cffi-bindings
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.10
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   black
     #   cookiecutter
@@ -52,10 +50,8 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
-cryptography==36.0.1
-    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 debugpy==1.5.1
@@ -74,28 +70,30 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-entrypoints==0.4
+entrypoints==0.3
     # via
     #   altair
     #   jupyter-client
     #   nbconvert
 executing==0.8.2
     # via stack-data
-flytekit==0.30.3
+flyteidl==0.21.24
+    # via flytekit
+flytekit==0.26.1
     # via flytekitplugins-great-expectations
-great-expectations==0.14.5
+great-expectations==0.14.3
     # via flytekitplugins-great-expectations
 greenlet==1.1.2
     # via sqlalchemy
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via
     #   great-expectations
     #   keyring
-ipykernel==6.9.1
+ipykernel==6.7.0
     # via
     #   ipywidgets
     #   notebook
@@ -112,10 +110,6 @@ ipywidgets==7.6.5
     # via great-expectations
 jedi==0.18.1
     # via ipython
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   altair
@@ -140,7 +134,7 @@ jupyter-client==7.1.2
     #   ipykernel
     #   nbclient
     #   notebook
-jupyter-core==4.9.2
+jupyter-core==4.9.1
     # via
     #   jupyter-client
     #   nbconvert
@@ -175,11 +169,11 @@ mypy-extensions==0.4.3
     # via
     #   black
     #   typing-inspect
-natsort==8.1.0
+natsort==8.0.2
     # via flytekit
-nbclient==0.5.11
+nbclient==0.5.10
     # via nbconvert
-nbconvert==6.4.2
+nbconvert==6.4.1
     # via notebook
 nbformat==5.1.3
     # via
@@ -195,7 +189,7 @@ nest-asyncio==1.5.4
     #   notebook
 notebook==6.4.8
     # via widgetsnbextension
-numpy==1.22.2
+numpy==1.22.1
     # via
     #   altair
     #   great-expectations
@@ -204,7 +198,7 @@ numpy==1.22.2
     #   scipy
 packaging==21.3
     # via bleach
-pandas==1.4.1
+pandas==1.4.0
     # via
     #   altair
     #   flytekit
@@ -219,16 +213,18 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.5.0
+platformdirs==2.4.1
     # via black
 poyo==0.5.0
     # via cookiecutter
 prometheus-client==0.13.1
     # via notebook
-prompt-toolkit==3.0.28
+prompt-toolkit==3.0.26
     # via ipython
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 ptyprocess==0.7.0
     # via
     #   pexpect
@@ -252,7 +248,7 @@ pyparsing==2.4.7
     #   packaging
 pyrsistent==0.18.1
     # via jsonschema
-python-dateutil==2.8.2
+python-dateutil==2.8.1
     # via
     #   arrow
     #   croniter
@@ -262,7 +258,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -285,16 +281,16 @@ requests==2.27.1
     #   flytekit
     #   great-expectations
     #   responses
-responses==0.18.0
+responses==0.17.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-ruamel.yaml==0.17.17
+ruamel-yaml==0.17.17
     # via great-expectations
-scipy==1.8.0
+ruamel-yaml-clib==0.2.6
+    # via ruamel-yaml
+scipy==1.7.3
     # via great-expectations
-secretstorage==3.3.1
-    # via keyring
 send2trash==1.8.0
     # via notebook
 six==1.16.0
@@ -302,15 +298,17 @@ six==1.16.0
     #   asttokens
     #   bleach
     #   cookiecutter
+    #   flytekit
     #   grpcio
     #   python-dateutil
+    #   responses
 sortedcontainers==2.4.0
     # via flytekit
 sqlalchemy==1.4.31
     # via
     #   -r requirements.in
     #   flytekitplugins-great-expectations
-stack-data==0.2.0
+stack-data==0.1.4
     # via ipython
 statsd==3.3.0
     # via flytekit
@@ -322,7 +320,7 @@ testpath==0.5.0
     # via nbconvert
 text-unidecode==1.3
     # via python-slugify
-tomli==2.0.1
+tomli==1.2.3
     # via black
 toolz==0.11.2
     # via altair
@@ -346,10 +344,9 @@ traitlets==5.1.1
     #   nbconvert
     #   nbformat
     #   notebook
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
-    #   flytekit
-    #   great-expectations
+    #   black
     #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
diff --git a/plugins/flytekit-hive/requirements.in b/plugins/flytekit-hive/requirements.in
index 7fb543577e..74c269d489 100644
--- a/plugins/flytekit-hive/requirements.in
+++ b/plugins/flytekit-hive/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-hive
diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt
index e59b52ac4f..3ae057f9a0 100644
--- a/plugins/flytekit-hive/requirements.txt
+++ b/plugins/flytekit-hive/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-hive
     # via -r requirements.in
 arrow==1.2.2
@@ -16,15 +12,13 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -32,7 +26,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -48,18 +42,16 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-hive
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -87,12 +79,14 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -107,7 +101,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -126,8 +120,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -139,7 +131,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-k8s-pod/requirements.in b/plugins/flytekit-k8s-pod/requirements.in
index 0e35b85b77..987e999ec5 100644
--- a/plugins/flytekit-k8s-pod/requirements.in
+++ b/plugins/flytekit-k8s-pod/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-pod
diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt
index 3918d181db..d60d9c46b6 100644
--- a/plugins/flytekit-k8s-pod/requirements.txt
+++ b/plugins/flytekit-k8s-pod/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-pod
     # via -r requirements.in
 arrow==1.2.2
@@ -20,15 +16,13 @@ certifi==2021.10.8
     # via
     #   kubernetes
     #   requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -36,7 +30,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -52,20 +46,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-pod
 google-auth==2.6.0
     # via kubernetes
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -74,7 +66,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-kubernetes==22.6.0
+kubernetes==21.7.0
     # via flytekitplugins-pod
 markupsafe==2.0.1
     # via jinja2
@@ -97,12 +89,14 @@ numpy==1.22.2
     #   pyarrow
 oauthlib==3.2.0
     # via requests-oauthlib
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -124,7 +118,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -151,8 +145,6 @@ retry==0.9.2
     # via flytekit
 rsa==4.8
     # via google-auth
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -166,7 +158,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-kf-mpi/requirements.in b/plugins/flytekit-kf-mpi/requirements.in
index 7eb70eb5d8..dc79a19e66 100644
--- a/plugins/flytekit-kf-mpi/requirements.in
+++ b/plugins/flytekit-kf-mpi/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-kfmpi
diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt
index de6d144057..c13f9f3b74 100644
--- a/plugins/flytekit-kf-mpi/requirements.txt
+++ b/plugins/flytekit-kf-mpi/requirements.txt
@@ -1,14 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
-    #   flytekitplugins-kfmpi
 -e file:.#egg=flytekitplugins-kfmpi
     # via -r requirements.in
 arrow==1.2.2
@@ -17,15 +12,13 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -33,7 +26,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -49,18 +42,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via
+    #   flytekit
+    #   flytekitplugins-kfmpi
+flytekit==0.30.0
     # via flytekitplugins-kfmpi
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -88,12 +81,14 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -108,7 +103,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -127,8 +122,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -140,7 +133,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-kf-pytorch/requirements.in b/plugins/flytekit-kf-pytorch/requirements.in
index 443f07f66c..7013f7ba68 100644
--- a/plugins/flytekit-kf-pytorch/requirements.in
+++ b/plugins/flytekit-kf-pytorch/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-kfpytorch
diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt
index 46ab5fd4a4..1696646c87 100644
--- a/plugins/flytekit-kf-pytorch/requirements.txt
+++ b/plugins/flytekit-kf-pytorch/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-kfpytorch
     # via -r requirements.in
 arrow==1.2.2
@@ -16,15 +12,13 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -32,7 +26,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -48,18 +42,16 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-kfpytorch
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -87,12 +79,14 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -107,7 +101,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -126,8 +120,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -139,7 +131,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-kf-tensorflow/requirements.in b/plugins/flytekit-kf-tensorflow/requirements.in
index 8b57632972..0f95a44bc5 100644
--- a/plugins/flytekit-kf-tensorflow/requirements.in
+++ b/plugins/flytekit-kf-tensorflow/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-kftensorflow
diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt
index dc90a674c2..d3b6253664 100644
--- a/plugins/flytekit-kf-tensorflow/requirements.txt
+++ b/plugins/flytekit-kf-tensorflow/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-kftensorflow
     # via -r requirements.in
 arrow==1.2.2
@@ -16,15 +12,13 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -32,7 +26,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -48,18 +42,16 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-kftensorflow
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -87,12 +79,14 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -107,7 +101,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -126,8 +120,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -139,7 +131,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-modin/requirements.in b/plugins/flytekit-modin/requirements.in
index e4754638cd..0248a83ad9 100644
--- a/plugins/flytekit-modin/requirements.in
+++ b/plugins/flytekit-modin/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-modin
diff --git a/plugins/flytekit-pandera/requirements.in b/plugins/flytekit-pandera/requirements.in
index 3ff03dd965..70e750d5e4 100644
--- a/plugins/flytekit-pandera/requirements.in
+++ b/plugins/flytekit-pandera/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-pandera
diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt
index facccfb564..f0b65cec17 100644
--- a/plugins/flytekit-pandera/requirements.txt
+++ b/plugins/flytekit-pandera/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-pandera
     # via -r requirements.in
 arrow==1.2.2
@@ -16,15 +12,13 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -32,7 +26,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -48,18 +42,16 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-pandera
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -90,16 +82,18 @@ numpy==1.22.2
     #   pyarrow
 packaging==21.3
     # via pandera
-pandas==1.4.1
+pandas==1.4.0
     # via
     #   flytekit
     #   pandera
-pandera==0.9.0
+pandera==0.8.1
     # via flytekitplugins-pandera
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -108,8 +102,6 @@ pyarrow==6.0.1
     #   pandera
 pycparser==2.21
     # via cffi
-pydantic==1.9.0
-    # via pandera
 pyparsing==3.0.7
     # via packaging
 python-dateutil==2.8.2
@@ -120,7 +112,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -139,8 +131,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -152,10 +142,9 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
-    #   pydantic
     #   typing-inspect
 typing-inspect==0.7.1
     # via
diff --git a/plugins/flytekit-papermill/requirements.in b/plugins/flytekit-papermill/requirements.in
index cbc5b6102b..a9dab93776 100644
--- a/plugins/flytekit-papermill/requirements.in
+++ b/plugins/flytekit-papermill/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-papermill
diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt
index 7449153b5a..8c3e4570e9 100644
--- a/plugins/flytekit-papermill/requirements.txt
+++ b/plugins/flytekit-papermill/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-papermill
     # via -r requirements.in
 ansiwrap==0.8.4
@@ -22,7 +18,7 @@ backcall==0.2.0
     # via ipython
 binaryornot==0.4.4
     # via cookiecutter
-black==22.1.0
+black==21.12b0
     # via ipython
 bleach==4.1.0
     # via nbconvert
@@ -32,11 +28,11 @@ cffi==1.15.0
     # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.10
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   black
     #   cookiecutter
@@ -46,7 +42,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -68,22 +64,24 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-entrypoints==0.4
+entrypoints==0.3
     # via
     #   jupyter-client
     #   nbconvert
     #   papermill
 executing==0.8.2
     # via stack-data
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.26.1
     # via flytekitplugins-papermill
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-ipykernel==6.9.1
+ipykernel==6.7.0
     # via flytekitplugins-papermill
 ipython==8.0.1
     # via ipykernel
@@ -108,7 +106,7 @@ jupyter-client==7.1.2
     # via
     #   ipykernel
     #   nbclient
-jupyter-core==4.9.2
+jupyter-core==4.9.1
     # via
     #   jupyter-client
     #   nbconvert
@@ -138,13 +136,13 @@ mypy-extensions==0.4.3
     # via
     #   black
     #   typing-inspect
-natsort==8.1.0
+natsort==8.0.2
     # via flytekit
-nbclient==0.5.11
+nbclient==0.5.10
     # via
     #   nbconvert
     #   papermill
-nbconvert==6.4.2
+nbconvert==6.4.1
     # via flytekitplugins-papermill
 nbformat==5.1.3
     # via
@@ -156,13 +154,13 @@ nest-asyncio==1.5.4
     #   ipykernel
     #   jupyter-client
     #   nbclient
-numpy==1.22.2
+numpy==1.22.1
     # via
     #   pandas
     #   pyarrow
 packaging==21.3
     # via bleach
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 pandocfilters==1.5.0
     # via nbconvert
@@ -176,14 +174,16 @@ pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.5.0
+platformdirs==2.4.1
     # via black
 poyo==0.5.0
     # via cookiecutter
-prompt-toolkit==3.0.28
+prompt-toolkit==3.0.26
     # via ipython
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
@@ -203,7 +203,7 @@ pyparsing==3.0.7
     # via packaging
 pyrsistent==0.18.1
     # via jsonschema
-python-dateutil==2.8.2
+python-dateutil==2.8.1
     # via
     #   arrow
     #   croniter
@@ -212,7 +212,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -232,7 +232,7 @@ requests==2.27.1
     #   flytekit
     #   papermill
     #   responses
-responses==0.18.0
+responses==0.17.0
     # via flytekit
 retry==0.9.2
     # via flytekit
@@ -243,11 +243,13 @@ six==1.16.0
     #   asttokens
     #   bleach
     #   cookiecutter
+    #   flytekit
     #   grpcio
     #   python-dateutil
+    #   responses
 sortedcontainers==2.4.0
     # via flytekit
-stack-data==0.2.0
+stack-data==0.1.4
     # via ipython
 statsd==3.3.0
     # via flytekit
@@ -259,7 +261,7 @@ text-unidecode==1.3
     # via python-slugify
 textwrap3==0.9.2
     # via ansiwrap
-tomli==2.0.1
+tomli==1.2.3
     # via black
 tornado==6.1
     # via
@@ -277,9 +279,9 @@ traitlets==5.1.1
     #   nbclient
     #   nbconvert
     #   nbformat
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
-    #   flytekit
+    #   black
     #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
diff --git a/plugins/flytekit-snowflake/requirements.in b/plugins/flytekit-snowflake/requirements.in
index f720269c73..2a118dd3bf 100644
--- a/plugins/flytekit-snowflake/requirements.in
+++ b/plugins/flytekit-snowflake/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-snowflake
diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt
index e9b2b125f2..53951100b5 100644
--- a/plugins/flytekit-snowflake/requirements.txt
+++ b/plugins/flytekit-snowflake/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-snowflake
     # via -r requirements.in
 arrow==1.2.2
@@ -16,15 +12,13 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -32,7 +26,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -48,18 +42,16 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-snowflake
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -87,12 +79,14 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -107,7 +101,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -126,8 +120,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -139,7 +131,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-spark/requirements.in b/plugins/flytekit-spark/requirements.in
index 9d2e62c77e..f112dbc8ef 100644
--- a/plugins/flytekit-spark/requirements.in
+++ b/plugins/flytekit-spark/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-spark
diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt
index 2ab33fe5f3..75d1e32d53 100644
--- a/plugins/flytekit-spark/requirements.txt
+++ b/plugins/flytekit-spark/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-spark
     # via -r requirements.in
 arrow==1.2.2
@@ -16,15 +12,13 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -32,7 +26,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -48,18 +42,16 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-spark
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -87,12 +79,14 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 py4j==0.10.9.3
@@ -111,7 +105,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -130,8 +124,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -143,7 +135,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-sqlalchemy/requirements.in b/plugins/flytekit-sqlalchemy/requirements.in
index e545653d01..21d9e97b47 100644
--- a/plugins/flytekit-sqlalchemy/requirements.in
+++ b/plugins/flytekit-sqlalchemy/requirements.in
@@ -1,3 +1,2 @@
 .
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekitplugins-sqlalchemy
diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt
index bced83587b..93dcc2790c 100644
--- a/plugins/flytekit-sqlalchemy/requirements.txt
+++ b/plugins/flytekit-sqlalchemy/requirements.txt
@@ -1,13 +1,9 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via
-    #   -r requirements.in
-    #   flytekit
 -e file:.#egg=flytekitplugins-sqlalchemy
     # via -r requirements.in
 arrow==1.2.2
@@ -16,15 +12,13 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
-cffi==1.15.0
-    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.11
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.3
+click==7.1.2
     # via
     #   cookiecutter
     #   flytekit
@@ -32,7 +26,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.1
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -48,20 +42,18 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flytekit==0.30.3
+flyteidl==0.22.0
+    # via flytekit
+flytekit==0.30.0
     # via flytekitplugins-sqlalchemy
 greenlet==1.1.2
     # via sqlalchemy
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.1
+importlib-metadata==4.10.1
     # via keyring
-jeepney==0.7.1
-    # via
-    #   keyring
-    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -89,12 +81,14 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -109,7 +103,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.0.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -128,8 +122,6 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-secretstorage==3.3.1
-    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -143,7 +135,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
     #   flytekit
     #   typing-inspect

From adaa44884f76bd4c7c2225e8e309132cb9111906 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Fri, 18 Feb 2022 14:49:28 -0800
Subject: [PATCH 22/35] wip - Add support for univariate lists

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/core/type_engine.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py
index e44f3fa737..3354dad84b 100644
--- a/flytekit/core/type_engine.py
+++ b/flytekit/core/type_engine.py
@@ -803,7 +803,9 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]:
     def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal:
         t = self.get_sub_type(python_type)
         lit_list = [TypeEngine.to_literal(ctx, x, t, expected.collection_type) for x in python_val]  # type: ignore
-        return Literal(collection=LiteralCollection(literals=lit_list))
+        hash = "|".join([literal.hash for literal in lit_list if literal.hash])
+        print(f"hash={hash}")
+        return Literal(collection=LiteralCollection(literals=lit_list), hash=hash if hash != "|" else None)
 
     def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[T]:
         st = self.get_sub_type(expected_python_type)

From 4b5f608f762a018eae1f42acc3bcf6c4cf12583a Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Fri, 18 Feb 2022 15:43:51 -0800
Subject: [PATCH 23/35] Add support for lists of annotated objects

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/core/type_engine.py                 |  9 ++++--
 tests/flytekit/unit/core/test_type_engine.py | 32 ++++++++++++++++++++
 tests/flytekit/unit/core/test_type_hints.py  | 26 ++++++++++++++++
 3 files changed, 65 insertions(+), 2 deletions(-)

diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py
index 3354dad84b..c50edc7dca 100644
--- a/flytekit/core/type_engine.py
+++ b/flytekit/core/type_engine.py
@@ -803,9 +803,14 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]:
     def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal:
         t = self.get_sub_type(python_type)
         lit_list = [TypeEngine.to_literal(ctx, x, t, expected.collection_type) for x in python_val]  # type: ignore
+        # In order to support caching of lists of offloaded objects we delegate the production of
+        # overridable hashes to each literal and concatenate them into a single string to represent
+        # the hash of the list.
         hash = "|".join([literal.hash for literal in lit_list if literal.hash])
-        print(f"hash={hash}")
-        return Literal(collection=LiteralCollection(literals=lit_list), hash=hash if hash != "|" else None)
+        return Literal(
+            collection=LiteralCollection(literals=lit_list),
+            hash=hash if hash != "|" else None,
+        )
 
     def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[T]:
         st = self.get_sub_type(expected_python_type)
diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py
index 56f3ca50af..59cce41998 100644
--- a/tests/flytekit/unit/core/test_type_engine.py
+++ b/tests/flytekit/unit/core/test_type_engine.py
@@ -914,6 +914,38 @@ def constant_hash(df: pd.DataFrame) -> str:
     assert expected_df.equals(python_df)
 
 
+def test_literal_hash_of_list_of_dataframes():
+    """
+    Test to confirm that lists of annotated dataframes override the hash of the collection
+    """
+    ctx = FlyteContext.current_context()
+
+    def hash_function(df: pd.DataFrame) -> str:
+        return str(sum(df["col1"]))
+
+    list_transformer = ListTransformer()
+    type_list_of_annotated = typing.List[Annotated[pd.DataFrame, HashMethod(hash_function)]]
+
+    df1 = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
+    df2 = pd.DataFrame(data={"col1": [2, 3], "col2": [4, 5]})
+    literal_collection_with_hash_set = TypeEngine.to_literal(
+        ctx,
+        [df1, df2],
+        type_list_of_annotated,
+        list_transformer.get_literal_type(type_list_of_annotated),
+    )
+    assert literal_collection_with_hash_set.hash == "3|5"
+    # Confirm tha the loaded dataframes are not affected
+    python_dfs = TypeEngine.to_python_value(ctx, literal_collection_with_hash_set, typing.List[pd.DataFrame])
+    expected_dfs = [
+        pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}),
+        pd.DataFrame(data={"col1": [2, 3], "col2": [4, 5]}),
+    ]
+    assert len(python_dfs) == len(expected_dfs)
+    for i in range(len(expected_dfs)):
+        assert expected_dfs[i].equals(python_dfs[i])
+
+
 def test_annotated_simple_types():
     def _check_annotation(t, annotation):
         lt = TypeEngine.to_literal_type(t)
diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py
index e5ed338599..293ba885fe 100644
--- a/tests/flytekit/unit/core/test_type_hints.py
+++ b/tests/flytekit/unit/core/test_type_hints.py
@@ -1647,6 +1647,32 @@ def t0() -> Annotated[pandas.DataFrame, HashMethod(constant_function)]:
     assert df.equals(expected_df)
 
 
+def test_task_hash_return_list_of_pandas_dataframe():
+    def hash_function(df: pandas.DataFrame) -> str:
+        return str(sum(df["col1"]) + sum(df["col2"]))
+
+    @task
+    def t0() -> typing.List[Annotated[pandas.DataFrame, HashMethod(hash_function)]]:
+        return [
+            pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}),
+            pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}),
+        ]
+
+    ctx = context_manager.FlyteContextManager.current_context()
+    output_lm = t0.dispatch_execute(ctx, _literal_models.LiteralMap(literals={}))
+    assert output_lm.literals["o0"].hash == "10|100"
+
+    # Confirm that the literal containing a hash does not have any effect on the scalar.
+    dfs = TypeEngine.to_python_value(ctx, output_lm.literals["o0"], typing.List[pandas.DataFrame])
+    expected_dfs = [
+        pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}),
+        pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}),
+    ]
+    assert len(dfs) == len(expected_dfs)
+    for i in range(len(dfs)):
+        assert dfs[i].equals(expected_dfs[i])
+
+
 def test_workflow_containing_multiple_annotated_tasks():
     def hash_function_t0(df: pandas.DataFrame) -> str:
         return "hash-0"

From ecffa04e5d061e56455b97c9363f8c55903015e5 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 23 Feb 2022 08:00:22 -0800
Subject: [PATCH 24/35] Revamp generation of cache key (to cover case of
 literals collections and maps)

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/core/local_cache.py                 | 32 ++++++++++++++----
 tests/flytekit/unit/core/test_local_cache.py | 34 ++++++++++++++++++++
 2 files changed, 59 insertions(+), 7 deletions(-)

diff --git a/flytekit/core/local_cache.py b/flytekit/core/local_cache.py
index 8c908f8138..4afab73955 100644
--- a/flytekit/core/local_cache.py
+++ b/flytekit/core/local_cache.py
@@ -1,24 +1,42 @@
+import base64
 from typing import Optional
 
+import cloudpickle
 from diskcache import Cache
 
-from flytekit.models.literals import Literal, LiteralMap
+from flytekit.models.literals import Literal, LiteralCollection, LiteralMap
 
 # Location on the filesystem where serialized objects will be stored
 # TODO: read from config
 CACHE_LOCATION = "~/.flyte/local-cache"
 
 
+def _recursive_hash_placement(literal: Literal) -> Literal:
+    if literal.collection is not None:
+        literals = [_recursive_hash_placement(literal) for literal in literal.collection.literals]
+        return Literal(collection=LiteralCollection(literals=literals))
+    elif literal.map is not None:
+        literal_map = {}
+        for key, literal in literal.map.literals.items():
+            literal_map[key] = _recursive_hash_placement(literal)
+        return Literal(map=LiteralMap(literal_map))
+
+    # Base case
+    if literal.hash is not None:
+        return Literal(hash=literal.hash)
+    else:
+        return literal
+
+
 def _calculate_cache_key(task_name: str, cache_version: str, input_literal_map: LiteralMap) -> str:
-    # TODO: document this.
+    # Traverse the literals and replace the literal with a new literal that only contains the hash
     literal_map_overridden = {}
     for key, literal in input_literal_map.literals.items():
-        if literal.hash is not None:
-            literal_map_overridden[key] = Literal(hash=literal.hash)
-        else:
-            literal_map_overridden[key] = literal
+        literal_map_overridden[key] = _recursive_hash_placement(literal)
 
-    return f"{task_name}-{cache_version}-{LiteralMap(literal_map_overridden)}"
+    # Pickle the literal map and use base64 encoding to generate a representation of it
+    b64_encoded = base64.b64encode(cloudpickle.dumps(LiteralMap(literal_map_overridden)))
+    return f"{task_name}-{cache_version}-{b64_encoded}"
 
 
 class LocalTaskCache(object):
diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py
index 922fdfcccd..f308cc88a2 100644
--- a/tests/flytekit/unit/core/test_local_cache.py
+++ b/tests/flytekit/unit/core/test_local_cache.py
@@ -1,6 +1,7 @@
 import datetime
 import typing
 from dataclasses import dataclass
+from typing import List
 
 import pandas
 from dataclasses_json import dataclass_json
@@ -351,6 +352,39 @@ def my_workflow():
     assert n_cached_task_calls == 1
 
 
+def test_list_of_pandas_dataframe_hash():
+    """
+    Test that cache is hit in the case of a list of pandas dataframes where we annotated dataframes to hash
+    the contents of the dataframes.
+    """
+
+    def hash_pandas_dataframe(df: pandas.DataFrame) -> str:
+        return str(pandas.util.hash_pandas_object(df))
+
+    @task
+    def uncached_data_reading_task() -> List[Annotated[pandas.DataFrame, HashMethod(hash_pandas_dataframe)]]:
+        return [pandas.DataFrame({"column_1": [1, 2, 3]}), pandas.DataFrame({"column_1": [10, 20, 30]})]
+
+    @task(cache=True, cache_version="0.1")
+    def cached_data_processing_task(data: List[pandas.DataFrame]) -> List[pandas.DataFrame]:
+        global n_cached_task_calls
+        n_cached_task_calls += 1
+        return [df * 2 for df in data]
+
+    @workflow
+    def my_workflow():
+        raw_data = uncached_data_reading_task()
+        cached_data_processing_task(data=raw_data)
+
+    assert n_cached_task_calls == 0
+    my_workflow()
+    assert n_cached_task_calls == 1
+
+    # Confirm that we see a cache hit in the case of annotated dataframes.
+    my_workflow()
+    assert n_cached_task_calls == 1
+
+
 """
 Update SD transformer so that it can to_python_value a Schema literal
   - If a Schema literal is detected, copy the uri and use the new decoder to unwrap the uri

From d4b0b4929eb34bcfcf1338df9a17ef2984b5b490 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 23 Feb 2022 08:01:18 -0800
Subject: [PATCH 25/35] Leave TODO for warning

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/core/type_engine.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py
index c50edc7dca..b00ee70556 100644
--- a/flytekit/core/type_engine.py
+++ b/flytekit/core/type_engine.py
@@ -881,6 +881,7 @@ def to_literal(
         for k, v in python_val.items():
             if type(k) != str:
                 raise ValueError("Flyte MapType expects all keys to be strings")
+            # TODO: log a warning for Annotated objects that contain HashMethod
             k_type, v_type = self.get_dict_types(python_type)
             lit_map[k] = TypeEngine.to_literal(ctx, v, v_type, expected.map_value_type)
         return Literal(map=LiteralMap(literals=lit_map))

From 4d54c595e29d5acbaba2359d3d01630f865b8265 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 23 Feb 2022 08:01:46 -0800
Subject: [PATCH 26/35] Revert "Add support for lists of annotated objects"

This reverts commit 4b5f608f762a018eae1f42acc3bcf6c4cf12583a.

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/core/type_engine.py                 |  9 ++----
 tests/flytekit/unit/core/test_type_engine.py | 32 --------------------
 tests/flytekit/unit/core/test_type_hints.py  | 26 ----------------
 3 files changed, 2 insertions(+), 65 deletions(-)

diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py
index b00ee70556..a931f0789b 100644
--- a/flytekit/core/type_engine.py
+++ b/flytekit/core/type_engine.py
@@ -803,14 +803,9 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]:
     def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal:
         t = self.get_sub_type(python_type)
         lit_list = [TypeEngine.to_literal(ctx, x, t, expected.collection_type) for x in python_val]  # type: ignore
-        # In order to support caching of lists of offloaded objects we delegate the production of
-        # overridable hashes to each literal and concatenate them into a single string to represent
-        # the hash of the list.
         hash = "|".join([literal.hash for literal in lit_list if literal.hash])
-        return Literal(
-            collection=LiteralCollection(literals=lit_list),
-            hash=hash if hash != "|" else None,
-        )
+        print(f"hash={hash}")
+        return Literal(collection=LiteralCollection(literals=lit_list), hash=hash if hash != "|" else None)
 
     def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[T]:
         st = self.get_sub_type(expected_python_type)
diff --git a/tests/flytekit/unit/core/test_type_engine.py b/tests/flytekit/unit/core/test_type_engine.py
index 59cce41998..56f3ca50af 100644
--- a/tests/flytekit/unit/core/test_type_engine.py
+++ b/tests/flytekit/unit/core/test_type_engine.py
@@ -914,38 +914,6 @@ def constant_hash(df: pd.DataFrame) -> str:
     assert expected_df.equals(python_df)
 
 
-def test_literal_hash_of_list_of_dataframes():
-    """
-    Test to confirm that lists of annotated dataframes override the hash of the collection
-    """
-    ctx = FlyteContext.current_context()
-
-    def hash_function(df: pd.DataFrame) -> str:
-        return str(sum(df["col1"]))
-
-    list_transformer = ListTransformer()
-    type_list_of_annotated = typing.List[Annotated[pd.DataFrame, HashMethod(hash_function)]]
-
-    df1 = pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]})
-    df2 = pd.DataFrame(data={"col1": [2, 3], "col2": [4, 5]})
-    literal_collection_with_hash_set = TypeEngine.to_literal(
-        ctx,
-        [df1, df2],
-        type_list_of_annotated,
-        list_transformer.get_literal_type(type_list_of_annotated),
-    )
-    assert literal_collection_with_hash_set.hash == "3|5"
-    # Confirm tha the loaded dataframes are not affected
-    python_dfs = TypeEngine.to_python_value(ctx, literal_collection_with_hash_set, typing.List[pd.DataFrame])
-    expected_dfs = [
-        pd.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}),
-        pd.DataFrame(data={"col1": [2, 3], "col2": [4, 5]}),
-    ]
-    assert len(python_dfs) == len(expected_dfs)
-    for i in range(len(expected_dfs)):
-        assert expected_dfs[i].equals(python_dfs[i])
-
-
 def test_annotated_simple_types():
     def _check_annotation(t, annotation):
         lt = TypeEngine.to_literal_type(t)
diff --git a/tests/flytekit/unit/core/test_type_hints.py b/tests/flytekit/unit/core/test_type_hints.py
index 293ba885fe..e5ed338599 100644
--- a/tests/flytekit/unit/core/test_type_hints.py
+++ b/tests/flytekit/unit/core/test_type_hints.py
@@ -1647,32 +1647,6 @@ def t0() -> Annotated[pandas.DataFrame, HashMethod(constant_function)]:
     assert df.equals(expected_df)
 
 
-def test_task_hash_return_list_of_pandas_dataframe():
-    def hash_function(df: pandas.DataFrame) -> str:
-        return str(sum(df["col1"]) + sum(df["col2"]))
-
-    @task
-    def t0() -> typing.List[Annotated[pandas.DataFrame, HashMethod(hash_function)]]:
-        return [
-            pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}),
-            pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}),
-        ]
-
-    ctx = context_manager.FlyteContextManager.current_context()
-    output_lm = t0.dispatch_execute(ctx, _literal_models.LiteralMap(literals={}))
-    assert output_lm.literals["o0"].hash == "10|100"
-
-    # Confirm that the literal containing a hash does not have any effect on the scalar.
-    dfs = TypeEngine.to_python_value(ctx, output_lm.literals["o0"], typing.List[pandas.DataFrame])
-    expected_dfs = [
-        pandas.DataFrame(data={"col1": [1, 2], "col2": [3, 4]}),
-        pandas.DataFrame(data={"col1": [10, 20], "col2": [30, 40]}),
-    ]
-    assert len(dfs) == len(expected_dfs)
-    for i in range(len(dfs)):
-        assert dfs[i].equals(expected_dfs[i])
-
-
 def test_workflow_containing_multiple_annotated_tasks():
     def hash_function_t0(df: pandas.DataFrame) -> str:
         return "hash-0"

From 82bbc1fbe07d5a32a25f68730c39f0e4408cd361 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 23 Feb 2022 08:02:06 -0800
Subject: [PATCH 27/35] Revert "wip - Add support for univariate lists"

This reverts commit adaa44884f76bd4c7c2225e8e309132cb9111906.

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 flytekit/core/type_engine.py | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/flytekit/core/type_engine.py b/flytekit/core/type_engine.py
index a931f0789b..1a84c744a8 100644
--- a/flytekit/core/type_engine.py
+++ b/flytekit/core/type_engine.py
@@ -803,9 +803,7 @@ def get_literal_type(self, t: Type[T]) -> Optional[LiteralType]:
     def to_literal(self, ctx: FlyteContext, python_val: T, python_type: Type[T], expected: LiteralType) -> Literal:
         t = self.get_sub_type(python_type)
         lit_list = [TypeEngine.to_literal(ctx, x, t, expected.collection_type) for x in python_val]  # type: ignore
-        hash = "|".join([literal.hash for literal in lit_list if literal.hash])
-        print(f"hash={hash}")
-        return Literal(collection=LiteralCollection(literals=lit_list), hash=hash if hash != "|" else None)
+        return Literal(collection=LiteralCollection(literals=lit_list))
 
     def to_python_value(self, ctx: FlyteContext, lv: Literal, expected_python_type: Type[T]) -> typing.List[T]:
         st = self.get_sub_type(expected_python_type)

From a21631b85db9a7b9bd46a88e4c45de567c6ddf0e Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Mon, 28 Feb 2022 17:29:48 -0800
Subject: [PATCH 28/35] Remove docstring

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 tests/flytekit/unit/core/test_local_cache.py | 12 ------------
 1 file changed, 12 deletions(-)

diff --git a/tests/flytekit/unit/core/test_local_cache.py b/tests/flytekit/unit/core/test_local_cache.py
index f308cc88a2..3f3e56de88 100644
--- a/tests/flytekit/unit/core/test_local_cache.py
+++ b/tests/flytekit/unit/core/test_local_cache.py
@@ -383,15 +383,3 @@ def my_workflow():
     # Confirm that we see a cache hit in the case of annotated dataframes.
     my_workflow()
     assert n_cached_task_calls == 1
-
-
-"""
-Update SD transformer so that it can to_python_value a Schema literal
-  - If a Schema literal is detected, copy the uri and use the new decoder to unwrap the uri
-Update FS transformer so that it can to_python_value a StructuredDataset literal
-  - If a StructuredDataset literal is detected, use the uri from that instead.
-
-Update all plugins that can take in a FlyteSchema to also be able to take in a StructuredDataset.
-
-All tests should work with the presence of SD imports.
-"""

From 3519149d718581d076ec2201651e092678732ff7 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Tue, 1 Mar 2022 17:06:17 -0800
Subject: [PATCH 29/35] Add flyteidl>=0.23.0

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 setup.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/setup.py b/setup.py
index 4e31ecc7fe..9a96657326 100644
--- a/setup.py
+++ b/setup.py
@@ -35,8 +35,7 @@
         ]
     },
     install_requires=[
-        # TODO: put the flyteidl constraint back
-        # "flyteidl>=0.21.4",
+        "flyteidl>=0.23.0",
         "wheel>=0.30.0,<1.0.0",
         "pandas>=1.0.0,<2.0.0",
         "pyarrow>=4.0.0,<7.0.0",

From 09e736d8847c4e447ae42cf0e2c5292f2c9f8fb9 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Tue, 1 Mar 2022 17:06:43 -0800
Subject: [PATCH 30/35] Remove mentions to branch flyteidl@add-hash-to-literal

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 .github/workflows/pythonbuild.yml                    |  1 -
 dev-requirements.txt                                 | 12 ++++++++++++
 doc-requirements.txt                                 | 12 +++++++++++-
 requirements-spark2.txt                              | 10 ++++++++--
 requirements.in                                      |  1 -
 requirements.txt                                     | 10 ++++++++--
 .../mock_flyte_repo/workflows/requirements.txt       |  2 +-
 7 files changed, 40 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/pythonbuild.yml b/.github/workflows/pythonbuild.yml
index 8972e0df4b..d38b666aee 100644
--- a/.github/workflows/pythonbuild.yml
+++ b/.github/workflows/pythonbuild.yml
@@ -106,7 +106,6 @@ jobs:
         pip install -e .
         if [ -f dev-requirements.txt ]; then pip install -r dev-requirements.txt; fi
         pip install --no-deps -U https://github.com/flyteorg/flytekit/archive/${{ github.sha }}.zip#egg=flytekit
-        pip install --no-deps -U "git+https://github.com/flyteorg/flyteidl@add-hash-to-literal#egg=flyteidl"
         pip freeze
     - name: Test with coverage
       run: |
diff --git a/dev-requirements.txt b/dev-requirements.txt
index b34300a82a..f8bd6af672 100644
--- a/dev-requirements.txt
+++ b/dev-requirements.txt
@@ -116,6 +116,10 @@ docstring-parser==0.13
     #   flytekit
 filelock==3.6.0
     # via virtualenv
+flyteidl==0.23.0
+    # via
+    #   -c requirements.txt
+    #   flytekit
 google-api-core[grpc]==2.5.0
     # via
     #   google-cloud-bigquery
@@ -137,6 +141,8 @@ google-resumable-media==2.3.0
     # via google-cloud-bigquery
 googleapis-common-protos==1.55.0
     # via
+    #   -c requirements.txt
+    #   flyteidl
     #   google-api-core
     #   grpcio-status
 grpcio==1.44.0
@@ -250,12 +256,18 @@ proto-plus==1.20.3
 protobuf==3.19.4
     # via
     #   -c requirements.txt
+    #   flyteidl
     #   flytekit
     #   google-api-core
     #   google-cloud-bigquery
     #   googleapis-common-protos
     #   grpcio-status
     #   proto-plus
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via
+    #   -c requirements.txt
+    #   flyteidl
 py==1.11.0
     # via
     #   -c requirements.txt
diff --git a/doc-requirements.txt b/doc-requirements.txt
index 71788c0b09..07ef3d0d5b 100644
--- a/doc-requirements.txt
+++ b/doc-requirements.txt
@@ -63,8 +63,12 @@ docutils==0.17.1
     # via
     #   sphinx
     #   sphinx-panels
+flyteidl==0.23.0
+    # via flytekit
 furo @ git+https://github.com/flyteorg/furo@main
     # via -r doc-requirements.in
+googleapis-common-protos==1.55.0
+    # via flyteidl
 grpcio==1.44.0
     # via
     #   -r doc-requirements.in
@@ -121,7 +125,13 @@ pandas==1.4.1
 poyo==0.5.0
     # via cookiecutter
 protobuf==3.19.4
-    # via flytekit
+    # via
+    #   flyteidl
+    #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
diff --git a/requirements-spark2.txt b/requirements-spark2.txt
index 90bfac547e..be50ade576 100644
--- a/requirements-spark2.txt
+++ b/requirements-spark2.txt
@@ -4,8 +4,6 @@
 #
 #    make requirements-spark2.txt
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via -r requirements.in
 -e file:.#egg=flytekit
     # via
     #   -r requirements-spark2.in
@@ -52,6 +50,10 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
+flyteidl==0.23.0
+    # via flytekit
+googleapis-common-protos==1.55.0
+    # via flyteidl
 grpcio==1.44.0
     # via flytekit
 idna==3.3
@@ -102,6 +104,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
diff --git a/requirements.in b/requirements.in
index 816f9301fe..0cf01a6f13 100644
--- a/requirements.in
+++ b/requirements.in
@@ -1,5 +1,4 @@
 .[all]
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
 -e file:.#egg=flytekit
 attrs<21
 # We need to restrict constrain the versions of both jsonschema and pyyaml because of docker-compose (which is
diff --git a/requirements.txt b/requirements.txt
index 54cb926700..2e51986bf4 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -4,8 +4,6 @@
 #
 #    make requirements.txt
 #
--e git+https://github.com/flyteorg/flyteidl.git@add-hash-to-literal#egg=flyteidl
-    # via -r requirements.in
 -e file:.#egg=flytekit
     # via -r requirements.in
 arrow==1.2.2
@@ -50,6 +48,10 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
+flyteidl==0.23.0
+    # via flytekit
+googleapis-common-protos==1.55.0
+    # via flyteidl
 grpcio==1.44.0
     # via flytekit
 idna==3.3
@@ -100,6 +102,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
diff --git a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
index e559b6762d..15a1f89700 100644
--- a/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
+++ b/tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.txt
@@ -44,7 +44,7 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.3
+flyteidl==0.23.0
     # via flytekit
 flytekit==0.30.3
     # via -r tests/flytekit/integration/remote/mock_flyte_repo/workflows/requirements.in

From 9ec103cf589b62dcf37a814cdace63f35ebeb026 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Tue, 1 Mar 2022 17:53:46 -0800
Subject: [PATCH 31/35] Bump flyteidl in plugins requirements

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 plugins/flytekit-aws-athena/requirements.txt  |  38 ++++---
 plugins/flytekit-aws-batch/requirements.txt   |  60 +++++++----
 .../flytekit-aws-sagemaker/requirements.txt   |  48 +++++----
 plugins/flytekit-bigquery/requirements.txt    |  38 ++++---
 plugins/flytekit-data-fsspec/requirements.txt |  46 +++++---
 plugins/flytekit-dolt/requirements.txt        |  38 ++++---
 .../requirements.txt                          | 102 +++++++++---------
 plugins/flytekit-hive/requirements.txt        |  38 ++++---
 plugins/flytekit-k8s-pod/requirements.txt     |  42 +++++---
 plugins/flytekit-kf-mpi/requirements.txt      |  38 ++++---
 plugins/flytekit-kf-pytorch/requirements.txt  |  38 ++++---
 .../flytekit-kf-tensorflow/requirements.txt   |  38 ++++---
 plugins/flytekit-pandera/requirements.txt     |  43 +++++---
 plugins/flytekit-papermill/requirements.txt   |  77 ++++++-------
 plugins/flytekit-snowflake/requirements.txt   |  38 ++++---
 plugins/flytekit-spark/requirements.txt       |  38 ++++---
 plugins/flytekit-sqlalchemy/requirements.txt  |  38 ++++---
 17 files changed, 499 insertions(+), 299 deletions(-)

diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt
index a1acae494f..6a2fc1f2d7 100644
--- a/plugins/flytekit-aws-athena/requirements.txt
+++ b/plugins/flytekit-aws-athena/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +44,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-athena
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -60,7 +68,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -79,7 +87,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -87,6 +95,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +113,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +132,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +145,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt
index 9181490234..ee76299714 100644
--- a/plugins/flytekit-aws-batch/requirements.txt
+++ b/plugins/flytekit-aws-batch/requirements.txt
@@ -1,24 +1,26 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
 #
 -e file:.#egg=flytekitplugins-awsbatch
     # via -r requirements.in
-arrow==1.2.1
+arrow==1.2.2
     # via jinja2-time
 binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.10
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,8 +28,10 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
+cryptography==36.0.1
+    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 decorator==5.1.1
@@ -40,16 +44,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.21.23
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.26.0
+flytekit==0.30.3
     # via flytekitplugins-awsbatch
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -58,7 +68,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -71,25 +81,31 @@ marshmallow-jsonschema==0.13.0
     # via flytekit
 mypy-extensions==0.4.3
     # via typing-inspect
-natsort==8.0.2
+natsort==8.1.0
     # via flytekit
-numpy==1.22.1
+numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.3.5
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
-protobuf==3.19.3
+protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
     # via flytekit
-python-dateutil==2.8.1
+pycparser==2.21
+    # via cffi
+python-dateutil==2.8.2
     # via
     #   arrow
     #   croniter
@@ -97,7 +113,7 @@ python-dateutil==2.8.1
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -112,25 +128,27 @@ requests==2.27.1
     #   cookiecutter
     #   flytekit
     #   responses
-responses==0.17.0
+responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
-    #   flytekit
     #   grpcio
     #   python-dateutil
-    #   responses
 sortedcontainers==2.4.0
     # via flytekit
 statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
-    # via typing-inspect
+typing-extensions==4.1.1
+    # via
+    #   flytekit
+    #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
 urllib3==1.26.8
diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt
index 11396f213e..acb982c9a2 100644
--- a/plugins/flytekit-aws-sagemaker/requirements.txt
+++ b/plugins/flytekit-aws-sagemaker/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,9 +12,9 @@ bcrypt==3.2.0
     # via paramiko
 binaryornot==0.4.4
     # via cookiecutter
-boto3==1.20.50
+boto3==1.21.10
     # via sagemaker-training
-botocore==1.23.50
+botocore==1.24.10
     # via
     #   boto3
     #   s3transfer
@@ -27,11 +27,11 @@ cffi==1.15.0
     #   pynacl
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -39,7 +39,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via
@@ -57,22 +57,28 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-awssagemaker
 gevent==21.12.0
     # via sagemaker-training
+googleapis-common-protos==1.55.0
+    # via flyteidl
 greenlet==1.1.2
     # via gevent
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
-inotify-simple==1.2.1
+inotify_simple==1.2.1
     # via sagemaker-training
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -85,7 +91,7 @@ jmespath==0.10.0
     #   botocore
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -106,7 +112,7 @@ numpy==1.22.2
     #   pyarrow
     #   sagemaker-training
     #   scipy
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 paramiko==2.9.2
     # via sagemaker-training
@@ -116,7 +122,11 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
     #   sagemaker-training
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 psutil==5.9.0
     # via sagemaker-training
 py==1.11.0
@@ -136,7 +146,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -157,12 +167,14 @@ retry==0.9.2
     # via flytekit
 retrying==1.3.3
     # via sagemaker-training
-s3transfer==0.5.1
+s3transfer==0.5.2
     # via boto3
 sagemaker-training==3.9.2
     # via flytekitplugins-awssagemaker
 scipy==1.8.0
     # via sagemaker-training
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   bcrypt
@@ -177,7 +189,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
@@ -199,9 +211,9 @@ wrapt==1.13.3
     #   flytekit
 zipp==3.7.0
     # via importlib-metadata
-zope-event==4.5.0
+zope.event==4.5.0
     # via gevent
-zope-interface==5.4.0
+zope.interface==5.4.0
     # via gevent
 
 # The following packages are considered to be unsafe in a requirements file:
diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt
index f0bc647453..b908f23e9e 100644
--- a/plugins/flytekit-bigquery/requirements.txt
+++ b/plugins/flytekit-bigquery/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -18,11 +18,11 @@ cffi==1.15.0
     # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -30,7 +30,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -46,9 +46,9 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-bigquery
 google-api-core[grpc]==2.5.0
     # via
@@ -58,29 +58,30 @@ google-auth==2.6.0
     # via
     #   google-api-core
     #   google-cloud-core
-google-cloud-bigquery==2.32.0
+google-cloud-bigquery==2.34.0
     # via flytekitplugins-bigquery
 google-cloud-core==2.2.2
     # via google-cloud-bigquery
 google-crc32c==1.3.0
     # via google-resumable-media
-google-resumable-media==2.2.0
+google-resumable-media==2.3.0
     # via google-cloud-bigquery
-googleapis-common-protos==1.54.0
+googleapis-common-protos==1.55.0
     # via
+    #   flyteidl
     #   google-api-core
     #   grpcio-status
-grpcio==1.43.0
+grpcio==1.44.0
     # via
     #   flytekit
     #   google-api-core
     #   google-cloud-bigquery
     #   grpcio-status
-grpcio-status==1.43.0
+grpcio-status==1.44.0
     # via google-api-core
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
 jeepney==0.7.1
     # via
@@ -94,7 +95,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -115,11 +116,11 @@ numpy==1.22.2
     #   pyarrow
 packaging==21.3
     # via google-cloud-bigquery
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
-proto-plus==1.20.0
+proto-plus==1.20.3
     # via google-cloud-bigquery
 protobuf==3.19.4
     # via
@@ -130,6 +131,9 @@ protobuf==3.19.4
     #   googleapis-common-protos
     #   grpcio-status
     #   proto-plus
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -153,7 +157,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -190,7 +194,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt
index 4dc89fa500..e728ce8676 100644
--- a/plugins/flytekit-data-fsspec/requirements.txt
+++ b/plugins/flytekit-data-fsspec/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -10,17 +10,19 @@ arrow==1.2.2
     # via jinja2-time
 binaryornot==0.4.4
     # via cookiecutter
-botocore==1.23.24
+botocore==1.24.10
     # via flytekitplugins-data-fsspec
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -28,8 +30,10 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
+cryptography==36.0.1
+    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 decorator==5.1.1
@@ -42,18 +46,24 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-data-fsspec
-fsspec==2022.1.0
+fsspec==2022.2.0
     # via flytekitplugins-data-fsspec
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -64,7 +74,7 @@ jmespath==0.10.0
     # via botocore
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -83,7 +93,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -91,10 +101,16 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
     # via flytekit
+pycparser==2.21
+    # via cffi
 python-dateutil==2.8.2
     # via
     #   arrow
@@ -104,7 +120,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -123,6 +139,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -134,7 +152,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt
index 01dfe2cb6b..940643fdf9 100644
--- a/plugins/flytekit-dolt/requirements.txt
+++ b/plugins/flytekit-dolt/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -48,16 +50,22 @@ dolt-integrations==0.1.5
     # via flytekitplugins-dolt
 doltcli==0.1.17
     # via dolt-integrations
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-dolt
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -66,7 +74,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -85,7 +93,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via
     #   dolt-integrations
     #   flytekit
@@ -95,6 +103,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -109,7 +121,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -128,6 +140,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -139,7 +153,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt
index 7aa0a867dd..7acb822eec 100644
--- a/plugins/flytekit-greatexpectations/requirements.txt
+++ b/plugins/flytekit-greatexpectations/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -8,10 +8,6 @@
     # via -r requirements.in
 altair==4.2.0
     # via great-expectations
-appnope==0.1.2
-    # via
-    #   ipykernel
-    #   ipython
 argon2-cffi==21.3.0
     # via notebook
 argon2-cffi-bindings==21.2.0
@@ -26,23 +22,22 @@ backcall==0.2.0
     # via ipython
 binaryornot==0.4.4
     # via cookiecutter
-black==21.12b0
-    # via ipython
 bleach==4.1.0
     # via nbconvert
 certifi==2021.10.8
     # via requests
 cffi==1.15.0
-    # via argon2-cffi-bindings
+    # via
+    #   argon2-cffi-bindings
+    #   cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.10
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
-    #   black
     #   cookiecutter
     #   flytekit
     #   great-expectations
@@ -50,8 +45,10 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
+cryptography==36.0.1
+    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 debugpy==1.5.1
@@ -70,34 +67,36 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-entrypoints==0.3
+entrypoints==0.4
     # via
     #   altair
     #   jupyter-client
     #   nbconvert
-executing==0.8.2
+executing==0.8.3
     # via stack-data
-flyteidl==0.21.24
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.26.1
+flytekit==0.30.3
     # via flytekitplugins-great-expectations
-great-expectations==0.14.3
+googleapis-common-protos==1.55.0
+    # via flyteidl
+great-expectations==0.14.5
     # via flytekitplugins-great-expectations
 greenlet==1.1.2
     # via sqlalchemy
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via
     #   great-expectations
     #   keyring
-ipykernel==6.7.0
+ipykernel==6.9.1
     # via
     #   ipywidgets
     #   notebook
-ipython==8.0.1
+ipython==8.1.0
     # via
     #   ipykernel
     #   ipywidgets
@@ -110,6 +109,10 @@ ipywidgets==7.6.5
     # via great-expectations
 jedi==0.18.1
     # via ipython
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   altair
@@ -134,7 +137,7 @@ jupyter-client==7.1.2
     #   ipykernel
     #   nbclient
     #   notebook
-jupyter-core==4.9.1
+jupyter-core==4.9.2
     # via
     #   jupyter-client
     #   nbconvert
@@ -146,7 +149,7 @@ jupyterlab-widgets==1.0.2
     # via ipywidgets
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -166,14 +169,12 @@ mistune==0.8.4
     #   great-expectations
     #   nbconvert
 mypy-extensions==0.4.3
-    # via
-    #   black
-    #   typing-inspect
-natsort==8.0.2
+    # via typing-inspect
+natsort==8.1.0
     # via flytekit
-nbclient==0.5.10
+nbclient==0.5.11
     # via nbconvert
-nbconvert==6.4.1
+nbconvert==6.4.2
     # via notebook
 nbformat==5.1.3
     # via
@@ -189,7 +190,7 @@ nest-asyncio==1.5.4
     #   notebook
 notebook==6.4.8
     # via widgetsnbextension
-numpy==1.22.1
+numpy==1.22.2
     # via
     #   altair
     #   great-expectations
@@ -198,7 +199,7 @@ numpy==1.22.1
     #   scipy
 packaging==21.3
     # via bleach
-pandas==1.4.0
+pandas==1.4.1
     # via
     #   altair
     #   flytekit
@@ -207,24 +208,24 @@ pandocfilters==1.5.0
     # via nbconvert
 parso==0.8.3
     # via jedi
-pathspec==0.9.0
-    # via black
 pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.4.1
-    # via black
 poyo==0.5.0
     # via cookiecutter
 prometheus-client==0.13.1
     # via notebook
-prompt-toolkit==3.0.26
+prompt-toolkit==3.0.28
     # via ipython
 protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 ptyprocess==0.7.0
     # via
     #   pexpect
@@ -248,7 +249,7 @@ pyparsing==2.4.7
     #   packaging
 pyrsistent==0.18.1
     # via jsonschema
-python-dateutil==2.8.1
+python-dateutil==2.8.2
     # via
     #   arrow
     #   croniter
@@ -258,7 +259,7 @@ python-dateutil==2.8.1
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -281,16 +282,16 @@ requests==2.27.1
     #   flytekit
     #   great-expectations
     #   responses
-responses==0.17.0
+responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
-ruamel-yaml==0.17.17
+ruamel.yaml==0.17.17
     # via great-expectations
-ruamel-yaml-clib==0.2.6
-    # via ruamel-yaml
-scipy==1.7.3
+scipy==1.8.0
     # via great-expectations
+secretstorage==3.3.1
+    # via keyring
 send2trash==1.8.0
     # via notebook
 six==1.16.0
@@ -298,17 +299,15 @@ six==1.16.0
     #   asttokens
     #   bleach
     #   cookiecutter
-    #   flytekit
     #   grpcio
     #   python-dateutil
-    #   responses
 sortedcontainers==2.4.0
     # via flytekit
 sqlalchemy==1.4.31
     # via
     #   -r requirements.in
     #   flytekitplugins-great-expectations
-stack-data==0.1.4
+stack-data==0.2.0
     # via ipython
 statsd==3.3.0
     # via flytekit
@@ -316,12 +315,10 @@ termcolor==1.1.0
     # via great-expectations
 terminado==0.13.1
     # via notebook
-testpath==0.5.0
+testpath==0.6.0
     # via nbconvert
 text-unidecode==1.3
     # via python-slugify
-tomli==1.2.3
-    # via black
 toolz==0.11.2
     # via altair
 tornado==6.1
@@ -330,7 +327,7 @@ tornado==6.1
     #   jupyter-client
     #   notebook
     #   terminado
-tqdm==4.62.3
+tqdm==4.63.0
     # via great-expectations
 traitlets==5.1.1
     # via
@@ -344,9 +341,10 @@ traitlets==5.1.1
     #   nbconvert
     #   nbformat
     #   notebook
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
-    #   black
+    #   flytekit
+    #   great-expectations
     #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt
index 3ae057f9a0..a7cf6dcbb2 100644
--- a/plugins/flytekit-hive/requirements.txt
+++ b/plugins/flytekit-hive/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +44,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-hive
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -60,7 +68,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -79,7 +87,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -87,6 +95,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +113,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +132,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +145,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt
index d60d9c46b6..7e7eea1284 100644
--- a/plugins/flytekit-k8s-pod/requirements.txt
+++ b/plugins/flytekit-k8s-pod/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -16,13 +16,15 @@ certifi==2021.10.8
     # via
     #   kubernetes
     #   requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -30,7 +32,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -46,18 +48,24 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-pod
 google-auth==2.6.0
     # via kubernetes
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -66,9 +74,9 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-kubernetes==21.7.0
+kubernetes==23.3.0
     # via flytekitplugins-pod
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -89,7 +97,7 @@ numpy==1.22.2
     #   pyarrow
 oauthlib==3.2.0
     # via requests-oauthlib
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -97,6 +105,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -118,7 +130,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -145,6 +157,8 @@ retry==0.9.2
     # via flytekit
 rsa==4.8
     # via google-auth
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -158,7 +172,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
@@ -170,7 +184,7 @@ urllib3==1.26.8
     #   kubernetes
     #   requests
     #   responses
-websocket-client==1.2.3
+websocket-client==1.3.1
     # via kubernetes
 wheel==0.37.1
     # via flytekit
diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt
index c13f9f3b74..4d5c66fcea 100644
--- a/plugins/flytekit-kf-mpi/requirements.txt
+++ b/plugins/flytekit-kf-mpi/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,18 +44,24 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via
     #   flytekit
     #   flytekitplugins-kfmpi
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-kfmpi
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -62,7 +70,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -81,7 +89,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -89,6 +97,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -103,7 +115,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -122,6 +134,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -133,7 +147,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt
index 1696646c87..6f4496e3b4 100644
--- a/plugins/flytekit-kf-pytorch/requirements.txt
+++ b/plugins/flytekit-kf-pytorch/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +44,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-kfpytorch
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -60,7 +68,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -79,7 +87,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -87,6 +95,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +113,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +132,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +145,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt
index d3b6253664..2af5b7c14b 100644
--- a/plugins/flytekit-kf-tensorflow/requirements.txt
+++ b/plugins/flytekit-kf-tensorflow/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +44,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-kftensorflow
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -60,7 +68,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -79,7 +87,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -87,6 +95,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +113,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +132,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +145,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt
index f0b65cec17..62c5e3319c 100644
--- a/plugins/flytekit-pandera/requirements.txt
+++ b/plugins/flytekit-pandera/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +44,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-pandera
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -60,7 +68,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -82,11 +90,11 @@ numpy==1.22.2
     #   pyarrow
 packaging==21.3
     # via pandera
-pandas==1.4.0
+pandas==1.4.1
     # via
     #   flytekit
     #   pandera
-pandera==0.8.1
+pandera==0.9.0
     # via flytekitplugins-pandera
 poyo==0.5.0
     # via cookiecutter
@@ -94,6 +102,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -102,6 +114,8 @@ pyarrow==6.0.1
     #   pandera
 pycparser==2.21
     # via cffi
+pydantic==1.9.0
+    # via pandera
 pyparsing==3.0.7
     # via packaging
 python-dateutil==2.8.2
@@ -112,7 +126,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -131,6 +145,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -142,9 +158,10 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
+    #   pydantic
     #   typing-inspect
 typing-inspect==0.7.1
     # via
diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt
index 8c3e4570e9..0a0cc6e49e 100644
--- a/plugins/flytekit-papermill/requirements.txt
+++ b/plugins/flytekit-papermill/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -18,8 +18,6 @@ backcall==0.2.0
     # via ipython
 binaryornot==0.4.4
     # via cookiecutter
-black==21.12b0
-    # via ipython
 bleach==4.1.0
     # via nbconvert
 certifi==2021.10.8
@@ -28,13 +26,12 @@ cffi==1.15.0
     # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.10
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
-    #   black
     #   cookiecutter
     #   flytekit
     #   papermill
@@ -42,7 +39,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -64,26 +61,28 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-entrypoints==0.3
+entrypoints==0.4
     # via
     #   jupyter-client
     #   nbconvert
     #   papermill
-executing==0.8.2
+executing==0.8.3
     # via stack-data
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.26.1
+flytekit==0.30.3
     # via flytekitplugins-papermill
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
-ipykernel==6.7.0
+ipykernel==6.9.1
     # via flytekitplugins-papermill
-ipython==8.0.1
+ipython==8.1.0
     # via ipykernel
 ipython-genutils==0.2.0
     # via nbformat
@@ -106,7 +105,7 @@ jupyter-client==7.1.2
     # via
     #   ipykernel
     #   nbclient
-jupyter-core==4.9.1
+jupyter-core==4.9.2
     # via
     #   jupyter-client
     #   nbconvert
@@ -115,7 +114,7 @@ jupyterlab-pygments==0.1.2
     # via nbconvert
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -133,16 +132,14 @@ matplotlib-inline==0.1.3
 mistune==0.8.4
     # via nbconvert
 mypy-extensions==0.4.3
-    # via
-    #   black
-    #   typing-inspect
-natsort==8.0.2
+    # via typing-inspect
+natsort==8.1.0
     # via flytekit
-nbclient==0.5.10
+nbclient==0.5.11
     # via
     #   nbconvert
     #   papermill
-nbconvert==6.4.1
+nbconvert==6.4.2
     # via flytekitplugins-papermill
 nbformat==5.1.3
     # via
@@ -154,13 +151,13 @@ nest-asyncio==1.5.4
     #   ipykernel
     #   jupyter-client
     #   nbclient
-numpy==1.22.1
+numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
 packaging==21.3
     # via bleach
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 pandocfilters==1.5.0
     # via nbconvert
@@ -168,22 +165,22 @@ papermill==2.3.4
     # via flytekitplugins-papermill
 parso==0.8.3
     # via jedi
-pathspec==0.9.0
-    # via black
 pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
-platformdirs==2.4.1
-    # via black
 poyo==0.5.0
     # via cookiecutter
-prompt-toolkit==3.0.26
+prompt-toolkit==3.0.28
     # via ipython
 protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
@@ -203,7 +200,7 @@ pyparsing==3.0.7
     # via packaging
 pyrsistent==0.18.1
     # via jsonschema
-python-dateutil==2.8.1
+python-dateutil==2.8.2
     # via
     #   arrow
     #   croniter
@@ -212,7 +209,7 @@ python-dateutil==2.8.1
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -232,7 +229,7 @@ requests==2.27.1
     #   flytekit
     #   papermill
     #   responses
-responses==0.17.0
+responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
@@ -243,31 +240,27 @@ six==1.16.0
     #   asttokens
     #   bleach
     #   cookiecutter
-    #   flytekit
     #   grpcio
     #   python-dateutil
-    #   responses
 sortedcontainers==2.4.0
     # via flytekit
-stack-data==0.1.4
+stack-data==0.2.0
     # via ipython
 statsd==3.3.0
     # via flytekit
 tenacity==8.0.1
     # via papermill
-testpath==0.5.0
+testpath==0.6.0
     # via nbconvert
 text-unidecode==1.3
     # via python-slugify
 textwrap3==0.9.2
     # via ansiwrap
-tomli==1.2.3
-    # via black
 tornado==6.1
     # via
     #   ipykernel
     #   jupyter-client
-tqdm==4.62.3
+tqdm==4.63.0
     # via papermill
 traitlets==5.1.1
     # via
@@ -279,9 +272,9 @@ traitlets==5.1.1
     #   nbclient
     #   nbconvert
     #   nbformat
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
-    #   black
+    #   flytekit
     #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json
diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt
index 53951100b5..4589cef6ef 100644
--- a/plugins/flytekit-snowflake/requirements.txt
+++ b/plugins/flytekit-snowflake/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +44,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-snowflake
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -60,7 +68,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -79,7 +87,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -87,6 +95,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -101,7 +113,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -120,6 +132,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -131,7 +145,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt
index 75d1e32d53..3b96470c9a 100644
--- a/plugins/flytekit-spark/requirements.txt
+++ b/plugins/flytekit-spark/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,16 +44,22 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-spark
-grpcio==1.43.0
+googleapis-common-protos==1.55.0
+    # via flyteidl
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -60,7 +68,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -79,7 +87,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -87,6 +95,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 py4j==0.10.9.3
@@ -105,7 +117,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -124,6 +136,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -135,7 +149,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect
diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt
index 93dcc2790c..5d0142168c 100644
--- a/plugins/flytekit-sqlalchemy/requirements.txt
+++ b/plugins/flytekit-sqlalchemy/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.9
+# This file is autogenerated by pip-compile with python 3.10
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -12,13 +12,15 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.11
+charset-normalizer==2.0.12
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==7.1.2
+click==8.0.4
     # via
     #   cookiecutter
     #   flytekit
@@ -26,7 +28,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.2.0
+croniter==1.3.4
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -42,18 +44,24 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
+flyteidl==0.23.0
     # via flytekit
-flytekit==0.30.0
+flytekit==0.30.3
     # via flytekitplugins-sqlalchemy
+googleapis-common-protos==1.55.0
+    # via flyteidl
 greenlet==1.1.2
     # via sqlalchemy
-grpcio==1.43.0
+grpcio==1.44.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.10.1
+importlib-metadata==4.11.2
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -62,7 +70,7 @@ jinja2-time==0.2.0
     # via cookiecutter
 keyring==23.5.0
     # via flytekit
-markupsafe==2.0.1
+markupsafe==2.1.0
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -81,7 +89,7 @@ numpy==1.22.2
     # via
     #   pandas
     #   pyarrow
-pandas==1.4.0
+pandas==1.4.1
     # via flytekit
 poyo==0.5.0
     # via cookiecutter
@@ -89,6 +97,10 @@ protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 pyarrow==6.0.1
@@ -103,7 +115,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==5.0.2
+python-slugify==6.1.1
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -122,6 +134,8 @@ responses==0.18.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter
@@ -135,7 +149,7 @@ statsd==3.3.0
     # via flytekit
 text-unidecode==1.3
     # via python-slugify
-typing-extensions==4.0.1
+typing-extensions==4.1.1
     # via
     #   flytekit
     #   typing-inspect

From fa4e3b254e3e6cd31bc4b0c65cb952c9e98389a8 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Tue, 1 Mar 2022 18:18:37 -0800
Subject: [PATCH 32/35] Regenerate plugins requirements again

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 plugins/flytekit-aws-athena/requirements.txt        | 2 +-
 plugins/flytekit-aws-batch/requirements.txt         | 2 +-
 plugins/flytekit-aws-sagemaker/requirements.txt     | 2 +-
 plugins/flytekit-bigquery/requirements.txt          | 2 +-
 plugins/flytekit-data-fsspec/requirements.txt       | 2 +-
 plugins/flytekit-dolt/requirements.txt              | 2 +-
 plugins/flytekit-greatexpectations/requirements.txt | 2 +-
 plugins/flytekit-hive/requirements.txt              | 2 +-
 plugins/flytekit-k8s-pod/requirements.txt           | 2 +-
 plugins/flytekit-kf-mpi/requirements.txt            | 2 +-
 plugins/flytekit-kf-pytorch/requirements.txt        | 2 +-
 plugins/flytekit-kf-tensorflow/requirements.txt     | 2 +-
 plugins/flytekit-pandera/requirements.txt           | 2 +-
 plugins/flytekit-papermill/requirements.txt         | 4 ++--
 plugins/flytekit-snowflake/requirements.txt         | 2 +-
 plugins/flytekit-spark/requirements.txt             | 2 +-
 plugins/flytekit-sqlalchemy/requirements.txt        | 2 +-
 17 files changed, 18 insertions(+), 18 deletions(-)

diff --git a/plugins/flytekit-aws-athena/requirements.txt b/plugins/flytekit-aws-athena/requirements.txt
index 6a2fc1f2d7..bb210d8c13 100644
--- a/plugins/flytekit-aws-athena/requirements.txt
+++ b/plugins/flytekit-aws-athena/requirements.txt
@@ -121,7 +121,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-aws-batch/requirements.txt b/plugins/flytekit-aws-batch/requirements.txt
index ee76299714..80a3fcf51f 100644
--- a/plugins/flytekit-aws-batch/requirements.txt
+++ b/plugins/flytekit-aws-batch/requirements.txt
@@ -121,7 +121,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-aws-sagemaker/requirements.txt b/plugins/flytekit-aws-sagemaker/requirements.txt
index acb982c9a2..f4b02da229 100644
--- a/plugins/flytekit-aws-sagemaker/requirements.txt
+++ b/plugins/flytekit-aws-sagemaker/requirements.txt
@@ -154,7 +154,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-bigquery/requirements.txt b/plugins/flytekit-bigquery/requirements.txt
index b908f23e9e..aef0a38759 100644
--- a/plugins/flytekit-bigquery/requirements.txt
+++ b/plugins/flytekit-bigquery/requirements.txt
@@ -165,7 +165,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-data-fsspec/requirements.txt b/plugins/flytekit-data-fsspec/requirements.txt
index e728ce8676..b3f506d572 100644
--- a/plugins/flytekit-data-fsspec/requirements.txt
+++ b/plugins/flytekit-data-fsspec/requirements.txt
@@ -128,7 +128,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-dolt/requirements.txt b/plugins/flytekit-dolt/requirements.txt
index 940643fdf9..fc9dc9f8b7 100644
--- a/plugins/flytekit-dolt/requirements.txt
+++ b/plugins/flytekit-dolt/requirements.txt
@@ -129,7 +129,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-greatexpectations/requirements.txt b/plugins/flytekit-greatexpectations/requirements.txt
index 7acb822eec..31b3be4b48 100644
--- a/plugins/flytekit-greatexpectations/requirements.txt
+++ b/plugins/flytekit-greatexpectations/requirements.txt
@@ -274,7 +274,7 @@ pyzmq==22.3.0
     # via
     #   jupyter-client
     #   notebook
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-hive/requirements.txt b/plugins/flytekit-hive/requirements.txt
index a7cf6dcbb2..69ed1e5f3e 100644
--- a/plugins/flytekit-hive/requirements.txt
+++ b/plugins/flytekit-hive/requirements.txt
@@ -121,7 +121,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-k8s-pod/requirements.txt b/plugins/flytekit-k8s-pod/requirements.txt
index 7e7eea1284..df398913af 100644
--- a/plugins/flytekit-k8s-pod/requirements.txt
+++ b/plugins/flytekit-k8s-pod/requirements.txt
@@ -140,7 +140,7 @@ pytz==2021.3
     #   pandas
 pyyaml==6.0
     # via kubernetes
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-kf-mpi/requirements.txt b/plugins/flytekit-kf-mpi/requirements.txt
index 4d5c66fcea..82a2317f2f 100644
--- a/plugins/flytekit-kf-mpi/requirements.txt
+++ b/plugins/flytekit-kf-mpi/requirements.txt
@@ -123,7 +123,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-kf-pytorch/requirements.txt b/plugins/flytekit-kf-pytorch/requirements.txt
index 6f4496e3b4..0a53a62333 100644
--- a/plugins/flytekit-kf-pytorch/requirements.txt
+++ b/plugins/flytekit-kf-pytorch/requirements.txt
@@ -121,7 +121,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-kf-tensorflow/requirements.txt b/plugins/flytekit-kf-tensorflow/requirements.txt
index 2af5b7c14b..b782c39630 100644
--- a/plugins/flytekit-kf-tensorflow/requirements.txt
+++ b/plugins/flytekit-kf-tensorflow/requirements.txt
@@ -121,7 +121,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-pandera/requirements.txt b/plugins/flytekit-pandera/requirements.txt
index 62c5e3319c..7b72a0b4c0 100644
--- a/plugins/flytekit-pandera/requirements.txt
+++ b/plugins/flytekit-pandera/requirements.txt
@@ -134,7 +134,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt
index 0a0cc6e49e..4e993563dc 100644
--- a/plugins/flytekit-papermill/requirements.txt
+++ b/plugins/flytekit-papermill/requirements.txt
@@ -1,5 +1,5 @@
 #
-# This file is autogenerated by pip-compile with python 3.10
+# This file is autogenerated by pip-compile with python 3.9
 # To update, run:
 #
 #    pip-compile requirements.in
@@ -221,7 +221,7 @@ pyyaml==6.0
     # via papermill
 pyzmq==22.3.0
     # via jupyter-client
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-snowflake/requirements.txt b/plugins/flytekit-snowflake/requirements.txt
index 4589cef6ef..2d34ec966f 100644
--- a/plugins/flytekit-snowflake/requirements.txt
+++ b/plugins/flytekit-snowflake/requirements.txt
@@ -121,7 +121,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-spark/requirements.txt b/plugins/flytekit-spark/requirements.txt
index 3b96470c9a..6e7c6e59f0 100644
--- a/plugins/flytekit-spark/requirements.txt
+++ b/plugins/flytekit-spark/requirements.txt
@@ -125,7 +125,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via
diff --git a/plugins/flytekit-sqlalchemy/requirements.txt b/plugins/flytekit-sqlalchemy/requirements.txt
index 5d0142168c..e617920549 100644
--- a/plugins/flytekit-sqlalchemy/requirements.txt
+++ b/plugins/flytekit-sqlalchemy/requirements.txt
@@ -123,7 +123,7 @@ pytz==2021.3
     # via
     #   flytekit
     #   pandas
-regex==2022.1.18
+regex==2022.3.2
     # via docker-image-py
 requests==2.27.1
     # via

From 373b3bbdb9c106e0b5e2788625e9b09ca30b4d55 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Tue, 1 Mar 2022 19:49:20 -0800
Subject: [PATCH 33/35] Restore papermill/requirements.txt

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 plugins/flytekit-papermill/requirements.txt | 77 +++++++++++----------
 1 file changed, 42 insertions(+), 35 deletions(-)

diff --git a/plugins/flytekit-papermill/requirements.txt b/plugins/flytekit-papermill/requirements.txt
index 4e993563dc..8c3e4570e9 100644
--- a/plugins/flytekit-papermill/requirements.txt
+++ b/plugins/flytekit-papermill/requirements.txt
@@ -18,6 +18,8 @@ backcall==0.2.0
     # via ipython
 binaryornot==0.4.4
     # via cookiecutter
+black==21.12b0
+    # via ipython
 bleach==4.1.0
     # via nbconvert
 certifi==2021.10.8
@@ -26,12 +28,13 @@ cffi==1.15.0
     # via cryptography
 chardet==4.0.0
     # via binaryornot
-charset-normalizer==2.0.12
+charset-normalizer==2.0.10
     # via requests
 checksumdir==1.2.0
     # via flytekit
-click==8.0.4
+click==7.1.2
     # via
+    #   black
     #   cookiecutter
     #   flytekit
     #   papermill
@@ -39,7 +42,7 @@ cloudpickle==2.0.0
     # via flytekit
 cookiecutter==1.7.3
     # via flytekit
-croniter==1.3.4
+croniter==1.2.0
     # via flytekit
 cryptography==36.0.1
     # via secretstorage
@@ -61,28 +64,26 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-entrypoints==0.4
+entrypoints==0.3
     # via
     #   jupyter-client
     #   nbconvert
     #   papermill
-executing==0.8.3
+executing==0.8.2
     # via stack-data
-flyteidl==0.23.0
+flyteidl==0.22.0
     # via flytekit
-flytekit==0.30.3
+flytekit==0.26.1
     # via flytekitplugins-papermill
-googleapis-common-protos==1.55.0
-    # via flyteidl
-grpcio==1.44.0
+grpcio==1.43.0
     # via flytekit
 idna==3.3
     # via requests
-importlib-metadata==4.11.2
+importlib-metadata==4.10.1
     # via keyring
-ipykernel==6.9.1
+ipykernel==6.7.0
     # via flytekitplugins-papermill
-ipython==8.1.0
+ipython==8.0.1
     # via ipykernel
 ipython-genutils==0.2.0
     # via nbformat
@@ -105,7 +106,7 @@ jupyter-client==7.1.2
     # via
     #   ipykernel
     #   nbclient
-jupyter-core==4.9.2
+jupyter-core==4.9.1
     # via
     #   jupyter-client
     #   nbconvert
@@ -114,7 +115,7 @@ jupyterlab-pygments==0.1.2
     # via nbconvert
 keyring==23.5.0
     # via flytekit
-markupsafe==2.1.0
+markupsafe==2.0.1
     # via jinja2
 marshmallow==3.14.1
     # via
@@ -132,14 +133,16 @@ matplotlib-inline==0.1.3
 mistune==0.8.4
     # via nbconvert
 mypy-extensions==0.4.3
-    # via typing-inspect
-natsort==8.1.0
+    # via
+    #   black
+    #   typing-inspect
+natsort==8.0.2
     # via flytekit
-nbclient==0.5.11
+nbclient==0.5.10
     # via
     #   nbconvert
     #   papermill
-nbconvert==6.4.2
+nbconvert==6.4.1
     # via flytekitplugins-papermill
 nbformat==5.1.3
     # via
@@ -151,13 +154,13 @@ nest-asyncio==1.5.4
     #   ipykernel
     #   jupyter-client
     #   nbclient
-numpy==1.22.2
+numpy==1.22.1
     # via
     #   pandas
     #   pyarrow
 packaging==21.3
     # via bleach
-pandas==1.4.1
+pandas==1.4.0
     # via flytekit
 pandocfilters==1.5.0
     # via nbconvert
@@ -165,22 +168,22 @@ papermill==2.3.4
     # via flytekitplugins-papermill
 parso==0.8.3
     # via jedi
+pathspec==0.9.0
+    # via black
 pexpect==4.8.0
     # via ipython
 pickleshare==0.7.5
     # via ipython
+platformdirs==2.4.1
+    # via black
 poyo==0.5.0
     # via cookiecutter
-prompt-toolkit==3.0.28
+prompt-toolkit==3.0.26
     # via ipython
 protobuf==3.19.4
     # via
     #   flyteidl
     #   flytekit
-    #   googleapis-common-protos
-    #   protoc-gen-swagger
-protoc-gen-swagger==0.1.0
-    # via flyteidl
 ptyprocess==0.7.0
     # via pexpect
 pure-eval==0.2.2
@@ -200,7 +203,7 @@ pyparsing==3.0.7
     # via packaging
 pyrsistent==0.18.1
     # via jsonschema
-python-dateutil==2.8.2
+python-dateutil==2.8.1
     # via
     #   arrow
     #   croniter
@@ -209,7 +212,7 @@ python-dateutil==2.8.2
     #   pandas
 python-json-logger==2.0.2
     # via flytekit
-python-slugify==6.1.1
+python-slugify==5.0.2
     # via cookiecutter
 pytimeparse==1.1.8
     # via flytekit
@@ -221,7 +224,7 @@ pyyaml==6.0
     # via papermill
 pyzmq==22.3.0
     # via jupyter-client
-regex==2022.3.2
+regex==2022.1.18
     # via docker-image-py
 requests==2.27.1
     # via
@@ -229,7 +232,7 @@ requests==2.27.1
     #   flytekit
     #   papermill
     #   responses
-responses==0.18.0
+responses==0.17.0
     # via flytekit
 retry==0.9.2
     # via flytekit
@@ -240,27 +243,31 @@ six==1.16.0
     #   asttokens
     #   bleach
     #   cookiecutter
+    #   flytekit
     #   grpcio
     #   python-dateutil
+    #   responses
 sortedcontainers==2.4.0
     # via flytekit
-stack-data==0.2.0
+stack-data==0.1.4
     # via ipython
 statsd==3.3.0
     # via flytekit
 tenacity==8.0.1
     # via papermill
-testpath==0.6.0
+testpath==0.5.0
     # via nbconvert
 text-unidecode==1.3
     # via python-slugify
 textwrap3==0.9.2
     # via ansiwrap
+tomli==1.2.3
+    # via black
 tornado==6.1
     # via
     #   ipykernel
     #   jupyter-client
-tqdm==4.63.0
+tqdm==4.62.3
     # via papermill
 traitlets==5.1.1
     # via
@@ -272,9 +279,9 @@ traitlets==5.1.1
     #   nbclient
     #   nbconvert
     #   nbformat
-typing-extensions==4.1.1
+typing-extensions==4.0.1
     # via
-    #   flytekit
+    #   black
     #   typing-inspect
 typing-inspect==0.7.1
     # via dataclasses-json

From 403830d4e1169fc9fd57cc18a7d586d49929ddd9 Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 2 Mar 2022 11:45:43 -0800
Subject: [PATCH 34/35] Point flytekitplugins-spark to the
 offloaded-objects-caching branch in papermill tests

Signed-off-by: Eduardo Apolinario <eapolinario@users.noreply.github.com>
---
 plugins/flytekit-papermill/dev-requirements.in  |  2 +-
 plugins/flytekit-papermill/dev-requirements.txt | 14 +++++++++++++-
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in
index 32404e0a2d..fe323f0279 100644
--- a/plugins/flytekit-papermill/dev-requirements.in
+++ b/plugins/flytekit-papermill/dev-requirements.in
@@ -1,2 +1,2 @@
-git+https://github.com/flyteorg/flytekit@add-sd-make-class-methods#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark
+git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark
 # vcs+protocol://repo_url/#egg=pkg&subdirectory=flyte
diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt
index ef30545338..bec6aec335 100644
--- a/plugins/flytekit-papermill/dev-requirements.txt
+++ b/plugins/flytekit-papermill/dev-requirements.txt
@@ -10,6 +10,8 @@ binaryornot==0.4.4
     # via cookiecutter
 certifi==2021.10.8
     # via requests
+cffi==1.15.0
+    # via cryptography
 chardet==4.0.0
     # via binaryornot
 charset-normalizer==2.0.10
@@ -26,6 +28,8 @@ cookiecutter==1.7.3
     # via flytekit
 croniter==1.2.0
     # via flytekit
+cryptography==36.0.1
+    # via secretstorage
 dataclasses-json==0.5.6
     # via flytekit
 decorator==5.1.1
@@ -42,7 +46,7 @@ flyteidl==0.22.0
     # via flytekit
 flytekit==0.30.0
     # via flytekitplugins-spark
-flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@add-sd-make-class-methods#subdirectory=plugins/flytekit-spark
+flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#subdirectory=plugins/flytekit-spark
     # via -r dev-requirements.in
 grpcio==1.43.0
     # via flytekit
@@ -50,6 +54,10 @@ idna==3.3
     # via requests
 importlib-metadata==4.10.1
     # via keyring
+jeepney==0.7.1
+    # via
+    #   keyring
+    #   secretstorage
 jinja2==3.0.3
     # via
     #   cookiecutter
@@ -91,6 +99,8 @@ py4j==0.10.9.3
     # via pyspark
 pyarrow==6.0.1
     # via flytekit
+pycparser==2.21
+    # via cffi
 pyspark==3.2.1
     # via flytekitplugins-spark
 python-dateutil==2.8.1
@@ -120,6 +130,8 @@ responses==0.17.0
     # via flytekit
 retry==0.9.2
     # via flytekit
+secretstorage==3.3.1
+    # via keyring
 six==1.16.0
     # via
     #   cookiecutter

From 9754a0f462f495dbd77df0572719885230bd909b Mon Sep 17 00:00:00 2001
From: Eduardo Apolinario <eapolinario@users.noreply.github.com>
Date: Wed, 2 Mar 2022 11:53:15 -0800
Subject: [PATCH 35/35] Set flyteidl>=0.23.0 in papermill dev-requirements

---
 plugins/flytekit-papermill/dev-requirements.in  |  1 +
 plugins/flytekit-papermill/dev-requirements.txt | 12 ++++++++++--
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/plugins/flytekit-papermill/dev-requirements.in b/plugins/flytekit-papermill/dev-requirements.in
index fe323f0279..7a47470868 100644
--- a/plugins/flytekit-papermill/dev-requirements.in
+++ b/plugins/flytekit-papermill/dev-requirements.in
@@ -1,2 +1,3 @@
+flyteidl>=0.23.0
 git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#egg=flytekitplugins-spark&subdirectory=plugins/flytekit-spark
 # vcs+protocol://repo_url/#egg=pkg&subdirectory=flyte
diff --git a/plugins/flytekit-papermill/dev-requirements.txt b/plugins/flytekit-papermill/dev-requirements.txt
index bec6aec335..f777991cc3 100644
--- a/plugins/flytekit-papermill/dev-requirements.txt
+++ b/plugins/flytekit-papermill/dev-requirements.txt
@@ -42,12 +42,16 @@ docker-image-py==0.1.12
     # via flytekit
 docstring-parser==0.13
     # via flytekit
-flyteidl==0.22.0
-    # via flytekit
+flyteidl==0.23.0
+    # via
+    #   -r dev-requirements.in
+    #   flytekit
 flytekit==0.30.0
     # via flytekitplugins-spark
 flytekitplugins-spark @ git+https://github.com/flyteorg/flytekit@offloaded-objects-caching#subdirectory=plugins/flytekit-spark
     # via -r dev-requirements.in
+googleapis-common-protos==1.55.0
+    # via flyteidl
 grpcio==1.43.0
     # via flytekit
 idna==3.3
@@ -93,6 +97,10 @@ protobuf==3.19.3
     # via
     #   flyteidl
     #   flytekit
+    #   googleapis-common-protos
+    #   protoc-gen-swagger
+protoc-gen-swagger==0.1.0
+    # via flyteidl
 py==1.11.0
     # via retry
 py4j==0.10.9.3
