From f254e22b2337d885f549f33254d40878ec22b46b Mon Sep 17 00:00:00 2001 From: David Pollock <20267387+dave-pollock@users.noreply.github.com> Date: Tue, 24 Oct 2023 16:34:17 +1100 Subject: [PATCH 1/4] Fix SalesforceHook compatiblity with Pandas 2.x --- airflow/providers/salesforce/hooks/salesforce.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/airflow/providers/salesforce/hooks/salesforce.py b/airflow/providers/salesforce/hooks/salesforce.py index bca3521226553..af62ab15aceaf 100644 --- a/airflow/providers/salesforce/hooks/salesforce.py +++ b/airflow/providers/salesforce/hooks/salesforce.py @@ -243,6 +243,7 @@ def _to_timestamp(cls, column: pd.Series) -> pd.Series: # if the column cannot be converted, # just return the original column untouched import pandas as pd + import numpy as np try: column = pd.to_datetime(column) @@ -259,7 +260,7 @@ def _to_timestamp(cls, column: pd.Series) -> pd.Series: try: converted.append(value.timestamp()) except (ValueError, AttributeError): - converted.append(pd.np.NaN) + converted.append(np.NaN) return pd.Series(converted, index=column.index) From a52fd9116122c7c2a0880943cc960549115e4699 Mon Sep 17 00:00:00 2001 From: David Pollock <20267387+dave-pollock@users.noreply.github.com> Date: Tue, 24 Oct 2023 19:32:03 +1100 Subject: [PATCH 2/4] Change order of imports --- airflow/providers/salesforce/hooks/salesforce.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/salesforce/hooks/salesforce.py b/airflow/providers/salesforce/hooks/salesforce.py index af62ab15aceaf..5bdbb6eaa815f 100644 --- a/airflow/providers/salesforce/hooks/salesforce.py +++ b/airflow/providers/salesforce/hooks/salesforce.py @@ -242,8 +242,8 @@ def _to_timestamp(cls, column: pd.Series) -> pd.Series: # between 0 and 10 are turned into timestamps # if the column cannot be converted, # just return the original column untouched - import pandas as pd import numpy as np + import pandas as pd try: column = pd.to_datetime(column) From 7f7934f131364fbbd0a6b62fc116c265b824ca08 Mon Sep 17 00:00:00 2001 From: David Pollock <20267387+dave-pollock@users.noreply.github.com> Date: Wed, 25 Oct 2023 17:17:24 +1100 Subject: [PATCH 3/4] Add test case --- tests/providers/salesforce/hooks/test_salesforce.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/providers/salesforce/hooks/test_salesforce.py b/tests/providers/salesforce/hooks/test_salesforce.py index aa3b03a38df22..2288cece55015 100644 --- a/tests/providers/salesforce/hooks/test_salesforce.py +++ b/tests/providers/salesforce/hooks/test_salesforce.py @@ -417,7 +417,7 @@ def test_write_object_to_file_ndjson_with_record_time(self, mock_data_frame, moc ) @patch( "pandas.DataFrame.from_records", - return_value=pd.DataFrame({"test": [1, 2, 3], "field_1": ["2019-01-01", "2019-01-02", "2019-01-03"]}), + return_value=pd.DataFrame({"test": [1, 2, 3, 4], "field_1": ["2019-01-01", "2019-01-02", "2019-01-03", 'NaT']}), ) def test_object_to_df_with_timestamp_conversion(self, mock_data_frame, mock_describe_object): obj_name = "obj_name" @@ -429,7 +429,7 @@ def test_object_to_df_with_timestamp_conversion(self, mock_data_frame, mock_desc mock_describe_object.assert_called_once_with(obj_name) pd.testing.assert_frame_equal( - data_frame, pd.DataFrame({"test": [1, 2, 3], "field_1": [1.546301e09, 1.546387e09, 1.546474e09]}) + data_frame, pd.DataFrame({"test": [1, 2, 3, 4], "field_1": [1.546301e09, 1.546387e09, 1.546474e09, np.nan]}) ) @patch("airflow.providers.salesforce.hooks.salesforce.time.time", return_value=1.23) From 8ff2ca52c3c9df0746ff328ab2e68bc9de17d717 Mon Sep 17 00:00:00 2001 From: David Pollock <20267387+dave-pollock@users.noreply.github.com> Date: Thu, 26 Oct 2023 09:59:30 +1100 Subject: [PATCH 4/4] Update test to match suggested --- tests/providers/salesforce/hooks/test_salesforce.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/providers/salesforce/hooks/test_salesforce.py b/tests/providers/salesforce/hooks/test_salesforce.py index 2288cece55015..82d13b9ca6d84 100644 --- a/tests/providers/salesforce/hooks/test_salesforce.py +++ b/tests/providers/salesforce/hooks/test_salesforce.py @@ -417,7 +417,9 @@ def test_write_object_to_file_ndjson_with_record_time(self, mock_data_frame, moc ) @patch( "pandas.DataFrame.from_records", - return_value=pd.DataFrame({"test": [1, 2, 3, 4], "field_1": ["2019-01-01", "2019-01-02", "2019-01-03", 'NaT']}), + return_value=pd.DataFrame( + {"test": [1, 2, 3, 4], "field_1": ["2019-01-01", "2019-01-02", "2019-01-03", "NaT"]} + ), ) def test_object_to_df_with_timestamp_conversion(self, mock_data_frame, mock_describe_object): obj_name = "obj_name" @@ -429,7 +431,8 @@ def test_object_to_df_with_timestamp_conversion(self, mock_data_frame, mock_desc mock_describe_object.assert_called_once_with(obj_name) pd.testing.assert_frame_equal( - data_frame, pd.DataFrame({"test": [1, 2, 3, 4], "field_1": [1.546301e09, 1.546387e09, 1.546474e09, np.nan]}) + data_frame, + pd.DataFrame({"test": [1, 2, 3, 4], "field_1": [1.546301e09, 1.546387e09, 1.546474e09, np.nan]}), ) @patch("airflow.providers.salesforce.hooks.salesforce.time.time", return_value=1.23)