From 1685e55e1420b52816734d19045be685fbdc8e0c Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 19:26:09 -0300 Subject: [PATCH 01/79] Transfer to upload sftp files to Azure Blob Storage sftp_transfer provides mechanism to extract file from a sftp path or files using a wildcard. --- .../microsoft/azure/transfers/stfp_to_wasb.py | 157 +++++++++++ .../azure/transfers/test_sftp_to_wasb.py | 243 ++++++++++++++++++ 2 files changed, 400 insertions(+) create mode 100644 airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py create mode 100644 tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py new file mode 100644 index 0000000000000..75031dbb1ebfd --- /dev/null +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -0,0 +1,157 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +This module contains SFTP to Azure Blob Storage operator. +""" +import os +from tempfile import NamedTemporaryFile +from typing import Optional, Union + +from airflow.exceptions import AirflowException +from airflow.models import BaseOperator +from airflow.providers.microsoft.azure.hooks.wasb import WasbHook +from airflow.providers.sftp.hooks.sftp import SFTPHook +from airflow.utils.decorators import apply_defaults + +WILDCARD = "*" +SFTP_FILE_PATH = "SFTP_FILE_PATH" +BLOB_NAME = "BLOB_NAME" + +class STFPToWasbOperator(BaseOperator): + """ + Transfer files to Azure Blob Storage from SFTP server. + + :param sftp_source_path: The sftp remote path. This is the specified file path + for downloading the single file or multiple files from the SFTP server. + You can use only one wildcard within your path. The wildcard can appear + inside the path or at the end of the path. + :type sftp_source_path: str + :param container_name: Name of the container. + :type container_name: str + :param blob_name: Name of the blob. + :type blob_name: str + :param sftp_conn_id: The sftp connection id. The name or identifier for + establishing a connection to the SFTP server. + :type sftp_conn_id: str + :param wasb_conn_id: Reference to the wasb connection. + :type wasb_conn_id: str + :param load_options: Optional keyword arguments that + `WasbHook.load_file()` takes. + :type load_options: dict + :param move_object: When move object is True, the object is moved instead + of copied to the new location. This is the equivalent of a mv command + as opposed to a cp command. + :type move_object: bool + """ + template_fields = ("sftp_source_path", "container_name", "blob_name") + + @apply_defaults + def __init__( + self, + sftp_source_path: str, + container_name: str, + blob_name: str, + sftp_conn_id: str = "ssh_default", + wasb_conn_id: str = 'wasb_default', + load_options: dict = None, + move_object: bool = False, + *args, + **kwargs + ) -> None: + super().__init__(*args, **kwargs) + + self.sftp_source_path = sftp_source_path + self.sftp_conn_id = sftp_conn_id + self.wasb_conn_id = wasb_conn_id + self.container_name = container_name + self.blob_name = blob_name + self.wasb_conn_id = wasb_conn_id + self.load_options = load_options if load_options is not None else {} + self.move_object = move_object + + def execute(self, context): + (sftp_hook, sftp_files) = self.get_sftp_files_map() + uploaded_files = self.upload_wasb(sftp_hook, sftp_files) + self.sftp_move(sftp_hook, uploaded_files) + + def get_sftp_files_map(self): + sftp_files = [] + sftp_hook = SFTPHook(self.sftp_conn_id) + + if WILDCARD in self.sftp_source_path: + self.check_wildcards_limit() + + prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) + sftp_complete_path = os.path.dirname(prefix) + + found_files, _, _ = sftp_hook.get_tree_map( + sftp_complete_path, prefix=prefix, delimiter=delimiter + ) + + self.log.info( + "Found %s files at sftp source path: %s", + str(len(found_files)), + self.sftp_source_path + ) + + for file in found_files: + future_blob_name = os.path.basename(file) + sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) + + else: + sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) + + return sftp_hook, sftp_files + + def check_wildcards_limit(self): + total_wildcards = self.sftp_source_path.count(WILDCARD) + if total_wildcards > 1: + raise AirflowException( + "Only one wildcard '*' is allowed in sftp_source_path parameter. " + "Found {} in {}.".format(total_wildcards, self.sftp_source_path) + ) + + def upload_wasb(self, sftp_hook, sftp_files): + uploaded_files = [] + wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) + for file in sftp_files: + + with NamedTemporaryFile("w") as tmp: + sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) + self.log.info( + 'Uploading %s to wasb://%s as %s', + file[SFTP_FILE_PATH], + self.container_name, + file[BLOB_NAME], + ) + wasb_hook.load_file(tmp.name, + self.container_name, + file[BLOB_NAME], + **self.load_options) + + uploaded_files.append(file[SFTP_FILE_PATH]) + + return uploaded_files + + def sftp_move(self, sftp_hook, uploaded_files): + if self.move_object: + for sftp_file_path in uploaded_files: + self.log.info("Executing delete of %s", sftp_file_path) + sftp_hook.delete_file(sftp_file_path) + + diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py new file mode 100644 index 0000000000000..17c2763b68228 --- /dev/null +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -0,0 +1,243 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +import datetime +import unittest +import os + +import mock + +from airflow import AirflowException +from airflow.models.dag import DAG +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME + + +TASK_ID = "test-gcs-to-sftp-operator" +WASB_CONN_ID = "wasb_default" +SFTP_CONN_ID = "ssh_default" + +CONTAINER_NAME = "test-container" +WILDCARD_PATH = "main_dir/*" +WILDCARD_FILE_NAME = "main_dir/test_object*.json" +SOURCE_OBJECT_NO_WILDCARD = "main_dir/test_object3.json" +SOURCE_OBJECT_MULTIPLE_WILDCARDS = "main_dir/csv/*/test_*.csv" +NEW_BLOB_NAME = "sponge-bob" +EXPECTED_BLOB_NAME = "test_object3.json" + + +# pylint: disable=unused-argument +class TestSFTPToWasbOperator(unittest.TestCase): + + + def test_init(self): + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=NEW_BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False + ) + self.assertEqual(operator.sftp_source_path, SOURCE_OBJECT_NO_WILDCARD) + self.assertEqual(operator.sftp_conn_id, SFTP_CONN_ID) + self.assertEqual(operator.container_name, CONTAINER_NAME) + self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) + + + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', + autospec=True) + def test_execute_more_than_one_wildcard_exception(self, mock_hook): + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False + ) + with self.assertRaises(AirflowException) as cm: + operator.check_wildcards_limit() + + err = cm.exception + self.assertIn("Only one wildcard '*' is allowed", str(err)) + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_get_sftp_files_map(self, sftp_hook, mock_hook): + sftp_hook.return_value.get_tree_map.return_value = [ + [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], + [], + [], + ] + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_PATH, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False + ) + _, files = operator.get_sftp_files_map() + + sftp_hook.assert_called_once_with(SFTP_CONN_ID) + + self.assertEquals(len(files), 2, "not matched at expected found files") + self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") + + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_get_sftp_files_map(self, sftp_hook, mock_hook): + sftp_hook.return_value.get_tree_map.return_value = [ + [SOURCE_OBJECT_NO_WILDCARD], + [], + [], + ] + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=True + ) + _, files = operator.get_sftp_files_map() + + sftp_hook.assert_called_once_with(SFTP_CONN_ID) + + self.assertEquals(len(files), 1, "no matched at expected found files") + self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_upload_wasb(self, sftp_hook, mock_hook): + + + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=True + ) + + sftp_files = [{SFTP_FILE_PATH: SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME: BLOB_NAME}] + files = operator.upload_wasb(sftp_hook, sftp_files) + + sftp_hook.retrieve_file.assert_has_calls( + [ + mock.call("main_dir/test_object3.json", mock.ANY) + ] + ) + + mock_hook.return_value.load_file.assert_called_once_with( + mock.ANY, CONTAINER_NAME, BLOB_NAME + ) + + self.assertEquals(len(files), 1, "no matched at expected uploaded files") + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_sftp_move(self, sftp_hook): + sftp_mock = sftp_hook.return_value + + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=True + ) + + sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] + operator.sftp_move(sftp_mock, sftp_file_paths) + + sftp_mock.delete_file.assert_has_calls( + [ + mock.call(SOURCE_OBJECT_NO_WILDCARD) + ] + ) + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_no_sftp_move(self, sftp_hook): + sftp_mock = sftp_hook.return_value + + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False + ) + + sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] + operator.sftp_move(sftp_mock, sftp_file_paths) + + sftp_mock.delete_file.assert_not_called() + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_execute(self, sftp_hook, mock_hook): + sftp_hook.return_value.get_tree_map.return_value = [ + [SOURCE_OBJECT_NO_WILDCARD], + [], + [], + ] + + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_FILE_NAME, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False + ) + + operator.execute(None) + + sftp_hook.return_value.get_tree_map.assert_called_with( + "main_dir", prefix="main_dir/test_object", delimiter=".json" + ) + + sftp_hook.return_value.retrieve_file.assert_has_calls( + [ + mock.call(SOURCE_OBJECT_NO_WILDCARD, mock.ANY) + ] + ) + + mock_hook.return_value.load_file.assert_called_once_with( + mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) + ) + + sftp_hook.delete_file.assert_not_called() + From c34d8a23afe007334266f2d38ad37fafde610de7 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:01:46 -0300 Subject: [PATCH 02/79] Fix tests names and adds sftp_to_wasb to operators-and-hooks-ref list Changes test_get_sftp_files_map to uses a proper name --- .../microsoft/azure/transfers/test_sftp_to_wasb.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 17c2763b68228..161f20a6e0882 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -47,7 +47,6 @@ # pylint: disable=unused-argument class TestSFTPToWasbOperator(unittest.TestCase): - def test_init(self): operator = STFPToWasbOperator( task_id=TASK_ID, @@ -63,8 +62,6 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): @@ -85,7 +82,7 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') - def test_get_sftp_files_map(self, sftp_hook, mock_hook): + def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], @@ -107,10 +104,9 @@ def test_get_sftp_files_map(self, sftp_hook, mock_hook): self.assertEquals(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') - def test_get_sftp_files_map(self, sftp_hook, mock_hook): + def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], From 69edc59abea6e4b825336c23f4900197252ec8da Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:28:33 -0300 Subject: [PATCH 03/79] Removes usage of deprecated method and unused imports at tests from sftp_to_wasb --- .../microsoft/azure/transfers/test_sftp_to_wasb.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 161f20a6e0882..8e05d4e8d2bea 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -18,14 +18,12 @@ # -import datetime import unittest import os import mock from airflow import AirflowException -from airflow.models.dag import DAG from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME @@ -101,7 +99,7 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEquals(len(files), 2, "not matched at expected found files") + self.assertEqual(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @@ -125,7 +123,7 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEquals(len(files), 1, "no matched at expected found files") + self.assertEqual(len(files), 1, "no matched at expected found files") self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @@ -156,7 +154,7 @@ def test_upload_wasb(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, BLOB_NAME ) - self.assertEquals(len(files), 1, "no matched at expected uploaded files") + self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_sftp_move(self, sftp_hook): From 9b788ce4f391d899e53566b5b1e8b8d1d6ec295c Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:43:13 -0300 Subject: [PATCH 04/79] Removes trailing newlines from both files Files with training lines: test_sftp_to_wasb and sftp_to_wasb --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 -- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 1 - 2 files changed, 3 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 75031dbb1ebfd..14eefba43c6ad 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -153,5 +153,3 @@ def sftp_move(self, sftp_hook, uploaded_files): for sftp_file_path in uploaded_files: self.log.info("Executing delete of %s", sftp_file_path) sftp_hook.delete_file(sftp_file_path) - - diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 8e05d4e8d2bea..ecbd5b480a1b4 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -234,4 +234,3 @@ def test_execute(self, sftp_hook, mock_hook): ) sftp_hook.delete_file.assert_not_called() - From 279461b9b0ad737ad3a93ff025fac8074647177e Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 22:57:31 -0300 Subject: [PATCH 05/79] Apply None as the default without type to load_options at sft_to_wasb --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 14eefba43c6ad..15f938ae92ffb 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -68,7 +68,7 @@ def __init__( blob_name: str, sftp_conn_id: str = "ssh_default", wasb_conn_id: str = 'wasb_default', - load_options: dict = None, + load_options=None, move_object: bool = False, *args, **kwargs From e391416a74da14f10541adfea110e3a08eddcd09 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 23:14:01 -0300 Subject: [PATCH 06/79] Removes unused import from sftp_to_wasb and excessive trailing lines --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 +- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 15f938ae92ffb..2ac8a7cc24cf8 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -20,7 +20,6 @@ """ import os from tempfile import NamedTemporaryFile -from typing import Optional, Union from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -32,6 +31,7 @@ SFTP_FILE_PATH = "SFTP_FILE_PATH" BLOB_NAME = "BLOB_NAME" + class STFPToWasbOperator(BaseOperator): """ Transfer files to Azure Blob Storage from SFTP server. diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index ecbd5b480a1b4..25079ee21a020 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -129,8 +129,6 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_upload_wasb(self, sftp_hook, mock_hook): - - operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, From d45663237b03d80b1f3e818f1311ec878599d831 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:00:41 -0300 Subject: [PATCH 07/79] Fix typo to sftp_to_wasb file name Adds the information from extra package to CONTRIBUTING.rst --- .../{stfp_to_wasb.py => sftp_to_wasb.py} | 7 ++- .../azure/transfers/test_sftp_to_wasb.py | 48 +++++++++---------- 2 files changed, 29 insertions(+), 26 deletions(-) rename airflow/providers/microsoft/azure/transfers/{stfp_to_wasb.py => sftp_to_wasb.py} (93%) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py similarity index 93% rename from airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py rename to airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 2ac8a7cc24cf8..b03bbd623fd42 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -32,7 +32,7 @@ BLOB_NAME = "BLOB_NAME" -class STFPToWasbOperator(BaseOperator): +class SFTPToWasbOperator(BaseOperator): """ Transfer files to Azure Blob Storage from SFTP server. @@ -85,11 +85,13 @@ def __init__( self.move_object = move_object def execute(self, context): + """Upload a file from SFTP to Azure Blob Storage.""" (sftp_hook, sftp_files) = self.get_sftp_files_map() uploaded_files = self.upload_wasb(sftp_hook, sftp_files) self.sftp_move(sftp_hook, uploaded_files) def get_sftp_files_map(self): + """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] sftp_hook = SFTPHook(self.sftp_conn_id) @@ -119,6 +121,7 @@ def get_sftp_files_map(self): return sftp_hook, sftp_files def check_wildcards_limit(self): + """Check if there is multiple Wildcard.""" total_wildcards = self.sftp_source_path.count(WILDCARD) if total_wildcards > 1: raise AirflowException( @@ -127,6 +130,7 @@ def check_wildcards_limit(self): ) def upload_wasb(self, sftp_hook, sftp_files): + """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) for file in sftp_files: @@ -149,6 +153,7 @@ def upload_wasb(self, sftp_hook, sftp_files): return uploaded_files def sftp_move(self, sftp_hook, uploaded_files): + """Performs a move of a list of files at SFTP to Azure Blob Storage.""" if self.move_object: for sftp_file_path in uploaded_files: self.log.info("Executing delete of %s", sftp_file_path) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 25079ee21a020..34b701ed9acd5 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -17,17 +17,15 @@ # under the License. # - -import unittest import os +import unittest import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME - +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator, \ + SFTP_FILE_PATH, \ + BLOB_NAME TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" @@ -46,7 +44,7 @@ class TestSFTPToWasbOperator(unittest.TestCase): def test_init(self): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -60,10 +58,10 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, @@ -78,15 +76,15 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_PATH, sftp_conn_id=SFTP_CONN_ID, @@ -102,15 +100,15 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): self.assertEqual(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -126,10 +124,10 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected found files") self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_upload_wasb(self, sftp_hook, mock_hook): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -154,11 +152,11 @@ def test_upload_wasb(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected uploaded files") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -177,11 +175,11 @@ def test_sftp_move(self, sftp_hook): ] ) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_no_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -196,8 +194,8 @@ def test_no_sftp_move(self, sftp_hook): sftp_mock.delete_file.assert_not_called() - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], @@ -205,7 +203,7 @@ def test_execute(self, sftp_hook, mock_hook): [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, From 31c440cd41e548a2bc2fa93de0869f3a013fee64 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:30:34 -0300 Subject: [PATCH 08/79] Adds dependency to dependencies.json Fix import at test_sftp_to_wasb. --- airflow/providers/dependencies.json | 3 ++- .../providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 5 ++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/airflow/providers/dependencies.json b/airflow/providers/dependencies.json index e4f100df684aa..c932cfa1a0deb 100644 --- a/airflow/providers/dependencies.json +++ b/airflow/providers/dependencies.json @@ -57,7 +57,8 @@ ], "microsoft.azure": [ "google", - "oracle" + "oracle", + "sftp" ], "microsoft.mssql": [ "odbc" diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 34b701ed9acd5..bac52a5839da4 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -23,9 +23,8 @@ import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator, \ - SFTP_FILE_PATH, \ - BLOB_NAME +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( + BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator) TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" From 376d04ee9586e7b2fb3de112e585af5c4f481fde Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:59:49 -0300 Subject: [PATCH 09/79] Fix changes made by hooks --- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index bac52a5839da4..e38b4df07aded 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -24,7 +24,8 @@ from airflow import AirflowException from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( - BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator) + BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator, +) TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" From 267648c5a1fe0460fadb93748f6b6818bd67620c Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 12 Jul 2020 13:53:06 -0300 Subject: [PATCH 10/79] Fix type hint --- .../microsoft/azure/transfers/sftp_to_wasb.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index b03bbd623fd42..8bae74c49c0bb 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -20,6 +20,7 @@ """ import os from tempfile import NamedTemporaryFile +from typing import Any, Dict, List, Optional, Tuple from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -68,7 +69,7 @@ def __init__( blob_name: str, sftp_conn_id: str = "ssh_default", wasb_conn_id: str = 'wasb_default', - load_options=None, + load_options: Optional[dict] = None, move_object: bool = False, *args, **kwargs @@ -84,13 +85,14 @@ def __init__( self.load_options = load_options if load_options is not None else {} self.move_object = move_object - def execute(self, context): + def execute(self, + context: Optional[Dict[Any, Any]]) -> None: """Upload a file from SFTP to Azure Blob Storage.""" (sftp_hook, sftp_files) = self.get_sftp_files_map() uploaded_files = self.upload_wasb(sftp_hook, sftp_files) self.sftp_move(sftp_hook, uploaded_files) - def get_sftp_files_map(self): + def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] sftp_hook = SFTPHook(self.sftp_conn_id) @@ -120,7 +122,7 @@ def get_sftp_files_map(self): return sftp_hook, sftp_files - def check_wildcards_limit(self): + def check_wildcards_limit(self) -> Any: """Check if there is multiple Wildcard.""" total_wildcards = self.sftp_source_path.count(WILDCARD) if total_wildcards > 1: @@ -129,7 +131,9 @@ def check_wildcards_limit(self): "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) - def upload_wasb(self, sftp_hook, sftp_files): + def upload_wasb(self, + sftp_hook: SFTPHook, + sftp_files: List[Dict[str, str]]) -> List[str]: """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) @@ -152,7 +156,9 @@ def upload_wasb(self, sftp_hook, sftp_files): return uploaded_files - def sftp_move(self, sftp_hook, uploaded_files): + def sftp_move(self, + sftp_hook: SFTPHook, + uploaded_files: List[str]) -> None: """Performs a move of a list of files at SFTP to Azure Blob Storage.""" if self.move_object: for sftp_file_path in uploaded_files: From 2974feea030bff74f79d50d74ad9b8e7a9232dfb Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 9 Aug 2020 18:56:13 -0300 Subject: [PATCH 11/79] Cleaning code --- .../microsoft/azure/transfers/sftp_to_wasb.py | 60 ++++++++++--------- .../azure/transfers/test_sftp_to_wasb.py | 55 +++++------------ 2 files changed, 48 insertions(+), 67 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 8bae74c49c0bb..4f4b51140f992 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -19,8 +19,11 @@ This module contains SFTP to Azure Blob Storage operator. """ import os +from collections import namedtuple from tempfile import NamedTemporaryFile -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional + +from cached_property import cached_property from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -29,8 +32,7 @@ from airflow.utils.decorators import apply_defaults WILDCARD = "*" -SFTP_FILE_PATH = "SFTP_FILE_PATH" -BLOB_NAME = "BLOB_NAME" +SftpFile = namedtuple('SftpFile', 'sftp_file_path, blob_name') class SFTPToWasbOperator(BaseOperator): @@ -67,7 +69,7 @@ def __init__( sftp_source_path: str, container_name: str, blob_name: str, - sftp_conn_id: str = "ssh_default", + sftp_conn_id: str = "sftp_default", wasb_conn_id: str = 'wasb_default', load_options: Optional[dict] = None, move_object: bool = False, @@ -88,14 +90,14 @@ def __init__( def execute(self, context: Optional[Dict[Any, Any]]) -> None: """Upload a file from SFTP to Azure Blob Storage.""" - (sftp_hook, sftp_files) = self.get_sftp_files_map() - uploaded_files = self.upload_wasb(sftp_hook, sftp_files) - self.sftp_move(sftp_hook, uploaded_files) + sftp_files: List[SftpFile] = self.get_sftp_files_map() + uploaded_files = self.copy_files_to_wasb(sftp_files) + if self.move_object: + self.delete_files(uploaded_files) - def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: + def get_sftp_files_map(self) -> List[SftpFile]: """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] - sftp_hook = SFTPHook(self.sftp_conn_id) if WILDCARD in self.sftp_source_path: self.check_wildcards_limit() @@ -103,7 +105,7 @@ def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) sftp_complete_path = os.path.dirname(prefix) - found_files, _, _ = sftp_hook.get_tree_map( + found_files, _, _ = self.sftp_hook.get_tree_map( sftp_complete_path, prefix=prefix, delimiter=delimiter ) @@ -115,12 +117,17 @@ def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: for file in found_files: future_blob_name = os.path.basename(file) - sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) + sftp_files.append(SftpFile(file, future_blob_name)) else: - sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) + sftp_files.append(SftpFile(self.sftp_source_path, self.blob_name)) - return sftp_hook, sftp_files + return sftp_files + + @cached_property + def sftp_hook(self): + """Property of sftp hook to be re-used.""" + return SFTPHook(self.sftp_conn_id) def check_wildcards_limit(self) -> Any: """Check if there is multiple Wildcard.""" @@ -131,36 +138,33 @@ def check_wildcards_limit(self) -> Any: "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) - def upload_wasb(self, - sftp_hook: SFTPHook, - sftp_files: List[Dict[str, str]]) -> List[str]: + def copy_files_to_wasb(self, + sftp_files: List[SftpFile]) -> List[str]: """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) for file in sftp_files: with NamedTemporaryFile("w") as tmp: - sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) + self.sftp_hook.retrieve_file(file.sftp_file_path, tmp.name) self.log.info( 'Uploading %s to wasb://%s as %s', - file[SFTP_FILE_PATH], + file.sftp_file_path, self.container_name, - file[BLOB_NAME], + file.blob_name, ) wasb_hook.load_file(tmp.name, self.container_name, - file[BLOB_NAME], + file.blob_name, **self.load_options) - uploaded_files.append(file[SFTP_FILE_PATH]) + uploaded_files.append(file.sftp_file_path) return uploaded_files - def sftp_move(self, - sftp_hook: SFTPHook, - uploaded_files: List[str]) -> None: + def delete_files(self, + uploaded_files: List[str]) -> None: """Performs a move of a list of files at SFTP to Azure Blob Storage.""" - if self.move_object: - for sftp_file_path in uploaded_files: - self.log.info("Executing delete of %s", sftp_file_path) - sftp_hook.delete_file(sftp_file_path) + for sftp_file_path in uploaded_files: + self.log.info("Executing delete of %s", sftp_file_path) + self.sftp_hook.delete_file(sftp_file_path) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index e38b4df07aded..8f02c8fff60e1 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -23,14 +23,13 @@ import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( - BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator, -) +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" SFTP_CONN_ID = "ssh_default" +BLOB_NAME = "BLOB_NAME" CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" @@ -93,12 +92,10 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=False ) - _, files = operator.get_sftp_files_map() - - sftp_hook.assert_called_once_with(SFTP_CONN_ID) + files = operator.get_sftp_files_map() self.assertEqual(len(files), 2, "not matched at expected found files") - self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') @@ -117,16 +114,14 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=True ) - _, files = operator.get_sftp_files_map() - - sftp_hook.assert_called_once_with(SFTP_CONN_ID) + files = operator.get_sftp_files_map() self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_upload_wasb(self, sftp_hook, mock_hook): + def test_copy_files_to_wasb(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, @@ -137,10 +132,10 @@ def test_upload_wasb(self, sftp_hook, mock_hook): move_object=True ) - sftp_files = [{SFTP_FILE_PATH: SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME: BLOB_NAME}] - files = operator.upload_wasb(sftp_hook, sftp_files) + sftp_files = [SftpFile(SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME)] + files = operator.copy_files_to_wasb(sftp_files) - sftp_hook.retrieve_file.assert_has_calls( + operator.sftp_hook.retrieve_file.assert_has_calls( [ mock.call("main_dir/test_object3.json", mock.ANY) ] @@ -153,7 +148,7 @@ def test_upload_wasb(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_sftp_move(self, sftp_hook): + def test_delete_files(self, sftp_hook): sftp_mock = sftp_hook.return_value operator = SFTPToWasbOperator( @@ -167,7 +162,7 @@ def test_sftp_move(self, sftp_hook): ) sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.sftp_move(sftp_mock, sftp_file_paths) + operator.delete_files(sftp_file_paths) sftp_mock.delete_file.assert_has_calls( [ @@ -175,13 +170,13 @@ def test_sftp_move(self, sftp_hook): ] ) + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_no_sftp_move(self, sftp_hook): - sftp_mock = sftp_hook.return_value + def test_execute(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, blob_name=BLOB_NAME, @@ -189,30 +184,12 @@ def test_no_sftp_move(self, sftp_hook): move_object=False ) - sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.sftp_move(sftp_mock, sftp_file_paths) - - sftp_mock.delete_file.assert_not_called() - - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( - task_id=TASK_ID, - sftp_source_path=WILDCARD_FILE_NAME, - sftp_conn_id=SFTP_CONN_ID, - container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, - wasb_conn_id=WASB_CONN_ID, - move_object=False - ) - operator.execute(None) sftp_hook.return_value.get_tree_map.assert_called_with( @@ -229,4 +206,4 @@ def test_execute(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) ) - sftp_hook.delete_file.assert_not_called() + operator.sftp_hook.delete_file.assert_not_called() From 64ee7f56434e9ce6ae3e14d85b289069d41338f3 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 19:26:09 -0300 Subject: [PATCH 12/79] Transfer to upload sftp files to Azure Blob Storage sftp_transfer provides mechanism to extract file from a sftp path or files using a wildcard. --- .../microsoft/azure/transfers/stfp_to_wasb.py | 157 ++++++++++++++++++ .../azure/transfers/test_sftp_to_wasb.py | 110 +++++++----- 2 files changed, 229 insertions(+), 38 deletions(-) create mode 100644 airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py new file mode 100644 index 0000000000000..75031dbb1ebfd --- /dev/null +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -0,0 +1,157 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +This module contains SFTP to Azure Blob Storage operator. +""" +import os +from tempfile import NamedTemporaryFile +from typing import Optional, Union + +from airflow.exceptions import AirflowException +from airflow.models import BaseOperator +from airflow.providers.microsoft.azure.hooks.wasb import WasbHook +from airflow.providers.sftp.hooks.sftp import SFTPHook +from airflow.utils.decorators import apply_defaults + +WILDCARD = "*" +SFTP_FILE_PATH = "SFTP_FILE_PATH" +BLOB_NAME = "BLOB_NAME" + +class STFPToWasbOperator(BaseOperator): + """ + Transfer files to Azure Blob Storage from SFTP server. + + :param sftp_source_path: The sftp remote path. This is the specified file path + for downloading the single file or multiple files from the SFTP server. + You can use only one wildcard within your path. The wildcard can appear + inside the path or at the end of the path. + :type sftp_source_path: str + :param container_name: Name of the container. + :type container_name: str + :param blob_name: Name of the blob. + :type blob_name: str + :param sftp_conn_id: The sftp connection id. The name or identifier for + establishing a connection to the SFTP server. + :type sftp_conn_id: str + :param wasb_conn_id: Reference to the wasb connection. + :type wasb_conn_id: str + :param load_options: Optional keyword arguments that + `WasbHook.load_file()` takes. + :type load_options: dict + :param move_object: When move object is True, the object is moved instead + of copied to the new location. This is the equivalent of a mv command + as opposed to a cp command. + :type move_object: bool + """ + template_fields = ("sftp_source_path", "container_name", "blob_name") + + @apply_defaults + def __init__( + self, + sftp_source_path: str, + container_name: str, + blob_name: str, + sftp_conn_id: str = "ssh_default", + wasb_conn_id: str = 'wasb_default', + load_options: dict = None, + move_object: bool = False, + *args, + **kwargs + ) -> None: + super().__init__(*args, **kwargs) + + self.sftp_source_path = sftp_source_path + self.sftp_conn_id = sftp_conn_id + self.wasb_conn_id = wasb_conn_id + self.container_name = container_name + self.blob_name = blob_name + self.wasb_conn_id = wasb_conn_id + self.load_options = load_options if load_options is not None else {} + self.move_object = move_object + + def execute(self, context): + (sftp_hook, sftp_files) = self.get_sftp_files_map() + uploaded_files = self.upload_wasb(sftp_hook, sftp_files) + self.sftp_move(sftp_hook, uploaded_files) + + def get_sftp_files_map(self): + sftp_files = [] + sftp_hook = SFTPHook(self.sftp_conn_id) + + if WILDCARD in self.sftp_source_path: + self.check_wildcards_limit() + + prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) + sftp_complete_path = os.path.dirname(prefix) + + found_files, _, _ = sftp_hook.get_tree_map( + sftp_complete_path, prefix=prefix, delimiter=delimiter + ) + + self.log.info( + "Found %s files at sftp source path: %s", + str(len(found_files)), + self.sftp_source_path + ) + + for file in found_files: + future_blob_name = os.path.basename(file) + sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) + + else: + sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) + + return sftp_hook, sftp_files + + def check_wildcards_limit(self): + total_wildcards = self.sftp_source_path.count(WILDCARD) + if total_wildcards > 1: + raise AirflowException( + "Only one wildcard '*' is allowed in sftp_source_path parameter. " + "Found {} in {}.".format(total_wildcards, self.sftp_source_path) + ) + + def upload_wasb(self, sftp_hook, sftp_files): + uploaded_files = [] + wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) + for file in sftp_files: + + with NamedTemporaryFile("w") as tmp: + sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) + self.log.info( + 'Uploading %s to wasb://%s as %s', + file[SFTP_FILE_PATH], + self.container_name, + file[BLOB_NAME], + ) + wasb_hook.load_file(tmp.name, + self.container_name, + file[BLOB_NAME], + **self.load_options) + + uploaded_files.append(file[SFTP_FILE_PATH]) + + return uploaded_files + + def sftp_move(self, sftp_hook, uploaded_files): + if self.move_object: + for sftp_file_path in uploaded_files: + self.log.info("Executing delete of %s", sftp_file_path) + sftp_hook.delete_file(sftp_file_path) + + diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 8f02c8fff60e1..17c2763b68228 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -17,19 +17,24 @@ # under the License. # -import os + +import datetime import unittest +import os import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator +from airflow.models.dag import DAG +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME + TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" SFTP_CONN_ID = "ssh_default" -BLOB_NAME = "BLOB_NAME" CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" @@ -42,8 +47,9 @@ # pylint: disable=unused-argument class TestSFTPToWasbOperator(unittest.TestCase): + def test_init(self): - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -57,10 +63,12 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', + + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, @@ -75,15 +83,15 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_get_sftp_files_map(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_PATH, sftp_conn_id=SFTP_CONN_ID, @@ -92,20 +100,23 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=False ) - files = operator.get_sftp_files_map() + _, files = operator.get_sftp_files_map() + + sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEqual(len(files), 2, "not matched at expected found files") - self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") + self.assertEquals(len(files), 2, "not matched at expected found files") + self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_get_sftp_files_map(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -114,15 +125,19 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=True ) - files = operator.get_sftp_files_map() + _, files = operator.get_sftp_files_map() + + sftp_hook.assert_called_once_with(SFTP_CONN_ID) + + self.assertEquals(len(files), 1, "no matched at expected found files") + self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_upload_wasb(self, sftp_hook, mock_hook): - self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0].blob_name, BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_copy_files_to_wasb(self, sftp_hook, mock_hook): - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -132,10 +147,10 @@ def test_copy_files_to_wasb(self, sftp_hook, mock_hook): move_object=True ) - sftp_files = [SftpFile(SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME)] - files = operator.copy_files_to_wasb(sftp_files) + sftp_files = [{SFTP_FILE_PATH: SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME: BLOB_NAME}] + files = operator.upload_wasb(sftp_hook, sftp_files) - operator.sftp_hook.retrieve_file.assert_has_calls( + sftp_hook.retrieve_file.assert_has_calls( [ mock.call("main_dir/test_object3.json", mock.ANY) ] @@ -145,13 +160,13 @@ def test_copy_files_to_wasb(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, BLOB_NAME ) - self.assertEqual(len(files), 1, "no matched at expected uploaded files") + self.assertEquals(len(files), 1, "no matched at expected uploaded files") - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_delete_files(self, sftp_hook): + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -162,7 +177,7 @@ def test_delete_files(self, sftp_hook): ) sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.delete_files(sftp_file_paths) + operator.sftp_move(sftp_mock, sftp_file_paths) sftp_mock.delete_file.assert_has_calls( [ @@ -170,13 +185,13 @@ def test_delete_files(self, sftp_hook): ] ) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_execute(self, sftp_hook, mock_hook): + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_no_sftp_move(self, sftp_hook): + sftp_mock = sftp_hook.return_value - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, - sftp_source_path=WILDCARD_FILE_NAME, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, blob_name=BLOB_NAME, @@ -184,12 +199,30 @@ def test_execute(self, sftp_hook, mock_hook): move_object=False ) + sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] + operator.sftp_move(sftp_mock, sftp_file_paths) + + sftp_mock.delete_file.assert_not_called() + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_FILE_NAME, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False + ) + operator.execute(None) sftp_hook.return_value.get_tree_map.assert_called_with( @@ -206,4 +239,5 @@ def test_execute(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) ) - operator.sftp_hook.delete_file.assert_not_called() + sftp_hook.delete_file.assert_not_called() + From a298447e00e5ac9a928b95f4da7a6eea9feb1e71 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 19:26:09 -0300 Subject: [PATCH 13/79] Add sftp_to_wasb to operators-and-hooks-ref --- docs/operators-and-hooks-ref.rst | 1646 ++++++++++++++++++++++++++++++ 1 file changed, 1646 insertions(+) create mode 100644 docs/operators-and-hooks-ref.rst diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst new file mode 100644 index 0000000000000..8c6a7d9338faa --- /dev/null +++ b/docs/operators-and-hooks-ref.rst @@ -0,0 +1,1646 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Operators and Hooks Reference +============================= + +.. contents:: Content + :local: + :depth: 1 + +.. _fundamentals: + +Fundamentals +------------ + +**Base:** + +.. list-table:: + :header-rows: 1 + + * - Module + - Guides + + * - :mod:`airflow.hooks.base_hook` + - + + * - :mod:`airflow.hooks.dbapi_hook` + - + + * - :mod:`airflow.models.baseoperator` + - + + * - :mod:`airflow.sensors.base_sensor_operator` + - + +**Operators:** + +.. list-table:: + :header-rows: 1 + + * - Operators + - Guides + + * - :mod:`airflow.operators.branch_operator` + - + * - :mod:`airflow.operators.dagrun_operator` + - + + * - :mod:`airflow.operators.dummy_operator` + - + + * - :mod:`airflow.operators.generic_transfer` + - + + * - :mod:`airflow.operators.latest_only_operator` + - + + * - :mod:`airflow.operators.subdag_operator` + - + + * - :mod:`airflow.operators.sql` + - + +**Sensors:** + +.. list-table:: + :header-rows: 1 + + * - Sensors + - Guides + + * - :mod:`airflow.sensors.weekday_sensor` + - + + * - :mod:`airflow.sensors.external_task_sensor` + - :doc:`How to use ` + + * - :mod:`airflow.sensors.sql_sensor` + - + + * - :mod:`airflow.sensors.time_delta_sensor` + - + + * - :mod:`airflow.sensors.time_sensor` + - + + +.. _Apache: + +ASF: Apache Software Foundation +------------------------------- + +Airflow supports various software created by `Apache Software Foundation `__. + +Software operators and hooks +'''''''''''''''''''''''''''' + +These integrations allow you to perform various operations within software developed by Apache Software +Foundation. + +.. list-table:: + :header-rows: 1 + + * - Service name + - Guides + - Hook + - Operator + - Sensor + + * - `Apache Cassandra `__ + - + - :mod:`airflow.providers.apache.cassandra.hooks.cassandra` + - + - :mod:`airflow.providers.apache.cassandra.sensors.record`, + :mod:`airflow.providers.apache.cassandra.sensors.table` + + * - `Apache Druid `__ + - + - :mod:`airflow.providers.apache.druid.hooks.druid` + - :mod:`airflow.providers.apache.druid.operators.druid`, + :mod:`airflow.providers.apache.druid.operators.druid_check` + - + + * - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.hooks.hive` + - :mod:`airflow.providers.apache.hive.operators.hive`, + :mod:`airflow.providers.apache.hive.operators.hive_stats` + - :mod:`airflow.providers.apache.hive.sensors.named_hive_partition`, + :mod:`airflow.providers.apache.hive.sensors.hive_partition`, + :mod:`airflow.providers.apache.hive.sensors.metastore_partition` + + * - `Apache Livy `__ + - + - :mod:`airflow.providers.apache.livy.hooks.livy` + - :mod:`airflow.providers.apache.livy.operators.livy` + - :mod:`airflow.providers.apache.livy.sensors.livy` + + * - `Apache Pig `__ + - + - :mod:`airflow.providers.apache.pig.hooks.pig` + - :mod:`airflow.providers.apache.pig.operators.pig` + - + + * - `Apache Pinot `__ + - + - :mod:`airflow.providers.apache.pinot.hooks.pinot` + - + - + + * - `Apache Spark `__ + - :doc:`How to use ` + - :mod:`airflow.providers.apache.spark.hooks.spark_jdbc`, + :mod:`airflow.providers.apache.spark.hooks.spark_jdbc_script`, + :mod:`airflow.providers.apache.spark.hooks.spark_sql`, + :mod:`airflow.providers.apache.spark.hooks.spark_submit` + - :mod:`airflow.providers.apache.spark.operators.spark_jdbc`, + :mod:`airflow.providers.apache.spark.operators.spark_sql`, + :mod:`airflow.providers.apache.spark.operators.spark_submit` + - + + * - `Apache Sqoop `__ + - + - :mod:`airflow.providers.apache.sqoop.hooks.sqoop` + - :mod:`airflow.providers.apache.sqoop.operators.sqoop` + - + + * - `Hadoop Distributed File System (HDFS) `__ + - + - :mod:`airflow.providers.apache.hdfs.hooks.hdfs` + - + - :mod:`airflow.providers.apache.hdfs.sensors.hdfs` + + * - `WebHDFS `__ + - + - :mod:`airflow.providers.apache.hdfs.hooks.webhdfs` + - + - :mod:`airflow.providers.apache.hdfs.sensors.web_hdfs` + +Transfer operators and hooks +'''''''''''''''''''''''''''' + +These integrations allow you to copy data from/to software developed by Apache Software +Foundation. + +.. list-table:: + :header-rows: 1 + + * - Source + - Destination + - Guide + - Operator + + * - `Amazon Simple Storage Service (S3) `_ + - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.transfers.s3_to_hive` + + * - `Amazon Simple Storage Service (S3) `_ + - `MySQL `__ + - + - :mod:`airflow.providers.mysql.transfers.s3_to_mysql` + + * - `Apache Cassandra `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.cassandra_to_gcs` + + * - `Apache Hive `__ + - `Amazon DynamoDB `__ + - + - :mod:`airflow.providers.amazon.aws.transfers.hive_to_dynamodb` + + * - `Apache Hive `__ + - `Apache Druid `__ + - + - :mod:`airflow.providers.apache.druid.transfers.hive_to_druid` + + * - `Apache Hive `__ + - `MySQL `__ + - + - :mod:`airflow.providers.apache.hive.transfers.hive_to_mysql` + + * - `Apache Hive `__ + - `Samba `__ + - + - :mod:`airflow.providers.apache.hive.transfers.hive_to_samba` + + * - `Microsoft SQL Server (MSSQL) `__ + - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.transfers.mssql_to_hive` + + * - `MySQL `__ + - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.transfers.mysql_to_hive` + + * - `Vertica `__ + - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.transfers.vertica_to_hive` + +.. _Azure: + +Azure: Microsoft Azure +---------------------- + +Airflow has limited support for `Microsoft Azure `__. + +Service operators and hooks +''''''''''''''''''''''''''' + +These integrations allow you to perform various operations within the Microsoft Azure. + + +.. list-table:: + :header-rows: 1 + + * - Service name + - Hook + - Operator + - Sensor + + * - `Azure Batch `__ + - :mod:`airflow.providers.microsoft.azure.hooks.azure_batch` + - :mod:`airflow.providers.microsoft.azure.operators.azure_batch` + - + + * - `Azure Blob Storage `__ + - :mod:`airflow.providers.microsoft.azure.hooks.wasb` + - :mod:`airflow.providers.microsoft.azure.operators.wasb_delete_blob` + - :mod:`airflow.providers.microsoft.azure.sensors.wasb` + + * - `Azure Container Instances `__ + - :mod:`airflow.providers.microsoft.azure.hooks.azure_container_instance`, + :mod:`airflow.providers.microsoft.azure.hooks.azure_container_registry`, + :mod:`airflow.providers.microsoft.azure.hooks.azure_container_volume` + - :mod:`airflow.providers.microsoft.azure.operators.azure_container_instances` + - + + * - `Azure Cosmos DB `__ + - :mod:`airflow.providers.microsoft.azure.hooks.azure_cosmos` + - :mod:`airflow.providers.microsoft.azure.operators.azure_cosmos` + - :mod:`airflow.providers.microsoft.azure.sensors.azure_cosmos` + + * - `Azure Data Lake Storage `__ + - :mod:`airflow.providers.microsoft.azure.hooks.azure_data_lake` + - :mod:`airflow.providers.microsoft.azure.operators.adls_list` + - + + * - `Azure Data Explorer `__ + - :mod:`airflow.providers.microsoft.azure.hooks.adx` + - :mod:`airflow.providers.microsoft.azure.operators.adx` + - + + * - `Azure Files `__ + - :mod:`airflow.providers.microsoft.azure.hooks.azure_fileshare` + - + - + +Transfer operators and hooks +'''''''''''''''''''''''''''' + +These integrations allow you to copy data from/to Microsoft Azure. + +.. list-table:: + :header-rows: 1 + + * - Source + - Destination + - Guide + - Operator + + * - `Azure Data Lake Storage `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.adls_to_gcs` + + * - Local + - `Azure Blob Storage `__ + - + - :mod:`airflow.providers.microsoft.azure.transfers.file_to_wasb` + + * - SFTP + - `Azure Blob Storage `__ + - + - :mod:`airflow.providers.microsoft.azure.transfers.sftp_to_wasb` + + + * - `Oracle `__ + - `Azure Data Lake Storage `__ + - + - :mod:`airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake` + + +.. _AWS: + +AWS: Amazon Web Services +------------------------ + +Airflow has support for `Amazon Web Services `__. + +All hooks are based on :mod:`airflow.providers.amazon.aws.hooks.base_aws`. + +Service operators and hooks +''''''''''''''''''''''''''' + +These integrations allow you to perform various operations within the Amazon Web Services. + +.. list-table:: + :header-rows: 1 + + * - Service name + - Guide + - Hook + - Operator + - Sensor + + * - `AWS Batch `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.batch_client`, + :mod:`airflow.providers.amazon.aws.hooks.batch_waiters` + - :mod:`airflow.providers.amazon.aws.operators.batch` + - + + * - `AWS DataSync `__ + - :doc:`How to use ` + - :mod:`airflow.providers.amazon.aws.hooks.datasync` + - :mod:`airflow.providers.amazon.aws.operators.datasync` + - + + * - `AWS Glue Catalog `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.glue_catalog` + - + - :mod:`airflow.providers.amazon.aws.sensors.glue_catalog_partition` + + * - `AWS Glue `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.glue` + - :mod:`airflow.providers.amazon.aws.operators.glue` + - :mod:`airflow.providers.amazon.aws.sensors.glue` + + * - `AWS Lambda `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.lambda_function` + - + - + + * - `Amazon Athena `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.athena` + - :mod:`airflow.providers.amazon.aws.operators.athena` + - :mod:`airflow.providers.amazon.aws.sensors.athena` + + * - `Amazon CloudFormation `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.cloud_formation` + - :mod:`airflow.providers.amazon.aws.operators.cloud_formation` + - :mod:`airflow.providers.amazon.aws.sensors.cloud_formation` + + * - `Amazon CloudWatch Logs `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.logs` + - + - + + * - `Amazon DynamoDB `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.aws_dynamodb` + - + - + + * - `Amazon EC2 `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.ec2` + - :mod:`airflow.providers.amazon.aws.operators.ec2_start_instance`, + :mod:`airflow.providers.amazon.aws.operators.ec2_stop_instance`, + - :mod:`airflow.providers.amazon.aws.sensors.ec2_instance_state` + + * - `Amazon ECS `__ + - :doc:`How to use ` + - + - :mod:`airflow.providers.amazon.aws.operators.ecs` + - + + * - `Amazon EMR `__ + - :doc:`How to use ` + - :mod:`airflow.providers.amazon.aws.hooks.emr` + - :mod:`airflow.providers.amazon.aws.operators.emr_add_steps`, + :mod:`airflow.providers.amazon.aws.operators.emr_create_job_flow`, + :mod:`airflow.providers.amazon.aws.operators.emr_terminate_job_flow`, + :mod:`airflow.providers.amazon.aws.operators.emr_modify_cluster` + - :mod:`airflow.providers.amazon.aws.sensors.emr_base`, + :mod:`airflow.providers.amazon.aws.sensors.emr_job_flow`, + :mod:`airflow.providers.amazon.aws.sensors.emr_step` + + * - `Amazon Kinesis Data Firehose `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.kinesis` + - + - + + * - `Amazon Redshift `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.redshift` + - + - :mod:`airflow.providers.amazon.aws.sensors.redshift` + + * - `Amazon SageMaker `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.sagemaker` + - :mod:`airflow.providers.amazon.aws.operators.sagemaker_base`, + :mod:`airflow.providers.amazon.aws.operators.sagemaker_endpoint_config`, + :mod:`airflow.providers.amazon.aws.operators.sagemaker_endpoint`, + :mod:`airflow.providers.amazon.aws.operators.sagemaker_model`, + :mod:`airflow.providers.amazon.aws.operators.sagemaker_training`, + :mod:`airflow.providers.amazon.aws.operators.sagemaker_transform`, + :mod:`airflow.providers.amazon.aws.operators.sagemaker_tuning` + - :mod:`airflow.providers.amazon.aws.sensors.sagemaker_base`, + :mod:`airflow.providers.amazon.aws.sensors.sagemaker_endpoint`, + :mod:`airflow.providers.amazon.aws.sensors.sagemaker_training`, + :mod:`airflow.providers.amazon.aws.sensors.sagemaker_transform`, + :mod:`airflow.providers.amazon.aws.sensors.sagemaker_tuning` + + * - `Amazon Simple Notification Service (SNS) `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.sns` + - :mod:`airflow.providers.amazon.aws.operators.sns` + - + + * - `Amazon Simple Queue Service (SQS) `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.sqs` + - :mod:`airflow.providers.amazon.aws.operators.sqs` + - :mod:`airflow.providers.amazon.aws.sensors.sqs` + + * - `Amazon Simple Storage Service (S3) `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.s3` + - :mod:`airflow.providers.amazon.aws.operators.s3_bucket`, + :mod:`airflow.providers.amazon.aws.operators.s3_file_transform`, + :mod:`airflow.providers.amazon.aws.operators.s3_copy_object`, + :mod:`airflow.providers.amazon.aws.operators.s3_delete_objects`, + :mod:`airflow.providers.amazon.aws.operators.s3_list` + - :mod:`airflow.providers.amazon.aws.sensors.s3_key`, + :mod:`airflow.providers.amazon.aws.sensors.s3_prefix` + + * - `AWS Step Functions `__ + - + - :mod:`airflow.providers.amazon.aws.hooks.step_function` + - :mod:`airflow.providers.amazon.aws.operators.step_function_start_execution`, + :mod:`airflow.providers.amazon.aws.operators.step_function_get_execution_output`, + - :mod:`airflow.providers.amazon.aws.sensors.step_function_execution`, + +Transfer operators and hooks +'''''''''''''''''''''''''''' + +These integrations allow you to copy data from/to Amazon Web Services. + +.. list-table:: + :header-rows: 1 + + * - Source + - Destination + - Guide + - Operator + + * - + .. _integration:AWS-Discovery-ref: + + All GCP services :ref:`[1] ` + - `Amazon Simple Storage Service (S3) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.amazon.aws.transfers.google_api_to_s3` + + * - `Amazon DataSync `__ + - `Amazon Simple Storage Service (S3) `_ + - :doc:`How to use ` + - :mod:`airflow.providers.amazon.aws.operators.datasync` + + * - `Amazon DynamoDB `__ + - `Amazon Simple Storage Service (S3) `_ + - + - :mod:`airflow.providers.amazon.aws.transfers.dynamodb_to_s3` + + * - `Amazon Redshift `__ + - `Amazon Simple Storage Service (S3) `_ + - + - :mod:`airflow.providers.amazon.aws.transfers.redshift_to_s3` + + * - `Amazon Simple Storage Service (S3) `_ + - `Amazon Redshift `__ + - :doc:`How to use ` + - :mod:`airflow.providers.amazon.aws.transfers.s3_to_redshift` + + * - `Amazon Simple Storage Service (S3) `_ + - `Snowflake `__ + - + - :mod:`airflow.providers.snowflake.transfers.s3_to_snowflake` + + * - `Amazon Simple Storage Service (S3) `_ + - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.transfers.s3_to_hive` + + * - `Amazon Simple Storage Service (S3) `__ + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.s3_to_gcs`, + :mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service` + + * - `Amazon Simple Storage Service (S3) `_ + - `SSH File Transfer Protocol (SFTP) `__ + - + - :mod:`airflow.providers.amazon.aws.transfers.s3_to_sftp` + + * - `Apache Hive `__ + - `Amazon DynamoDB `__ + - + - :mod:`airflow.providers.amazon.aws.transfers.hive_to_dynamodb` + + * - `Google Cloud Storage (GCS) `__ + - `Amazon Simple Storage Service (S3) `__ + - + - :mod:`airflow.providers.amazon.aws.transfers.gcs_to_s3` + + * - `Internet Message Access Protocol (IMAP) `__ + - `Amazon Simple Storage Service (S3) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.amazon.aws.transfers.imap_attachment_to_s3` + + * - `MongoDB `__ + - `Amazon Simple Storage Service (S3) `__ + - + - :mod:`airflow.providers.amazon.aws.transfers.mongo_to_s3` + + * - `SSH File Transfer Protocol (SFTP) `__ + - `Amazon Simple Storage Service (S3) `_ + - + - :mod:`airflow.providers.amazon.aws.transfers.sftp_to_s3` + + * - `MySQL `__ + - `Amazon Simple Storage Service (S3) `_ + - + - :mod:`airflow.providers.amazon.aws.transfers.mysql_to_s3` + +:ref:`[1] ` Those discovery-based operators use +:class:`~airflow.providers.google.common.hooks.discovery_api.GoogleDiscoveryApiHook` to communicate with Google +Services via the `Google API Python Client `__. +Please note that this library is in maintenance mode hence it won't fully support GCP in the future. +Therefore it is recommended that you use the custom GCP Service Operators for working with the Google +Cloud Platform. + +.. _Google: + +Google +------ + +Airflow has support for the `Google service `__. + +All hooks are based on :class:`airflow.providers.google.common.hooks.base_google.GoogleBaseHook`. Some integration +also use :mod:`airflow.providers.google.common.hooks.discovery_api`. + +See the :doc:`GCP connection type ` documentation to +configure connections to Google services. + +.. _GCP: + +GCP: Google Cloud Platform +'''''''''''''''''''''''''' + +Airflow has extensive support for the `Google Cloud Platform `__. + +.. note:: + You can learn how to use Google Cloud Platform integrations by analyzing the + `source code of the Google Cloud Platform example DAGs + `_ + + +Service operators and hooks +""""""""""""""""""""""""""" + +These integrations allow you to perform various operations within the Google Cloud Platform. + +.. + PLEASE KEEP THE ALPHABETICAL ORDER OF THE LIST BELOW, BUT OMIT THE "Cloud" PREFIX + +.. list-table:: + :header-rows: 1 + + * - Service name + - Guide + - Hook + - Operator + - Sensor + + + * - `AutoML `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.automl` + - :mod:`airflow.providers.google.cloud.operators.automl` + - + + * - `BigQuery `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.bigquery` + - :mod:`airflow.providers.google.cloud.operators.bigquery` + - :mod:`airflow.providers.google.cloud.sensors.bigquery` + + * - `BigQuery Data Transfer Service `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.bigquery_dts` + - :mod:`airflow.providers.google.cloud.operators.bigquery_dts` + - :mod:`airflow.providers.google.cloud.sensors.bigquery_dts` + + * - `Bigtable `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.bigtable` + - :mod:`airflow.providers.google.cloud.operators.bigtable` + - :mod:`airflow.providers.google.cloud.sensors.bigtable` + + * - `Cloud Build `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.cloud_build` + - :mod:`airflow.providers.google.cloud.operators.cloud_build` + - + + * - `Compute Engine `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.compute` + - :mod:`airflow.providers.google.cloud.operators.compute` + - + + * - `Cloud Data Loss Prevention (DLP) `__ + - + - :mod:`airflow.providers.google.cloud.hooks.dlp` + - :mod:`airflow.providers.google.cloud.operators.dlp` + - + + * - `DataFusion `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.datafusion` + - :mod:`airflow.providers.google.cloud.operators.datafusion` + - + + * - `Datacatalog `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.datacatalog` + - :mod:`airflow.providers.google.cloud.operators.datacatalog` + - + + * - `Dataflow `__ + - + - :mod:`airflow.providers.google.cloud.hooks.dataflow` + - :mod:`airflow.providers.google.cloud.operators.dataflow` + - + + * - `Dataproc `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.dataproc` + - :mod:`airflow.providers.google.cloud.operators.dataproc` + - + + * - `Datastore `__ + - + - :mod:`airflow.providers.google.cloud.hooks.datastore` + - :mod:`airflow.providers.google.cloud.operators.datastore` + - + + * - `Cloud Functions `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.functions` + - :mod:`airflow.providers.google.cloud.operators.functions` + - + + * - `Cloud Firestore `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.firebase.hooks.firestore` + - :mod:`airflow.providers.google.firebase.operators.firestore` + - + + * - `Cloud Key Management Service (KMS) `__ + - + - :mod:`airflow.providers.google.cloud.hooks.kms` + - + - + * - `Cloud Life Sciences `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.life_sciences` + - :mod:`airflow.providers.google.cloud.operators.life_sciences` + - + + * - `Kubernetes Engine `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.kubernetes_engine` + - :mod:`airflow.providers.google.cloud.operators.kubernetes_engine` + - + + * - `Machine Learning Engine `__ + - + - :mod:`airflow.providers.google.cloud.hooks.mlengine` + - :mod:`airflow.providers.google.cloud.operators.mlengine` + - + + * - `Cloud Memorystore `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.cloud_memorystore` + - :mod:`airflow.providers.google.cloud.operators.cloud_memorystore` + - + + * - `Natural Language `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.natural_language` + - :mod:`airflow.providers.google.cloud.operators.natural_language` + - + + * - `Cloud Pub/Sub `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.pubsub` + - :mod:`airflow.providers.google.cloud.operators.pubsub` + - :mod:`airflow.providers.google.cloud.sensors.pubsub` + + * - `Cloud Secret Manager `__ + - + - :mod:`airflow.providers.google.cloud.hooks.secret_manager` + - + - + + * - `Cloud Spanner `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.spanner` + - :mod:`airflow.providers.google.cloud.operators.spanner` + - + + * - `Cloud Speech-to-Text `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.speech_to_text` + - :mod:`airflow.providers.google.cloud.operators.speech_to_text` + - + + * - `Cloud SQL `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.cloud_sql` + - :mod:`airflow.providers.google.cloud.operators.cloud_sql` + - + + * - `Cloud Stackdriver `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.stackdriver` + - :mod:`airflow.providers.google.cloud.operators.stackdriver` + - + + * - `Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.gcs` + - :mod:`airflow.providers.google.cloud.operators.gcs` + - :mod:`airflow.providers.google.cloud.sensors.gcs` + + * - `Storage Transfer Service `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.cloud_storage_transfer_service` + - :mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service` + - :mod:`airflow.providers.google.cloud.sensors.cloud_storage_transfer_service` + + * - `Cloud Tasks `__ + - + - :mod:`airflow.providers.google.cloud.hooks.tasks` + - :mod:`airflow.providers.google.cloud.operators.tasks` + - + + * - `Cloud Text-to-Speech `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.text_to_speech` + - :mod:`airflow.providers.google.cloud.operators.text_to_speech` + - + + * - `Cloud Translation `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.translate` + - :mod:`airflow.providers.google.cloud.operators.translate` + - + + * - `Cloud Video Intelligence `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.video_intelligence` + - :mod:`airflow.providers.google.cloud.operators.video_intelligence` + - + + * - `Cloud Vision `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.hooks.vision` + - :mod:`airflow.providers.google.cloud.operators.vision` + - + +Transfer operators and hooks +"""""""""""""""""""""""""""" + +These integrations allow you to copy data from/to Google Cloud Platform. + +.. list-table:: + :header-rows: 1 + + * - Source + - Destination + - Guide + - Operator + + * - + .. _integration:GCP-Discovery-ref: + + All services :ref:`[1] ` + - `Amazon Simple Storage Service (S3) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.amazon.aws.transfers.google_api_to_s3` + + * - `Amazon Simple Storage Service (S3) `__ + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.s3_to_gcs`, + :mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service` + + * - `Apache Cassandra `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.cassandra_to_gcs` + + * - `Azure Data Lake Storage `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.adls_to_gcs` + + * - `Facebook Ads `__ + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.facebook_ads_to_gcs` + + + * - `Google Ads `__ + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.ads.transfers.ads_to_gcs` + + * - `Google BigQuery `__ + - `MySQL `__ + - + - :mod:`airflow.providers.google.cloud.transfers.bigquery_to_mysql` + + * - `Google BigQuery `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.bigquery_to_gcs` + + * - `Google BigQuery `__ + - `Google BigQuery `__ + - + - :mod:`airflow.providers.google.cloud.transfers.bigquery_to_bigquery` + + * - `Cloud Firestore `__ + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.firebase.operators.firestore` + + * - `Google Cloud Storage (GCS) `__ + - `Amazon Simple Storage Service (S3) `__ + - + - :mod:`airflow.providers.amazon.aws.transfers.gcs_to_s3` + + * - `Google Cloud Storage (GCS) `__ + - `Google BigQuery `__ + - + - :mod:`airflow.providers.google.cloud.transfers.gcs_to_bigquery` + + * - `Google Cloud Storage (GCS) `__ + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use `, + :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.gcs_to_gcs`, + :mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service` + + * - `Google Cloud Storage (GCS) `__ + - Local + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.gcs_to_local` + + * - `Google Cloud Storage (GCS) `__ + - `Google Drive `__ + - + - :mod:`airflow.providers.google.suite.transfers.gcs_to_gdrive` + + * - `Google Cloud Storage (GCS) `__ + - SFTP + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.gcs_to_sftp` + + * - Local + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.local_to_gcs` + + * - `Microsoft SQL Server (MSSQL) `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.mssql_to_gcs` + + * - `MySQL `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.mysql_to_gcs` + + * - `PostgresSQL `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.postgres_to_gcs` + + * - `Presto `__ + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.presto_to_gcs` + + * - SFTP + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.sftp_to_gcs` + + * - SQL + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.sql_to_gcs` + + * - `Google Spreadsheet `__ + - `Google Cloud Storage (GCS) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.transfers.sheets_to_gcs` + + * - `Google Cloud Storage (GCS) `__ + - `Google Spreadsheet `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.suite.transfers.gcs_to_sheets` + +.. _integration:GCP-Discovery: + +:ref:`[1] ` Those discovery-based operators use +:class:`~airflow.providers.google.common.hooks.discovery_api.GoogleDiscoveryApiHook` to communicate with Google +Services via the `Google API Python Client `__. +Please note that this library is in maintenance mode hence it won't fully support Google in the future. +Therefore it is recommended that you use the custom Google Service Operators for working with the Google +services. + +Other operators and hooks +""""""""""""""""""""""""" + +.. list-table:: + :header-rows: 1 + + * - Guide + - Operator + - Hook + + * - :doc:`How to use ` + - :mod:`airflow.providers.google.cloud.operators.translate_speech` + - + +Google Marketing Platform +''''''''''''''''''''''''' + +.. note:: + You can learn how to use Google Marketing Platform integrations by analyzing the + `source code `_ + of the example DAGs. + + +.. list-table:: + :header-rows: 1 + + * - Source + - Destination + - Guide + - Operator + - Sensor + + * - `Analytics360 `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.marketing_platform.hooks.analytics` + - :mod:`airflow.providers.google.marketing_platform.operators.analytics` + - + + * - `Google Campaign Manager `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.marketing_platform.hooks.campaign_manager` + - :mod:`airflow.providers.google.marketing_platform.operators.campaign_manager` + - :mod:`airflow.providers.google.marketing_platform.sensors.campaign_manager` + + * - `Google Display&Video 360 `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.marketing_platform.hooks.display_video` + - :mod:`airflow.providers.google.marketing_platform.operators.display_video` + - :mod:`airflow.providers.google.marketing_platform.sensors.display_video` + + * - `Google Search Ads 360 `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.marketing_platform.hooks.search_ads` + - :mod:`airflow.providers.google.marketing_platform.operators.search_ads` + - :mod:`airflow.providers.google.marketing_platform.sensors.search_ads` + +Other Google operators and hooks +'''''''''''''''''''''''''''''''' + +.. list-table:: + :header-rows: 1 + + * - Service name + - Guide + - Hook + - Operator + + * - `Google Ads `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.ads.hooks.ads` + - :mod:`airflow.providers.google.ads.operators.ads` + + * - `Google Drive `__ + - + - :mod:`airflow.providers.google.suite.hooks.drive` + - + + * - `Cloud Firestore `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.firebase.hooks.firestore` + - :mod:`airflow.providers.google.firebase.operators.firestore` + + * - `Google Spreadsheet `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.suite.hooks.sheets` + - :mod:`airflow.providers.google.suite.operators.sheets` + +.. _yc_service: + +Yandex.Cloud +-------------------------- + +Airflow has a limited support for the `Yandex.Cloud `__. + +See the :doc:`Yandex.Cloud connection type ` documentation to +configure connections to Yandex.Cloud. + +All hooks are based on :class:`airflow.providers.yandex.hooks.yandex.YandexCloudBaseHook`. + +.. note:: + You can learn how to use Yandex.Cloud integrations by analyzing the + `example DAG `_ + +Service operators and hooks +''''''''''''''''''''''''''' + +These integrations allow you to perform various operations within the Yandex.Cloud. + +.. + PLEASE KEEP THE ALPHABETICAL ORDER OF THE LIST BELOW, BUT OMIT THE "Cloud" PREFIX + +.. list-table:: + :header-rows: 1 + + * - Service name + - Guide + - Hook + - Operator + - Sensor + + * - `Base Classes `__ + - :doc:`How to use ` + - :mod:`airflow.providers.yandex.hooks.yandex` + - + - + + * - `Data Proc `__ + - :doc:`How to use ` + - :mod:`airflow.providers.yandex.hooks.yandexcloud_dataproc` + - :mod:`airflow.providers.yandex.operators.yandexcloud_dataproc` + - + + +.. _service: + +Service integrations +-------------------- + +Service operators and hooks +''''''''''''''''''''''''''' + +These integrations allow you to perform various operations within various services. + +.. list-table:: + :header-rows: 1 + + * - Service name + - Guide + - Hook + - Operator + - Sensor + + * - `Atlassian Jira `__ + - + - :mod:`airflow.providers.jira.hooks.jira` + - :mod:`airflow.providers.jira.operators.jira` + - :mod:`airflow.providers.jira.sensors.jira` + + * - `Databricks `__ + - + - :mod:`airflow.providers.databricks.hooks.databricks` + - :mod:`airflow.providers.databricks.operators.databricks` + - + + * - `Datadog `__ + - + - :mod:`airflow.providers.datadog.hooks.datadog` + - + - :mod:`airflow.providers.datadog.sensors.datadog` + + * - `Pagerduty `__ + - + - :mod:`airflow.providers.pagerduty.hooks.pagerduty` + - + - + + * - `Dingding `__ + - :doc:`How to use ` + - :mod:`airflow.providers.dingding.hooks.dingding` + - :mod:`airflow.providers.dingding.operators.dingding` + - + + * - `Discord `__ + - + - :mod:`airflow.providers.discord.hooks.discord_webhook` + - :mod:`airflow.providers.discord.operators.discord_webhook` + - + + * - `Facebook Ads `__ + - + - :mod:`airflow.providers.facebook.ads.hooks.ads` + - + - + + * - `IBM Cloudant `__ + - + - :mod:`airflow.providers.cloudant.hooks.cloudant` + - + - + + * - `Jenkins `__ + - + - :mod:`airflow.providers.jenkins.hooks.jenkins` + - :mod:`airflow.providers.jenkins.operators.jenkins_job_trigger` + - + + * - `Opsgenie `__ + - + - :mod:`airflow.providers.opsgenie.hooks.opsgenie_alert` + - :mod:`airflow.providers.opsgenie.operators.opsgenie_alert` + - + + * - `Qubole `__ + - + - :mod:`airflow.providers.qubole.hooks.qubole`, + :mod:`airflow.providers.qubole.hooks.qubole_check` + - :mod:`airflow.providers.qubole.operators.qubole`, + :mod:`airflow.providers.qubole.operators.qubole_check` + - :mod:`airflow.providers.qubole.sensors.qubole` + + * - `Salesforce `__ + - + - :mod:`airflow.providers.salesforce.hooks.salesforce`, + :mod:`airflow.providers.salesforce.hooks.tableau` + - :mod:`airflow.providers.salesforce.operators.tableau_refresh_workbook` + - :mod:`airflow.providers.salesforce.sensors.tableau_job_status` + + * - `Segment `__ + - + - :mod:`airflow.providers.segment.hooks.segment` + - :mod:`airflow.providers.segment.operators.segment_track_event` + - + + * - `Slack `__ + - + - :mod:`airflow.providers.slack.hooks.slack`, + :mod:`airflow.providers.slack.hooks.slack_webhook` + - :mod:`airflow.providers.slack.operators.slack`, + :mod:`airflow.providers.slack.operators.slack_webhook` + - + + * - `Snowflake `__ + - + - :mod:`airflow.providers.snowflake.hooks.snowflake` + - :mod:`airflow.providers.snowflake.operators.snowflake`, + :mod:`airflow.providers.snowflake.transfers.snowflake_to_slack` + - + + * - `Vertica `__ + - + - :mod:`airflow.providers.vertica.hooks.vertica` + - :mod:`airflow.providers.vertica.operators.vertica` + - + + * - `Zendesk `__ + - + - :mod:`airflow.providers.zendesk.hooks.zendesk` + - + - + + +Transfer operators and hooks +'''''''''''''''''''''''''''' + +These integrations allow you to perform various operations within various services. + +.. list-table:: + :header-rows: 1 + + * - Source + - Destination + - Guide + - Operator + + * - `Google Cloud Storage (GCS) `__ + - `Google Drive `__ + - :doc:`How to use ` + - :mod:`airflow.providers.google.suite.transfers.gcs_to_gdrive` + + * - `Vertica `__ + - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.transfers.vertica_to_hive` + + * - `Vertica `__ + - `MySQL `__ + - + - :mod:`airflow.providers.mysql.transfers.vertica_to_mysql` + +.. _software: + +Software integrations +--------------------- + +Software operators and hooks +'''''''''''''''''''''''''''' + +These integrations allow you to perform various operations using various software. + +.. list-table:: + :header-rows: 1 + + * - Service name + - Guide + - Hook + - Operator + - Sensor + + * - `Celery `__ + - + - + - + - :mod:`airflow.providers.celery.sensors.celery_queue` + + * - `Docker `__ + - + - :mod:`airflow.providers.docker.hooks.docker` + - :mod:`airflow.providers.docker.operators.docker`, + :mod:`airflow.providers.docker.operators.docker_swarm` + - + + * - `Elasticsearch `__ + - + - :mod:`airflow.providers.elasticsearch.hooks.elasticsearch` + - + - + + * - `Exasol `__ + - + - :mod:`airflow.providers.exasol.hooks.exasol` + - :mod:`airflow.providers.exasol.operators.exasol` + - + + * - `GNU Bash `__ + - :doc:`How to use ` + - + - :mod:`airflow.operators.bash` + - :mod:`airflow.sensors.bash` + + * - `Hashicorp Vault `__ + - + - :mod:`airflow.providers.hashicorp.hooks.vault` + - + - + + * - `Kubernetes `__ + - :doc:`How to use ` + - :mod:`airflow.providers.cncf.kubernetes.hooks.kubernetes` + - :mod:`airflow.providers.cncf.kubernetes.operators.kubernetes_pod` + :mod:`airflow.providers.cncf.kubernetes.operators.spark_kubernetes` + - :mod:`airflow.providers.cncf.kubernetes.sensors.spark_kubernetes` + + + * - `Microsoft SQL Server (MSSQL) `__ + - + - :mod:`airflow.providers.microsoft.mssql.hooks.mssql`, + :mod:`airflow.providers.odbc.hooks.odbc` + - :mod:`airflow.providers.microsoft.mssql.operators.mssql` + - + + + * - `ODBC `__ + - + - :mod:`airflow.providers.odbc.hooks.odbc` + - + - + + * - `MongoDB `__ + - + - :mod:`airflow.providers.mongo.hooks.mongo` + - + - :mod:`airflow.providers.mongo.sensors.mongo` + + + * - `MySQL `__ + - + - :mod:`airflow.providers.mysql.hooks.mysql` + - :mod:`airflow.providers.mysql.operators.mysql` + - + + * - `OpenFaaS `__ + - + - :mod:`airflow.providers.openfaas.hooks.openfaas` + - + - + + * - `Oracle `__ + - + - :mod:`airflow.providers.oracle.hooks.oracle` + - :mod:`airflow.providers.oracle.operators.oracle` + - + + * - `Papermill `__ + - :doc:`How to use ` + - + - :mod:`airflow.providers.papermill.operators.papermill` + - + + * - `PostgresSQL `__ + - + - :mod:`airflow.providers.postgres.hooks.postgres` + - :mod:`airflow.providers.postgres.operators.postgres` + - + + * - `Presto `__ + - + - :mod:`airflow.providers.presto.hooks.presto` + - + - + + * - `Python `__ + - + - :doc:`How to use ` + - :mod:`airflow.operators.python` + - :mod:`airflow.sensors.python` + + * - `Redis `__ + - + - :mod:`airflow.providers.redis.hooks.redis` + - :mod:`airflow.providers.redis.operators.redis_publish` + - :mod:`airflow.providers.redis.sensors.redis_pub_sub`, + :mod:`airflow.providers.redis.sensors.redis_key` + + * - `Samba `__ + - + - :mod:`airflow.providers.samba.hooks.samba` + - + - + + * - `Singularity `__ + - + - + - :mod:`airflow.providers.singularity.operators.singularity` + - + + * - `SQLite `__ + - + - :mod:`airflow.providers.sqlite.hooks.sqlite` + - :mod:`airflow.providers.sqlite.operators.sqlite` + - + + +Transfer operators and hooks +'''''''''''''''''''''''''''' + +These integrations allow you to copy data. + +.. list-table:: + :header-rows: 1 + + * - Source + - Destination + - Guide + - Operator + + * - `Apache Hive `__ + - `Samba `__ + - + - :mod:`airflow.providers.apache.hive.transfers.hive_to_samba` + + * - `BigQuery `__ + - `MySQL `__ + - + - :mod:`airflow.providers.google.cloud.transfers.bigquery_to_mysql` + + * - `Microsoft SQL Server (MSSQL) `__ + - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.transfers.mssql_to_hive` + + * - `Microsoft SQL Server (MSSQL) `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.mssql_to_gcs` + + * - `MongoDB `__ + - `Amazon Simple Storage Service (S3) `__ + - + - :mod:`airflow.providers.amazon.aws.transfers.mongo_to_s3` + + * - `MySQL `__ + - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.transfers.mysql_to_hive` + + * - `MySQL `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.mysql_to_gcs` + + * - `Oracle `__ + - `Azure Data Lake Storage `__ + - + - :mod:`airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake` + + * - `Oracle `__ + - `Oracle `__ + - + - :mod:`airflow.providers.oracle.transfers.oracle_to_oracle` + + * - `PostgresSQL `__ + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.postgres_to_gcs` + + * - `Presto `__ + - `MySQL `__ + - + - :mod:`airflow.providers.mysql.transfers.presto_to_mysql` + + * - SQL + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.sql_to_gcs` + + * - `Vertica `__ + - `Apache Hive `__ + - + - :mod:`airflow.providers.apache.hive.transfers.vertica_to_hive` + + * - `Vertica `__ + - `MySQL `__ + - + - :mod:`airflow.providers.mysql.transfers.vertica_to_mysql` + +.. _protocol: + +Protocol integrations +--------------------- + +Protocol operators and hooks +'''''''''''''''''''''''''''' + +These integrations allow you to perform various operations within various services using standardized +communication protocols or interface. + +.. list-table:: + :header-rows: 1 + + * - Service name + - Guide + - Hooks + - Operator + - Sensor + + * - `File Transfer Protocol (FTP) `__ + - + - :mod:`airflow.providers.ftp.hooks.ftp` + - + - :mod:`airflow.providers.ftp.sensors.ftp` + + * - Filesystem + - + - :mod:`airflow.hooks.filesystem` + - + - :mod:`airflow.sensors.filesystem` + + * - `Hypertext Transfer Protocol (HTTP) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.http.hooks.http` + - :mod:`airflow.providers.http.operators.http` + - :mod:`airflow.providers.http.sensors.http` + + * - `Internet Message Access Protocol (IMAP) `__ + - + - :mod:`airflow.providers.imap.hooks.imap` + - + - :mod:`airflow.providers.imap.sensors.imap_attachment` + + * - `Java Database Connectivity (JDBC) `__ + - + - :mod:`airflow.providers.jdbc.hooks.jdbc` + - :mod:`airflow.providers.jdbc.operators.jdbc` + - + + * - `SSH File Transfer Protocol (SFTP) `__ + - + - :mod:`airflow.providers.sftp.hooks.sftp` + - :mod:`airflow.providers.sftp.operators.sftp` + - :mod:`airflow.providers.sftp.sensors.sftp` + + * - `Secure Shell (SSH) `__ + - + - :mod:`airflow.providers.ssh.hooks.ssh` + - :mod:`airflow.providers.ssh.operators.ssh` + - + + * - `Simple Mail Transfer Protocol (SMTP) `__ + - + - + - :mod:`airflow.providers.email.operators.email` + - + + * - `Windows Remote Management (WinRM) `__ + - + - :mod:`airflow.providers.microsoft.winrm.hooks.winrm` + - :mod:`airflow.providers.microsoft.winrm.operators.winrm` + - + + * - `gRPC `__ + - + - :mod:`airflow.providers.grpc.hooks.grpc` + - :mod:`airflow.providers.grpc.operators.grpc` + - + +Transfer operators and hooks +'''''''''''''''''''''''''''' + +These integrations allow you to copy data. + +.. list-table:: + :header-rows: 1 + + * - Source + - Destination + - Guide + - Operator + + * - `Amazon Simple Storage Service (S3) `_ + - `SSH File Transfer Protocol (SFTP) `__ + - + - :mod:`airflow.providers.amazon.aws.transfers.s3_to_sftp` + + * - Filesystem + - `Azure Blob Storage `__ + - + - :mod:`airflow.providers.microsoft.azure.transfers.file_to_wasb` + + * - Filesystem + - `Google Cloud Storage (GCS) `__ + - + - :mod:`airflow.providers.google.cloud.transfers.local_to_gcs` + + * - `Internet Message Access Protocol (IMAP) `__ + - `Amazon Simple Storage Service (S3) `__ + - :doc:`How to use ` + - :mod:`airflow.providers.amazon.aws.transfers.imap_attachment_to_s3` + + * - `SSH File Transfer Protocol (SFTP) `__ + - `Amazon Simple Storage Service (S3) `_ + - + - :mod:`airflow.providers.amazon.aws.transfers.sftp_to_s3` From 2bca3ab2533d12124d417fce28fbdee1dc48e1f0 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:01:46 -0300 Subject: [PATCH 14/79] Fix tests names and adds sftp_to_wasb to operators-and-hooks-ref list Changes test_get_sftp_files_map to uses a proper name --- docs/operators-and-hooks-ref.rst | 6 ++++++ .../microsoft/azure/transfers/test_sftp_to_wasb.py | 8 ++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index 8c6a7d9338faa..c7f9096398c50 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -1630,6 +1630,12 @@ These integrations allow you to copy data. - - :mod:`airflow.providers.microsoft.azure.transfers.file_to_wasb` + * - SFTP + - `Azure Blob Storage `__ + - + - :mod:`airflow.providers.microsoft.azure.transfers.stfp_to_wasb` + + * - Filesystem - `Google Cloud Storage (GCS) `__ - diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 17c2763b68228..161f20a6e0882 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -47,7 +47,6 @@ # pylint: disable=unused-argument class TestSFTPToWasbOperator(unittest.TestCase): - def test_init(self): operator = STFPToWasbOperator( task_id=TASK_ID, @@ -63,8 +62,6 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): @@ -85,7 +82,7 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') - def test_get_sftp_files_map(self, sftp_hook, mock_hook): + def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], @@ -107,10 +104,9 @@ def test_get_sftp_files_map(self, sftp_hook, mock_hook): self.assertEquals(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') - def test_get_sftp_files_map(self, sftp_hook, mock_hook): + def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], From fb97f2274a87d80093dbc8e5f8b739f8a0b4acaa Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:28:33 -0300 Subject: [PATCH 15/79] Removes usage of deprecated method and unused imports at tests from sftp_to_wasb --- .../microsoft/azure/transfers/test_sftp_to_wasb.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 161f20a6e0882..8e05d4e8d2bea 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -18,14 +18,12 @@ # -import datetime import unittest import os import mock from airflow import AirflowException -from airflow.models.dag import DAG from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME @@ -101,7 +99,7 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEquals(len(files), 2, "not matched at expected found files") + self.assertEqual(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @@ -125,7 +123,7 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEquals(len(files), 1, "no matched at expected found files") + self.assertEqual(len(files), 1, "no matched at expected found files") self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @@ -156,7 +154,7 @@ def test_upload_wasb(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, BLOB_NAME ) - self.assertEquals(len(files), 1, "no matched at expected uploaded files") + self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_sftp_move(self, sftp_hook): From d868c586ec92effda3e205d1af0bbc2157553485 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:43:13 -0300 Subject: [PATCH 16/79] Removes trailing newlines from both files Files with training lines: test_sftp_to_wasb and sftp_to_wasb --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 -- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 1 - 2 files changed, 3 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 75031dbb1ebfd..14eefba43c6ad 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -153,5 +153,3 @@ def sftp_move(self, sftp_hook, uploaded_files): for sftp_file_path in uploaded_files: self.log.info("Executing delete of %s", sftp_file_path) sftp_hook.delete_file(sftp_file_path) - - diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 8e05d4e8d2bea..ecbd5b480a1b4 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -234,4 +234,3 @@ def test_execute(self, sftp_hook, mock_hook): ) sftp_hook.delete_file.assert_not_called() - From 00ca90e623860953cb1c65ee1dc68071824defd2 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 22:57:31 -0300 Subject: [PATCH 17/79] Apply None as the default without type to load_options at sft_to_wasb --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 14eefba43c6ad..15f938ae92ffb 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -68,7 +68,7 @@ def __init__( blob_name: str, sftp_conn_id: str = "ssh_default", wasb_conn_id: str = 'wasb_default', - load_options: dict = None, + load_options=None, move_object: bool = False, *args, **kwargs From 79054974651a57b4e4c29129749815e5e7d78f31 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 23:14:01 -0300 Subject: [PATCH 18/79] Removes unused import from sftp_to_wasb and excessive trailing lines --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 +- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 15f938ae92ffb..2ac8a7cc24cf8 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -20,7 +20,6 @@ """ import os from tempfile import NamedTemporaryFile -from typing import Optional, Union from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -32,6 +31,7 @@ SFTP_FILE_PATH = "SFTP_FILE_PATH" BLOB_NAME = "BLOB_NAME" + class STFPToWasbOperator(BaseOperator): """ Transfer files to Azure Blob Storage from SFTP server. diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index ecbd5b480a1b4..25079ee21a020 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -129,8 +129,6 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_upload_wasb(self, sftp_hook, mock_hook): - - operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, From 401560fff0143696ae5c0f91659e6456ebebc234 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:00:41 -0300 Subject: [PATCH 19/79] Fix typo to sftp_to_wasb file name Adds the information from extra package to CONTRIBUTING.rst --- CONTRIBUTING.rst | 594 ++++++++++-------- .../microsoft/azure/transfers/sftp_to_wasb.py | 62 +- .../microsoft/azure/transfers/stfp_to_wasb.py | 155 ----- docs/operators-and-hooks-ref.rst | 6 - .../azure/transfers/test_sftp_to_wasb.py | 48 +- 5 files changed, 377 insertions(+), 488 deletions(-) delete mode 100644 airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 90be9975b3e90..4f0f6a757919c 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,4 +1,4 @@ -.. Licensed to the Apache Software Foundation (ASF) under one + .. Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file @@ -23,16 +23,6 @@ Contributions Contributions are welcome and are greatly appreciated! Every little bit helps, and credit will always be given. -This document aims to explain the subject of contributions if you have not contributed to -any Open Source project, but it will also help people who have contributed to other projects learn about the -rules of that community. - -New Contributor ---------------- -If you are a new contributor, please follow the `Contributors Quick Start `__ guide to get a gentle step-by-step introduction to setting up the development -environment and making your first contribution. - Get Mentoring Support --------------------- @@ -102,30 +92,24 @@ If you are proposing a new feature: - Remember that this is a volunteer-driven project, and that contributions are welcome :) - -Roles +Documentation ============= -There are several roles within the Airflow Open-Source community. +The latest API documentation is usually available +`here `__. -For detailed information for each role, see: `Committers and PMC's <./COMMITTERS.rst>`__. +To generate a local version: -PMC Member ------------ +1. Set up an Airflow development environment. -The PMC (Project Management Committee) is a group of maintainers that drives changes in the way that -Airflow is managed as a project. +2. Install the ``doc`` extra. -Considering Apache, the role of the PMC is primarily to ensure that Airflow conforms to Apache's processes -and guidelines. +.. code-block:: bash -Committers/Maintainers ----------------------- + pip install -e '.[doc]' -Committers are community members that have write access to the project’s repositories, i.e., they can modify the code, -documentation, and website by themselves and also accept other contributions. -The official list of committers can be found `here `__. +3. Generate and serve the documentation as follows: Additionally, committers are listed in a few other places (some of these may only be visible to existing committers): @@ -220,19 +204,23 @@ You can configure the Docker-based Breeze development environment as follows: .. code-block:: bash - mkvirtualenv myenv --python=python3.6 + cd docs + ./build + ./start_doc_server.sh -5. Initialize the created environment: +.. note:: + The docs build script ``build`` requires Python 3.6 or greater. -.. code-block:: bash +**Known issues:** - ./breeze initialize-local-virtualenv --python 3.6 +If you are creating a new directory for new integration in the ``airflow.providers`` package, +you should also update the ``docs/autoapi_templates/index.rst`` file. -6. Open your IDE (for example, PyCharm) and select the virtualenv you created - as the project's default virtualenv in your IDE. +If you are creating a ``hooks``, ``sensors``, ``operators`` directory in +the ``airflow.providers`` package, you should also update +the ``docs/operators-and-hooks-ref.rst`` file. -Step 3: Connect with People ---------------------------- +If you are creating ``example_dags`` directory, you need to create ``example_dags/__init__.py`` with Apache license or copy another ``__init__.py`` file that contains the necessary license. For effective collaboration, make sure to join the following Airflow groups: @@ -356,7 +344,7 @@ these guidelines: - Include tests, either as doctests, unit tests, or both, to your pull request. - The airflow repo uses `GitHub Actions `__ to + The airflow repo uses `Github Actions `__ to run the tests and `codecov `__ to track coverage. You can set up both for free on your fork. It will help you make sure you do not break the build with your PR and that you help increase coverage. @@ -396,39 +384,16 @@ Airflow Git Branches ==================== All new development in Airflow happens in the ``master`` branch. All PRs should target that branch. - - -We also have a ``v2-0-test`` branch that is used to test ``2.0.x`` series of Airflow and where committers +We also have a ``v1-10-test`` branch that is used to test ``1.10.x`` series of Airflow and where committers cherry-pick selected commits from the master branch. - Cherry-picking is done with the ``-x`` flag. -The ``v2-0-test`` branch might be broken at times during testing. Expect force-pushes there so -committers should coordinate between themselves on who is working on the ``v2-0-test`` branch - +The ``v1-10-test`` branch might be broken at times during testing. Expect force-pushes there so +committers should coordinate between themselves on who is working on the ``v1-10-test`` branch - usually these are developers with the release manager permissions. -The ``v2-0-stable`` branch is rather stable - there are minimum changes coming from approved PRs that -passed the tests. This means that the branch is rather, well, "stable". - -Once the ``v2-0-test`` branch stabilises, the ``v2-0-stable`` branch is synchronized with ``v2-0-test``. -The ``v2-0-stable`` branch is used to release ``2.0.x`` releases. - -The general approach is that cherry-picking a commit that has already had a PR and unit tests run -against main is done to ``v2-0-test`` branch, but PRs from contributors towards 2.0 should target -``v2-0-stable`` branch. - -The ``v2-0-test`` branch and ``v2-0-stable`` ones are merged just before the release and that's the -time when they converge. - -The production images are build in DockerHub from: - -* master branch for development -* v2-0-test branch for testing 2.0.x release -* ``2.0.*``, ``2.0.*rc*`` releases from the ``v2-0-stable`` branch when we prepare release candidates and - final releases. There are no production images prepared from v2-0-stable branch. - -Similar rules apply to ``1.10.x`` releases until June 2020. We have ``v1-10-test`` and ``v1-10-stable`` -branches there. +Once the branch is stable, the ``v1-10-stable`` branch is synchronized with ``v1-10-test``. +The ``v1-10-stable`` branch is used to release ``1.10.x`` releases. Development Environments ======================== @@ -526,7 +491,7 @@ Benefits: - Breeze environment is almost the same as used in the CI automated builds. So, if the tests run in your Breeze environment, they will work in the CI as well. - See ``_ for details about Airflow CI. + See ``_ for details about Airflow CI. Limitations: @@ -547,31 +512,6 @@ Limitations: They are optimized for repeatability of tests, maintainability and speed of building rather than production performance. The production images are not yet officially published. - -Airflow dependencies -==================== - -.. note:: - - On November 2020, new version of PIP (20.3) has been released with a new, 2020 resolver. This resolver - might work with Apache Airflow as of 20.3.3, but it might lead to errors in installation. It might - depend on your choice of extras. In order to install Airflow you might need to either downgrade - pip to version 20.2.4 ``pip install --upgrade pip==20.2.4`` or, in case you use Pip 20.3, - you need to add option ``--use-deprecated legacy-resolver`` to your pip install command. - - While ``pip 20.3.3`` solved most of the ``teething`` problems of 20.3, this note will remain here until we - set ``pip 20.3`` as official version in our CI pipeline where we are testing the installation as well. - Due to those constraints, only ``pip`` installation is currently officially supported. - - While they are some successes with using other tools like `poetry `_ or - `pip-tools `_, they do not share the same workflow as - ``pip`` - especially when it comes to constraint vs. requirements management. - Installing via ``Poetry`` or ``pip-tools`` is not currently supported. - - If you wish to install airflow using those tools you should use the constraint files and convert - them to appropriate format and workflow that your tool requires. - - Extras ------ @@ -599,38 +539,112 @@ virtualenv, webhdfs, winrm, yandex, zendesk .. END EXTRAS HERE -Provider packages ------------------ -Airflow 2.0 is split into core and providers. They are delivered as separate packages: +Airflow dependencies +-------------------- + +Airflow is not a standard python project. Most of the python projects fall into one of two types - +application or library. As described in +[StackOverflow Question](https://stackoverflow.com/questions/28509481/should-i-pin-my-python-dependencies-versions) +decision whether to pin (freeze) requirements for a python project depdends on the type. For +applications, dependencies should be pinned, but for libraries, they should be open. + +For application, pinning the dependencies makes it more stable to install in the future - because new +(even transitive) dependencies might cause installation to fail. For libraries - the dependencies should +be open to allow several different libraries with the same requirements to be installed at the same time. + +The problem is that Apache Airflow is a bit of both - application to install and library to be used when +you are developing your own operators and DAGs. + +This - seemingly unsolvable - puzzle is solved by having pinned requirement files. Those are available +as of airflow 1.10.10. + +Pinned requirement files +------------------------ + +By default when you install ``apache-airflow`` package - the dependencies are as open as possible while +still allowing the apache-airflow package to install. This means that 'apache-airflow' package might fail to +install in case a direct or transitive dependency is released that breaks the installation. In such case +when installing ``apache-airflow``, you might need to provide additional constraints (for +example ``pip install apache-airflow==1.10.2 Werkzeug<1.0.0``) + +However we now have ``requirements-python.txt`` file generated +automatically and committed in the requirements folder based on the set of all latest working and tested +requirement versions. Those ``requirement-python.txt`` files can be used as +constraints file when installing Apache Airflow - either from the sources + +.. code-block:: bash + + pip install -e . --constraint requirements/requirements-python3.6.txt + + +or from the pypi package + +.. code-block:: bash + + pip install apache-airflow --constraint requirements/requirements-python3.6.txt + + +This works also with extras - for example: + +.. code-block:: bash -* ``apache-airflow`` - core of Apache Airflow -* ``apache-airflow-providers-*`` - More than 50 provider packages to communicate with external services + pip install .[ssh] --constraint requirements/requirements-python3.6.txt + + +It is also possible to use constraints directly from github using tag/version name: + +.. code-block:: bash -In Airflow 1.10 all those providers were installed together within one single package and when you installed -airflow locally, from sources, they were also installed. In Airflow 2.0, providers are separated out, -and not packaged together with the core, unless you set ``INSTALL_PROVIDERS_FROM_SOURCES`` environment -variable to ``true``. + pip install apache-airflow[ssh]==1.10.10 \ + --constraint https://raw.githubusercontent.com/apache/airflow/1.10.10/requirements/requirements-python3.6.txt -In Breeze - which is a development environment, ``INSTALL_PROVIDERS_FROM_SOURCES`` variable is set to true, -but you can add ``--skip-installing-airflow-providers-from-sources`` flag to Breeze to skip installing providers when -building the images. +There are different set of fixed requirements for different python major/minor versions and you should +use the right requirements file for the right python version. -One watch-out - providers are still always installed (or rather available) if you install airflow from -sources using ``-e`` (or ``--editable``) flag. In such case airflow is read directly from the sources -without copying airflow packages to the usual installation location, and since 'providers' folder is -in this airflow folder - the providers package is importable. +The ``requirements-python.txt`` file MUST be regenerated every time after +the ``setup.py`` is updated. This is checked automatically in the CI build. There are separate +jobs for each python version that checks if the requirements should be updated. + +If they are not updated, you should regenerate the requirements locally using Breeze as described below. + +Generating requirement files +---------------------------- + +This should be done every time after you modify setup.py file. You can generate requirement files +using `Breeze `_ . Simply use those commands: + +.. code-block:: bash + + breeze generate-requirements --python 3.7 + +.. code-block:: bash + + breeze generate-requirements --python 3.6 + +Note that when you generate requirements this way, you might update to latest version of requirements +that were released since the last time so during tests you might get errors unrelated to your change. +In this case the easiest way to fix it is to limit the culprit dependency to the previous version +with ```` constraint added in setup.py. + +Backport providers packages +--------------------------- + +Since we are developing new operators in the master branch, we prepared backport packages ready to be +installed for Airflow 1.10.* series. Those backport operators (the tested ones) are going to be released +in PyPi and we are going to maintain the list at +`Backported providers package page `_ Some of the packages have cross-dependencies with other providers packages. This typically happens for transfer operators where operators use hooks from the other providers in case they are transferring data between the providers. The list of dependencies is maintained (automatically with pre-commits) in the ``airflow/providers/dependencies.json``. Pre-commits are also used to generate dependencies. -The dependency list is automatically used during PyPI packages generation. +The dependency list is automatically used during pypi packages generation. Cross-dependencies between provider packages are converted into extras - if you need functionality from the other provider package you can install it adding [extra] after the -``apache-airflow-backport-providers-PROVIDER`` for example: -``pip install apache-airflow-backport-providers-google[amazon]`` in case you want to use GCP +apache-airflow-backport-providers-PROVIDER for example ``pip install +apache-airflow-backport-providers-google[amazon]`` in case you want to use GCP transfer operators from Amazon ECS. If you add a new dependency between different providers packages, it will be detected automatically during @@ -661,9 +675,9 @@ apache.hive amazon,microsoft.mssql,mysql,presto,samba,vertica apache.livy http dingding http discord http -google amazon,apache.beam,apache.cassandra,cncf.kubernetes,facebook,microsoft.azure,microsoft.mssql,mysql,oracle,postgres,presto,salesforce,sftp,ssh +google amazon,apache.cassandra,cncf.kubernetes,facebook,microsoft.azure,microsoft.mssql,mysql,postgres,presto,sftp hashicorp google -microsoft.azure google,oracle +microsoft.azure oracle microsoft.mssql odbc mysql amazon,presto,vertica opsgenie http @@ -987,46 +1001,24 @@ If this function is designed to be called by "end-users" (i.e. DAG authors) then Don't use time() for duration calculations ----------------------------------------- -If you wish to compute the time difference between two events with in the same process, use -``time.monotonic()``, not ``time.time()`` nor ``timzeone.utcnow()``. - -If you are measuring duration for performance reasons, then ``time.perf_counter()`` should be used. (On many -platforms, this uses the same underlying clock mechanism as monotonic, but ``perf_counter`` is guaranteed to be -the highest accuracy clock on the system, monotonic is simply "guaranteed" to not go backwards.) - -If you wish to time how long a block of code takes, use ``Stats.timer()`` -- either with a metric name, which -will be timed and submitted automatically: - -.. code-block:: python - - from airflow.stats import Stats - - ... - - with Stats.timer("my_timer_metric"): - ... - -or to time but not send a metric: - -.. code-block:: python - - from airflow.stats import Stats +We support the following types of tests: - ... +* **Unit tests** are Python tests launched with ``pytest``. + Unit tests are available both in the `Breeze environment `_ + and `local virtualenv `_. - with Stats.timer() as timer: - ... +* **Integration tests** are available in the Breeze development environment + that is also used for Airflow's CI tests. Integration test are special tests that require + additional services running, such as Postgres, Mysql, Kerberos, etc. - log.info("Code took %.3f seconds", timer.duration) +* **System tests** are automatic tests that use external systems like + Google Cloud Platform. These tests are intended for an end-to-end DAG execution. -For full docs on ``timer()`` check out `airflow/stats.py`_. +For details on running different types of Airflow tests, see `TESTING.rst `_. -If the start_date of a duration calculation needs to be stored in a database, then this has to be done using -datetime objects. In all other cases, using datetime for duration calculation MUST be avoided as creating and -diffing datetime operations are (comparatively) slow. Naming Conventions for provider packages ----------------------------------------- +======================================== In Airflow 2.0 we standardized and enforced naming for provider packages, modules and classes. those rules (introduced as AIP-21) were not only introduced but enforced using automated checks @@ -1054,18 +1046,18 @@ The rules are as follows: * secrets -> secret backends are stored here * transfers -> transfer operators are stored here -* Module names do not contain word "hooks", "operators" etc. The right type comes from +* Module names do not contain word "hooks" , "operators" etc. The right type comes from the package. For example 'hooks.datastore' module contains DataStore hook and 'operators.datastore' contains DataStore operators. * Class names contain 'Operator', 'Hook', 'Sensor' - for example DataStoreHook, DataStoreExportOperator -* Operator name usually follows the convention: ``Operator`` +* Operator name usually follows the convention: Operator (BigQueryExecuteQueryOperator) is a good example * Transfer Operators are those that actively push data from one service/provider and send it to another service (might be for the same or another provider). This usually involves two hooks. The convention - for those ``ToOperator``. They are not named *TransferOperator nor *Transfer. + for those ToOperator. They are not named *TransferOperator nor *Transfer. * Operators that use external service to perform transfer (for example CloudDataTransferService operators are not placed in "transfers" package and do not have to follow the naming convention for @@ -1079,32 +1071,14 @@ The rules are as follows: * For Cloud Providers or Service providers that usually means that the transfer operators should land at the "target" side of the transfer -* Secret Backend name follows the convention: ``Backend``. +* Secret Backend name follows the convention: Backend. -* Tests are grouped in parallel packages under "tests.providers" top level package. Module name is usually - ``test_.py``, +* Tests are grouped in parallel packages under "tests.providers" top level package. Module name is usually + "test_.py', * System tests (not yet fully automated but allowing to run e2e testing of particular provider) are named with _system.py suffix. -Test Infrastructure -=================== - -We support the following types of tests: - -* **Unit tests** are Python tests launched with ``pytest``. - Unit tests are available both in the `Breeze environment `_ - and `local virtualenv `_. - -* **Integration tests** are available in the Breeze development environment - that is also used for Airflow's CI tests. Integration test are special tests that require - additional services running, such as Postgres, Mysql, Kerberos, etc. - -* **System tests** are automatic tests that use external systems like - Google Cloud. These tests are intended for an end-to-end DAG execution. - -For details on running different types of Airflow tests, see `TESTING.rst `_. - Metadata Database Updates ========================= @@ -1147,7 +1121,7 @@ To install yarn on macOS: .. code-block:: bash - brew install node + brew install node --without-npm brew install yarn yarn config set prefix ~/.yarn @@ -1200,13 +1174,13 @@ commands: yarn run dev -Follow JavaScript Style Guide +Follow Javascript Style Guide ----------------------------- We try to enforce a more consistent style and follow the JS community guidelines. -Once you add or modify any JavaScript code in the project, please make sure it +Once you add or modify any javascript code in the project, please make sure it follows the guidelines defined in `Airbnb JavaScript Style Guide `__. @@ -1222,125 +1196,226 @@ commands: # Check JS code in .js and .html files, report any errors/warnings and fix them if possible yarn run lint:fix -How to sync your fork -===================== +Contribution Workflow Example +============================== -When you have your fork, you should periodically synchronize the master of your fork with the -Apache Airflow master. In order to do that you can ``git pull --rebase`` to your local git repository from -apache remote and push the master (often with ``--force`` to your fork). There is also an easy -way using ``Force sync master from apache/airflow`` workflow. You can go to "Actions" in your repository and -choose the workflow and manually trigger the workflow using "Run workflow" command. +Typically, you start your first contribution by reviewing open tickets +at `GitHub issues `__. -This will force-push the master from apache/airflow to the master in your fork. Note that in case you -modified the master in your fork, you might loose those changes. +If you create pull-request, you don't have to create an issue first, but if you want, you can do it. +Creating an issue will allow you to collect feedback or share plans with other people. +For example, you want to have the following sample ticket assigned to you: +`#7782: Add extra CC: to the emails sent by Aiflow `_. -How to rebase PR -================ +In general, your contribution includes the following stages: -A lot of people are unfamiliar with the rebase workflow in Git, but we think it is an excellent workflow, -providing a better alternative to the merge workflow. We've therefore written a short guide for those who would like to learn it. +.. image:: images/workflow.png + :align: center + :alt: Contribution Workflow -As opposed to the merge workflow, the rebase workflow allows us to -clearly separate your changes from the changes of others. It puts the responsibility of rebasing on the -author of the change. It also produces a "single-line" series of commits on the master branch. This -makes it easier to understand what was going on and to find reasons for problems (it is especially -useful for "bisecting" when looking for a commit that introduced some bugs). +1. Make your own `fork `__ of + the Apache Airflow `main repository `__. -First of all, we suggest you read about the rebase workflow here: -`Merging vs. rebasing `_. This is an -excellent article that describes all the ins/outs of the rebase workflow. I recommend keeping it for future reference. +2. Create a `local virtualenv `_, + initialize the `Breeze environment `__, and + install `pre-commit framework `__. + If you want to add more changes in the future, set up your fork and enable Github Actions. -The goal of rebasing your PR on top of ``apache/master`` is to "transplant" your change on top of -the latest changes that are merged by others. It also allows you to fix all the conflicts -that arise as a result of other people changing the same files as you and merging the changes to ``apache/master``. +3. Join `devlist `__ + and set up a `Slack account `__. -Here is how rebase looks in practice (you can find a summary below these detailed steps): +4. Make the change and create a `Pull Request from your fork `__. -1. You first need to add the Apache project remote to your git repository. This is only necessary once, -so if it's not the first time you are following this tutorial you can skip this step. In this example, -we will be adding the remote -as "apache" so you can refer to it easily: +5. Ping @ #development slack, comment @people. Be annoying. Be considerate. -* If you use ssh: ``git remote add apache git@github.com:apache/airflow.git`` -* If you use https: ``git remote add apache https://github.com/apache/airflow.git`` +Step 1: Fork the Apache Repo +---------------------------- +From the `apache/airflow `_ repo, +`create a fork `_: -2. You then need to make sure that you have the latest master fetched from the ``apache`` repository. You can do this - via: +.. image:: images/fork.png + :align: center + :alt: Creating a fork - ``git fetch apache`` (to fetch apache remote) - ``git fetch --all`` (to fetch all remotes) +Step 2: Configure Your Environment +---------------------------------- +Configure the Docker-based Breeze development environment and run tests. -3. Assuming that your feature is in a branch in your repository called ``my-branch`` you can easily check - what is the base commit you should rebase from by: +You can use the default Breeze configuration as follows: - ``git merge-base my-branch apache/master`` +1. Install the latest versions of the Docker Community Edition + and Docker Compose and add them to the PATH. - This will print the HASH of the base commit which you should use to rebase your feature from. - For example: ``5abce471e0690c6b8d06ca25685b0845c5fd270f``. Copy that HASH and go to the next step. +2. Enter Breeze: ``./breeze`` - Optionally, if you want better control you can also find this commit hash manually. + Breeze starts with downloading the Airflow CI image from + the Docker Hub and installing all required dependencies. - Run: +3. Enter the Docker environment and mount your local sources + to make them immediately visible in the environment. - ``git log`` +4. Create a local virtualenv, for example: - And find the first commit that you DO NOT want to "transplant". +.. code-block:: bash - Performing: + mkvirtualenv myenv --python=python3.6 - ``git rebase HASH`` +5. Initialize the created environment: - Will "transplant" all commits after the commit with the HASH. +.. code-block:: bash -4. Providing that you weren't already working on your branch, check out your feature branch locally via: + ./breeze --initialize-local-virtualenv - ``git checkout my-branch`` +6. Open your IDE (for example, PyCharm) and select the virtualenv you created + as the project's default virtualenv in your IDE. -5. Rebase: +Step 3: Connect with People +--------------------------- - ``git rebase HASH --onto apache/master`` +For effective collaboration, make sure to join the following Airflow groups: - For example: +- Mailing lists: - ``git rebase 5abce471e0690c6b8d06ca25685b0845c5fd270f --onto apache/master`` + - Developer’s mailing list ``_ + (quite substantial traffic on this list) -6. If you have no conflicts - that's cool. You rebased. You can now run ``git push --force-with-lease`` to - push your changes to your repository. That should trigger the build in our CI if you have a - Pull Request (PR) opened already. + - All commits mailing list: ``_ + (very high traffic on this list) -7. While rebasing you might have conflicts. Read carefully what git tells you when it prints information - about the conflicts. You need to solve the conflicts manually. This is sometimes the most difficult - part and requires deliberately correcting your code and looking at what has changed since you developed your - changes. + - Airflow users mailing list: ``_ + (reasonably small traffic on this list) - There are various tools that can help you with this. You can use: +- `Issues on GitHub `__ - ``git mergetool`` +- `Slack (chat) `__ - You can configure different merge tools with it. You can also use IntelliJ/PyCharm's excellent merge tool. - When you open a project in PyCharm which has conflicts, you can go to VCS > Git > Resolve Conflicts and there - you have a very intuitive and helpful merge tool. For more information, see - `Resolve conflicts `_. +Step 4: Prepare PR +------------------ -8. After you've solved your conflict run: +1. Update the local sources to address the issue. + + For example, to address this example issue, do the following: - ``git rebase --continue`` + * Read about `email configuration in Airflow `__. - And go either to point 6. or 7, depending on whether you have more commits that cause conflicts in your PR (rebasing applies each - commit from your PR one-by-one). + * Find the class you should modify. For the example ticket, + this is `email.py `__. + + * Find the test class where you should add tests. For the example ticket, + this is `test_email.py `__. -Summary -------------- + * Create a local branch for your development. Make sure to use latest + ``apache/master`` as base for the branch. See `How to Rebase PR <#how-to-rebase-pr>`_ for some details + on setting up the ``apache`` remote. Note - some people develop their changes directy in their own + ``master`` branches - this is OK and you can make PR from your master to ``apache/master`` but we + recommend to always create a local branch for your development. This allows you to easily compare + changes, have several changes that you work on at the same time and many more. + If you have ``apache`` set as remote then you can make sure that you have latest changes in your master + by ``git pull apache master`` when you are in the local ``master`` branch. If you have conflicts and + want to override your locally changed master you can override your local changes with + ``git fetch apache; git reset --hard apache/master``. -Useful when you understand the flow but don't remember the steps and want a quick reference. + * Modify the class and add necessary code and unit tests. + + * Run the unit tests from the `IDE `__ + or `local virtualenv `__ as you see fit. + + * Run the tests in `Breeze `__. + + * Run and fix all the `static checks `__. If you have + `pre-commits installed `__, + this step is automatically run while you are committing your code. If not, you can do it manually + via ``git add`` and then ``pre-commit run``. + +2. Rebase your fork, squash commits, and resolve all conflicts. See `How to rebase PR <#how-to-rebase-pr>`_ + if you need help with rebasing your change. Remember to rebase often if your PR takes a lot of time to + review/fix. This will make rebase process much easier and less painful - and the more often you do it, + the more comfortable you will feel doing it. + +3. Re-run static code checks again. + +4. Create a pull request with the following title for the sample ticket: + ``[AIRFLOW-5934] Added extra CC: field to the Airflow emails.`` + +Make sure to follow other PR guidelines described in `this document <#pull-request-guidelines>`_. + + +Step 5: Pass PR Review +---------------------- + +.. image:: images/review.png + :align: center + :alt: PR Review + +Note that committers will use **Squash and Merge** instead of **Rebase and Merge** +when merging PRs and your commit will be squashed to single commit. + +How to rebase PR +================ + +A lot of people are unfamiliar with rebase workflow in Git, but we think it is an excellent workflow, +much better than merge workflow, so here is a short guide for those who would like to learn it. It's really +worth to spend a few minutes learning it. As opposed to merge workflow, the rebase workflow allows to +clearly separate your changes from changes of others, puts responsibility of proper rebase on the +author of the change. It also produces a "single-line" series of commits in master branch which +makes it much easier to understand what was going on and to find reasons for problems (it is especially +useful for "bisecting" when looking for a commit that introduced some bugs. + + +First of all - you can read about rebase workflow here: +`Merging vs. rebasing `_ - this is an +excellent article that describes all ins/outs of rebase. I recommend reading it and keeping it as reference. + +The goal of rebasing your PR on top of ``apache/master`` is to "transplant" your change on top of +the latest changes that are merged by others. It also allows you to fix all the conflicts +that are result of other people changing the same files as you and merging the changes to ``apache/master``. + +Here is how rebase looks in practice: + +1. You need to add Apache remote to your git repository. You can add it as "apache" remote so that + you can refer to it easily: + +``git remote add apache git@github.com:apache/airflow.git`` if you use ssh or +``git remote add apache https://github.com/apache/airflow.git`` if you use https. + +Later on + +2. You need to make sure that you have the latest master fetched from ``apache`` repository. You can do it + by ``git fetch apache`` for apache remote or ``git fetch --all`` to fetch all remotes. + +3. Assuming that your feature is in a branch in your repository called ``my-branch`` you can check easily + what is the base commit you should rebase from by: ``git merge-base my-branch apache/master``. + This will print the HASH of the base commit which you should use to rebase your feature from - + for example: ``5abce471e0690c6b8d06ca25685b0845c5fd270f``. You can also find this commit hash manually - + if you want better control. Run ``git log`` and find the first commit that you DO NOT want to "transplant". + ``git rebase HASH`` will "trasplant" all commits after the commit with the HASH. + +4. Make sure you checked out your branch locally: -``git fetch --all`` -``git merge-base my-branch apache/master`` ``git checkout my-branch`` -``git rebase HASH --onto apache/master`` -``git push --force-with-lease`` + +5. Rebase: + Run: ``git rebase HASH --onto apache/master`` + for example: ``git rebase 5abce471e0690c6b8d06ca25685b0845c5fd270f --onto apache/master`` + +6. If you have no conflicts - that's cool. You rebased. You can now run ``git push --force-with-lease`` to + push your changes to your repository. That should trigger the build in our CI if you have a + Pull Request opened already. + +7. While rebasing you might have conflicts. Read carefully what git tells you when it prints information + about the conflicts. You need to solve the conflicts manually. This is sometimes the most difficult + part and requires deliberate correcting your code looking what has changed since you developed your + changes. There are various tools that can help you with that. You can use ``git mergetool`` (and you can + configure different merge tools with it). Also you can use IntelliJ/PyCharm excellent merge tool. + When you open project in PyCharm which has conflict you can go to VCS->Git->Resolve Conflicts and there + you have a very intuitive and helpful merge tool. You can see more information + about it in `Resolve conflicts `_ + +8. After you solved conflicts simply run ``git rebase --continue`` and go either to point 6. or 7. + above depending if you have more commits that cause conflicts in your PR (rebasing applies each + commit from your PR one-by-one). How to communicate ================== @@ -1368,17 +1443,17 @@ You can join the channels via links at the `Airflow Community page `_ for: +* Github `Pull Requests (PRs) `_ for: * discussing implementation details of PRs * not for architectural discussions (use the devlist for that) * The deprecated `JIRA issues `_ for: - * checking out old but still valuable issues that are not on GitHub yet - * mentioning the JIRA issue number in the title of the related PR you would like to open on GitHub + * checking out old but still valuable issues that are not on Github yet + * mentioning the JIRA issue number in the title of the related PR you would like to open on Github **IMPORTANT** -We don't create new issues on JIRA anymore. The reason we still look at JIRA issues is that there are valuable tickets inside of it. However, each new PR should be created on `GitHub issues `_ as stated in `Contribution Workflow Example `_ +We don't create new issues on JIRA anymore. The reason we still look at JIRA issues is that there are valuable tickets inside of it. However, each new PR should be created on `Github issues `_ as stated in `Contribution Workflow Example `_ -* The `Apache Airflow Slack `_ for: +* The `Apache Airflow Slack `_ for: * ad-hoc questions related to development (#development channel) * asking for review (#development channel) * asking for help with PRs (#how-to-pr channel) @@ -1451,21 +1526,8 @@ Here are a few rules that are important to keep in mind when you enter our commu * It’s OK to express your own emotions while communicating - it helps other people to understand you * Be considerate for feelings of others. Tell about how you feel not what you think of others - - -Commit Policy -============= - -The following commit policy passed by a vote 8(binding FOR) to 0 against on May 27, 2016 on the dev list -and slightly modified and consensus reached in October 2020: - -* Commits need a +1 vote from a committer who is not the author -* Do not merge a PR that regresses linting or does not pass CI tests (unless we have - justification such as clearly transient error). -* When we do AIP voting, both PMC and committer +1s are considered as binding vote. - Resources & Links ================= -- `Airflow’s official documentation `__ +- `Airflow’s official documentation `__ - `More resources and links to Airflow related content on the Wiki `__ diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 4f4b51140f992..b03bbd623fd42 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -19,11 +19,7 @@ This module contains SFTP to Azure Blob Storage operator. """ import os -from collections import namedtuple from tempfile import NamedTemporaryFile -from typing import Any, Dict, List, Optional - -from cached_property import cached_property from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -32,7 +28,8 @@ from airflow.utils.decorators import apply_defaults WILDCARD = "*" -SftpFile = namedtuple('SftpFile', 'sftp_file_path, blob_name') +SFTP_FILE_PATH = "SFTP_FILE_PATH" +BLOB_NAME = "BLOB_NAME" class SFTPToWasbOperator(BaseOperator): @@ -69,9 +66,9 @@ def __init__( sftp_source_path: str, container_name: str, blob_name: str, - sftp_conn_id: str = "sftp_default", + sftp_conn_id: str = "ssh_default", wasb_conn_id: str = 'wasb_default', - load_options: Optional[dict] = None, + load_options=None, move_object: bool = False, *args, **kwargs @@ -87,17 +84,16 @@ def __init__( self.load_options = load_options if load_options is not None else {} self.move_object = move_object - def execute(self, - context: Optional[Dict[Any, Any]]) -> None: + def execute(self, context): """Upload a file from SFTP to Azure Blob Storage.""" - sftp_files: List[SftpFile] = self.get_sftp_files_map() - uploaded_files = self.copy_files_to_wasb(sftp_files) - if self.move_object: - self.delete_files(uploaded_files) + (sftp_hook, sftp_files) = self.get_sftp_files_map() + uploaded_files = self.upload_wasb(sftp_hook, sftp_files) + self.sftp_move(sftp_hook, uploaded_files) - def get_sftp_files_map(self) -> List[SftpFile]: + def get_sftp_files_map(self): """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] + sftp_hook = SFTPHook(self.sftp_conn_id) if WILDCARD in self.sftp_source_path: self.check_wildcards_limit() @@ -105,7 +101,7 @@ def get_sftp_files_map(self) -> List[SftpFile]: prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) sftp_complete_path = os.path.dirname(prefix) - found_files, _, _ = self.sftp_hook.get_tree_map( + found_files, _, _ = sftp_hook.get_tree_map( sftp_complete_path, prefix=prefix, delimiter=delimiter ) @@ -117,19 +113,14 @@ def get_sftp_files_map(self) -> List[SftpFile]: for file in found_files: future_blob_name = os.path.basename(file) - sftp_files.append(SftpFile(file, future_blob_name)) + sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) else: - sftp_files.append(SftpFile(self.sftp_source_path, self.blob_name)) + sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) - return sftp_files + return sftp_hook, sftp_files - @cached_property - def sftp_hook(self): - """Property of sftp hook to be re-used.""" - return SFTPHook(self.sftp_conn_id) - - def check_wildcards_limit(self) -> Any: + def check_wildcards_limit(self): """Check if there is multiple Wildcard.""" total_wildcards = self.sftp_source_path.count(WILDCARD) if total_wildcards > 1: @@ -138,33 +129,32 @@ def check_wildcards_limit(self) -> Any: "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) - def copy_files_to_wasb(self, - sftp_files: List[SftpFile]) -> List[str]: + def upload_wasb(self, sftp_hook, sftp_files): """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) for file in sftp_files: with NamedTemporaryFile("w") as tmp: - self.sftp_hook.retrieve_file(file.sftp_file_path, tmp.name) + sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) self.log.info( 'Uploading %s to wasb://%s as %s', - file.sftp_file_path, + file[SFTP_FILE_PATH], self.container_name, - file.blob_name, + file[BLOB_NAME], ) wasb_hook.load_file(tmp.name, self.container_name, - file.blob_name, + file[BLOB_NAME], **self.load_options) - uploaded_files.append(file.sftp_file_path) + uploaded_files.append(file[SFTP_FILE_PATH]) return uploaded_files - def delete_files(self, - uploaded_files: List[str]) -> None: + def sftp_move(self, sftp_hook, uploaded_files): """Performs a move of a list of files at SFTP to Azure Blob Storage.""" - for sftp_file_path in uploaded_files: - self.log.info("Executing delete of %s", sftp_file_path) - self.sftp_hook.delete_file(sftp_file_path) + if self.move_object: + for sftp_file_path in uploaded_files: + self.log.info("Executing delete of %s", sftp_file_path) + sftp_hook.delete_file(sftp_file_path) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py deleted file mode 100644 index 2ac8a7cc24cf8..0000000000000 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ /dev/null @@ -1,155 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -This module contains SFTP to Azure Blob Storage operator. -""" -import os -from tempfile import NamedTemporaryFile - -from airflow.exceptions import AirflowException -from airflow.models import BaseOperator -from airflow.providers.microsoft.azure.hooks.wasb import WasbHook -from airflow.providers.sftp.hooks.sftp import SFTPHook -from airflow.utils.decorators import apply_defaults - -WILDCARD = "*" -SFTP_FILE_PATH = "SFTP_FILE_PATH" -BLOB_NAME = "BLOB_NAME" - - -class STFPToWasbOperator(BaseOperator): - """ - Transfer files to Azure Blob Storage from SFTP server. - - :param sftp_source_path: The sftp remote path. This is the specified file path - for downloading the single file or multiple files from the SFTP server. - You can use only one wildcard within your path. The wildcard can appear - inside the path or at the end of the path. - :type sftp_source_path: str - :param container_name: Name of the container. - :type container_name: str - :param blob_name: Name of the blob. - :type blob_name: str - :param sftp_conn_id: The sftp connection id. The name or identifier for - establishing a connection to the SFTP server. - :type sftp_conn_id: str - :param wasb_conn_id: Reference to the wasb connection. - :type wasb_conn_id: str - :param load_options: Optional keyword arguments that - `WasbHook.load_file()` takes. - :type load_options: dict - :param move_object: When move object is True, the object is moved instead - of copied to the new location. This is the equivalent of a mv command - as opposed to a cp command. - :type move_object: bool - """ - template_fields = ("sftp_source_path", "container_name", "blob_name") - - @apply_defaults - def __init__( - self, - sftp_source_path: str, - container_name: str, - blob_name: str, - sftp_conn_id: str = "ssh_default", - wasb_conn_id: str = 'wasb_default', - load_options=None, - move_object: bool = False, - *args, - **kwargs - ) -> None: - super().__init__(*args, **kwargs) - - self.sftp_source_path = sftp_source_path - self.sftp_conn_id = sftp_conn_id - self.wasb_conn_id = wasb_conn_id - self.container_name = container_name - self.blob_name = blob_name - self.wasb_conn_id = wasb_conn_id - self.load_options = load_options if load_options is not None else {} - self.move_object = move_object - - def execute(self, context): - (sftp_hook, sftp_files) = self.get_sftp_files_map() - uploaded_files = self.upload_wasb(sftp_hook, sftp_files) - self.sftp_move(sftp_hook, uploaded_files) - - def get_sftp_files_map(self): - sftp_files = [] - sftp_hook = SFTPHook(self.sftp_conn_id) - - if WILDCARD in self.sftp_source_path: - self.check_wildcards_limit() - - prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) - sftp_complete_path = os.path.dirname(prefix) - - found_files, _, _ = sftp_hook.get_tree_map( - sftp_complete_path, prefix=prefix, delimiter=delimiter - ) - - self.log.info( - "Found %s files at sftp source path: %s", - str(len(found_files)), - self.sftp_source_path - ) - - for file in found_files: - future_blob_name = os.path.basename(file) - sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) - - else: - sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) - - return sftp_hook, sftp_files - - def check_wildcards_limit(self): - total_wildcards = self.sftp_source_path.count(WILDCARD) - if total_wildcards > 1: - raise AirflowException( - "Only one wildcard '*' is allowed in sftp_source_path parameter. " - "Found {} in {}.".format(total_wildcards, self.sftp_source_path) - ) - - def upload_wasb(self, sftp_hook, sftp_files): - uploaded_files = [] - wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) - for file in sftp_files: - - with NamedTemporaryFile("w") as tmp: - sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) - self.log.info( - 'Uploading %s to wasb://%s as %s', - file[SFTP_FILE_PATH], - self.container_name, - file[BLOB_NAME], - ) - wasb_hook.load_file(tmp.name, - self.container_name, - file[BLOB_NAME], - **self.load_options) - - uploaded_files.append(file[SFTP_FILE_PATH]) - - return uploaded_files - - def sftp_move(self, sftp_hook, uploaded_files): - if self.move_object: - for sftp_file_path in uploaded_files: - self.log.info("Executing delete of %s", sftp_file_path) - sftp_hook.delete_file(sftp_file_path) diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index c7f9096398c50..8c6a7d9338faa 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -1630,12 +1630,6 @@ These integrations allow you to copy data. - - :mod:`airflow.providers.microsoft.azure.transfers.file_to_wasb` - * - SFTP - - `Azure Blob Storage `__ - - - - :mod:`airflow.providers.microsoft.azure.transfers.stfp_to_wasb` - - * - Filesystem - `Google Cloud Storage (GCS) `__ - diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 25079ee21a020..34b701ed9acd5 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -17,17 +17,15 @@ # under the License. # - -import unittest import os +import unittest import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME - +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator, \ + SFTP_FILE_PATH, \ + BLOB_NAME TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" @@ -46,7 +44,7 @@ class TestSFTPToWasbOperator(unittest.TestCase): def test_init(self): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -60,10 +58,10 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, @@ -78,15 +76,15 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_PATH, sftp_conn_id=SFTP_CONN_ID, @@ -102,15 +100,15 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): self.assertEqual(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -126,10 +124,10 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected found files") self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_upload_wasb(self, sftp_hook, mock_hook): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -154,11 +152,11 @@ def test_upload_wasb(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected uploaded files") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -177,11 +175,11 @@ def test_sftp_move(self, sftp_hook): ] ) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_no_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -196,8 +194,8 @@ def test_no_sftp_move(self, sftp_hook): sftp_mock.delete_file.assert_not_called() - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], @@ -205,7 +203,7 @@ def test_execute(self, sftp_hook, mock_hook): [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, From c2d26b9467c4597e7b2cb765f511c051b0e30c2c Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:30:34 -0300 Subject: [PATCH 20/79] Adds dependency to dependencies.json Fix import at test_sftp_to_wasb. --- .../providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 34b701ed9acd5..bac52a5839da4 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -23,9 +23,8 @@ import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator, \ - SFTP_FILE_PATH, \ - BLOB_NAME +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( + BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator) TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" From 35e4f01327019b02970f89dfc272f5e52068b866 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:59:49 -0300 Subject: [PATCH 21/79] Fix changes made by hooks --- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index bac52a5839da4..e38b4df07aded 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -24,7 +24,8 @@ from airflow import AirflowException from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( - BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator) + BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator, +) TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" From 0b161c8befb29c9f63de763b0f17038f4cdbdca5 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 12 Jul 2020 13:53:06 -0300 Subject: [PATCH 22/79] Fix type hint --- .../microsoft/azure/transfers/sftp_to_wasb.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index b03bbd623fd42..8bae74c49c0bb 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -20,6 +20,7 @@ """ import os from tempfile import NamedTemporaryFile +from typing import Any, Dict, List, Optional, Tuple from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -68,7 +69,7 @@ def __init__( blob_name: str, sftp_conn_id: str = "ssh_default", wasb_conn_id: str = 'wasb_default', - load_options=None, + load_options: Optional[dict] = None, move_object: bool = False, *args, **kwargs @@ -84,13 +85,14 @@ def __init__( self.load_options = load_options if load_options is not None else {} self.move_object = move_object - def execute(self, context): + def execute(self, + context: Optional[Dict[Any, Any]]) -> None: """Upload a file from SFTP to Azure Blob Storage.""" (sftp_hook, sftp_files) = self.get_sftp_files_map() uploaded_files = self.upload_wasb(sftp_hook, sftp_files) self.sftp_move(sftp_hook, uploaded_files) - def get_sftp_files_map(self): + def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] sftp_hook = SFTPHook(self.sftp_conn_id) @@ -120,7 +122,7 @@ def get_sftp_files_map(self): return sftp_hook, sftp_files - def check_wildcards_limit(self): + def check_wildcards_limit(self) -> Any: """Check if there is multiple Wildcard.""" total_wildcards = self.sftp_source_path.count(WILDCARD) if total_wildcards > 1: @@ -129,7 +131,9 @@ def check_wildcards_limit(self): "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) - def upload_wasb(self, sftp_hook, sftp_files): + def upload_wasb(self, + sftp_hook: SFTPHook, + sftp_files: List[Dict[str, str]]) -> List[str]: """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) @@ -152,7 +156,9 @@ def upload_wasb(self, sftp_hook, sftp_files): return uploaded_files - def sftp_move(self, sftp_hook, uploaded_files): + def sftp_move(self, + sftp_hook: SFTPHook, + uploaded_files: List[str]) -> None: """Performs a move of a list of files at SFTP to Azure Blob Storage.""" if self.move_object: for sftp_file_path in uploaded_files: From ec85cb2c4d93dd8533f239516636db2a187479db Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 9 Aug 2020 18:56:13 -0300 Subject: [PATCH 23/79] Cleaning code --- .../microsoft/azure/transfers/sftp_to_wasb.py | 60 ++++++++++--------- .../azure/transfers/test_sftp_to_wasb.py | 55 +++++------------ 2 files changed, 48 insertions(+), 67 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 8bae74c49c0bb..4f4b51140f992 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -19,8 +19,11 @@ This module contains SFTP to Azure Blob Storage operator. """ import os +from collections import namedtuple from tempfile import NamedTemporaryFile -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional + +from cached_property import cached_property from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -29,8 +32,7 @@ from airflow.utils.decorators import apply_defaults WILDCARD = "*" -SFTP_FILE_PATH = "SFTP_FILE_PATH" -BLOB_NAME = "BLOB_NAME" +SftpFile = namedtuple('SftpFile', 'sftp_file_path, blob_name') class SFTPToWasbOperator(BaseOperator): @@ -67,7 +69,7 @@ def __init__( sftp_source_path: str, container_name: str, blob_name: str, - sftp_conn_id: str = "ssh_default", + sftp_conn_id: str = "sftp_default", wasb_conn_id: str = 'wasb_default', load_options: Optional[dict] = None, move_object: bool = False, @@ -88,14 +90,14 @@ def __init__( def execute(self, context: Optional[Dict[Any, Any]]) -> None: """Upload a file from SFTP to Azure Blob Storage.""" - (sftp_hook, sftp_files) = self.get_sftp_files_map() - uploaded_files = self.upload_wasb(sftp_hook, sftp_files) - self.sftp_move(sftp_hook, uploaded_files) + sftp_files: List[SftpFile] = self.get_sftp_files_map() + uploaded_files = self.copy_files_to_wasb(sftp_files) + if self.move_object: + self.delete_files(uploaded_files) - def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: + def get_sftp_files_map(self) -> List[SftpFile]: """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] - sftp_hook = SFTPHook(self.sftp_conn_id) if WILDCARD in self.sftp_source_path: self.check_wildcards_limit() @@ -103,7 +105,7 @@ def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) sftp_complete_path = os.path.dirname(prefix) - found_files, _, _ = sftp_hook.get_tree_map( + found_files, _, _ = self.sftp_hook.get_tree_map( sftp_complete_path, prefix=prefix, delimiter=delimiter ) @@ -115,12 +117,17 @@ def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: for file in found_files: future_blob_name = os.path.basename(file) - sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) + sftp_files.append(SftpFile(file, future_blob_name)) else: - sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) + sftp_files.append(SftpFile(self.sftp_source_path, self.blob_name)) - return sftp_hook, sftp_files + return sftp_files + + @cached_property + def sftp_hook(self): + """Property of sftp hook to be re-used.""" + return SFTPHook(self.sftp_conn_id) def check_wildcards_limit(self) -> Any: """Check if there is multiple Wildcard.""" @@ -131,36 +138,33 @@ def check_wildcards_limit(self) -> Any: "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) - def upload_wasb(self, - sftp_hook: SFTPHook, - sftp_files: List[Dict[str, str]]) -> List[str]: + def copy_files_to_wasb(self, + sftp_files: List[SftpFile]) -> List[str]: """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) for file in sftp_files: with NamedTemporaryFile("w") as tmp: - sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) + self.sftp_hook.retrieve_file(file.sftp_file_path, tmp.name) self.log.info( 'Uploading %s to wasb://%s as %s', - file[SFTP_FILE_PATH], + file.sftp_file_path, self.container_name, - file[BLOB_NAME], + file.blob_name, ) wasb_hook.load_file(tmp.name, self.container_name, - file[BLOB_NAME], + file.blob_name, **self.load_options) - uploaded_files.append(file[SFTP_FILE_PATH]) + uploaded_files.append(file.sftp_file_path) return uploaded_files - def sftp_move(self, - sftp_hook: SFTPHook, - uploaded_files: List[str]) -> None: + def delete_files(self, + uploaded_files: List[str]) -> None: """Performs a move of a list of files at SFTP to Azure Blob Storage.""" - if self.move_object: - for sftp_file_path in uploaded_files: - self.log.info("Executing delete of %s", sftp_file_path) - sftp_hook.delete_file(sftp_file_path) + for sftp_file_path in uploaded_files: + self.log.info("Executing delete of %s", sftp_file_path) + self.sftp_hook.delete_file(sftp_file_path) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index e38b4df07aded..8f02c8fff60e1 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -23,14 +23,13 @@ import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( - BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator, -) +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" SFTP_CONN_ID = "ssh_default" +BLOB_NAME = "BLOB_NAME" CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" @@ -93,12 +92,10 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=False ) - _, files = operator.get_sftp_files_map() - - sftp_hook.assert_called_once_with(SFTP_CONN_ID) + files = operator.get_sftp_files_map() self.assertEqual(len(files), 2, "not matched at expected found files") - self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') @@ -117,16 +114,14 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=True ) - _, files = operator.get_sftp_files_map() - - sftp_hook.assert_called_once_with(SFTP_CONN_ID) + files = operator.get_sftp_files_map() self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_upload_wasb(self, sftp_hook, mock_hook): + def test_copy_files_to_wasb(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, @@ -137,10 +132,10 @@ def test_upload_wasb(self, sftp_hook, mock_hook): move_object=True ) - sftp_files = [{SFTP_FILE_PATH: SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME: BLOB_NAME}] - files = operator.upload_wasb(sftp_hook, sftp_files) + sftp_files = [SftpFile(SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME)] + files = operator.copy_files_to_wasb(sftp_files) - sftp_hook.retrieve_file.assert_has_calls( + operator.sftp_hook.retrieve_file.assert_has_calls( [ mock.call("main_dir/test_object3.json", mock.ANY) ] @@ -153,7 +148,7 @@ def test_upload_wasb(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_sftp_move(self, sftp_hook): + def test_delete_files(self, sftp_hook): sftp_mock = sftp_hook.return_value operator = SFTPToWasbOperator( @@ -167,7 +162,7 @@ def test_sftp_move(self, sftp_hook): ) sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.sftp_move(sftp_mock, sftp_file_paths) + operator.delete_files(sftp_file_paths) sftp_mock.delete_file.assert_has_calls( [ @@ -175,13 +170,13 @@ def test_sftp_move(self, sftp_hook): ] ) + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_no_sftp_move(self, sftp_hook): - sftp_mock = sftp_hook.return_value + def test_execute(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, blob_name=BLOB_NAME, @@ -189,30 +184,12 @@ def test_no_sftp_move(self, sftp_hook): move_object=False ) - sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.sftp_move(sftp_mock, sftp_file_paths) - - sftp_mock.delete_file.assert_not_called() - - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( - task_id=TASK_ID, - sftp_source_path=WILDCARD_FILE_NAME, - sftp_conn_id=SFTP_CONN_ID, - container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, - wasb_conn_id=WASB_CONN_ID, - move_object=False - ) - operator.execute(None) sftp_hook.return_value.get_tree_map.assert_called_with( @@ -229,4 +206,4 @@ def test_execute(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) ) - sftp_hook.delete_file.assert_not_called() + operator.sftp_hook.delete_file.assert_not_called() From b22cbb0c712ae894e19700dd1973c71f019115b5 Mon Sep 17 00:00:00 2001 From: guilherme Date: Mon, 15 Feb 2021 21:51:58 -0300 Subject: [PATCH 24/79] Resolves issues from MR --- CONTRIBUTING.rst | 2 +- .../example_dags/example_sftp_to_wasb.py | 45 +++++ .../providers/microsoft/azure/provider.yaml | 4 + .../operators/sftp_to_azure_blob.rst | 65 +++++++ .../azure/transfers/test_sftp_to_wasb.py | 177 +++++++++++------- 5 files changed, 221 insertions(+), 72 deletions(-) create mode 100644 airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py create mode 100644 docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_azure_blob.rst diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 4f0f6a757919c..abd00c0c92853 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -677,7 +677,7 @@ dingding http discord http google amazon,apache.cassandra,cncf.kubernetes,facebook,microsoft.azure,microsoft.mssql,mysql,postgres,presto,sftp hashicorp google -microsoft.azure oracle +microsoft.azure google,oracle,sftp microsoft.mssql odbc mysql amazon,presto,vertica opsgenie http diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py new file mode 100644 index 0000000000000..959b131b870a4 --- /dev/null +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -0,0 +1,45 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import os + +from airflow import DAG +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator +from airflow.utils.dates import days_ago + +AZURE_CONTAINER_NAME = os.environ.get("AZURE_CONTAINER_NAME", "airflow") +BLOB_PREFIX = os.environ.get("AZURE_BLOB_PREFIX", "airflow") +SFTP_SRC_PATH = os.environ.get("AZURE_SFTP_SRC_PATH", "test-sftp-azure") + +with DAG( + "example_sftp_to_azure_blob", + schedule_interval=None, + start_date=days_ago(1), # Override to match your needs +) as dag: + + # [START how_to_sftp_to_azure_blob] + transfer_files_to_azure = SFTPToWasbOperator( + task_id="transfer_files_to_gcs", + # SFTP args + sftp_conn_id="sftp_default", + sftp_source_path=SFTP_SRC_PATH, + # AZURE args + wasb_conn_id="wasb_default", + container_name=AZURE_CONTAINER_NAME, + blob_prefix=BLOB_PREFIX, + ) + # [END how_to_sftp_to_azure_blob] diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index 6829470e66921..963cd73d9c335 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -145,6 +145,10 @@ transfers: target-integration-name: Google Cloud Storage (GCS) how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/azure_blob_to_gcs.rst python-module: airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs + - source-integration-name: SSH File Transfer Protocol (SFTP) + target-integration-name: Microsoft Azure Data Lake Storage + how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_azure_blob.rst + python-module: airflow.providers.microsoft.azure.transfers.sftp_to_wasb hook-class-names: - airflow.providers.microsoft.azure.hooks.base_azure.AzureBaseHook diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_azure_blob.rst b/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_azure_blob.rst new file mode 100644 index 0000000000000..42ca7d792ceff --- /dev/null +++ b/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_azure_blob.rst @@ -0,0 +1,65 @@ + + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +Azure Blob Storage Transfer Operator +==================================== +The Blob service stores text and binary data as objects in the cloud. +The Blob service offers the following three resources: the storage account, containers, and blobs. +Within your storage account, containers provide a way to organize sets of blobs. +For more information about the service visit `Azure Blob Storage API documentation `_. + +Before you begin +^^^^^^^^^^^^^^^^ +Before using Blob Storage within Airflow you need to authenticate your account with Token, Login and Password. +Please follow Azure +`instructions `_ +to do it. + +TOKEN should be added to the Connection in Airflow in JSON format, Login and Password as plain text. +You can check `how to do such connection `_. + +See following example. +Set values for these fields: + +.. code-block:: + + SFTP Conn Id: sftp_default + WASB Conn Id: wasb_default + Login: Storage Account Name + Password: KEY1 + Extra: {"sas_token": "TOKEN"} + +.. contents:: + :depth: 1 + :local: + +.. _howto/operator:SFTPToWasbOperator: + +Transfer Data from SFTP Source Path to Blob Storage +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Operator transfers data from SFTP Source Path to specified container in Azure Blob Storage + +To get information about jobs within a Azure Blob Storage use: +:class:`~airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPToWasbOperator` +Example usage: + +.. exampleinclude:: /../../airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py + :language: python + :dedent: 4 + :start-after: [START how_to_sftp_to_azure_blob] + :end-before: [END how_to_sftp_to_azure_blob] diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 8f02c8fff60e1..9c06f20961231 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -17,57 +17,53 @@ # under the License. # -import os import unittest - -import mock +from unittest import mock from airflow import AirflowException from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator +BLOB_PREFIX = "sponge-bob" +CONTAINER_NAME = "test-container" +SOURCE_PATH_NO_WILDCARD = "main_dir/" +EXPECTED_BLOB_NAME = "test_object3.json" +EXPECTED_FILES = [SOURCE_PATH_NO_WILDCARD + EXPECTED_BLOB_NAME] +SOURCE_PATH_MULTIPLE_WILDCARDS = "main_dir/csv/*/test_*" +SFTP_CONN_ID = "ssh_default" TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" -SFTP_CONN_ID = "ssh_default" - -BLOB_NAME = "BLOB_NAME" -CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" -SOURCE_OBJECT_NO_WILDCARD = "main_dir/test_object3.json" -SOURCE_OBJECT_MULTIPLE_WILDCARDS = "main_dir/csv/*/test_*.csv" -NEW_BLOB_NAME = "sponge-bob" -EXPECTED_BLOB_NAME = "test_object3.json" # pylint: disable=unused-argument class TestSFTPToWasbOperator(unittest.TestCase): - def test_init(self): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=NEW_BLOB_NAME, + blob_prefix=BLOB_PREFIX, wasb_conn_id=WASB_CONN_ID, - move_object=False + move_object=False, ) - self.assertEqual(operator.sftp_source_path, SOURCE_OBJECT_NO_WILDCARD) + self.assertEqual(operator.sftp_source_path, SOURCE_PATH_NO_WILDCARD) self.assertEqual(operator.sftp_conn_id, SFTP_CONN_ID) self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) + self.assertEqual(operator.blob_prefix, BLOB_PREFIX) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', - autospec=True) + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, + sftp_source_path=SOURCE_PATH_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, + blob_prefix=BLOB_PREFIX, wasb_conn_id=WASB_CONN_ID, - move_object=False + move_object=False, ) with self.assertRaises(AirflowException) as cm: operator.check_wildcards_limit() @@ -75,121 +71,162 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): - sftp_hook.return_value.get_tree_map.return_value = [ - [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], - [], - [], - ] + def test_source_path_contains_wildcard(self): operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_PATH, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, + blob_prefix=BLOB_PREFIX, wasb_conn_id=WASB_CONN_ID, - move_object=False + move_object=False, ) - files = operator.get_sftp_files_map() - self.assertEqual(len(files), 2, "not matched at expected found files") - self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") + output = operator.source_path_contains_wildcard + + self.assertTrue(output, "This source path contains wildcard") + + def test_source_path_not_contains_wildcard(self): + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_prefix=BLOB_PREFIX, + wasb_conn_id=WASB_CONN_ID, + move_object=False, + ) + + output = operator.source_path_contains_wildcard + + self.assertFalse(output, "This source path does not contains wildcard") + + def test_get_sftp_tree_behavior(self): + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_PATH, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False, + ) + sftp_complete_path, prefix, delimiter = operator.get_tree_behavior() + + self.assertEqual(sftp_complete_path, "main_dir", "not matched at expected complete path") + self.assertEqual(prefix, "main_dir/", "Prefix must be before the wildcard") + self.assertEqual(delimiter, "", "Delimiter must be empty") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): + def test_get_sftp_files_map(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ - [SOURCE_OBJECT_NO_WILDCARD], + EXPECTED_FILES, [], [], ] operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=True + move_object=True, ) files = operator.get_sftp_files_map() self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0].blob_name, BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_copy_files_to_wasb(self, sftp_hook, mock_hook): + def test_upload_wasb(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=True + move_object=True, ) - sftp_files = [SftpFile(SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME)] + sftp_files = [SftpFile(EXPECTED_FILES[0], EXPECTED_BLOB_NAME)] files = operator.copy_files_to_wasb(sftp_files) - operator.sftp_hook.retrieve_file.assert_has_calls( - [ - mock.call("main_dir/test_object3.json", mock.ANY) - ] - ) + operator.sftp_hook.retrieve_file.assert_has_calls([mock.call("main_dir/test_object3.json", mock.ANY)]) - mock_hook.return_value.load_file.assert_called_once_with( - mock.ANY, CONTAINER_NAME, BLOB_NAME - ) + mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, EXPECTED_BLOB_NAME) self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_delete_files(self, sftp_hook): + def test_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=True + move_object=True, ) - sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] + sftp_file_paths = EXPECTED_FILES operator.delete_files(sftp_file_paths) - sftp_mock.delete_file.assert_has_calls( - [ - mock.call(SOURCE_OBJECT_NO_WILDCARD) - ] - ) + sftp_mock.delete_file.assert_has_calls([mock.call(EXPECTED_FILES[0])]) @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_execute(self, sftp_hook, mock_hook): + sftp_hook.return_value.get_tree_map.return_value = [ + ["main_dir/test_object.json"], + [], + [], + ] operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=False + move_object=False, ) + operator.execute(None) + + sftp_hook.return_value.get_tree_map.assert_called_with( + "main_dir", prefix="main_dir/test_object", delimiter=".json" + ) + + sftp_hook.return_value.retrieve_file.assert_has_calls( + [mock.call("main_dir/test_object.json", mock.ANY)] + ) + + mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") + + sftp_hook.delete_file.assert_not_called() + + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') + def test_execute_moving_files(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ - [SOURCE_OBJECT_NO_WILDCARD], + ["main_dir/test_object.json"], [], [], ] + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_FILE_NAME, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + wasb_conn_id=WASB_CONN_ID, + blob_prefix=BLOB_PREFIX, + move_object=True, + ) + operator.execute(None) sftp_hook.return_value.get_tree_map.assert_called_with( @@ -197,13 +234,11 @@ def test_execute(self, sftp_hook, mock_hook): ) sftp_hook.return_value.retrieve_file.assert_has_calls( - [ - mock.call(SOURCE_OBJECT_NO_WILDCARD, mock.ANY) - ] + [mock.call("main_dir/test_object.json", mock.ANY)] ) mock_hook.return_value.load_file.assert_called_once_with( - mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) + mock.ANY, CONTAINER_NAME, BLOB_PREFIX + "test_object.json" ) - operator.sftp_hook.delete_file.assert_not_called() + self.assertTrue(sftp_hook.return_value.delete_file.called, "It did perform move of file") From 02ab631a4ee1a4bd8b446729b011260fe4168c0a Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 19:26:09 -0300 Subject: [PATCH 25/79] Transfer to upload sftp files to Azure Blob Storage sftp_transfer provides mechanism to extract file from a sftp path or files using a wildcard. --- .../microsoft/azure/transfers/stfp_to_wasb.py | 157 ++++++++++++ .../azure/transfers/test_sftp_to_wasb.py | 227 +++++++++--------- 2 files changed, 270 insertions(+), 114 deletions(-) create mode 100644 airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py new file mode 100644 index 0000000000000..75031dbb1ebfd --- /dev/null +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -0,0 +1,157 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +This module contains SFTP to Azure Blob Storage operator. +""" +import os +from tempfile import NamedTemporaryFile +from typing import Optional, Union + +from airflow.exceptions import AirflowException +from airflow.models import BaseOperator +from airflow.providers.microsoft.azure.hooks.wasb import WasbHook +from airflow.providers.sftp.hooks.sftp import SFTPHook +from airflow.utils.decorators import apply_defaults + +WILDCARD = "*" +SFTP_FILE_PATH = "SFTP_FILE_PATH" +BLOB_NAME = "BLOB_NAME" + +class STFPToWasbOperator(BaseOperator): + """ + Transfer files to Azure Blob Storage from SFTP server. + + :param sftp_source_path: The sftp remote path. This is the specified file path + for downloading the single file or multiple files from the SFTP server. + You can use only one wildcard within your path. The wildcard can appear + inside the path or at the end of the path. + :type sftp_source_path: str + :param container_name: Name of the container. + :type container_name: str + :param blob_name: Name of the blob. + :type blob_name: str + :param sftp_conn_id: The sftp connection id. The name or identifier for + establishing a connection to the SFTP server. + :type sftp_conn_id: str + :param wasb_conn_id: Reference to the wasb connection. + :type wasb_conn_id: str + :param load_options: Optional keyword arguments that + `WasbHook.load_file()` takes. + :type load_options: dict + :param move_object: When move object is True, the object is moved instead + of copied to the new location. This is the equivalent of a mv command + as opposed to a cp command. + :type move_object: bool + """ + template_fields = ("sftp_source_path", "container_name", "blob_name") + + @apply_defaults + def __init__( + self, + sftp_source_path: str, + container_name: str, + blob_name: str, + sftp_conn_id: str = "ssh_default", + wasb_conn_id: str = 'wasb_default', + load_options: dict = None, + move_object: bool = False, + *args, + **kwargs + ) -> None: + super().__init__(*args, **kwargs) + + self.sftp_source_path = sftp_source_path + self.sftp_conn_id = sftp_conn_id + self.wasb_conn_id = wasb_conn_id + self.container_name = container_name + self.blob_name = blob_name + self.wasb_conn_id = wasb_conn_id + self.load_options = load_options if load_options is not None else {} + self.move_object = move_object + + def execute(self, context): + (sftp_hook, sftp_files) = self.get_sftp_files_map() + uploaded_files = self.upload_wasb(sftp_hook, sftp_files) + self.sftp_move(sftp_hook, uploaded_files) + + def get_sftp_files_map(self): + sftp_files = [] + sftp_hook = SFTPHook(self.sftp_conn_id) + + if WILDCARD in self.sftp_source_path: + self.check_wildcards_limit() + + prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) + sftp_complete_path = os.path.dirname(prefix) + + found_files, _, _ = sftp_hook.get_tree_map( + sftp_complete_path, prefix=prefix, delimiter=delimiter + ) + + self.log.info( + "Found %s files at sftp source path: %s", + str(len(found_files)), + self.sftp_source_path + ) + + for file in found_files: + future_blob_name = os.path.basename(file) + sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) + + else: + sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) + + return sftp_hook, sftp_files + + def check_wildcards_limit(self): + total_wildcards = self.sftp_source_path.count(WILDCARD) + if total_wildcards > 1: + raise AirflowException( + "Only one wildcard '*' is allowed in sftp_source_path parameter. " + "Found {} in {}.".format(total_wildcards, self.sftp_source_path) + ) + + def upload_wasb(self, sftp_hook, sftp_files): + uploaded_files = [] + wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) + for file in sftp_files: + + with NamedTemporaryFile("w") as tmp: + sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) + self.log.info( + 'Uploading %s to wasb://%s as %s', + file[SFTP_FILE_PATH], + self.container_name, + file[BLOB_NAME], + ) + wasb_hook.load_file(tmp.name, + self.container_name, + file[BLOB_NAME], + **self.load_options) + + uploaded_files.append(file[SFTP_FILE_PATH]) + + return uploaded_files + + def sftp_move(self, sftp_hook, uploaded_files): + if self.move_object: + for sftp_file_path in uploaded_files: + self.log.info("Executing delete of %s", sftp_file_path) + sftp_hook.delete_file(sftp_file_path) + + diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 9c06f20961231..17c2763b68228 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -17,53 +17,65 @@ # under the License. # + +import datetime import unittest -from unittest import mock +import os + +import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator +from airflow.models.dag import DAG +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME + -BLOB_PREFIX = "sponge-bob" -CONTAINER_NAME = "test-container" -SOURCE_PATH_NO_WILDCARD = "main_dir/" -EXPECTED_BLOB_NAME = "test_object3.json" -EXPECTED_FILES = [SOURCE_PATH_NO_WILDCARD + EXPECTED_BLOB_NAME] -SOURCE_PATH_MULTIPLE_WILDCARDS = "main_dir/csv/*/test_*" -SFTP_CONN_ID = "ssh_default" TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" +SFTP_CONN_ID = "ssh_default" + +CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" +SOURCE_OBJECT_NO_WILDCARD = "main_dir/test_object3.json" +SOURCE_OBJECT_MULTIPLE_WILDCARDS = "main_dir/csv/*/test_*.csv" +NEW_BLOB_NAME = "sponge-bob" +EXPECTED_BLOB_NAME = "test_object3.json" # pylint: disable=unused-argument class TestSFTPToWasbOperator(unittest.TestCase): + + def test_init(self): - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_PATH_NO_WILDCARD, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_prefix=BLOB_PREFIX, + blob_name=NEW_BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=False, + move_object=False ) - self.assertEqual(operator.sftp_source_path, SOURCE_PATH_NO_WILDCARD) + self.assertEqual(operator.sftp_source_path, SOURCE_OBJECT_NO_WILDCARD) self.assertEqual(operator.sftp_conn_id, SFTP_CONN_ID) self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - self.assertEqual(operator.blob_prefix, BLOB_PREFIX) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', autospec=True) + + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', + autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_PATH_MULTIPLE_WILDCARDS, + sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_prefix=BLOB_PREFIX, + blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=False, + move_object=False ) with self.assertRaises(AirflowException) as cm: operator.check_wildcards_limit() @@ -71,160 +83,144 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - def test_source_path_contains_wildcard(self): - operator = SFTPToWasbOperator( + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_get_sftp_files_map(self, sftp_hook, mock_hook): + sftp_hook.return_value.get_tree_map.return_value = [ + [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], + [], + [], + ] + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_PATH, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_prefix=BLOB_PREFIX, + blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=False, + move_object=False ) + _, files = operator.get_sftp_files_map() - output = operator.source_path_contains_wildcard + sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertTrue(output, "This source path contains wildcard") + self.assertEquals(len(files), 2, "not matched at expected found files") + self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - def test_source_path_not_contains_wildcard(self): - operator = SFTPToWasbOperator( - task_id=TASK_ID, - sftp_source_path=SOURCE_PATH_NO_WILDCARD, - sftp_conn_id=SFTP_CONN_ID, - container_name=CONTAINER_NAME, - blob_prefix=BLOB_PREFIX, - wasb_conn_id=WASB_CONN_ID, - move_object=False, - ) - output = operator.source_path_contains_wildcard - - self.assertFalse(output, "This source path does not contains wildcard") - - def test_get_sftp_tree_behavior(self): - operator = SFTPToWasbOperator( - task_id=TASK_ID, - sftp_source_path=WILDCARD_PATH, - sftp_conn_id=SFTP_CONN_ID, - container_name=CONTAINER_NAME, - wasb_conn_id=WASB_CONN_ID, - move_object=False, - ) - sftp_complete_path, prefix, delimiter = operator.get_tree_behavior() - - self.assertEqual(sftp_complete_path, "main_dir", "not matched at expected complete path") - self.assertEqual(prefix, "main_dir/", "Prefix must be before the wildcard") - self.assertEqual(delimiter, "", "Delimiter must be empty") - - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_get_sftp_files_map(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ - EXPECTED_FILES, + [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_PATH_NO_WILDCARD, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=True, + move_object=True ) - files = operator.get_sftp_files_map() + _, files = operator.get_sftp_files_map() + + sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") + self.assertEquals(len(files), 1, "no matched at expected found files") + self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_upload_wasb(self, sftp_hook, mock_hook): - operator = SFTPToWasbOperator( + + + operator = STFPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_PATH_NO_WILDCARD, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=True, + move_object=True ) - sftp_files = [SftpFile(EXPECTED_FILES[0], EXPECTED_BLOB_NAME)] - files = operator.copy_files_to_wasb(sftp_files) + sftp_files = [{SFTP_FILE_PATH: SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME: BLOB_NAME}] + files = operator.upload_wasb(sftp_hook, sftp_files) - operator.sftp_hook.retrieve_file.assert_has_calls([mock.call("main_dir/test_object3.json", mock.ANY)]) + sftp_hook.retrieve_file.assert_has_calls( + [ + mock.call("main_dir/test_object3.json", mock.ANY) + ] + ) - mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, EXPECTED_BLOB_NAME) + mock_hook.return_value.load_file.assert_called_once_with( + mock.ANY, CONTAINER_NAME, BLOB_NAME + ) - self.assertEqual(len(files), 1, "no matched at expected uploaded files") + self.assertEquals(len(files), 1, "no matched at expected uploaded files") - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_PATH_NO_WILDCARD, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=True, + move_object=True ) - sftp_file_paths = EXPECTED_FILES - operator.delete_files(sftp_file_paths) + sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] + operator.sftp_move(sftp_mock, sftp_file_paths) - sftp_mock.delete_file.assert_has_calls([mock.call(EXPECTED_FILES[0])]) + sftp_mock.delete_file.assert_has_calls( + [ + mock.call(SOURCE_OBJECT_NO_WILDCARD) + ] + ) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_execute(self, sftp_hook, mock_hook): - sftp_hook.return_value.get_tree_map.return_value = [ - ["main_dir/test_object.json"], - [], - [], - ] + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_no_sftp_move(self, sftp_hook): + sftp_mock = sftp_hook.return_value - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, - sftp_source_path=WILDCARD_FILE_NAME, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=False, + move_object=False ) - operator.execute(None) + sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] + operator.sftp_move(sftp_mock, sftp_file_paths) - sftp_hook.return_value.get_tree_map.assert_called_with( - "main_dir", prefix="main_dir/test_object", delimiter=".json" - ) + sftp_mock.delete_file.assert_not_called() - sftp_hook.return_value.retrieve_file.assert_has_calls( - [mock.call("main_dir/test_object.json", mock.ANY)] - ) - - mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") - - sftp_hook.delete_file.assert_not_called() - - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_execute_moving_files(self, sftp_hook, mock_hook): + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ - ["main_dir/test_object.json"], + [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - blob_prefix=BLOB_PREFIX, - move_object=True, + move_object=False ) operator.execute(None) @@ -234,11 +230,14 @@ def test_execute_moving_files(self, sftp_hook, mock_hook): ) sftp_hook.return_value.retrieve_file.assert_has_calls( - [mock.call("main_dir/test_object.json", mock.ANY)] + [ + mock.call(SOURCE_OBJECT_NO_WILDCARD, mock.ANY) + ] ) mock_hook.return_value.load_file.assert_called_once_with( - mock.ANY, CONTAINER_NAME, BLOB_PREFIX + "test_object.json" + mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) ) - self.assertTrue(sftp_hook.return_value.delete_file.called, "It did perform move of file") + sftp_hook.delete_file.assert_not_called() + From 3fef6ceca40bb1a88b35451162d4555329ac71bc Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:01:46 -0300 Subject: [PATCH 26/79] Fix tests names and adds sftp_to_wasb to operators-and-hooks-ref list Changes test_get_sftp_files_map to uses a proper name --- .../microsoft/azure/transfers/test_sftp_to_wasb.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 17c2763b68228..161f20a6e0882 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -47,7 +47,6 @@ # pylint: disable=unused-argument class TestSFTPToWasbOperator(unittest.TestCase): - def test_init(self): operator = STFPToWasbOperator( task_id=TASK_ID, @@ -63,8 +62,6 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): @@ -85,7 +82,7 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') - def test_get_sftp_files_map(self, sftp_hook, mock_hook): + def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], @@ -107,10 +104,9 @@ def test_get_sftp_files_map(self, sftp_hook, mock_hook): self.assertEquals(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') - def test_get_sftp_files_map(self, sftp_hook, mock_hook): + def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], From c79e1a9093721e2a85e0027645a031df6241e8c8 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:28:33 -0300 Subject: [PATCH 27/79] Removes usage of deprecated method and unused imports at tests from sftp_to_wasb --- .../microsoft/azure/transfers/test_sftp_to_wasb.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 161f20a6e0882..8e05d4e8d2bea 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -18,14 +18,12 @@ # -import datetime import unittest import os import mock from airflow import AirflowException -from airflow.models.dag import DAG from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME @@ -101,7 +99,7 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEquals(len(files), 2, "not matched at expected found files") + self.assertEqual(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @@ -125,7 +123,7 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEquals(len(files), 1, "no matched at expected found files") + self.assertEqual(len(files), 1, "no matched at expected found files") self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @@ -156,7 +154,7 @@ def test_upload_wasb(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, BLOB_NAME ) - self.assertEquals(len(files), 1, "no matched at expected uploaded files") + self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_sftp_move(self, sftp_hook): From 9e27c8b1314e6885480303bdbab28e87a56faeaa Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:43:13 -0300 Subject: [PATCH 28/79] Removes trailing newlines from both files Files with training lines: test_sftp_to_wasb and sftp_to_wasb --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 -- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 1 - 2 files changed, 3 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 75031dbb1ebfd..14eefba43c6ad 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -153,5 +153,3 @@ def sftp_move(self, sftp_hook, uploaded_files): for sftp_file_path in uploaded_files: self.log.info("Executing delete of %s", sftp_file_path) sftp_hook.delete_file(sftp_file_path) - - diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 8e05d4e8d2bea..ecbd5b480a1b4 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -234,4 +234,3 @@ def test_execute(self, sftp_hook, mock_hook): ) sftp_hook.delete_file.assert_not_called() - From 33d9a03f10b3336d38a2ae269b915dee8188691f Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 22:57:31 -0300 Subject: [PATCH 29/79] Apply None as the default without type to load_options at sft_to_wasb --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 14eefba43c6ad..15f938ae92ffb 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -68,7 +68,7 @@ def __init__( blob_name: str, sftp_conn_id: str = "ssh_default", wasb_conn_id: str = 'wasb_default', - load_options: dict = None, + load_options=None, move_object: bool = False, *args, **kwargs From bb26cfaa4dfb939e0c89671cca2538e11c1666b0 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 23:14:01 -0300 Subject: [PATCH 30/79] Removes unused import from sftp_to_wasb and excessive trailing lines --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 +- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 15f938ae92ffb..2ac8a7cc24cf8 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -20,7 +20,6 @@ """ import os from tempfile import NamedTemporaryFile -from typing import Optional, Union from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -32,6 +31,7 @@ SFTP_FILE_PATH = "SFTP_FILE_PATH" BLOB_NAME = "BLOB_NAME" + class STFPToWasbOperator(BaseOperator): """ Transfer files to Azure Blob Storage from SFTP server. diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index ecbd5b480a1b4..25079ee21a020 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -129,8 +129,6 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_upload_wasb(self, sftp_hook, mock_hook): - - operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, From 705b9b32e61ae30d0ed56ffe0eb5e4a932cb7889 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:00:41 -0300 Subject: [PATCH 31/79] Fix typo to sftp_to_wasb file name Adds the information from extra package to CONTRIBUTING.rst --- .../microsoft/azure/transfers/stfp_to_wasb.py | 155 ------------------ .../azure/transfers/test_sftp_to_wasb.py | 48 +++--- 2 files changed, 23 insertions(+), 180 deletions(-) delete mode 100644 airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py deleted file mode 100644 index 2ac8a7cc24cf8..0000000000000 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ /dev/null @@ -1,155 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -This module contains SFTP to Azure Blob Storage operator. -""" -import os -from tempfile import NamedTemporaryFile - -from airflow.exceptions import AirflowException -from airflow.models import BaseOperator -from airflow.providers.microsoft.azure.hooks.wasb import WasbHook -from airflow.providers.sftp.hooks.sftp import SFTPHook -from airflow.utils.decorators import apply_defaults - -WILDCARD = "*" -SFTP_FILE_PATH = "SFTP_FILE_PATH" -BLOB_NAME = "BLOB_NAME" - - -class STFPToWasbOperator(BaseOperator): - """ - Transfer files to Azure Blob Storage from SFTP server. - - :param sftp_source_path: The sftp remote path. This is the specified file path - for downloading the single file or multiple files from the SFTP server. - You can use only one wildcard within your path. The wildcard can appear - inside the path or at the end of the path. - :type sftp_source_path: str - :param container_name: Name of the container. - :type container_name: str - :param blob_name: Name of the blob. - :type blob_name: str - :param sftp_conn_id: The sftp connection id. The name or identifier for - establishing a connection to the SFTP server. - :type sftp_conn_id: str - :param wasb_conn_id: Reference to the wasb connection. - :type wasb_conn_id: str - :param load_options: Optional keyword arguments that - `WasbHook.load_file()` takes. - :type load_options: dict - :param move_object: When move object is True, the object is moved instead - of copied to the new location. This is the equivalent of a mv command - as opposed to a cp command. - :type move_object: bool - """ - template_fields = ("sftp_source_path", "container_name", "blob_name") - - @apply_defaults - def __init__( - self, - sftp_source_path: str, - container_name: str, - blob_name: str, - sftp_conn_id: str = "ssh_default", - wasb_conn_id: str = 'wasb_default', - load_options=None, - move_object: bool = False, - *args, - **kwargs - ) -> None: - super().__init__(*args, **kwargs) - - self.sftp_source_path = sftp_source_path - self.sftp_conn_id = sftp_conn_id - self.wasb_conn_id = wasb_conn_id - self.container_name = container_name - self.blob_name = blob_name - self.wasb_conn_id = wasb_conn_id - self.load_options = load_options if load_options is not None else {} - self.move_object = move_object - - def execute(self, context): - (sftp_hook, sftp_files) = self.get_sftp_files_map() - uploaded_files = self.upload_wasb(sftp_hook, sftp_files) - self.sftp_move(sftp_hook, uploaded_files) - - def get_sftp_files_map(self): - sftp_files = [] - sftp_hook = SFTPHook(self.sftp_conn_id) - - if WILDCARD in self.sftp_source_path: - self.check_wildcards_limit() - - prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) - sftp_complete_path = os.path.dirname(prefix) - - found_files, _, _ = sftp_hook.get_tree_map( - sftp_complete_path, prefix=prefix, delimiter=delimiter - ) - - self.log.info( - "Found %s files at sftp source path: %s", - str(len(found_files)), - self.sftp_source_path - ) - - for file in found_files: - future_blob_name = os.path.basename(file) - sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) - - else: - sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) - - return sftp_hook, sftp_files - - def check_wildcards_limit(self): - total_wildcards = self.sftp_source_path.count(WILDCARD) - if total_wildcards > 1: - raise AirflowException( - "Only one wildcard '*' is allowed in sftp_source_path parameter. " - "Found {} in {}.".format(total_wildcards, self.sftp_source_path) - ) - - def upload_wasb(self, sftp_hook, sftp_files): - uploaded_files = [] - wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) - for file in sftp_files: - - with NamedTemporaryFile("w") as tmp: - sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) - self.log.info( - 'Uploading %s to wasb://%s as %s', - file[SFTP_FILE_PATH], - self.container_name, - file[BLOB_NAME], - ) - wasb_hook.load_file(tmp.name, - self.container_name, - file[BLOB_NAME], - **self.load_options) - - uploaded_files.append(file[SFTP_FILE_PATH]) - - return uploaded_files - - def sftp_move(self, sftp_hook, uploaded_files): - if self.move_object: - for sftp_file_path in uploaded_files: - self.log.info("Executing delete of %s", sftp_file_path) - sftp_hook.delete_file(sftp_file_path) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 25079ee21a020..34b701ed9acd5 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -17,17 +17,15 @@ # under the License. # - -import unittest import os +import unittest import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME - +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator, \ + SFTP_FILE_PATH, \ + BLOB_NAME TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" @@ -46,7 +44,7 @@ class TestSFTPToWasbOperator(unittest.TestCase): def test_init(self): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -60,10 +58,10 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, @@ -78,15 +76,15 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_PATH, sftp_conn_id=SFTP_CONN_ID, @@ -102,15 +100,15 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): self.assertEqual(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -126,10 +124,10 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected found files") self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_upload_wasb(self, sftp_hook, mock_hook): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -154,11 +152,11 @@ def test_upload_wasb(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected uploaded files") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -177,11 +175,11 @@ def test_sftp_move(self, sftp_hook): ] ) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_no_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -196,8 +194,8 @@ def test_no_sftp_move(self, sftp_hook): sftp_mock.delete_file.assert_not_called() - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], @@ -205,7 +203,7 @@ def test_execute(self, sftp_hook, mock_hook): [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, From 83ee2190c6f087327f69c8bf8848807d9acea029 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:30:34 -0300 Subject: [PATCH 32/79] Adds dependency to dependencies.json Fix import at test_sftp_to_wasb. --- .../providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 34b701ed9acd5..bac52a5839da4 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -23,9 +23,8 @@ import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator, \ - SFTP_FILE_PATH, \ - BLOB_NAME +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( + BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator) TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" From e4b6e9fda0809c3a397f341d724dc4cfa0c2d34a Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:59:49 -0300 Subject: [PATCH 33/79] Fix changes made by hooks --- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index bac52a5839da4..e38b4df07aded 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -24,7 +24,8 @@ from airflow import AirflowException from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( - BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator) + BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator, +) TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" From 5ff0a310cbc00deb7f14d33f85c5fca1b5b369d1 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 9 Aug 2020 18:56:13 -0300 Subject: [PATCH 34/79] Cleaning code --- .../azure/transfers/test_sftp_to_wasb.py | 55 ++++++------------- 1 file changed, 16 insertions(+), 39 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index e38b4df07aded..8f02c8fff60e1 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -23,14 +23,13 @@ import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( - BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator, -) +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" SFTP_CONN_ID = "ssh_default" +BLOB_NAME = "BLOB_NAME" CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" @@ -93,12 +92,10 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=False ) - _, files = operator.get_sftp_files_map() - - sftp_hook.assert_called_once_with(SFTP_CONN_ID) + files = operator.get_sftp_files_map() self.assertEqual(len(files), 2, "not matched at expected found files") - self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') @@ -117,16 +114,14 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=True ) - _, files = operator.get_sftp_files_map() - - sftp_hook.assert_called_once_with(SFTP_CONN_ID) + files = operator.get_sftp_files_map() self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_upload_wasb(self, sftp_hook, mock_hook): + def test_copy_files_to_wasb(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, @@ -137,10 +132,10 @@ def test_upload_wasb(self, sftp_hook, mock_hook): move_object=True ) - sftp_files = [{SFTP_FILE_PATH: SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME: BLOB_NAME}] - files = operator.upload_wasb(sftp_hook, sftp_files) + sftp_files = [SftpFile(SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME)] + files = operator.copy_files_to_wasb(sftp_files) - sftp_hook.retrieve_file.assert_has_calls( + operator.sftp_hook.retrieve_file.assert_has_calls( [ mock.call("main_dir/test_object3.json", mock.ANY) ] @@ -153,7 +148,7 @@ def test_upload_wasb(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_sftp_move(self, sftp_hook): + def test_delete_files(self, sftp_hook): sftp_mock = sftp_hook.return_value operator = SFTPToWasbOperator( @@ -167,7 +162,7 @@ def test_sftp_move(self, sftp_hook): ) sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.sftp_move(sftp_mock, sftp_file_paths) + operator.delete_files(sftp_file_paths) sftp_mock.delete_file.assert_has_calls( [ @@ -175,13 +170,13 @@ def test_sftp_move(self, sftp_hook): ] ) + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_no_sftp_move(self, sftp_hook): - sftp_mock = sftp_hook.return_value + def test_execute(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, blob_name=BLOB_NAME, @@ -189,30 +184,12 @@ def test_no_sftp_move(self, sftp_hook): move_object=False ) - sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.sftp_move(sftp_mock, sftp_file_paths) - - sftp_mock.delete_file.assert_not_called() - - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( - task_id=TASK_ID, - sftp_source_path=WILDCARD_FILE_NAME, - sftp_conn_id=SFTP_CONN_ID, - container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, - wasb_conn_id=WASB_CONN_ID, - move_object=False - ) - operator.execute(None) sftp_hook.return_value.get_tree_map.assert_called_with( @@ -229,4 +206,4 @@ def test_execute(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) ) - sftp_hook.delete_file.assert_not_called() + operator.sftp_hook.delete_file.assert_not_called() From da6540abeccc74f786e1158396d50fa01d0bb76b Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 19:26:09 -0300 Subject: [PATCH 35/79] Transfer to upload sftp files to Azure Blob Storage sftp_transfer provides mechanism to extract file from a sftp path or files using a wildcard. --- .../microsoft/azure/transfers/stfp_to_wasb.py | 157 ++++++++++++++++++ .../azure/transfers/test_sftp_to_wasb.py | 110 +++++++----- 2 files changed, 229 insertions(+), 38 deletions(-) create mode 100644 airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py new file mode 100644 index 0000000000000..75031dbb1ebfd --- /dev/null +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -0,0 +1,157 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +This module contains SFTP to Azure Blob Storage operator. +""" +import os +from tempfile import NamedTemporaryFile +from typing import Optional, Union + +from airflow.exceptions import AirflowException +from airflow.models import BaseOperator +from airflow.providers.microsoft.azure.hooks.wasb import WasbHook +from airflow.providers.sftp.hooks.sftp import SFTPHook +from airflow.utils.decorators import apply_defaults + +WILDCARD = "*" +SFTP_FILE_PATH = "SFTP_FILE_PATH" +BLOB_NAME = "BLOB_NAME" + +class STFPToWasbOperator(BaseOperator): + """ + Transfer files to Azure Blob Storage from SFTP server. + + :param sftp_source_path: The sftp remote path. This is the specified file path + for downloading the single file or multiple files from the SFTP server. + You can use only one wildcard within your path. The wildcard can appear + inside the path or at the end of the path. + :type sftp_source_path: str + :param container_name: Name of the container. + :type container_name: str + :param blob_name: Name of the blob. + :type blob_name: str + :param sftp_conn_id: The sftp connection id. The name or identifier for + establishing a connection to the SFTP server. + :type sftp_conn_id: str + :param wasb_conn_id: Reference to the wasb connection. + :type wasb_conn_id: str + :param load_options: Optional keyword arguments that + `WasbHook.load_file()` takes. + :type load_options: dict + :param move_object: When move object is True, the object is moved instead + of copied to the new location. This is the equivalent of a mv command + as opposed to a cp command. + :type move_object: bool + """ + template_fields = ("sftp_source_path", "container_name", "blob_name") + + @apply_defaults + def __init__( + self, + sftp_source_path: str, + container_name: str, + blob_name: str, + sftp_conn_id: str = "ssh_default", + wasb_conn_id: str = 'wasb_default', + load_options: dict = None, + move_object: bool = False, + *args, + **kwargs + ) -> None: + super().__init__(*args, **kwargs) + + self.sftp_source_path = sftp_source_path + self.sftp_conn_id = sftp_conn_id + self.wasb_conn_id = wasb_conn_id + self.container_name = container_name + self.blob_name = blob_name + self.wasb_conn_id = wasb_conn_id + self.load_options = load_options if load_options is not None else {} + self.move_object = move_object + + def execute(self, context): + (sftp_hook, sftp_files) = self.get_sftp_files_map() + uploaded_files = self.upload_wasb(sftp_hook, sftp_files) + self.sftp_move(sftp_hook, uploaded_files) + + def get_sftp_files_map(self): + sftp_files = [] + sftp_hook = SFTPHook(self.sftp_conn_id) + + if WILDCARD in self.sftp_source_path: + self.check_wildcards_limit() + + prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) + sftp_complete_path = os.path.dirname(prefix) + + found_files, _, _ = sftp_hook.get_tree_map( + sftp_complete_path, prefix=prefix, delimiter=delimiter + ) + + self.log.info( + "Found %s files at sftp source path: %s", + str(len(found_files)), + self.sftp_source_path + ) + + for file in found_files: + future_blob_name = os.path.basename(file) + sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) + + else: + sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) + + return sftp_hook, sftp_files + + def check_wildcards_limit(self): + total_wildcards = self.sftp_source_path.count(WILDCARD) + if total_wildcards > 1: + raise AirflowException( + "Only one wildcard '*' is allowed in sftp_source_path parameter. " + "Found {} in {}.".format(total_wildcards, self.sftp_source_path) + ) + + def upload_wasb(self, sftp_hook, sftp_files): + uploaded_files = [] + wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) + for file in sftp_files: + + with NamedTemporaryFile("w") as tmp: + sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) + self.log.info( + 'Uploading %s to wasb://%s as %s', + file[SFTP_FILE_PATH], + self.container_name, + file[BLOB_NAME], + ) + wasb_hook.load_file(tmp.name, + self.container_name, + file[BLOB_NAME], + **self.load_options) + + uploaded_files.append(file[SFTP_FILE_PATH]) + + return uploaded_files + + def sftp_move(self, sftp_hook, uploaded_files): + if self.move_object: + for sftp_file_path in uploaded_files: + self.log.info("Executing delete of %s", sftp_file_path) + sftp_hook.delete_file(sftp_file_path) + + diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 8f02c8fff60e1..17c2763b68228 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -17,19 +17,24 @@ # under the License. # -import os + +import datetime import unittest +import os import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator +from airflow.models.dag import DAG +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH +from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME + TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" SFTP_CONN_ID = "ssh_default" -BLOB_NAME = "BLOB_NAME" CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" @@ -42,8 +47,9 @@ # pylint: disable=unused-argument class TestSFTPToWasbOperator(unittest.TestCase): + def test_init(self): - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -57,10 +63,12 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', + + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, @@ -75,15 +83,15 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_get_sftp_files_map(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_PATH, sftp_conn_id=SFTP_CONN_ID, @@ -92,20 +100,23 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=False ) - files = operator.get_sftp_files_map() + _, files = operator.get_sftp_files_map() + + sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEqual(len(files), 2, "not matched at expected found files") - self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") + self.assertEquals(len(files), 2, "not matched at expected found files") + self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_get_sftp_files_map(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -114,15 +125,19 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=True ) - files = operator.get_sftp_files_map() + _, files = operator.get_sftp_files_map() + + sftp_hook.assert_called_once_with(SFTP_CONN_ID) + + self.assertEquals(len(files), 1, "no matched at expected found files") + self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_upload_wasb(self, sftp_hook, mock_hook): - self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0].blob_name, BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_copy_files_to_wasb(self, sftp_hook, mock_hook): - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -132,10 +147,10 @@ def test_copy_files_to_wasb(self, sftp_hook, mock_hook): move_object=True ) - sftp_files = [SftpFile(SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME)] - files = operator.copy_files_to_wasb(sftp_files) + sftp_files = [{SFTP_FILE_PATH: SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME: BLOB_NAME}] + files = operator.upload_wasb(sftp_hook, sftp_files) - operator.sftp_hook.retrieve_file.assert_has_calls( + sftp_hook.retrieve_file.assert_has_calls( [ mock.call("main_dir/test_object3.json", mock.ANY) ] @@ -145,13 +160,13 @@ def test_copy_files_to_wasb(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, BLOB_NAME ) - self.assertEqual(len(files), 1, "no matched at expected uploaded files") + self.assertEquals(len(files), 1, "no matched at expected uploaded files") - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_delete_files(self, sftp_hook): + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -162,7 +177,7 @@ def test_delete_files(self, sftp_hook): ) sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.delete_files(sftp_file_paths) + operator.sftp_move(sftp_mock, sftp_file_paths) sftp_mock.delete_file.assert_has_calls( [ @@ -170,13 +185,13 @@ def test_delete_files(self, sftp_hook): ] ) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_execute(self, sftp_hook, mock_hook): + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_no_sftp_move(self, sftp_hook): + sftp_mock = sftp_hook.return_value - operator = SFTPToWasbOperator( + operator = STFPToWasbOperator( task_id=TASK_ID, - sftp_source_path=WILDCARD_FILE_NAME, + sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, blob_name=BLOB_NAME, @@ -184,12 +199,30 @@ def test_execute(self, sftp_hook, mock_hook): move_object=False ) + sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] + operator.sftp_move(sftp_mock, sftp_file_paths) + + sftp_mock.delete_file.assert_not_called() + + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] + operator = STFPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_FILE_NAME, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_name=BLOB_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False + ) + operator.execute(None) sftp_hook.return_value.get_tree_map.assert_called_with( @@ -206,4 +239,5 @@ def test_execute(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) ) - operator.sftp_hook.delete_file.assert_not_called() + sftp_hook.delete_file.assert_not_called() + From a3b2f68c9f8a8812f7342248565d52b2647ede2f Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:01:46 -0300 Subject: [PATCH 36/79] Fix tests names and adds sftp_to_wasb to operators-and-hooks-ref list Changes test_get_sftp_files_map to uses a proper name --- docs/operators-and-hooks-ref.rst | 6 ++++++ .../microsoft/azure/transfers/test_sftp_to_wasb.py | 8 ++------ 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index 8c6a7d9338faa..c7f9096398c50 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -1630,6 +1630,12 @@ These integrations allow you to copy data. - - :mod:`airflow.providers.microsoft.azure.transfers.file_to_wasb` + * - SFTP + - `Azure Blob Storage `__ + - + - :mod:`airflow.providers.microsoft.azure.transfers.stfp_to_wasb` + + * - Filesystem - `Google Cloud Storage (GCS) `__ - diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 17c2763b68228..161f20a6e0882 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -47,7 +47,6 @@ # pylint: disable=unused-argument class TestSFTPToWasbOperator(unittest.TestCase): - def test_init(self): operator = STFPToWasbOperator( task_id=TASK_ID, @@ -63,8 +62,6 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): @@ -85,7 +82,7 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') - def test_get_sftp_files_map(self, sftp_hook, mock_hook): + def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], @@ -107,10 +104,9 @@ def test_get_sftp_files_map(self, sftp_hook, mock_hook): self.assertEquals(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') - def test_get_sftp_files_map(self, sftp_hook, mock_hook): + def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], From ed726efba300d70fbc08ed972f9d3a001413e014 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:28:33 -0300 Subject: [PATCH 37/79] Removes usage of deprecated method and unused imports at tests from sftp_to_wasb --- .../microsoft/azure/transfers/test_sftp_to_wasb.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 161f20a6e0882..8e05d4e8d2bea 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -18,14 +18,12 @@ # -import datetime import unittest import os import mock from airflow import AirflowException -from airflow.models.dag import DAG from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME @@ -101,7 +99,7 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEquals(len(files), 2, "not matched at expected found files") + self.assertEqual(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @@ -125,7 +123,7 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.assert_called_once_with(SFTP_CONN_ID) - self.assertEquals(len(files), 1, "no matched at expected found files") + self.assertEqual(len(files), 1, "no matched at expected found files") self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @@ -156,7 +154,7 @@ def test_upload_wasb(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, BLOB_NAME ) - self.assertEquals(len(files), 1, "no matched at expected uploaded files") + self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_sftp_move(self, sftp_hook): From 0a825d818b121c02fd4707c5bf944186929754c0 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 21:43:13 -0300 Subject: [PATCH 38/79] Removes trailing newlines from both files Files with training lines: test_sftp_to_wasb and sftp_to_wasb --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 -- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 1 - 2 files changed, 3 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 75031dbb1ebfd..14eefba43c6ad 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -153,5 +153,3 @@ def sftp_move(self, sftp_hook, uploaded_files): for sftp_file_path in uploaded_files: self.log.info("Executing delete of %s", sftp_file_path) sftp_hook.delete_file(sftp_file_path) - - diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 8e05d4e8d2bea..ecbd5b480a1b4 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -234,4 +234,3 @@ def test_execute(self, sftp_hook, mock_hook): ) sftp_hook.delete_file.assert_not_called() - From ac98f6b327e875166f8cb0327e64ae8e9e6a31bf Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 22:57:31 -0300 Subject: [PATCH 39/79] Apply None as the default without type to load_options at sft_to_wasb --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 14eefba43c6ad..15f938ae92ffb 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -68,7 +68,7 @@ def __init__( blob_name: str, sftp_conn_id: str = "ssh_default", wasb_conn_id: str = 'wasb_default', - load_options: dict = None, + load_options=None, move_object: bool = False, *args, **kwargs From d0a383ad28cb7253b42aa3528cdb5e1044263fb2 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 5 Jul 2020 23:14:01 -0300 Subject: [PATCH 40/79] Removes unused import from sftp_to_wasb and excessive trailing lines --- airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py | 2 +- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py index 15f938ae92ffb..2ac8a7cc24cf8 100644 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py @@ -20,7 +20,6 @@ """ import os from tempfile import NamedTemporaryFile -from typing import Optional, Union from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -32,6 +31,7 @@ SFTP_FILE_PATH = "SFTP_FILE_PATH" BLOB_NAME = "BLOB_NAME" + class STFPToWasbOperator(BaseOperator): """ Transfer files to Azure Blob Storage from SFTP server. diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index ecbd5b480a1b4..25079ee21a020 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -129,8 +129,6 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') def test_upload_wasb(self, sftp_hook, mock_hook): - - operator = STFPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, From 7a9d4abbb5826a14b19fd695d72febc6eb135889 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:00:41 -0300 Subject: [PATCH 41/79] Fix typo to sftp_to_wasb file name Adds the information from extra package to CONTRIBUTING.rst --- .../microsoft/azure/transfers/sftp_to_wasb.py | 62 +++---- .../microsoft/azure/transfers/stfp_to_wasb.py | 155 ------------------ docs/operators-and-hooks-ref.rst | 6 - .../azure/transfers/test_sftp_to_wasb.py | 48 +++--- 4 files changed, 49 insertions(+), 222 deletions(-) delete mode 100644 airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 4f4b51140f992..b03bbd623fd42 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -19,11 +19,7 @@ This module contains SFTP to Azure Blob Storage operator. """ import os -from collections import namedtuple from tempfile import NamedTemporaryFile -from typing import Any, Dict, List, Optional - -from cached_property import cached_property from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -32,7 +28,8 @@ from airflow.utils.decorators import apply_defaults WILDCARD = "*" -SftpFile = namedtuple('SftpFile', 'sftp_file_path, blob_name') +SFTP_FILE_PATH = "SFTP_FILE_PATH" +BLOB_NAME = "BLOB_NAME" class SFTPToWasbOperator(BaseOperator): @@ -69,9 +66,9 @@ def __init__( sftp_source_path: str, container_name: str, blob_name: str, - sftp_conn_id: str = "sftp_default", + sftp_conn_id: str = "ssh_default", wasb_conn_id: str = 'wasb_default', - load_options: Optional[dict] = None, + load_options=None, move_object: bool = False, *args, **kwargs @@ -87,17 +84,16 @@ def __init__( self.load_options = load_options if load_options is not None else {} self.move_object = move_object - def execute(self, - context: Optional[Dict[Any, Any]]) -> None: + def execute(self, context): """Upload a file from SFTP to Azure Blob Storage.""" - sftp_files: List[SftpFile] = self.get_sftp_files_map() - uploaded_files = self.copy_files_to_wasb(sftp_files) - if self.move_object: - self.delete_files(uploaded_files) + (sftp_hook, sftp_files) = self.get_sftp_files_map() + uploaded_files = self.upload_wasb(sftp_hook, sftp_files) + self.sftp_move(sftp_hook, uploaded_files) - def get_sftp_files_map(self) -> List[SftpFile]: + def get_sftp_files_map(self): """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] + sftp_hook = SFTPHook(self.sftp_conn_id) if WILDCARD in self.sftp_source_path: self.check_wildcards_limit() @@ -105,7 +101,7 @@ def get_sftp_files_map(self) -> List[SftpFile]: prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) sftp_complete_path = os.path.dirname(prefix) - found_files, _, _ = self.sftp_hook.get_tree_map( + found_files, _, _ = sftp_hook.get_tree_map( sftp_complete_path, prefix=prefix, delimiter=delimiter ) @@ -117,19 +113,14 @@ def get_sftp_files_map(self) -> List[SftpFile]: for file in found_files: future_blob_name = os.path.basename(file) - sftp_files.append(SftpFile(file, future_blob_name)) + sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) else: - sftp_files.append(SftpFile(self.sftp_source_path, self.blob_name)) + sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) - return sftp_files + return sftp_hook, sftp_files - @cached_property - def sftp_hook(self): - """Property of sftp hook to be re-used.""" - return SFTPHook(self.sftp_conn_id) - - def check_wildcards_limit(self) -> Any: + def check_wildcards_limit(self): """Check if there is multiple Wildcard.""" total_wildcards = self.sftp_source_path.count(WILDCARD) if total_wildcards > 1: @@ -138,33 +129,32 @@ def check_wildcards_limit(self) -> Any: "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) - def copy_files_to_wasb(self, - sftp_files: List[SftpFile]) -> List[str]: + def upload_wasb(self, sftp_hook, sftp_files): """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) for file in sftp_files: with NamedTemporaryFile("w") as tmp: - self.sftp_hook.retrieve_file(file.sftp_file_path, tmp.name) + sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) self.log.info( 'Uploading %s to wasb://%s as %s', - file.sftp_file_path, + file[SFTP_FILE_PATH], self.container_name, - file.blob_name, + file[BLOB_NAME], ) wasb_hook.load_file(tmp.name, self.container_name, - file.blob_name, + file[BLOB_NAME], **self.load_options) - uploaded_files.append(file.sftp_file_path) + uploaded_files.append(file[SFTP_FILE_PATH]) return uploaded_files - def delete_files(self, - uploaded_files: List[str]) -> None: + def sftp_move(self, sftp_hook, uploaded_files): """Performs a move of a list of files at SFTP to Azure Blob Storage.""" - for sftp_file_path in uploaded_files: - self.log.info("Executing delete of %s", sftp_file_path) - self.sftp_hook.delete_file(sftp_file_path) + if self.move_object: + for sftp_file_path in uploaded_files: + self.log.info("Executing delete of %s", sftp_file_path) + sftp_hook.delete_file(sftp_file_path) diff --git a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py deleted file mode 100644 index 2ac8a7cc24cf8..0000000000000 --- a/airflow/providers/microsoft/azure/transfers/stfp_to_wasb.py +++ /dev/null @@ -1,155 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. -""" -This module contains SFTP to Azure Blob Storage operator. -""" -import os -from tempfile import NamedTemporaryFile - -from airflow.exceptions import AirflowException -from airflow.models import BaseOperator -from airflow.providers.microsoft.azure.hooks.wasb import WasbHook -from airflow.providers.sftp.hooks.sftp import SFTPHook -from airflow.utils.decorators import apply_defaults - -WILDCARD = "*" -SFTP_FILE_PATH = "SFTP_FILE_PATH" -BLOB_NAME = "BLOB_NAME" - - -class STFPToWasbOperator(BaseOperator): - """ - Transfer files to Azure Blob Storage from SFTP server. - - :param sftp_source_path: The sftp remote path. This is the specified file path - for downloading the single file or multiple files from the SFTP server. - You can use only one wildcard within your path. The wildcard can appear - inside the path or at the end of the path. - :type sftp_source_path: str - :param container_name: Name of the container. - :type container_name: str - :param blob_name: Name of the blob. - :type blob_name: str - :param sftp_conn_id: The sftp connection id. The name or identifier for - establishing a connection to the SFTP server. - :type sftp_conn_id: str - :param wasb_conn_id: Reference to the wasb connection. - :type wasb_conn_id: str - :param load_options: Optional keyword arguments that - `WasbHook.load_file()` takes. - :type load_options: dict - :param move_object: When move object is True, the object is moved instead - of copied to the new location. This is the equivalent of a mv command - as opposed to a cp command. - :type move_object: bool - """ - template_fields = ("sftp_source_path", "container_name", "blob_name") - - @apply_defaults - def __init__( - self, - sftp_source_path: str, - container_name: str, - blob_name: str, - sftp_conn_id: str = "ssh_default", - wasb_conn_id: str = 'wasb_default', - load_options=None, - move_object: bool = False, - *args, - **kwargs - ) -> None: - super().__init__(*args, **kwargs) - - self.sftp_source_path = sftp_source_path - self.sftp_conn_id = sftp_conn_id - self.wasb_conn_id = wasb_conn_id - self.container_name = container_name - self.blob_name = blob_name - self.wasb_conn_id = wasb_conn_id - self.load_options = load_options if load_options is not None else {} - self.move_object = move_object - - def execute(self, context): - (sftp_hook, sftp_files) = self.get_sftp_files_map() - uploaded_files = self.upload_wasb(sftp_hook, sftp_files) - self.sftp_move(sftp_hook, uploaded_files) - - def get_sftp_files_map(self): - sftp_files = [] - sftp_hook = SFTPHook(self.sftp_conn_id) - - if WILDCARD in self.sftp_source_path: - self.check_wildcards_limit() - - prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) - sftp_complete_path = os.path.dirname(prefix) - - found_files, _, _ = sftp_hook.get_tree_map( - sftp_complete_path, prefix=prefix, delimiter=delimiter - ) - - self.log.info( - "Found %s files at sftp source path: %s", - str(len(found_files)), - self.sftp_source_path - ) - - for file in found_files: - future_blob_name = os.path.basename(file) - sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) - - else: - sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) - - return sftp_hook, sftp_files - - def check_wildcards_limit(self): - total_wildcards = self.sftp_source_path.count(WILDCARD) - if total_wildcards > 1: - raise AirflowException( - "Only one wildcard '*' is allowed in sftp_source_path parameter. " - "Found {} in {}.".format(total_wildcards, self.sftp_source_path) - ) - - def upload_wasb(self, sftp_hook, sftp_files): - uploaded_files = [] - wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) - for file in sftp_files: - - with NamedTemporaryFile("w") as tmp: - sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) - self.log.info( - 'Uploading %s to wasb://%s as %s', - file[SFTP_FILE_PATH], - self.container_name, - file[BLOB_NAME], - ) - wasb_hook.load_file(tmp.name, - self.container_name, - file[BLOB_NAME], - **self.load_options) - - uploaded_files.append(file[SFTP_FILE_PATH]) - - return uploaded_files - - def sftp_move(self, sftp_hook, uploaded_files): - if self.move_object: - for sftp_file_path in uploaded_files: - self.log.info("Executing delete of %s", sftp_file_path) - sftp_hook.delete_file(sftp_file_path) diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst index c7f9096398c50..8c6a7d9338faa 100644 --- a/docs/operators-and-hooks-ref.rst +++ b/docs/operators-and-hooks-ref.rst @@ -1630,12 +1630,6 @@ These integrations allow you to copy data. - - :mod:`airflow.providers.microsoft.azure.transfers.file_to_wasb` - * - SFTP - - `Azure Blob Storage `__ - - - - :mod:`airflow.providers.microsoft.azure.transfers.stfp_to_wasb` - - * - Filesystem - `Google Cloud Storage (GCS) `__ - diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 25079ee21a020..34b701ed9acd5 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -17,17 +17,15 @@ # under the License. # - -import unittest import os +import unittest import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import STFPToWasbOperator -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import SFTP_FILE_PATH -from airflow.providers.microsoft.azure.transfers.stfp_to_wasb import BLOB_NAME - +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator, \ + SFTP_FILE_PATH, \ + BLOB_NAME TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" @@ -46,7 +44,7 @@ class TestSFTPToWasbOperator(unittest.TestCase): def test_init(self): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -60,10 +58,10 @@ def test_init(self): self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook', + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, @@ -78,15 +76,15 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_PATH, sftp_conn_id=SFTP_CONN_ID, @@ -102,15 +100,15 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): self.assertEqual(len(files), 2, "not matched at expected found files") self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -126,10 +124,10 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected found files") self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_upload_wasb(self, sftp_hook, mock_hook): - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -154,11 +152,11 @@ def test_upload_wasb(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected uploaded files") - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -177,11 +175,11 @@ def test_sftp_move(self, sftp_hook): ] ) - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_no_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, @@ -196,8 +194,8 @@ def test_no_sftp_move(self, sftp_hook): sftp_mock.delete_file.assert_not_called() - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.stfp_to_wasb.SFTPHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], @@ -205,7 +203,7 @@ def test_execute(self, sftp_hook, mock_hook): [], ] - operator = STFPToWasbOperator( + operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, From daa647a445e8a6171c1446a632f6f2435e703e28 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:30:34 -0300 Subject: [PATCH 42/79] Adds dependency to dependencies.json Fix import at test_sftp_to_wasb. --- .../providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 34b701ed9acd5..bac52a5839da4 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -23,9 +23,8 @@ import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator, \ - SFTP_FILE_PATH, \ - BLOB_NAME +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( + BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator) TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" From 42f318a5b857962776285aa0f651c10291c9abb0 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Mon, 6 Jul 2020 00:59:49 -0300 Subject: [PATCH 43/79] Fix changes made by hooks --- tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index bac52a5839da4..e38b4df07aded 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -24,7 +24,8 @@ from airflow import AirflowException from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( - BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator) + BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator, +) TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" From 548b281e8fef98c8fdc246c3a723942c196180a1 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 12 Jul 2020 13:53:06 -0300 Subject: [PATCH 44/79] Fix type hint --- .../microsoft/azure/transfers/sftp_to_wasb.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index b03bbd623fd42..8bae74c49c0bb 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -20,6 +20,7 @@ """ import os from tempfile import NamedTemporaryFile +from typing import Any, Dict, List, Optional, Tuple from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -68,7 +69,7 @@ def __init__( blob_name: str, sftp_conn_id: str = "ssh_default", wasb_conn_id: str = 'wasb_default', - load_options=None, + load_options: Optional[dict] = None, move_object: bool = False, *args, **kwargs @@ -84,13 +85,14 @@ def __init__( self.load_options = load_options if load_options is not None else {} self.move_object = move_object - def execute(self, context): + def execute(self, + context: Optional[Dict[Any, Any]]) -> None: """Upload a file from SFTP to Azure Blob Storage.""" (sftp_hook, sftp_files) = self.get_sftp_files_map() uploaded_files = self.upload_wasb(sftp_hook, sftp_files) self.sftp_move(sftp_hook, uploaded_files) - def get_sftp_files_map(self): + def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] sftp_hook = SFTPHook(self.sftp_conn_id) @@ -120,7 +122,7 @@ def get_sftp_files_map(self): return sftp_hook, sftp_files - def check_wildcards_limit(self): + def check_wildcards_limit(self) -> Any: """Check if there is multiple Wildcard.""" total_wildcards = self.sftp_source_path.count(WILDCARD) if total_wildcards > 1: @@ -129,7 +131,9 @@ def check_wildcards_limit(self): "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) - def upload_wasb(self, sftp_hook, sftp_files): + def upload_wasb(self, + sftp_hook: SFTPHook, + sftp_files: List[Dict[str, str]]) -> List[str]: """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) @@ -152,7 +156,9 @@ def upload_wasb(self, sftp_hook, sftp_files): return uploaded_files - def sftp_move(self, sftp_hook, uploaded_files): + def sftp_move(self, + sftp_hook: SFTPHook, + uploaded_files: List[str]) -> None: """Performs a move of a list of files at SFTP to Azure Blob Storage.""" if self.move_object: for sftp_file_path in uploaded_files: From 80f6d13d020c115b2eccc64bc31e63221e83f544 Mon Sep 17 00:00:00 2001 From: WOLVERY Date: Sun, 9 Aug 2020 18:56:13 -0300 Subject: [PATCH 45/79] Cleaning code --- .../microsoft/azure/transfers/sftp_to_wasb.py | 60 ++++++++++--------- .../azure/transfers/test_sftp_to_wasb.py | 55 +++++------------ 2 files changed, 48 insertions(+), 67 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 8bae74c49c0bb..4f4b51140f992 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -19,8 +19,11 @@ This module contains SFTP to Azure Blob Storage operator. """ import os +from collections import namedtuple from tempfile import NamedTemporaryFile -from typing import Any, Dict, List, Optional, Tuple +from typing import Any, Dict, List, Optional + +from cached_property import cached_property from airflow.exceptions import AirflowException from airflow.models import BaseOperator @@ -29,8 +32,7 @@ from airflow.utils.decorators import apply_defaults WILDCARD = "*" -SFTP_FILE_PATH = "SFTP_FILE_PATH" -BLOB_NAME = "BLOB_NAME" +SftpFile = namedtuple('SftpFile', 'sftp_file_path, blob_name') class SFTPToWasbOperator(BaseOperator): @@ -67,7 +69,7 @@ def __init__( sftp_source_path: str, container_name: str, blob_name: str, - sftp_conn_id: str = "ssh_default", + sftp_conn_id: str = "sftp_default", wasb_conn_id: str = 'wasb_default', load_options: Optional[dict] = None, move_object: bool = False, @@ -88,14 +90,14 @@ def __init__( def execute(self, context: Optional[Dict[Any, Any]]) -> None: """Upload a file from SFTP to Azure Blob Storage.""" - (sftp_hook, sftp_files) = self.get_sftp_files_map() - uploaded_files = self.upload_wasb(sftp_hook, sftp_files) - self.sftp_move(sftp_hook, uploaded_files) + sftp_files: List[SftpFile] = self.get_sftp_files_map() + uploaded_files = self.copy_files_to_wasb(sftp_files) + if self.move_object: + self.delete_files(uploaded_files) - def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: + def get_sftp_files_map(self) -> List[SftpFile]: """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] - sftp_hook = SFTPHook(self.sftp_conn_id) if WILDCARD in self.sftp_source_path: self.check_wildcards_limit() @@ -103,7 +105,7 @@ def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) sftp_complete_path = os.path.dirname(prefix) - found_files, _, _ = sftp_hook.get_tree_map( + found_files, _, _ = self.sftp_hook.get_tree_map( sftp_complete_path, prefix=prefix, delimiter=delimiter ) @@ -115,12 +117,17 @@ def get_sftp_files_map(self) -> Tuple[SFTPHook, List[Dict[str, str]]]: for file in found_files: future_blob_name = os.path.basename(file) - sftp_files.append({SFTP_FILE_PATH: file, BLOB_NAME: future_blob_name}) + sftp_files.append(SftpFile(file, future_blob_name)) else: - sftp_files.append({SFTP_FILE_PATH: self.sftp_source_path, BLOB_NAME: self.blob_name}) + sftp_files.append(SftpFile(self.sftp_source_path, self.blob_name)) - return sftp_hook, sftp_files + return sftp_files + + @cached_property + def sftp_hook(self): + """Property of sftp hook to be re-used.""" + return SFTPHook(self.sftp_conn_id) def check_wildcards_limit(self) -> Any: """Check if there is multiple Wildcard.""" @@ -131,36 +138,33 @@ def check_wildcards_limit(self) -> Any: "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) - def upload_wasb(self, - sftp_hook: SFTPHook, - sftp_files: List[Dict[str, str]]) -> List[str]: + def copy_files_to_wasb(self, + sftp_files: List[SftpFile]) -> List[str]: """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) for file in sftp_files: with NamedTemporaryFile("w") as tmp: - sftp_hook.retrieve_file(file[SFTP_FILE_PATH], tmp.name) + self.sftp_hook.retrieve_file(file.sftp_file_path, tmp.name) self.log.info( 'Uploading %s to wasb://%s as %s', - file[SFTP_FILE_PATH], + file.sftp_file_path, self.container_name, - file[BLOB_NAME], + file.blob_name, ) wasb_hook.load_file(tmp.name, self.container_name, - file[BLOB_NAME], + file.blob_name, **self.load_options) - uploaded_files.append(file[SFTP_FILE_PATH]) + uploaded_files.append(file.sftp_file_path) return uploaded_files - def sftp_move(self, - sftp_hook: SFTPHook, - uploaded_files: List[str]) -> None: + def delete_files(self, + uploaded_files: List[str]) -> None: """Performs a move of a list of files at SFTP to Azure Blob Storage.""" - if self.move_object: - for sftp_file_path in uploaded_files: - self.log.info("Executing delete of %s", sftp_file_path) - sftp_hook.delete_file(sftp_file_path) + for sftp_file_path in uploaded_files: + self.log.info("Executing delete of %s", sftp_file_path) + self.sftp_hook.delete_file(sftp_file_path) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index e38b4df07aded..8f02c8fff60e1 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -23,14 +23,13 @@ import mock from airflow import AirflowException -from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import ( - BLOB_NAME, SFTP_FILE_PATH, SFTPToWasbOperator, -) +from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" SFTP_CONN_ID = "ssh_default" +BLOB_NAME = "BLOB_NAME" CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" @@ -93,12 +92,10 @@ def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=False ) - _, files = operator.get_sftp_files_map() - - sftp_hook.assert_called_once_with(SFTP_CONN_ID) + files = operator.get_sftp_files_map() self.assertEqual(len(files), 2, "not matched at expected found files") - self.assertEqual(files[0][BLOB_NAME], EXPECTED_BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') @@ -117,16 +114,14 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): wasb_conn_id=WASB_CONN_ID, move_object=True ) - _, files = operator.get_sftp_files_map() - - sftp_hook.assert_called_once_with(SFTP_CONN_ID) + files = operator.get_sftp_files_map() self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0][BLOB_NAME], BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_upload_wasb(self, sftp_hook, mock_hook): + def test_copy_files_to_wasb(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, @@ -137,10 +132,10 @@ def test_upload_wasb(self, sftp_hook, mock_hook): move_object=True ) - sftp_files = [{SFTP_FILE_PATH: SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME: BLOB_NAME}] - files = operator.upload_wasb(sftp_hook, sftp_files) + sftp_files = [SftpFile(SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME)] + files = operator.copy_files_to_wasb(sftp_files) - sftp_hook.retrieve_file.assert_has_calls( + operator.sftp_hook.retrieve_file.assert_has_calls( [ mock.call("main_dir/test_object3.json", mock.ANY) ] @@ -153,7 +148,7 @@ def test_upload_wasb(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_sftp_move(self, sftp_hook): + def test_delete_files(self, sftp_hook): sftp_mock = sftp_hook.return_value operator = SFTPToWasbOperator( @@ -167,7 +162,7 @@ def test_sftp_move(self, sftp_hook): ) sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.sftp_move(sftp_mock, sftp_file_paths) + operator.delete_files(sftp_file_paths) sftp_mock.delete_file.assert_has_calls( [ @@ -175,13 +170,13 @@ def test_sftp_move(self, sftp_hook): ] ) + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_no_sftp_move(self, sftp_hook): - sftp_mock = sftp_hook.return_value + def test_execute(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, blob_name=BLOB_NAME, @@ -189,30 +184,12 @@ def test_no_sftp_move(self, sftp_hook): move_object=False ) - sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] - operator.sftp_move(sftp_mock, sftp_file_paths) - - sftp_mock.delete_file.assert_not_called() - - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_execute(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ [SOURCE_OBJECT_NO_WILDCARD], [], [], ] - operator = SFTPToWasbOperator( - task_id=TASK_ID, - sftp_source_path=WILDCARD_FILE_NAME, - sftp_conn_id=SFTP_CONN_ID, - container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, - wasb_conn_id=WASB_CONN_ID, - move_object=False - ) - operator.execute(None) sftp_hook.return_value.get_tree_map.assert_called_with( @@ -229,4 +206,4 @@ def test_execute(self, sftp_hook, mock_hook): mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) ) - sftp_hook.delete_file.assert_not_called() + operator.sftp_hook.delete_file.assert_not_called() From 3cbe63bf520b87a8e18fe4ce2fe07f4505f59007 Mon Sep 17 00:00:00 2001 From: guilherme Date: Mon, 15 Feb 2021 21:51:58 -0300 Subject: [PATCH 46/79] Resolves issues from MR --- .../azure/transfers/test_sftp_to_wasb.py | 177 +++++++++++------- 1 file changed, 106 insertions(+), 71 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 8f02c8fff60e1..9c06f20961231 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -17,57 +17,53 @@ # under the License. # -import os import unittest - -import mock +from unittest import mock from airflow import AirflowException from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator +BLOB_PREFIX = "sponge-bob" +CONTAINER_NAME = "test-container" +SOURCE_PATH_NO_WILDCARD = "main_dir/" +EXPECTED_BLOB_NAME = "test_object3.json" +EXPECTED_FILES = [SOURCE_PATH_NO_WILDCARD + EXPECTED_BLOB_NAME] +SOURCE_PATH_MULTIPLE_WILDCARDS = "main_dir/csv/*/test_*" +SFTP_CONN_ID = "ssh_default" TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" -SFTP_CONN_ID = "ssh_default" - -BLOB_NAME = "BLOB_NAME" -CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" -SOURCE_OBJECT_NO_WILDCARD = "main_dir/test_object3.json" -SOURCE_OBJECT_MULTIPLE_WILDCARDS = "main_dir/csv/*/test_*.csv" -NEW_BLOB_NAME = "sponge-bob" -EXPECTED_BLOB_NAME = "test_object3.json" # pylint: disable=unused-argument class TestSFTPToWasbOperator(unittest.TestCase): - def test_init(self): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=NEW_BLOB_NAME, + blob_prefix=BLOB_PREFIX, wasb_conn_id=WASB_CONN_ID, - move_object=False + move_object=False, ) - self.assertEqual(operator.sftp_source_path, SOURCE_OBJECT_NO_WILDCARD) + self.assertEqual(operator.sftp_source_path, SOURCE_PATH_NO_WILDCARD) self.assertEqual(operator.sftp_conn_id, SFTP_CONN_ID) self.assertEqual(operator.container_name, CONTAINER_NAME) self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) + self.assertEqual(operator.blob_prefix, BLOB_PREFIX) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', - autospec=True) + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, + sftp_source_path=SOURCE_PATH_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, + blob_prefix=BLOB_PREFIX, wasb_conn_id=WASB_CONN_ID, - move_object=False + move_object=False, ) with self.assertRaises(AirflowException) as cm: operator.check_wildcards_limit() @@ -75,121 +71,162 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_get_sftp_files_map_with_wildcard(self, sftp_hook, mock_hook): - sftp_hook.return_value.get_tree_map.return_value = [ - [SOURCE_OBJECT_NO_WILDCARD, SOURCE_OBJECT_NO_WILDCARD], - [], - [], - ] + def test_source_path_contains_wildcard(self): operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_PATH, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, + blob_prefix=BLOB_PREFIX, wasb_conn_id=WASB_CONN_ID, - move_object=False + move_object=False, ) - files = operator.get_sftp_files_map() - self.assertEqual(len(files), 2, "not matched at expected found files") - self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") + output = operator.source_path_contains_wildcard + + self.assertTrue(output, "This source path contains wildcard") + + def test_source_path_not_contains_wildcard(self): + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + blob_prefix=BLOB_PREFIX, + wasb_conn_id=WASB_CONN_ID, + move_object=False, + ) + + output = operator.source_path_contains_wildcard + + self.assertFalse(output, "This source path does not contains wildcard") + + def test_get_sftp_tree_behavior(self): + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_PATH, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False, + ) + sftp_complete_path, prefix, delimiter = operator.get_tree_behavior() + + self.assertEqual(sftp_complete_path, "main_dir", "not matched at expected complete path") + self.assertEqual(prefix, "main_dir/", "Prefix must be before the wildcard") + self.assertEqual(delimiter, "", "Delimiter must be empty") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): + def test_get_sftp_files_map(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ - [SOURCE_OBJECT_NO_WILDCARD], + EXPECTED_FILES, [], [], ] operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=True + move_object=True, ) files = operator.get_sftp_files_map() self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0].blob_name, BLOB_NAME, "expected blob name not matched") + self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_copy_files_to_wasb(self, sftp_hook, mock_hook): + def test_upload_wasb(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=True + move_object=True, ) - sftp_files = [SftpFile(SOURCE_OBJECT_NO_WILDCARD, BLOB_NAME)] + sftp_files = [SftpFile(EXPECTED_FILES[0], EXPECTED_BLOB_NAME)] files = operator.copy_files_to_wasb(sftp_files) - operator.sftp_hook.retrieve_file.assert_has_calls( - [ - mock.call("main_dir/test_object3.json", mock.ANY) - ] - ) + operator.sftp_hook.retrieve_file.assert_has_calls([mock.call("main_dir/test_object3.json", mock.ANY)]) - mock_hook.return_value.load_file.assert_called_once_with( - mock.ANY, CONTAINER_NAME, BLOB_NAME - ) + mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, EXPECTED_BLOB_NAME) self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_delete_files(self, sftp_hook): + def test_sftp_move(self, sftp_hook): sftp_mock = sftp_hook.return_value operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_OBJECT_NO_WILDCARD, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=True + move_object=True, ) - sftp_file_paths = [SOURCE_OBJECT_NO_WILDCARD] + sftp_file_paths = EXPECTED_FILES operator.delete_files(sftp_file_paths) - sftp_mock.delete_file.assert_has_calls( - [ - mock.call(SOURCE_OBJECT_NO_WILDCARD) - ] - ) + sftp_mock.delete_file.assert_has_calls([mock.call(EXPECTED_FILES[0])]) @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_execute(self, sftp_hook, mock_hook): + sftp_hook.return_value.get_tree_map.return_value = [ + ["main_dir/test_object.json"], + [], + [], + ] operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_FILE_NAME, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, - blob_name=BLOB_NAME, wasb_conn_id=WASB_CONN_ID, - move_object=False + move_object=False, ) + operator.execute(None) + + sftp_hook.return_value.get_tree_map.assert_called_with( + "main_dir", prefix="main_dir/test_object", delimiter=".json" + ) + + sftp_hook.return_value.retrieve_file.assert_has_calls( + [mock.call("main_dir/test_object.json", mock.ANY)] + ) + + mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") + + sftp_hook.delete_file.assert_not_called() + + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') + def test_execute_moving_files(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ - [SOURCE_OBJECT_NO_WILDCARD], + ["main_dir/test_object.json"], [], [], ] + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_FILE_NAME, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + wasb_conn_id=WASB_CONN_ID, + blob_prefix=BLOB_PREFIX, + move_object=True, + ) + operator.execute(None) sftp_hook.return_value.get_tree_map.assert_called_with( @@ -197,13 +234,11 @@ def test_execute(self, sftp_hook, mock_hook): ) sftp_hook.return_value.retrieve_file.assert_has_calls( - [ - mock.call(SOURCE_OBJECT_NO_WILDCARD, mock.ANY) - ] + [mock.call("main_dir/test_object.json", mock.ANY)] ) mock_hook.return_value.load_file.assert_called_once_with( - mock.ANY, CONTAINER_NAME, os.path.basename(SOURCE_OBJECT_NO_WILDCARD) + mock.ANY, CONTAINER_NAME, BLOB_PREFIX + "test_object.json" ) - operator.sftp_hook.delete_file.assert_not_called() + self.assertTrue(sftp_hook.return_value.delete_file.called, "It did perform move of file") From a7b526ebcf88c9bc591e69c5afcb83b498ab1a03 Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 16 Feb 2021 00:43:31 -0300 Subject: [PATCH 47/79] Fix files --- CONTRIBUTING.rst | 594 ++++++++---------- airflow/providers/dependencies.json | 3 +- .../microsoft/azure/transfers/sftp_to_wasb.py | 21 +- 3 files changed, 274 insertions(+), 344 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index abd00c0c92853..90be9975b3e90 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -1,4 +1,4 @@ - .. Licensed to the Apache Software Foundation (ASF) under one +.. Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information regarding copyright ownership. The ASF licenses this file @@ -23,6 +23,16 @@ Contributions Contributions are welcome and are greatly appreciated! Every little bit helps, and credit will always be given. +This document aims to explain the subject of contributions if you have not contributed to +any Open Source project, but it will also help people who have contributed to other projects learn about the +rules of that community. + +New Contributor +--------------- +If you are a new contributor, please follow the `Contributors Quick Start `__ guide to get a gentle step-by-step introduction to setting up the development +environment and making your first contribution. + Get Mentoring Support --------------------- @@ -92,24 +102,30 @@ If you are proposing a new feature: - Remember that this is a volunteer-driven project, and that contributions are welcome :) -Documentation + +Roles ============= -The latest API documentation is usually available -`here `__. +There are several roles within the Airflow Open-Source community. -To generate a local version: +For detailed information for each role, see: `Committers and PMC's <./COMMITTERS.rst>`__. -1. Set up an Airflow development environment. +PMC Member +----------- -2. Install the ``doc`` extra. +The PMC (Project Management Committee) is a group of maintainers that drives changes in the way that +Airflow is managed as a project. -.. code-block:: bash +Considering Apache, the role of the PMC is primarily to ensure that Airflow conforms to Apache's processes +and guidelines. - pip install -e '.[doc]' +Committers/Maintainers +---------------------- +Committers are community members that have write access to the project’s repositories, i.e., they can modify the code, +documentation, and website by themselves and also accept other contributions. -3. Generate and serve the documentation as follows: +The official list of committers can be found `here `__. Additionally, committers are listed in a few other places (some of these may only be visible to existing committers): @@ -204,23 +220,19 @@ You can configure the Docker-based Breeze development environment as follows: .. code-block:: bash - cd docs - ./build - ./start_doc_server.sh + mkvirtualenv myenv --python=python3.6 -.. note:: - The docs build script ``build`` requires Python 3.6 or greater. +5. Initialize the created environment: -**Known issues:** +.. code-block:: bash -If you are creating a new directory for new integration in the ``airflow.providers`` package, -you should also update the ``docs/autoapi_templates/index.rst`` file. + ./breeze initialize-local-virtualenv --python 3.6 -If you are creating a ``hooks``, ``sensors``, ``operators`` directory in -the ``airflow.providers`` package, you should also update -the ``docs/operators-and-hooks-ref.rst`` file. +6. Open your IDE (for example, PyCharm) and select the virtualenv you created + as the project's default virtualenv in your IDE. -If you are creating ``example_dags`` directory, you need to create ``example_dags/__init__.py`` with Apache license or copy another ``__init__.py`` file that contains the necessary license. +Step 3: Connect with People +--------------------------- For effective collaboration, make sure to join the following Airflow groups: @@ -344,7 +356,7 @@ these guidelines: - Include tests, either as doctests, unit tests, or both, to your pull request. - The airflow repo uses `Github Actions `__ to + The airflow repo uses `GitHub Actions `__ to run the tests and `codecov `__ to track coverage. You can set up both for free on your fork. It will help you make sure you do not break the build with your PR and that you help increase coverage. @@ -384,16 +396,39 @@ Airflow Git Branches ==================== All new development in Airflow happens in the ``master`` branch. All PRs should target that branch. -We also have a ``v1-10-test`` branch that is used to test ``1.10.x`` series of Airflow and where committers + + +We also have a ``v2-0-test`` branch that is used to test ``2.0.x`` series of Airflow and where committers cherry-pick selected commits from the master branch. + Cherry-picking is done with the ``-x`` flag. -The ``v1-10-test`` branch might be broken at times during testing. Expect force-pushes there so -committers should coordinate between themselves on who is working on the ``v1-10-test`` branch - +The ``v2-0-test`` branch might be broken at times during testing. Expect force-pushes there so +committers should coordinate between themselves on who is working on the ``v2-0-test`` branch - usually these are developers with the release manager permissions. -Once the branch is stable, the ``v1-10-stable`` branch is synchronized with ``v1-10-test``. -The ``v1-10-stable`` branch is used to release ``1.10.x`` releases. +The ``v2-0-stable`` branch is rather stable - there are minimum changes coming from approved PRs that +passed the tests. This means that the branch is rather, well, "stable". + +Once the ``v2-0-test`` branch stabilises, the ``v2-0-stable`` branch is synchronized with ``v2-0-test``. +The ``v2-0-stable`` branch is used to release ``2.0.x`` releases. + +The general approach is that cherry-picking a commit that has already had a PR and unit tests run +against main is done to ``v2-0-test`` branch, but PRs from contributors towards 2.0 should target +``v2-0-stable`` branch. + +The ``v2-0-test`` branch and ``v2-0-stable`` ones are merged just before the release and that's the +time when they converge. + +The production images are build in DockerHub from: + +* master branch for development +* v2-0-test branch for testing 2.0.x release +* ``2.0.*``, ``2.0.*rc*`` releases from the ``v2-0-stable`` branch when we prepare release candidates and + final releases. There are no production images prepared from v2-0-stable branch. + +Similar rules apply to ``1.10.x`` releases until June 2020. We have ``v1-10-test`` and ``v1-10-stable`` +branches there. Development Environments ======================== @@ -491,7 +526,7 @@ Benefits: - Breeze environment is almost the same as used in the CI automated builds. So, if the tests run in your Breeze environment, they will work in the CI as well. - See ``_ for details about Airflow CI. + See ``_ for details about Airflow CI. Limitations: @@ -512,6 +547,31 @@ Limitations: They are optimized for repeatability of tests, maintainability and speed of building rather than production performance. The production images are not yet officially published. + +Airflow dependencies +==================== + +.. note:: + + On November 2020, new version of PIP (20.3) has been released with a new, 2020 resolver. This resolver + might work with Apache Airflow as of 20.3.3, but it might lead to errors in installation. It might + depend on your choice of extras. In order to install Airflow you might need to either downgrade + pip to version 20.2.4 ``pip install --upgrade pip==20.2.4`` or, in case you use Pip 20.3, + you need to add option ``--use-deprecated legacy-resolver`` to your pip install command. + + While ``pip 20.3.3`` solved most of the ``teething`` problems of 20.3, this note will remain here until we + set ``pip 20.3`` as official version in our CI pipeline where we are testing the installation as well. + Due to those constraints, only ``pip`` installation is currently officially supported. + + While they are some successes with using other tools like `poetry `_ or + `pip-tools `_, they do not share the same workflow as + ``pip`` - especially when it comes to constraint vs. requirements management. + Installing via ``Poetry`` or ``pip-tools`` is not currently supported. + + If you wish to install airflow using those tools you should use the constraint files and convert + them to appropriate format and workflow that your tool requires. + + Extras ------ @@ -539,112 +599,38 @@ virtualenv, webhdfs, winrm, yandex, zendesk .. END EXTRAS HERE +Provider packages +----------------- -Airflow dependencies --------------------- - -Airflow is not a standard python project. Most of the python projects fall into one of two types - -application or library. As described in -[StackOverflow Question](https://stackoverflow.com/questions/28509481/should-i-pin-my-python-dependencies-versions) -decision whether to pin (freeze) requirements for a python project depdends on the type. For -applications, dependencies should be pinned, but for libraries, they should be open. - -For application, pinning the dependencies makes it more stable to install in the future - because new -(even transitive) dependencies might cause installation to fail. For libraries - the dependencies should -be open to allow several different libraries with the same requirements to be installed at the same time. - -The problem is that Apache Airflow is a bit of both - application to install and library to be used when -you are developing your own operators and DAGs. - -This - seemingly unsolvable - puzzle is solved by having pinned requirement files. Those are available -as of airflow 1.10.10. - -Pinned requirement files ------------------------- - -By default when you install ``apache-airflow`` package - the dependencies are as open as possible while -still allowing the apache-airflow package to install. This means that 'apache-airflow' package might fail to -install in case a direct or transitive dependency is released that breaks the installation. In such case -when installing ``apache-airflow``, you might need to provide additional constraints (for -example ``pip install apache-airflow==1.10.2 Werkzeug<1.0.0``) - -However we now have ``requirements-python.txt`` file generated -automatically and committed in the requirements folder based on the set of all latest working and tested -requirement versions. Those ``requirement-python.txt`` files can be used as -constraints file when installing Apache Airflow - either from the sources - -.. code-block:: bash - - pip install -e . --constraint requirements/requirements-python3.6.txt - - -or from the pypi package - -.. code-block:: bash - - pip install apache-airflow --constraint requirements/requirements-python3.6.txt - - -This works also with extras - for example: - -.. code-block:: bash +Airflow 2.0 is split into core and providers. They are delivered as separate packages: - pip install .[ssh] --constraint requirements/requirements-python3.6.txt - - -It is also possible to use constraints directly from github using tag/version name: - -.. code-block:: bash +* ``apache-airflow`` - core of Apache Airflow +* ``apache-airflow-providers-*`` - More than 50 provider packages to communicate with external services - pip install apache-airflow[ssh]==1.10.10 \ - --constraint https://raw.githubusercontent.com/apache/airflow/1.10.10/requirements/requirements-python3.6.txt +In Airflow 1.10 all those providers were installed together within one single package and when you installed +airflow locally, from sources, they were also installed. In Airflow 2.0, providers are separated out, +and not packaged together with the core, unless you set ``INSTALL_PROVIDERS_FROM_SOURCES`` environment +variable to ``true``. -There are different set of fixed requirements for different python major/minor versions and you should -use the right requirements file for the right python version. +In Breeze - which is a development environment, ``INSTALL_PROVIDERS_FROM_SOURCES`` variable is set to true, +but you can add ``--skip-installing-airflow-providers-from-sources`` flag to Breeze to skip installing providers when +building the images. -The ``requirements-python.txt`` file MUST be regenerated every time after -the ``setup.py`` is updated. This is checked automatically in the CI build. There are separate -jobs for each python version that checks if the requirements should be updated. - -If they are not updated, you should regenerate the requirements locally using Breeze as described below. - -Generating requirement files ----------------------------- - -This should be done every time after you modify setup.py file. You can generate requirement files -using `Breeze `_ . Simply use those commands: - -.. code-block:: bash - - breeze generate-requirements --python 3.7 - -.. code-block:: bash - - breeze generate-requirements --python 3.6 - -Note that when you generate requirements this way, you might update to latest version of requirements -that were released since the last time so during tests you might get errors unrelated to your change. -In this case the easiest way to fix it is to limit the culprit dependency to the previous version -with ```` constraint added in setup.py. - -Backport providers packages ---------------------------- - -Since we are developing new operators in the master branch, we prepared backport packages ready to be -installed for Airflow 1.10.* series. Those backport operators (the tested ones) are going to be released -in PyPi and we are going to maintain the list at -`Backported providers package page `_ +One watch-out - providers are still always installed (or rather available) if you install airflow from +sources using ``-e`` (or ``--editable``) flag. In such case airflow is read directly from the sources +without copying airflow packages to the usual installation location, and since 'providers' folder is +in this airflow folder - the providers package is importable. Some of the packages have cross-dependencies with other providers packages. This typically happens for transfer operators where operators use hooks from the other providers in case they are transferring data between the providers. The list of dependencies is maintained (automatically with pre-commits) in the ``airflow/providers/dependencies.json``. Pre-commits are also used to generate dependencies. -The dependency list is automatically used during pypi packages generation. +The dependency list is automatically used during PyPI packages generation. Cross-dependencies between provider packages are converted into extras - if you need functionality from the other provider package you can install it adding [extra] after the -apache-airflow-backport-providers-PROVIDER for example ``pip install -apache-airflow-backport-providers-google[amazon]`` in case you want to use GCP +``apache-airflow-backport-providers-PROVIDER`` for example: +``pip install apache-airflow-backport-providers-google[amazon]`` in case you want to use GCP transfer operators from Amazon ECS. If you add a new dependency between different providers packages, it will be detected automatically during @@ -675,9 +661,9 @@ apache.hive amazon,microsoft.mssql,mysql,presto,samba,vertica apache.livy http dingding http discord http -google amazon,apache.cassandra,cncf.kubernetes,facebook,microsoft.azure,microsoft.mssql,mysql,postgres,presto,sftp +google amazon,apache.beam,apache.cassandra,cncf.kubernetes,facebook,microsoft.azure,microsoft.mssql,mysql,oracle,postgres,presto,salesforce,sftp,ssh hashicorp google -microsoft.azure google,oracle,sftp +microsoft.azure google,oracle microsoft.mssql odbc mysql amazon,presto,vertica opsgenie http @@ -1001,24 +987,46 @@ If this function is designed to be called by "end-users" (i.e. DAG authors) then Don't use time() for duration calculations ----------------------------------------- -We support the following types of tests: +If you wish to compute the time difference between two events with in the same process, use +``time.monotonic()``, not ``time.time()`` nor ``timzeone.utcnow()``. -* **Unit tests** are Python tests launched with ``pytest``. - Unit tests are available both in the `Breeze environment `_ - and `local virtualenv `_. +If you are measuring duration for performance reasons, then ``time.perf_counter()`` should be used. (On many +platforms, this uses the same underlying clock mechanism as monotonic, but ``perf_counter`` is guaranteed to be +the highest accuracy clock on the system, monotonic is simply "guaranteed" to not go backwards.) -* **Integration tests** are available in the Breeze development environment - that is also used for Airflow's CI tests. Integration test are special tests that require - additional services running, such as Postgres, Mysql, Kerberos, etc. +If you wish to time how long a block of code takes, use ``Stats.timer()`` -- either with a metric name, which +will be timed and submitted automatically: -* **System tests** are automatic tests that use external systems like - Google Cloud Platform. These tests are intended for an end-to-end DAG execution. +.. code-block:: python -For details on running different types of Airflow tests, see `TESTING.rst `_. + from airflow.stats import Stats + + ... + + with Stats.timer("my_timer_metric"): + ... + +or to time but not send a metric: + +.. code-block:: python + + from airflow.stats import Stats + + ... + + with Stats.timer() as timer: + ... + + log.info("Code took %.3f seconds", timer.duration) +For full docs on ``timer()`` check out `airflow/stats.py`_. + +If the start_date of a duration calculation needs to be stored in a database, then this has to be done using +datetime objects. In all other cases, using datetime for duration calculation MUST be avoided as creating and +diffing datetime operations are (comparatively) slow. Naming Conventions for provider packages -======================================== +---------------------------------------- In Airflow 2.0 we standardized and enforced naming for provider packages, modules and classes. those rules (introduced as AIP-21) were not only introduced but enforced using automated checks @@ -1046,18 +1054,18 @@ The rules are as follows: * secrets -> secret backends are stored here * transfers -> transfer operators are stored here -* Module names do not contain word "hooks" , "operators" etc. The right type comes from +* Module names do not contain word "hooks", "operators" etc. The right type comes from the package. For example 'hooks.datastore' module contains DataStore hook and 'operators.datastore' contains DataStore operators. * Class names contain 'Operator', 'Hook', 'Sensor' - for example DataStoreHook, DataStoreExportOperator -* Operator name usually follows the convention: Operator +* Operator name usually follows the convention: ``Operator`` (BigQueryExecuteQueryOperator) is a good example * Transfer Operators are those that actively push data from one service/provider and send it to another service (might be for the same or another provider). This usually involves two hooks. The convention - for those ToOperator. They are not named *TransferOperator nor *Transfer. + for those ``ToOperator``. They are not named *TransferOperator nor *Transfer. * Operators that use external service to perform transfer (for example CloudDataTransferService operators are not placed in "transfers" package and do not have to follow the naming convention for @@ -1071,14 +1079,32 @@ The rules are as follows: * For Cloud Providers or Service providers that usually means that the transfer operators should land at the "target" side of the transfer -* Secret Backend name follows the convention: Backend. +* Secret Backend name follows the convention: ``Backend``. -* Tests are grouped in parallel packages under "tests.providers" top level package. Module name is usually - "test_.py', +* Tests are grouped in parallel packages under "tests.providers" top level package. Module name is usually + ``test_.py``, * System tests (not yet fully automated but allowing to run e2e testing of particular provider) are named with _system.py suffix. +Test Infrastructure +=================== + +We support the following types of tests: + +* **Unit tests** are Python tests launched with ``pytest``. + Unit tests are available both in the `Breeze environment `_ + and `local virtualenv `_. + +* **Integration tests** are available in the Breeze development environment + that is also used for Airflow's CI tests. Integration test are special tests that require + additional services running, such as Postgres, Mysql, Kerberos, etc. + +* **System tests** are automatic tests that use external systems like + Google Cloud. These tests are intended for an end-to-end DAG execution. + +For details on running different types of Airflow tests, see `TESTING.rst `_. + Metadata Database Updates ========================= @@ -1121,7 +1147,7 @@ To install yarn on macOS: .. code-block:: bash - brew install node --without-npm + brew install node brew install yarn yarn config set prefix ~/.yarn @@ -1174,13 +1200,13 @@ commands: yarn run dev -Follow Javascript Style Guide +Follow JavaScript Style Guide ----------------------------- We try to enforce a more consistent style and follow the JS community guidelines. -Once you add or modify any javascript code in the project, please make sure it +Once you add or modify any JavaScript code in the project, please make sure it follows the guidelines defined in `Airbnb JavaScript Style Guide `__. @@ -1196,226 +1222,125 @@ commands: # Check JS code in .js and .html files, report any errors/warnings and fix them if possible yarn run lint:fix -Contribution Workflow Example -============================== - -Typically, you start your first contribution by reviewing open tickets -at `GitHub issues `__. - -If you create pull-request, you don't have to create an issue first, but if you want, you can do it. -Creating an issue will allow you to collect feedback or share plans with other people. - -For example, you want to have the following sample ticket assigned to you: -`#7782: Add extra CC: to the emails sent by Aiflow `_. - -In general, your contribution includes the following stages: - -.. image:: images/workflow.png - :align: center - :alt: Contribution Workflow - -1. Make your own `fork `__ of - the Apache Airflow `main repository `__. - -2. Create a `local virtualenv `_, - initialize the `Breeze environment `__, and - install `pre-commit framework `__. - If you want to add more changes in the future, set up your fork and enable Github Actions. - -3. Join `devlist `__ - and set up a `Slack account `__. - -4. Make the change and create a `Pull Request from your fork `__. - -5. Ping @ #development slack, comment @people. Be annoying. Be considerate. - -Step 1: Fork the Apache Repo ----------------------------- -From the `apache/airflow `_ repo, -`create a fork `_: - -.. image:: images/fork.png - :align: center - :alt: Creating a fork - - -Step 2: Configure Your Environment ----------------------------------- -Configure the Docker-based Breeze development environment and run tests. - -You can use the default Breeze configuration as follows: - -1. Install the latest versions of the Docker Community Edition - and Docker Compose and add them to the PATH. - -2. Enter Breeze: ``./breeze`` - - Breeze starts with downloading the Airflow CI image from - the Docker Hub and installing all required dependencies. - -3. Enter the Docker environment and mount your local sources - to make them immediately visible in the environment. - -4. Create a local virtualenv, for example: - -.. code-block:: bash - - mkvirtualenv myenv --python=python3.6 - -5. Initialize the created environment: - -.. code-block:: bash - - ./breeze --initialize-local-virtualenv +How to sync your fork +===================== -6. Open your IDE (for example, PyCharm) and select the virtualenv you created - as the project's default virtualenv in your IDE. +When you have your fork, you should periodically synchronize the master of your fork with the +Apache Airflow master. In order to do that you can ``git pull --rebase`` to your local git repository from +apache remote and push the master (often with ``--force`` to your fork). There is also an easy +way using ``Force sync master from apache/airflow`` workflow. You can go to "Actions" in your repository and +choose the workflow and manually trigger the workflow using "Run workflow" command. -Step 3: Connect with People ---------------------------- +This will force-push the master from apache/airflow to the master in your fork. Note that in case you +modified the master in your fork, you might loose those changes. -For effective collaboration, make sure to join the following Airflow groups: -- Mailing lists: +How to rebase PR +================ - - Developer’s mailing list ``_ - (quite substantial traffic on this list) +A lot of people are unfamiliar with the rebase workflow in Git, but we think it is an excellent workflow, +providing a better alternative to the merge workflow. We've therefore written a short guide for those who would like to learn it. - - All commits mailing list: ``_ - (very high traffic on this list) +As opposed to the merge workflow, the rebase workflow allows us to +clearly separate your changes from the changes of others. It puts the responsibility of rebasing on the +author of the change. It also produces a "single-line" series of commits on the master branch. This +makes it easier to understand what was going on and to find reasons for problems (it is especially +useful for "bisecting" when looking for a commit that introduced some bugs). - - Airflow users mailing list: ``_ - (reasonably small traffic on this list) +First of all, we suggest you read about the rebase workflow here: +`Merging vs. rebasing `_. This is an +excellent article that describes all the ins/outs of the rebase workflow. I recommend keeping it for future reference. -- `Issues on GitHub `__ +The goal of rebasing your PR on top of ``apache/master`` is to "transplant" your change on top of +the latest changes that are merged by others. It also allows you to fix all the conflicts +that arise as a result of other people changing the same files as you and merging the changes to ``apache/master``. -- `Slack (chat) `__ +Here is how rebase looks in practice (you can find a summary below these detailed steps): -Step 4: Prepare PR ------------------- +1. You first need to add the Apache project remote to your git repository. This is only necessary once, +so if it's not the first time you are following this tutorial you can skip this step. In this example, +we will be adding the remote +as "apache" so you can refer to it easily: -1. Update the local sources to address the issue. +* If you use ssh: ``git remote add apache git@github.com:apache/airflow.git`` +* If you use https: ``git remote add apache https://github.com/apache/airflow.git`` - For example, to address this example issue, do the following: +2. You then need to make sure that you have the latest master fetched from the ``apache`` repository. You can do this + via: - * Read about `email configuration in Airflow `__. + ``git fetch apache`` (to fetch apache remote) - * Find the class you should modify. For the example ticket, - this is `email.py `__. + ``git fetch --all`` (to fetch all remotes) - * Find the test class where you should add tests. For the example ticket, - this is `test_email.py `__. +3. Assuming that your feature is in a branch in your repository called ``my-branch`` you can easily check + what is the base commit you should rebase from by: - * Create a local branch for your development. Make sure to use latest - ``apache/master`` as base for the branch. See `How to Rebase PR <#how-to-rebase-pr>`_ for some details - on setting up the ``apache`` remote. Note - some people develop their changes directy in their own - ``master`` branches - this is OK and you can make PR from your master to ``apache/master`` but we - recommend to always create a local branch for your development. This allows you to easily compare - changes, have several changes that you work on at the same time and many more. - If you have ``apache`` set as remote then you can make sure that you have latest changes in your master - by ``git pull apache master`` when you are in the local ``master`` branch. If you have conflicts and - want to override your locally changed master you can override your local changes with - ``git fetch apache; git reset --hard apache/master``. + ``git merge-base my-branch apache/master`` - * Modify the class and add necessary code and unit tests. + This will print the HASH of the base commit which you should use to rebase your feature from. + For example: ``5abce471e0690c6b8d06ca25685b0845c5fd270f``. Copy that HASH and go to the next step. - * Run the unit tests from the `IDE `__ - or `local virtualenv `__ as you see fit. + Optionally, if you want better control you can also find this commit hash manually. - * Run the tests in `Breeze `__. + Run: - * Run and fix all the `static checks `__. If you have - `pre-commits installed `__, - this step is automatically run while you are committing your code. If not, you can do it manually - via ``git add`` and then ``pre-commit run``. + ``git log`` -2. Rebase your fork, squash commits, and resolve all conflicts. See `How to rebase PR <#how-to-rebase-pr>`_ - if you need help with rebasing your change. Remember to rebase often if your PR takes a lot of time to - review/fix. This will make rebase process much easier and less painful - and the more often you do it, - the more comfortable you will feel doing it. + And find the first commit that you DO NOT want to "transplant". -3. Re-run static code checks again. + Performing: -4. Create a pull request with the following title for the sample ticket: - ``[AIRFLOW-5934] Added extra CC: field to the Airflow emails.`` + ``git rebase HASH`` -Make sure to follow other PR guidelines described in `this document <#pull-request-guidelines>`_. + Will "transplant" all commits after the commit with the HASH. +4. Providing that you weren't already working on your branch, check out your feature branch locally via: -Step 5: Pass PR Review ----------------------- + ``git checkout my-branch`` -.. image:: images/review.png - :align: center - :alt: PR Review +5. Rebase: -Note that committers will use **Squash and Merge** instead of **Rebase and Merge** -when merging PRs and your commit will be squashed to single commit. + ``git rebase HASH --onto apache/master`` -How to rebase PR -================ + For example: -A lot of people are unfamiliar with rebase workflow in Git, but we think it is an excellent workflow, -much better than merge workflow, so here is a short guide for those who would like to learn it. It's really -worth to spend a few minutes learning it. As opposed to merge workflow, the rebase workflow allows to -clearly separate your changes from changes of others, puts responsibility of proper rebase on the -author of the change. It also produces a "single-line" series of commits in master branch which -makes it much easier to understand what was going on and to find reasons for problems (it is especially -useful for "bisecting" when looking for a commit that introduced some bugs. + ``git rebase 5abce471e0690c6b8d06ca25685b0845c5fd270f --onto apache/master`` +6. If you have no conflicts - that's cool. You rebased. You can now run ``git push --force-with-lease`` to + push your changes to your repository. That should trigger the build in our CI if you have a + Pull Request (PR) opened already. -First of all - you can read about rebase workflow here: -`Merging vs. rebasing `_ - this is an -excellent article that describes all ins/outs of rebase. I recommend reading it and keeping it as reference. +7. While rebasing you might have conflicts. Read carefully what git tells you when it prints information + about the conflicts. You need to solve the conflicts manually. This is sometimes the most difficult + part and requires deliberately correcting your code and looking at what has changed since you developed your + changes. -The goal of rebasing your PR on top of ``apache/master`` is to "transplant" your change on top of -the latest changes that are merged by others. It also allows you to fix all the conflicts -that are result of other people changing the same files as you and merging the changes to ``apache/master``. + There are various tools that can help you with this. You can use: -Here is how rebase looks in practice: + ``git mergetool`` -1. You need to add Apache remote to your git repository. You can add it as "apache" remote so that - you can refer to it easily: + You can configure different merge tools with it. You can also use IntelliJ/PyCharm's excellent merge tool. + When you open a project in PyCharm which has conflicts, you can go to VCS > Git > Resolve Conflicts and there + you have a very intuitive and helpful merge tool. For more information, see + `Resolve conflicts `_. -``git remote add apache git@github.com:apache/airflow.git`` if you use ssh or -``git remote add apache https://github.com/apache/airflow.git`` if you use https. +8. After you've solved your conflict run: -Later on + ``git rebase --continue`` -2. You need to make sure that you have the latest master fetched from ``apache`` repository. You can do it - by ``git fetch apache`` for apache remote or ``git fetch --all`` to fetch all remotes. + And go either to point 6. or 7, depending on whether you have more commits that cause conflicts in your PR (rebasing applies each + commit from your PR one-by-one). -3. Assuming that your feature is in a branch in your repository called ``my-branch`` you can check easily - what is the base commit you should rebase from by: ``git merge-base my-branch apache/master``. - This will print the HASH of the base commit which you should use to rebase your feature from - - for example: ``5abce471e0690c6b8d06ca25685b0845c5fd270f``. You can also find this commit hash manually - - if you want better control. Run ``git log`` and find the first commit that you DO NOT want to "transplant". - ``git rebase HASH`` will "trasplant" all commits after the commit with the HASH. +Summary +------------- -4. Make sure you checked out your branch locally: +Useful when you understand the flow but don't remember the steps and want a quick reference. +``git fetch --all`` +``git merge-base my-branch apache/master`` ``git checkout my-branch`` - -5. Rebase: - Run: ``git rebase HASH --onto apache/master`` - for example: ``git rebase 5abce471e0690c6b8d06ca25685b0845c5fd270f --onto apache/master`` - -6. If you have no conflicts - that's cool. You rebased. You can now run ``git push --force-with-lease`` to - push your changes to your repository. That should trigger the build in our CI if you have a - Pull Request opened already. - -7. While rebasing you might have conflicts. Read carefully what git tells you when it prints information - about the conflicts. You need to solve the conflicts manually. This is sometimes the most difficult - part and requires deliberate correcting your code looking what has changed since you developed your - changes. There are various tools that can help you with that. You can use ``git mergetool`` (and you can - configure different merge tools with it). Also you can use IntelliJ/PyCharm excellent merge tool. - When you open project in PyCharm which has conflict you can go to VCS->Git->Resolve Conflicts and there - you have a very intuitive and helpful merge tool. You can see more information - about it in `Resolve conflicts `_ - -8. After you solved conflicts simply run ``git rebase --continue`` and go either to point 6. or 7. - above depending if you have more commits that cause conflicts in your PR (rebasing applies each - commit from your PR one-by-one). +``git rebase HASH --onto apache/master`` +``git push --force-with-lease`` How to communicate ================== @@ -1443,17 +1368,17 @@ You can join the channels via links at the `Airflow Community page `_ for: +* GitHub `Pull Requests (PRs) `_ for: * discussing implementation details of PRs * not for architectural discussions (use the devlist for that) * The deprecated `JIRA issues `_ for: - * checking out old but still valuable issues that are not on Github yet - * mentioning the JIRA issue number in the title of the related PR you would like to open on Github + * checking out old but still valuable issues that are not on GitHub yet + * mentioning the JIRA issue number in the title of the related PR you would like to open on GitHub **IMPORTANT** -We don't create new issues on JIRA anymore. The reason we still look at JIRA issues is that there are valuable tickets inside of it. However, each new PR should be created on `Github issues `_ as stated in `Contribution Workflow Example `_ +We don't create new issues on JIRA anymore. The reason we still look at JIRA issues is that there are valuable tickets inside of it. However, each new PR should be created on `GitHub issues `_ as stated in `Contribution Workflow Example `_ -* The `Apache Airflow Slack `_ for: +* The `Apache Airflow Slack `_ for: * ad-hoc questions related to development (#development channel) * asking for review (#development channel) * asking for help with PRs (#how-to-pr channel) @@ -1526,8 +1451,21 @@ Here are a few rules that are important to keep in mind when you enter our commu * It’s OK to express your own emotions while communicating - it helps other people to understand you * Be considerate for feelings of others. Tell about how you feel not what you think of others + + +Commit Policy +============= + +The following commit policy passed by a vote 8(binding FOR) to 0 against on May 27, 2016 on the dev list +and slightly modified and consensus reached in October 2020: + +* Commits need a +1 vote from a committer who is not the author +* Do not merge a PR that regresses linting or does not pass CI tests (unless we have + justification such as clearly transient error). +* When we do AIP voting, both PMC and committer +1s are considered as binding vote. + Resources & Links ================= -- `Airflow’s official documentation `__ +- `Airflow’s official documentation `__ - `More resources and links to Airflow related content on the Wiki `__ diff --git a/airflow/providers/dependencies.json b/airflow/providers/dependencies.json index c932cfa1a0deb..e4f100df684aa 100644 --- a/airflow/providers/dependencies.json +++ b/airflow/providers/dependencies.json @@ -57,8 +57,7 @@ ], "microsoft.azure": [ "google", - "oracle", - "sftp" + "oracle" ], "microsoft.mssql": [ "odbc" diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 4f4b51140f992..4bdfbad34458b 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -61,6 +61,7 @@ class SFTPToWasbOperator(BaseOperator): as opposed to a cp command. :type move_object: bool """ + template_fields = ("sftp_source_path", "container_name", "blob_name") @apply_defaults @@ -74,7 +75,7 @@ def __init__( load_options: Optional[dict] = None, move_object: bool = False, *args, - **kwargs + **kwargs, ) -> None: super().__init__(*args, **kwargs) @@ -87,8 +88,7 @@ def __init__( self.load_options = load_options if load_options is not None else {} self.move_object = move_object - def execute(self, - context: Optional[Dict[Any, Any]]) -> None: + def execute(self, context: Optional[Dict[Any, Any]]) -> None: """Upload a file from SFTP to Azure Blob Storage.""" sftp_files: List[SftpFile] = self.get_sftp_files_map() uploaded_files = self.copy_files_to_wasb(sftp_files) @@ -110,9 +110,7 @@ def get_sftp_files_map(self) -> List[SftpFile]: ) self.log.info( - "Found %s files at sftp source path: %s", - str(len(found_files)), - self.sftp_source_path + "Found %s files at sftp source path: %s", str(len(found_files)), self.sftp_source_path ) for file in found_files: @@ -138,8 +136,7 @@ def check_wildcards_limit(self) -> Any: "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) - def copy_files_to_wasb(self, - sftp_files: List[SftpFile]) -> List[str]: + def copy_files_to_wasb(self, sftp_files: List[SftpFile]) -> List[str]: """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) @@ -153,17 +150,13 @@ def copy_files_to_wasb(self, self.container_name, file.blob_name, ) - wasb_hook.load_file(tmp.name, - self.container_name, - file.blob_name, - **self.load_options) + wasb_hook.load_file(tmp.name, self.container_name, file.blob_name, **self.load_options) uploaded_files.append(file.sftp_file_path) return uploaded_files - def delete_files(self, - uploaded_files: List[str]) -> None: + def delete_files(self, uploaded_files: List[str]) -> None: """Performs a move of a list of files at SFTP to Azure Blob Storage.""" for sftp_file_path in uploaded_files: self.log.info("Executing delete of %s", sftp_file_path) From 0221102b68afedfd20b292af60b13e258573930e Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 16 Feb 2021 00:45:27 -0300 Subject: [PATCH 48/79] Removes old file --- docs/operators-and-hooks-ref.rst | 1646 ------------------------------ 1 file changed, 1646 deletions(-) delete mode 100644 docs/operators-and-hooks-ref.rst diff --git a/docs/operators-and-hooks-ref.rst b/docs/operators-and-hooks-ref.rst deleted file mode 100644 index 8c6a7d9338faa..0000000000000 --- a/docs/operators-and-hooks-ref.rst +++ /dev/null @@ -1,1646 +0,0 @@ - .. Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - .. http://www.apache.org/licenses/LICENSE-2.0 - - .. Unless required by applicable law or agreed to in writing, - software distributed under the License is distributed on an - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - KIND, either express or implied. See the License for the - specific language governing permissions and limitations - under the License. - -Operators and Hooks Reference -============================= - -.. contents:: Content - :local: - :depth: 1 - -.. _fundamentals: - -Fundamentals ------------- - -**Base:** - -.. list-table:: - :header-rows: 1 - - * - Module - - Guides - - * - :mod:`airflow.hooks.base_hook` - - - - * - :mod:`airflow.hooks.dbapi_hook` - - - - * - :mod:`airflow.models.baseoperator` - - - - * - :mod:`airflow.sensors.base_sensor_operator` - - - -**Operators:** - -.. list-table:: - :header-rows: 1 - - * - Operators - - Guides - - * - :mod:`airflow.operators.branch_operator` - - - * - :mod:`airflow.operators.dagrun_operator` - - - - * - :mod:`airflow.operators.dummy_operator` - - - - * - :mod:`airflow.operators.generic_transfer` - - - - * - :mod:`airflow.operators.latest_only_operator` - - - - * - :mod:`airflow.operators.subdag_operator` - - - - * - :mod:`airflow.operators.sql` - - - -**Sensors:** - -.. list-table:: - :header-rows: 1 - - * - Sensors - - Guides - - * - :mod:`airflow.sensors.weekday_sensor` - - - - * - :mod:`airflow.sensors.external_task_sensor` - - :doc:`How to use ` - - * - :mod:`airflow.sensors.sql_sensor` - - - - * - :mod:`airflow.sensors.time_delta_sensor` - - - - * - :mod:`airflow.sensors.time_sensor` - - - - -.. _Apache: - -ASF: Apache Software Foundation -------------------------------- - -Airflow supports various software created by `Apache Software Foundation `__. - -Software operators and hooks -'''''''''''''''''''''''''''' - -These integrations allow you to perform various operations within software developed by Apache Software -Foundation. - -.. list-table:: - :header-rows: 1 - - * - Service name - - Guides - - Hook - - Operator - - Sensor - - * - `Apache Cassandra `__ - - - - :mod:`airflow.providers.apache.cassandra.hooks.cassandra` - - - - :mod:`airflow.providers.apache.cassandra.sensors.record`, - :mod:`airflow.providers.apache.cassandra.sensors.table` - - * - `Apache Druid `__ - - - - :mod:`airflow.providers.apache.druid.hooks.druid` - - :mod:`airflow.providers.apache.druid.operators.druid`, - :mod:`airflow.providers.apache.druid.operators.druid_check` - - - - * - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.hooks.hive` - - :mod:`airflow.providers.apache.hive.operators.hive`, - :mod:`airflow.providers.apache.hive.operators.hive_stats` - - :mod:`airflow.providers.apache.hive.sensors.named_hive_partition`, - :mod:`airflow.providers.apache.hive.sensors.hive_partition`, - :mod:`airflow.providers.apache.hive.sensors.metastore_partition` - - * - `Apache Livy `__ - - - - :mod:`airflow.providers.apache.livy.hooks.livy` - - :mod:`airflow.providers.apache.livy.operators.livy` - - :mod:`airflow.providers.apache.livy.sensors.livy` - - * - `Apache Pig `__ - - - - :mod:`airflow.providers.apache.pig.hooks.pig` - - :mod:`airflow.providers.apache.pig.operators.pig` - - - - * - `Apache Pinot `__ - - - - :mod:`airflow.providers.apache.pinot.hooks.pinot` - - - - - - * - `Apache Spark `__ - - :doc:`How to use ` - - :mod:`airflow.providers.apache.spark.hooks.spark_jdbc`, - :mod:`airflow.providers.apache.spark.hooks.spark_jdbc_script`, - :mod:`airflow.providers.apache.spark.hooks.spark_sql`, - :mod:`airflow.providers.apache.spark.hooks.spark_submit` - - :mod:`airflow.providers.apache.spark.operators.spark_jdbc`, - :mod:`airflow.providers.apache.spark.operators.spark_sql`, - :mod:`airflow.providers.apache.spark.operators.spark_submit` - - - - * - `Apache Sqoop `__ - - - - :mod:`airflow.providers.apache.sqoop.hooks.sqoop` - - :mod:`airflow.providers.apache.sqoop.operators.sqoop` - - - - * - `Hadoop Distributed File System (HDFS) `__ - - - - :mod:`airflow.providers.apache.hdfs.hooks.hdfs` - - - - :mod:`airflow.providers.apache.hdfs.sensors.hdfs` - - * - `WebHDFS `__ - - - - :mod:`airflow.providers.apache.hdfs.hooks.webhdfs` - - - - :mod:`airflow.providers.apache.hdfs.sensors.web_hdfs` - -Transfer operators and hooks -'''''''''''''''''''''''''''' - -These integrations allow you to copy data from/to software developed by Apache Software -Foundation. - -.. list-table:: - :header-rows: 1 - - * - Source - - Destination - - Guide - - Operator - - * - `Amazon Simple Storage Service (S3) `_ - - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.transfers.s3_to_hive` - - * - `Amazon Simple Storage Service (S3) `_ - - `MySQL `__ - - - - :mod:`airflow.providers.mysql.transfers.s3_to_mysql` - - * - `Apache Cassandra `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.cassandra_to_gcs` - - * - `Apache Hive `__ - - `Amazon DynamoDB `__ - - - - :mod:`airflow.providers.amazon.aws.transfers.hive_to_dynamodb` - - * - `Apache Hive `__ - - `Apache Druid `__ - - - - :mod:`airflow.providers.apache.druid.transfers.hive_to_druid` - - * - `Apache Hive `__ - - `MySQL `__ - - - - :mod:`airflow.providers.apache.hive.transfers.hive_to_mysql` - - * - `Apache Hive `__ - - `Samba `__ - - - - :mod:`airflow.providers.apache.hive.transfers.hive_to_samba` - - * - `Microsoft SQL Server (MSSQL) `__ - - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.transfers.mssql_to_hive` - - * - `MySQL `__ - - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.transfers.mysql_to_hive` - - * - `Vertica `__ - - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.transfers.vertica_to_hive` - -.. _Azure: - -Azure: Microsoft Azure ----------------------- - -Airflow has limited support for `Microsoft Azure `__. - -Service operators and hooks -''''''''''''''''''''''''''' - -These integrations allow you to perform various operations within the Microsoft Azure. - - -.. list-table:: - :header-rows: 1 - - * - Service name - - Hook - - Operator - - Sensor - - * - `Azure Batch `__ - - :mod:`airflow.providers.microsoft.azure.hooks.azure_batch` - - :mod:`airflow.providers.microsoft.azure.operators.azure_batch` - - - - * - `Azure Blob Storage `__ - - :mod:`airflow.providers.microsoft.azure.hooks.wasb` - - :mod:`airflow.providers.microsoft.azure.operators.wasb_delete_blob` - - :mod:`airflow.providers.microsoft.azure.sensors.wasb` - - * - `Azure Container Instances `__ - - :mod:`airflow.providers.microsoft.azure.hooks.azure_container_instance`, - :mod:`airflow.providers.microsoft.azure.hooks.azure_container_registry`, - :mod:`airflow.providers.microsoft.azure.hooks.azure_container_volume` - - :mod:`airflow.providers.microsoft.azure.operators.azure_container_instances` - - - - * - `Azure Cosmos DB `__ - - :mod:`airflow.providers.microsoft.azure.hooks.azure_cosmos` - - :mod:`airflow.providers.microsoft.azure.operators.azure_cosmos` - - :mod:`airflow.providers.microsoft.azure.sensors.azure_cosmos` - - * - `Azure Data Lake Storage `__ - - :mod:`airflow.providers.microsoft.azure.hooks.azure_data_lake` - - :mod:`airflow.providers.microsoft.azure.operators.adls_list` - - - - * - `Azure Data Explorer `__ - - :mod:`airflow.providers.microsoft.azure.hooks.adx` - - :mod:`airflow.providers.microsoft.azure.operators.adx` - - - - * - `Azure Files `__ - - :mod:`airflow.providers.microsoft.azure.hooks.azure_fileshare` - - - - - -Transfer operators and hooks -'''''''''''''''''''''''''''' - -These integrations allow you to copy data from/to Microsoft Azure. - -.. list-table:: - :header-rows: 1 - - * - Source - - Destination - - Guide - - Operator - - * - `Azure Data Lake Storage `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.adls_to_gcs` - - * - Local - - `Azure Blob Storage `__ - - - - :mod:`airflow.providers.microsoft.azure.transfers.file_to_wasb` - - * - SFTP - - `Azure Blob Storage `__ - - - - :mod:`airflow.providers.microsoft.azure.transfers.sftp_to_wasb` - - - * - `Oracle `__ - - `Azure Data Lake Storage `__ - - - - :mod:`airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake` - - -.. _AWS: - -AWS: Amazon Web Services ------------------------- - -Airflow has support for `Amazon Web Services `__. - -All hooks are based on :mod:`airflow.providers.amazon.aws.hooks.base_aws`. - -Service operators and hooks -''''''''''''''''''''''''''' - -These integrations allow you to perform various operations within the Amazon Web Services. - -.. list-table:: - :header-rows: 1 - - * - Service name - - Guide - - Hook - - Operator - - Sensor - - * - `AWS Batch `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.batch_client`, - :mod:`airflow.providers.amazon.aws.hooks.batch_waiters` - - :mod:`airflow.providers.amazon.aws.operators.batch` - - - - * - `AWS DataSync `__ - - :doc:`How to use ` - - :mod:`airflow.providers.amazon.aws.hooks.datasync` - - :mod:`airflow.providers.amazon.aws.operators.datasync` - - - - * - `AWS Glue Catalog `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.glue_catalog` - - - - :mod:`airflow.providers.amazon.aws.sensors.glue_catalog_partition` - - * - `AWS Glue `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.glue` - - :mod:`airflow.providers.amazon.aws.operators.glue` - - :mod:`airflow.providers.amazon.aws.sensors.glue` - - * - `AWS Lambda `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.lambda_function` - - - - - - * - `Amazon Athena `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.athena` - - :mod:`airflow.providers.amazon.aws.operators.athena` - - :mod:`airflow.providers.amazon.aws.sensors.athena` - - * - `Amazon CloudFormation `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.cloud_formation` - - :mod:`airflow.providers.amazon.aws.operators.cloud_formation` - - :mod:`airflow.providers.amazon.aws.sensors.cloud_formation` - - * - `Amazon CloudWatch Logs `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.logs` - - - - - - * - `Amazon DynamoDB `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.aws_dynamodb` - - - - - - * - `Amazon EC2 `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.ec2` - - :mod:`airflow.providers.amazon.aws.operators.ec2_start_instance`, - :mod:`airflow.providers.amazon.aws.operators.ec2_stop_instance`, - - :mod:`airflow.providers.amazon.aws.sensors.ec2_instance_state` - - * - `Amazon ECS `__ - - :doc:`How to use ` - - - - :mod:`airflow.providers.amazon.aws.operators.ecs` - - - - * - `Amazon EMR `__ - - :doc:`How to use ` - - :mod:`airflow.providers.amazon.aws.hooks.emr` - - :mod:`airflow.providers.amazon.aws.operators.emr_add_steps`, - :mod:`airflow.providers.amazon.aws.operators.emr_create_job_flow`, - :mod:`airflow.providers.amazon.aws.operators.emr_terminate_job_flow`, - :mod:`airflow.providers.amazon.aws.operators.emr_modify_cluster` - - :mod:`airflow.providers.amazon.aws.sensors.emr_base`, - :mod:`airflow.providers.amazon.aws.sensors.emr_job_flow`, - :mod:`airflow.providers.amazon.aws.sensors.emr_step` - - * - `Amazon Kinesis Data Firehose `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.kinesis` - - - - - - * - `Amazon Redshift `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.redshift` - - - - :mod:`airflow.providers.amazon.aws.sensors.redshift` - - * - `Amazon SageMaker `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.sagemaker` - - :mod:`airflow.providers.amazon.aws.operators.sagemaker_base`, - :mod:`airflow.providers.amazon.aws.operators.sagemaker_endpoint_config`, - :mod:`airflow.providers.amazon.aws.operators.sagemaker_endpoint`, - :mod:`airflow.providers.amazon.aws.operators.sagemaker_model`, - :mod:`airflow.providers.amazon.aws.operators.sagemaker_training`, - :mod:`airflow.providers.amazon.aws.operators.sagemaker_transform`, - :mod:`airflow.providers.amazon.aws.operators.sagemaker_tuning` - - :mod:`airflow.providers.amazon.aws.sensors.sagemaker_base`, - :mod:`airflow.providers.amazon.aws.sensors.sagemaker_endpoint`, - :mod:`airflow.providers.amazon.aws.sensors.sagemaker_training`, - :mod:`airflow.providers.amazon.aws.sensors.sagemaker_transform`, - :mod:`airflow.providers.amazon.aws.sensors.sagemaker_tuning` - - * - `Amazon Simple Notification Service (SNS) `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.sns` - - :mod:`airflow.providers.amazon.aws.operators.sns` - - - - * - `Amazon Simple Queue Service (SQS) `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.sqs` - - :mod:`airflow.providers.amazon.aws.operators.sqs` - - :mod:`airflow.providers.amazon.aws.sensors.sqs` - - * - `Amazon Simple Storage Service (S3) `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.s3` - - :mod:`airflow.providers.amazon.aws.operators.s3_bucket`, - :mod:`airflow.providers.amazon.aws.operators.s3_file_transform`, - :mod:`airflow.providers.amazon.aws.operators.s3_copy_object`, - :mod:`airflow.providers.amazon.aws.operators.s3_delete_objects`, - :mod:`airflow.providers.amazon.aws.operators.s3_list` - - :mod:`airflow.providers.amazon.aws.sensors.s3_key`, - :mod:`airflow.providers.amazon.aws.sensors.s3_prefix` - - * - `AWS Step Functions `__ - - - - :mod:`airflow.providers.amazon.aws.hooks.step_function` - - :mod:`airflow.providers.amazon.aws.operators.step_function_start_execution`, - :mod:`airflow.providers.amazon.aws.operators.step_function_get_execution_output`, - - :mod:`airflow.providers.amazon.aws.sensors.step_function_execution`, - -Transfer operators and hooks -'''''''''''''''''''''''''''' - -These integrations allow you to copy data from/to Amazon Web Services. - -.. list-table:: - :header-rows: 1 - - * - Source - - Destination - - Guide - - Operator - - * - - .. _integration:AWS-Discovery-ref: - - All GCP services :ref:`[1] ` - - `Amazon Simple Storage Service (S3) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.amazon.aws.transfers.google_api_to_s3` - - * - `Amazon DataSync `__ - - `Amazon Simple Storage Service (S3) `_ - - :doc:`How to use ` - - :mod:`airflow.providers.amazon.aws.operators.datasync` - - * - `Amazon DynamoDB `__ - - `Amazon Simple Storage Service (S3) `_ - - - - :mod:`airflow.providers.amazon.aws.transfers.dynamodb_to_s3` - - * - `Amazon Redshift `__ - - `Amazon Simple Storage Service (S3) `_ - - - - :mod:`airflow.providers.amazon.aws.transfers.redshift_to_s3` - - * - `Amazon Simple Storage Service (S3) `_ - - `Amazon Redshift `__ - - :doc:`How to use ` - - :mod:`airflow.providers.amazon.aws.transfers.s3_to_redshift` - - * - `Amazon Simple Storage Service (S3) `_ - - `Snowflake `__ - - - - :mod:`airflow.providers.snowflake.transfers.s3_to_snowflake` - - * - `Amazon Simple Storage Service (S3) `_ - - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.transfers.s3_to_hive` - - * - `Amazon Simple Storage Service (S3) `__ - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.s3_to_gcs`, - :mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service` - - * - `Amazon Simple Storage Service (S3) `_ - - `SSH File Transfer Protocol (SFTP) `__ - - - - :mod:`airflow.providers.amazon.aws.transfers.s3_to_sftp` - - * - `Apache Hive `__ - - `Amazon DynamoDB `__ - - - - :mod:`airflow.providers.amazon.aws.transfers.hive_to_dynamodb` - - * - `Google Cloud Storage (GCS) `__ - - `Amazon Simple Storage Service (S3) `__ - - - - :mod:`airflow.providers.amazon.aws.transfers.gcs_to_s3` - - * - `Internet Message Access Protocol (IMAP) `__ - - `Amazon Simple Storage Service (S3) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.amazon.aws.transfers.imap_attachment_to_s3` - - * - `MongoDB `__ - - `Amazon Simple Storage Service (S3) `__ - - - - :mod:`airflow.providers.amazon.aws.transfers.mongo_to_s3` - - * - `SSH File Transfer Protocol (SFTP) `__ - - `Amazon Simple Storage Service (S3) `_ - - - - :mod:`airflow.providers.amazon.aws.transfers.sftp_to_s3` - - * - `MySQL `__ - - `Amazon Simple Storage Service (S3) `_ - - - - :mod:`airflow.providers.amazon.aws.transfers.mysql_to_s3` - -:ref:`[1] ` Those discovery-based operators use -:class:`~airflow.providers.google.common.hooks.discovery_api.GoogleDiscoveryApiHook` to communicate with Google -Services via the `Google API Python Client `__. -Please note that this library is in maintenance mode hence it won't fully support GCP in the future. -Therefore it is recommended that you use the custom GCP Service Operators for working with the Google -Cloud Platform. - -.. _Google: - -Google ------- - -Airflow has support for the `Google service `__. - -All hooks are based on :class:`airflow.providers.google.common.hooks.base_google.GoogleBaseHook`. Some integration -also use :mod:`airflow.providers.google.common.hooks.discovery_api`. - -See the :doc:`GCP connection type ` documentation to -configure connections to Google services. - -.. _GCP: - -GCP: Google Cloud Platform -'''''''''''''''''''''''''' - -Airflow has extensive support for the `Google Cloud Platform `__. - -.. note:: - You can learn how to use Google Cloud Platform integrations by analyzing the - `source code of the Google Cloud Platform example DAGs - `_ - - -Service operators and hooks -""""""""""""""""""""""""""" - -These integrations allow you to perform various operations within the Google Cloud Platform. - -.. - PLEASE KEEP THE ALPHABETICAL ORDER OF THE LIST BELOW, BUT OMIT THE "Cloud" PREFIX - -.. list-table:: - :header-rows: 1 - - * - Service name - - Guide - - Hook - - Operator - - Sensor - - - * - `AutoML `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.automl` - - :mod:`airflow.providers.google.cloud.operators.automl` - - - - * - `BigQuery `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.bigquery` - - :mod:`airflow.providers.google.cloud.operators.bigquery` - - :mod:`airflow.providers.google.cloud.sensors.bigquery` - - * - `BigQuery Data Transfer Service `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.bigquery_dts` - - :mod:`airflow.providers.google.cloud.operators.bigquery_dts` - - :mod:`airflow.providers.google.cloud.sensors.bigquery_dts` - - * - `Bigtable `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.bigtable` - - :mod:`airflow.providers.google.cloud.operators.bigtable` - - :mod:`airflow.providers.google.cloud.sensors.bigtable` - - * - `Cloud Build `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.cloud_build` - - :mod:`airflow.providers.google.cloud.operators.cloud_build` - - - - * - `Compute Engine `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.compute` - - :mod:`airflow.providers.google.cloud.operators.compute` - - - - * - `Cloud Data Loss Prevention (DLP) `__ - - - - :mod:`airflow.providers.google.cloud.hooks.dlp` - - :mod:`airflow.providers.google.cloud.operators.dlp` - - - - * - `DataFusion `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.datafusion` - - :mod:`airflow.providers.google.cloud.operators.datafusion` - - - - * - `Datacatalog `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.datacatalog` - - :mod:`airflow.providers.google.cloud.operators.datacatalog` - - - - * - `Dataflow `__ - - - - :mod:`airflow.providers.google.cloud.hooks.dataflow` - - :mod:`airflow.providers.google.cloud.operators.dataflow` - - - - * - `Dataproc `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.dataproc` - - :mod:`airflow.providers.google.cloud.operators.dataproc` - - - - * - `Datastore `__ - - - - :mod:`airflow.providers.google.cloud.hooks.datastore` - - :mod:`airflow.providers.google.cloud.operators.datastore` - - - - * - `Cloud Functions `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.functions` - - :mod:`airflow.providers.google.cloud.operators.functions` - - - - * - `Cloud Firestore `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.firebase.hooks.firestore` - - :mod:`airflow.providers.google.firebase.operators.firestore` - - - - * - `Cloud Key Management Service (KMS) `__ - - - - :mod:`airflow.providers.google.cloud.hooks.kms` - - - - - * - `Cloud Life Sciences `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.life_sciences` - - :mod:`airflow.providers.google.cloud.operators.life_sciences` - - - - * - `Kubernetes Engine `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.kubernetes_engine` - - :mod:`airflow.providers.google.cloud.operators.kubernetes_engine` - - - - * - `Machine Learning Engine `__ - - - - :mod:`airflow.providers.google.cloud.hooks.mlengine` - - :mod:`airflow.providers.google.cloud.operators.mlengine` - - - - * - `Cloud Memorystore `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.cloud_memorystore` - - :mod:`airflow.providers.google.cloud.operators.cloud_memorystore` - - - - * - `Natural Language `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.natural_language` - - :mod:`airflow.providers.google.cloud.operators.natural_language` - - - - * - `Cloud Pub/Sub `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.pubsub` - - :mod:`airflow.providers.google.cloud.operators.pubsub` - - :mod:`airflow.providers.google.cloud.sensors.pubsub` - - * - `Cloud Secret Manager `__ - - - - :mod:`airflow.providers.google.cloud.hooks.secret_manager` - - - - - - * - `Cloud Spanner `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.spanner` - - :mod:`airflow.providers.google.cloud.operators.spanner` - - - - * - `Cloud Speech-to-Text `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.speech_to_text` - - :mod:`airflow.providers.google.cloud.operators.speech_to_text` - - - - * - `Cloud SQL `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.cloud_sql` - - :mod:`airflow.providers.google.cloud.operators.cloud_sql` - - - - * - `Cloud Stackdriver `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.stackdriver` - - :mod:`airflow.providers.google.cloud.operators.stackdriver` - - - - * - `Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.gcs` - - :mod:`airflow.providers.google.cloud.operators.gcs` - - :mod:`airflow.providers.google.cloud.sensors.gcs` - - * - `Storage Transfer Service `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.cloud_storage_transfer_service` - - :mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service` - - :mod:`airflow.providers.google.cloud.sensors.cloud_storage_transfer_service` - - * - `Cloud Tasks `__ - - - - :mod:`airflow.providers.google.cloud.hooks.tasks` - - :mod:`airflow.providers.google.cloud.operators.tasks` - - - - * - `Cloud Text-to-Speech `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.text_to_speech` - - :mod:`airflow.providers.google.cloud.operators.text_to_speech` - - - - * - `Cloud Translation `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.translate` - - :mod:`airflow.providers.google.cloud.operators.translate` - - - - * - `Cloud Video Intelligence `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.video_intelligence` - - :mod:`airflow.providers.google.cloud.operators.video_intelligence` - - - - * - `Cloud Vision `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.hooks.vision` - - :mod:`airflow.providers.google.cloud.operators.vision` - - - -Transfer operators and hooks -"""""""""""""""""""""""""""" - -These integrations allow you to copy data from/to Google Cloud Platform. - -.. list-table:: - :header-rows: 1 - - * - Source - - Destination - - Guide - - Operator - - * - - .. _integration:GCP-Discovery-ref: - - All services :ref:`[1] ` - - `Amazon Simple Storage Service (S3) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.amazon.aws.transfers.google_api_to_s3` - - * - `Amazon Simple Storage Service (S3) `__ - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.s3_to_gcs`, - :mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service` - - * - `Apache Cassandra `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.cassandra_to_gcs` - - * - `Azure Data Lake Storage `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.adls_to_gcs` - - * - `Facebook Ads `__ - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.facebook_ads_to_gcs` - - - * - `Google Ads `__ - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.ads.transfers.ads_to_gcs` - - * - `Google BigQuery `__ - - `MySQL `__ - - - - :mod:`airflow.providers.google.cloud.transfers.bigquery_to_mysql` - - * - `Google BigQuery `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.bigquery_to_gcs` - - * - `Google BigQuery `__ - - `Google BigQuery `__ - - - - :mod:`airflow.providers.google.cloud.transfers.bigquery_to_bigquery` - - * - `Cloud Firestore `__ - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.firebase.operators.firestore` - - * - `Google Cloud Storage (GCS) `__ - - `Amazon Simple Storage Service (S3) `__ - - - - :mod:`airflow.providers.amazon.aws.transfers.gcs_to_s3` - - * - `Google Cloud Storage (GCS) `__ - - `Google BigQuery `__ - - - - :mod:`airflow.providers.google.cloud.transfers.gcs_to_bigquery` - - * - `Google Cloud Storage (GCS) `__ - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use `, - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.gcs_to_gcs`, - :mod:`airflow.providers.google.cloud.operators.cloud_storage_transfer_service` - - * - `Google Cloud Storage (GCS) `__ - - Local - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.gcs_to_local` - - * - `Google Cloud Storage (GCS) `__ - - `Google Drive `__ - - - - :mod:`airflow.providers.google.suite.transfers.gcs_to_gdrive` - - * - `Google Cloud Storage (GCS) `__ - - SFTP - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.gcs_to_sftp` - - * - Local - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.local_to_gcs` - - * - `Microsoft SQL Server (MSSQL) `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.mssql_to_gcs` - - * - `MySQL `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.mysql_to_gcs` - - * - `PostgresSQL `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.postgres_to_gcs` - - * - `Presto `__ - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.presto_to_gcs` - - * - SFTP - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.sftp_to_gcs` - - * - SQL - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.sql_to_gcs` - - * - `Google Spreadsheet `__ - - `Google Cloud Storage (GCS) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.transfers.sheets_to_gcs` - - * - `Google Cloud Storage (GCS) `__ - - `Google Spreadsheet `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.suite.transfers.gcs_to_sheets` - -.. _integration:GCP-Discovery: - -:ref:`[1] ` Those discovery-based operators use -:class:`~airflow.providers.google.common.hooks.discovery_api.GoogleDiscoveryApiHook` to communicate with Google -Services via the `Google API Python Client `__. -Please note that this library is in maintenance mode hence it won't fully support Google in the future. -Therefore it is recommended that you use the custom Google Service Operators for working with the Google -services. - -Other operators and hooks -""""""""""""""""""""""""" - -.. list-table:: - :header-rows: 1 - - * - Guide - - Operator - - Hook - - * - :doc:`How to use ` - - :mod:`airflow.providers.google.cloud.operators.translate_speech` - - - -Google Marketing Platform -''''''''''''''''''''''''' - -.. note:: - You can learn how to use Google Marketing Platform integrations by analyzing the - `source code `_ - of the example DAGs. - - -.. list-table:: - :header-rows: 1 - - * - Source - - Destination - - Guide - - Operator - - Sensor - - * - `Analytics360 `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.marketing_platform.hooks.analytics` - - :mod:`airflow.providers.google.marketing_platform.operators.analytics` - - - - * - `Google Campaign Manager `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.marketing_platform.hooks.campaign_manager` - - :mod:`airflow.providers.google.marketing_platform.operators.campaign_manager` - - :mod:`airflow.providers.google.marketing_platform.sensors.campaign_manager` - - * - `Google Display&Video 360 `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.marketing_platform.hooks.display_video` - - :mod:`airflow.providers.google.marketing_platform.operators.display_video` - - :mod:`airflow.providers.google.marketing_platform.sensors.display_video` - - * - `Google Search Ads 360 `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.marketing_platform.hooks.search_ads` - - :mod:`airflow.providers.google.marketing_platform.operators.search_ads` - - :mod:`airflow.providers.google.marketing_platform.sensors.search_ads` - -Other Google operators and hooks -'''''''''''''''''''''''''''''''' - -.. list-table:: - :header-rows: 1 - - * - Service name - - Guide - - Hook - - Operator - - * - `Google Ads `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.ads.hooks.ads` - - :mod:`airflow.providers.google.ads.operators.ads` - - * - `Google Drive `__ - - - - :mod:`airflow.providers.google.suite.hooks.drive` - - - - * - `Cloud Firestore `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.firebase.hooks.firestore` - - :mod:`airflow.providers.google.firebase.operators.firestore` - - * - `Google Spreadsheet `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.suite.hooks.sheets` - - :mod:`airflow.providers.google.suite.operators.sheets` - -.. _yc_service: - -Yandex.Cloud --------------------------- - -Airflow has a limited support for the `Yandex.Cloud `__. - -See the :doc:`Yandex.Cloud connection type ` documentation to -configure connections to Yandex.Cloud. - -All hooks are based on :class:`airflow.providers.yandex.hooks.yandex.YandexCloudBaseHook`. - -.. note:: - You can learn how to use Yandex.Cloud integrations by analyzing the - `example DAG `_ - -Service operators and hooks -''''''''''''''''''''''''''' - -These integrations allow you to perform various operations within the Yandex.Cloud. - -.. - PLEASE KEEP THE ALPHABETICAL ORDER OF THE LIST BELOW, BUT OMIT THE "Cloud" PREFIX - -.. list-table:: - :header-rows: 1 - - * - Service name - - Guide - - Hook - - Operator - - Sensor - - * - `Base Classes `__ - - :doc:`How to use ` - - :mod:`airflow.providers.yandex.hooks.yandex` - - - - - - * - `Data Proc `__ - - :doc:`How to use ` - - :mod:`airflow.providers.yandex.hooks.yandexcloud_dataproc` - - :mod:`airflow.providers.yandex.operators.yandexcloud_dataproc` - - - - -.. _service: - -Service integrations --------------------- - -Service operators and hooks -''''''''''''''''''''''''''' - -These integrations allow you to perform various operations within various services. - -.. list-table:: - :header-rows: 1 - - * - Service name - - Guide - - Hook - - Operator - - Sensor - - * - `Atlassian Jira `__ - - - - :mod:`airflow.providers.jira.hooks.jira` - - :mod:`airflow.providers.jira.operators.jira` - - :mod:`airflow.providers.jira.sensors.jira` - - * - `Databricks `__ - - - - :mod:`airflow.providers.databricks.hooks.databricks` - - :mod:`airflow.providers.databricks.operators.databricks` - - - - * - `Datadog `__ - - - - :mod:`airflow.providers.datadog.hooks.datadog` - - - - :mod:`airflow.providers.datadog.sensors.datadog` - - * - `Pagerduty `__ - - - - :mod:`airflow.providers.pagerduty.hooks.pagerduty` - - - - - - * - `Dingding `__ - - :doc:`How to use ` - - :mod:`airflow.providers.dingding.hooks.dingding` - - :mod:`airflow.providers.dingding.operators.dingding` - - - - * - `Discord `__ - - - - :mod:`airflow.providers.discord.hooks.discord_webhook` - - :mod:`airflow.providers.discord.operators.discord_webhook` - - - - * - `Facebook Ads `__ - - - - :mod:`airflow.providers.facebook.ads.hooks.ads` - - - - - - * - `IBM Cloudant `__ - - - - :mod:`airflow.providers.cloudant.hooks.cloudant` - - - - - - * - `Jenkins `__ - - - - :mod:`airflow.providers.jenkins.hooks.jenkins` - - :mod:`airflow.providers.jenkins.operators.jenkins_job_trigger` - - - - * - `Opsgenie `__ - - - - :mod:`airflow.providers.opsgenie.hooks.opsgenie_alert` - - :mod:`airflow.providers.opsgenie.operators.opsgenie_alert` - - - - * - `Qubole `__ - - - - :mod:`airflow.providers.qubole.hooks.qubole`, - :mod:`airflow.providers.qubole.hooks.qubole_check` - - :mod:`airflow.providers.qubole.operators.qubole`, - :mod:`airflow.providers.qubole.operators.qubole_check` - - :mod:`airflow.providers.qubole.sensors.qubole` - - * - `Salesforce `__ - - - - :mod:`airflow.providers.salesforce.hooks.salesforce`, - :mod:`airflow.providers.salesforce.hooks.tableau` - - :mod:`airflow.providers.salesforce.operators.tableau_refresh_workbook` - - :mod:`airflow.providers.salesforce.sensors.tableau_job_status` - - * - `Segment `__ - - - - :mod:`airflow.providers.segment.hooks.segment` - - :mod:`airflow.providers.segment.operators.segment_track_event` - - - - * - `Slack `__ - - - - :mod:`airflow.providers.slack.hooks.slack`, - :mod:`airflow.providers.slack.hooks.slack_webhook` - - :mod:`airflow.providers.slack.operators.slack`, - :mod:`airflow.providers.slack.operators.slack_webhook` - - - - * - `Snowflake `__ - - - - :mod:`airflow.providers.snowflake.hooks.snowflake` - - :mod:`airflow.providers.snowflake.operators.snowflake`, - :mod:`airflow.providers.snowflake.transfers.snowflake_to_slack` - - - - * - `Vertica `__ - - - - :mod:`airflow.providers.vertica.hooks.vertica` - - :mod:`airflow.providers.vertica.operators.vertica` - - - - * - `Zendesk `__ - - - - :mod:`airflow.providers.zendesk.hooks.zendesk` - - - - - - -Transfer operators and hooks -'''''''''''''''''''''''''''' - -These integrations allow you to perform various operations within various services. - -.. list-table:: - :header-rows: 1 - - * - Source - - Destination - - Guide - - Operator - - * - `Google Cloud Storage (GCS) `__ - - `Google Drive `__ - - :doc:`How to use ` - - :mod:`airflow.providers.google.suite.transfers.gcs_to_gdrive` - - * - `Vertica `__ - - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.transfers.vertica_to_hive` - - * - `Vertica `__ - - `MySQL `__ - - - - :mod:`airflow.providers.mysql.transfers.vertica_to_mysql` - -.. _software: - -Software integrations ---------------------- - -Software operators and hooks -'''''''''''''''''''''''''''' - -These integrations allow you to perform various operations using various software. - -.. list-table:: - :header-rows: 1 - - * - Service name - - Guide - - Hook - - Operator - - Sensor - - * - `Celery `__ - - - - - - - - :mod:`airflow.providers.celery.sensors.celery_queue` - - * - `Docker `__ - - - - :mod:`airflow.providers.docker.hooks.docker` - - :mod:`airflow.providers.docker.operators.docker`, - :mod:`airflow.providers.docker.operators.docker_swarm` - - - - * - `Elasticsearch `__ - - - - :mod:`airflow.providers.elasticsearch.hooks.elasticsearch` - - - - - - * - `Exasol `__ - - - - :mod:`airflow.providers.exasol.hooks.exasol` - - :mod:`airflow.providers.exasol.operators.exasol` - - - - * - `GNU Bash `__ - - :doc:`How to use ` - - - - :mod:`airflow.operators.bash` - - :mod:`airflow.sensors.bash` - - * - `Hashicorp Vault `__ - - - - :mod:`airflow.providers.hashicorp.hooks.vault` - - - - - - * - `Kubernetes `__ - - :doc:`How to use ` - - :mod:`airflow.providers.cncf.kubernetes.hooks.kubernetes` - - :mod:`airflow.providers.cncf.kubernetes.operators.kubernetes_pod` - :mod:`airflow.providers.cncf.kubernetes.operators.spark_kubernetes` - - :mod:`airflow.providers.cncf.kubernetes.sensors.spark_kubernetes` - - - * - `Microsoft SQL Server (MSSQL) `__ - - - - :mod:`airflow.providers.microsoft.mssql.hooks.mssql`, - :mod:`airflow.providers.odbc.hooks.odbc` - - :mod:`airflow.providers.microsoft.mssql.operators.mssql` - - - - - * - `ODBC `__ - - - - :mod:`airflow.providers.odbc.hooks.odbc` - - - - - - * - `MongoDB `__ - - - - :mod:`airflow.providers.mongo.hooks.mongo` - - - - :mod:`airflow.providers.mongo.sensors.mongo` - - - * - `MySQL `__ - - - - :mod:`airflow.providers.mysql.hooks.mysql` - - :mod:`airflow.providers.mysql.operators.mysql` - - - - * - `OpenFaaS `__ - - - - :mod:`airflow.providers.openfaas.hooks.openfaas` - - - - - - * - `Oracle `__ - - - - :mod:`airflow.providers.oracle.hooks.oracle` - - :mod:`airflow.providers.oracle.operators.oracle` - - - - * - `Papermill `__ - - :doc:`How to use ` - - - - :mod:`airflow.providers.papermill.operators.papermill` - - - - * - `PostgresSQL `__ - - - - :mod:`airflow.providers.postgres.hooks.postgres` - - :mod:`airflow.providers.postgres.operators.postgres` - - - - * - `Presto `__ - - - - :mod:`airflow.providers.presto.hooks.presto` - - - - - - * - `Python `__ - - - - :doc:`How to use ` - - :mod:`airflow.operators.python` - - :mod:`airflow.sensors.python` - - * - `Redis `__ - - - - :mod:`airflow.providers.redis.hooks.redis` - - :mod:`airflow.providers.redis.operators.redis_publish` - - :mod:`airflow.providers.redis.sensors.redis_pub_sub`, - :mod:`airflow.providers.redis.sensors.redis_key` - - * - `Samba `__ - - - - :mod:`airflow.providers.samba.hooks.samba` - - - - - - * - `Singularity `__ - - - - - - :mod:`airflow.providers.singularity.operators.singularity` - - - - * - `SQLite `__ - - - - :mod:`airflow.providers.sqlite.hooks.sqlite` - - :mod:`airflow.providers.sqlite.operators.sqlite` - - - - -Transfer operators and hooks -'''''''''''''''''''''''''''' - -These integrations allow you to copy data. - -.. list-table:: - :header-rows: 1 - - * - Source - - Destination - - Guide - - Operator - - * - `Apache Hive `__ - - `Samba `__ - - - - :mod:`airflow.providers.apache.hive.transfers.hive_to_samba` - - * - `BigQuery `__ - - `MySQL `__ - - - - :mod:`airflow.providers.google.cloud.transfers.bigquery_to_mysql` - - * - `Microsoft SQL Server (MSSQL) `__ - - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.transfers.mssql_to_hive` - - * - `Microsoft SQL Server (MSSQL) `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.mssql_to_gcs` - - * - `MongoDB `__ - - `Amazon Simple Storage Service (S3) `__ - - - - :mod:`airflow.providers.amazon.aws.transfers.mongo_to_s3` - - * - `MySQL `__ - - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.transfers.mysql_to_hive` - - * - `MySQL `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.mysql_to_gcs` - - * - `Oracle `__ - - `Azure Data Lake Storage `__ - - - - :mod:`airflow.providers.microsoft.azure.transfers.oracle_to_azure_data_lake` - - * - `Oracle `__ - - `Oracle `__ - - - - :mod:`airflow.providers.oracle.transfers.oracle_to_oracle` - - * - `PostgresSQL `__ - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.postgres_to_gcs` - - * - `Presto `__ - - `MySQL `__ - - - - :mod:`airflow.providers.mysql.transfers.presto_to_mysql` - - * - SQL - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.sql_to_gcs` - - * - `Vertica `__ - - `Apache Hive `__ - - - - :mod:`airflow.providers.apache.hive.transfers.vertica_to_hive` - - * - `Vertica `__ - - `MySQL `__ - - - - :mod:`airflow.providers.mysql.transfers.vertica_to_mysql` - -.. _protocol: - -Protocol integrations ---------------------- - -Protocol operators and hooks -'''''''''''''''''''''''''''' - -These integrations allow you to perform various operations within various services using standardized -communication protocols or interface. - -.. list-table:: - :header-rows: 1 - - * - Service name - - Guide - - Hooks - - Operator - - Sensor - - * - `File Transfer Protocol (FTP) `__ - - - - :mod:`airflow.providers.ftp.hooks.ftp` - - - - :mod:`airflow.providers.ftp.sensors.ftp` - - * - Filesystem - - - - :mod:`airflow.hooks.filesystem` - - - - :mod:`airflow.sensors.filesystem` - - * - `Hypertext Transfer Protocol (HTTP) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.http.hooks.http` - - :mod:`airflow.providers.http.operators.http` - - :mod:`airflow.providers.http.sensors.http` - - * - `Internet Message Access Protocol (IMAP) `__ - - - - :mod:`airflow.providers.imap.hooks.imap` - - - - :mod:`airflow.providers.imap.sensors.imap_attachment` - - * - `Java Database Connectivity (JDBC) `__ - - - - :mod:`airflow.providers.jdbc.hooks.jdbc` - - :mod:`airflow.providers.jdbc.operators.jdbc` - - - - * - `SSH File Transfer Protocol (SFTP) `__ - - - - :mod:`airflow.providers.sftp.hooks.sftp` - - :mod:`airflow.providers.sftp.operators.sftp` - - :mod:`airflow.providers.sftp.sensors.sftp` - - * - `Secure Shell (SSH) `__ - - - - :mod:`airflow.providers.ssh.hooks.ssh` - - :mod:`airflow.providers.ssh.operators.ssh` - - - - * - `Simple Mail Transfer Protocol (SMTP) `__ - - - - - - :mod:`airflow.providers.email.operators.email` - - - - * - `Windows Remote Management (WinRM) `__ - - - - :mod:`airflow.providers.microsoft.winrm.hooks.winrm` - - :mod:`airflow.providers.microsoft.winrm.operators.winrm` - - - - * - `gRPC `__ - - - - :mod:`airflow.providers.grpc.hooks.grpc` - - :mod:`airflow.providers.grpc.operators.grpc` - - - -Transfer operators and hooks -'''''''''''''''''''''''''''' - -These integrations allow you to copy data. - -.. list-table:: - :header-rows: 1 - - * - Source - - Destination - - Guide - - Operator - - * - `Amazon Simple Storage Service (S3) `_ - - `SSH File Transfer Protocol (SFTP) `__ - - - - :mod:`airflow.providers.amazon.aws.transfers.s3_to_sftp` - - * - Filesystem - - `Azure Blob Storage `__ - - - - :mod:`airflow.providers.microsoft.azure.transfers.file_to_wasb` - - * - Filesystem - - `Google Cloud Storage (GCS) `__ - - - - :mod:`airflow.providers.google.cloud.transfers.local_to_gcs` - - * - `Internet Message Access Protocol (IMAP) `__ - - `Amazon Simple Storage Service (S3) `__ - - :doc:`How to use ` - - :mod:`airflow.providers.amazon.aws.transfers.imap_attachment_to_s3` - - * - `SSH File Transfer Protocol (SFTP) `__ - - `Amazon Simple Storage Service (S3) `_ - - - - :mod:`airflow.providers.amazon.aws.transfers.sftp_to_s3` From e2376e6d19bea5c1c754cd887de998a8f40a8e49 Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 16 Feb 2021 01:07:40 -0300 Subject: [PATCH 49/79] Fix issues from old MR --- airflow/providers/dependencies.json | 3 +- .../microsoft/azure/transfers/sftp_to_wasb.py | 70 +++++++------ .../azure/transfers/test_sftp_to_wasb.py | 99 ++++--------------- 3 files changed, 61 insertions(+), 111 deletions(-) diff --git a/airflow/providers/dependencies.json b/airflow/providers/dependencies.json index e4f100df684aa..c932cfa1a0deb 100644 --- a/airflow/providers/dependencies.json +++ b/airflow/providers/dependencies.json @@ -57,7 +57,8 @@ ], "microsoft.azure": [ "google", - "oracle" + "oracle", + "sftp" ], "microsoft.mssql": [ "odbc" diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 4bdfbad34458b..8a4c86170f9d6 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -15,13 +15,11 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -""" -This module contains SFTP to Azure Blob Storage operator. -""" +"""This module contains SFTP to Azure Blob Storage operator.""" import os from collections import namedtuple from tempfile import NamedTemporaryFile -from typing import Any, Dict, List, Optional +from typing import Any, Dict, List, Optional, Tuple from cached_property import cached_property @@ -46,8 +44,8 @@ class SFTPToWasbOperator(BaseOperator): :type sftp_source_path: str :param container_name: Name of the container. :type container_name: str - :param blob_name: Name of the blob. - :type blob_name: str + :param blob_prefix: Prefix to name a blob. + :type blob_prefix: str :param sftp_conn_id: The sftp connection id. The name or identifier for establishing a connection to the SFTP server. :type sftp_conn_id: str @@ -69,7 +67,7 @@ def __init__( self, sftp_source_path: str, container_name: str, - blob_name: str, + blob_prefix: str = "", sftp_conn_id: str = "sftp_default", wasb_conn_id: str = 'wasb_default', load_options: Optional[dict] = None, @@ -80,10 +78,10 @@ def __init__( super().__init__(*args, **kwargs) self.sftp_source_path = sftp_source_path + self.blob_prefix = blob_prefix self.sftp_conn_id = sftp_conn_id self.wasb_conn_id = wasb_conn_id self.container_name = container_name - self.blob_name = blob_name self.wasb_conn_id = wasb_conn_id self.load_options = load_options if load_options is not None else {} self.move_object = move_object @@ -99,33 +97,34 @@ def get_sftp_files_map(self) -> List[SftpFile]: """Get SFTP files from the source path, it may use a WILDCARD to this end.""" sftp_files = [] - if WILDCARD in self.sftp_source_path: - self.check_wildcards_limit() + sftp_complete_path, prefix, delimiter = self.get_tree_behavior() - prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) - sftp_complete_path = os.path.dirname(prefix) + found_files, _, _ = self.sftp_hook.get_tree_map( + sftp_complete_path, prefix=prefix, delimiter=delimiter + ) - found_files, _, _ = self.sftp_hook.get_tree_map( - sftp_complete_path, prefix=prefix, delimiter=delimiter - ) + self.log.info("Found %s files at sftp source path: %s", str(len(found_files)), self.sftp_source_path) - self.log.info( - "Found %s files at sftp source path: %s", str(len(found_files)), self.sftp_source_path - ) + for file in found_files: + future_blob_name = self.get_future_blob_name(file) + sftp_files.append(SftpFile(file, future_blob_name)) - for file in found_files: - future_blob_name = os.path.basename(file) - sftp_files.append(SftpFile(file, future_blob_name)) + return sftp_files - else: - sftp_files.append(SftpFile(self.sftp_source_path, self.blob_name)) + @staticmethod + def get_tree_behavior(self) -> Tuple[str, Optional[str], Optional[str]]: + """Extracts from source path the tree behavior to interact with the remote folder""" + self.check_wildcards_limit() - return sftp_files + if self.source_path_contains_wildcard: - @cached_property - def sftp_hook(self): - """Property of sftp hook to be re-used.""" - return SFTPHook(self.sftp_conn_id) + prefix, delimiter = self.sftp_source_path.split(WILDCARD, 1) + + sftp_complete_path = os.path.dirname(prefix) + + return sftp_complete_path, prefix, delimiter + + return self.sftp_source_path, None, None def check_wildcards_limit(self) -> Any: """Check if there is multiple Wildcard.""" @@ -136,12 +135,25 @@ def check_wildcards_limit(self) -> Any: "Found {} in {}.".format(total_wildcards, self.sftp_source_path) ) + @property + def source_path_contains_wildcard(self) -> bool: + """Does source path contains a wildcard""" + return WILDCARD in self.sftp_source_path + + @cached_property + def sftp_hook(self) -> SFTPHook: + """Property of sftp hook to be re-used.""" + return SFTPHook(self.sftp_conn_id) + + def get_future_blob_name(self, file: str) -> str: + """Get a blob name based by the previous name and a blob_prefix variable""" + return self.blob_prefix + os.path.basename(file) + def copy_files_to_wasb(self, sftp_files: List[SftpFile]) -> List[str]: """Upload a list of files from sftp_files to Azure Blob Storage with a new Blob Name.""" uploaded_files = [] wasb_hook = WasbHook(wasb_conn_id=self.wasb_conn_id) for file in sftp_files: - with NamedTemporaryFile("w") as tmp: self.sftp_hook.retrieve_file(file.sftp_file_path, tmp.name) self.log.info( diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 9c06f20961231..626ec9b14df33 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -23,17 +23,18 @@ from airflow import AirflowException from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SftpFile, SFTPToWasbOperator -BLOB_PREFIX = "sponge-bob" -CONTAINER_NAME = "test-container" -SOURCE_PATH_NO_WILDCARD = "main_dir/" -EXPECTED_BLOB_NAME = "test_object3.json" -EXPECTED_FILES = [SOURCE_PATH_NO_WILDCARD + EXPECTED_BLOB_NAME] -SOURCE_PATH_MULTIPLE_WILDCARDS = "main_dir/csv/*/test_*" -SFTP_CONN_ID = "ssh_default" TASK_ID = "test-gcs-to-sftp-operator" WASB_CONN_ID = "wasb_default" +SFTP_CONN_ID = "ssh_default" + +CONTAINER_NAME = "test-container" WILDCARD_PATH = "main_dir/*" WILDCARD_FILE_NAME = "main_dir/test_object*.json" +SOURCE_PATH_NO_WILDCARD = "main_dir/" +SOURCE_OBJECT_MULTIPLE_WILDCARDS = "main_dir/csv/*/test_*.csv" +BLOB_PREFIX = "sponge-bob" +EXPECTED_BLOB_NAME = "test_object3.json" +EXPECTED_FILES = [SOURCE_PATH_NO_WILDCARD + EXPECTED_BLOB_NAME] # pylint: disable=unused-argument @@ -58,7 +59,7 @@ def test_init(self): def test_execute_more_than_one_wildcard_exception(self, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, - sftp_source_path=SOURCE_PATH_MULTIPLE_WILDCARDS, + sftp_source_path=SOURCE_OBJECT_MULTIPLE_WILDCARDS, sftp_conn_id=SFTP_CONN_ID, container_name=CONTAINER_NAME, blob_prefix=BLOB_PREFIX, @@ -71,36 +72,6 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): err = cm.exception self.assertIn("Only one wildcard '*' is allowed", str(err)) - def test_source_path_contains_wildcard(self): - operator = SFTPToWasbOperator( - task_id=TASK_ID, - sftp_source_path=WILDCARD_PATH, - sftp_conn_id=SFTP_CONN_ID, - container_name=CONTAINER_NAME, - blob_prefix=BLOB_PREFIX, - wasb_conn_id=WASB_CONN_ID, - move_object=False, - ) - - output = operator.source_path_contains_wildcard - - self.assertTrue(output, "This source path contains wildcard") - - def test_source_path_not_contains_wildcard(self): - operator = SFTPToWasbOperator( - task_id=TASK_ID, - sftp_source_path=SOURCE_PATH_NO_WILDCARD, - sftp_conn_id=SFTP_CONN_ID, - container_name=CONTAINER_NAME, - blob_prefix=BLOB_PREFIX, - wasb_conn_id=WASB_CONN_ID, - move_object=False, - ) - - output = operator.source_path_contains_wildcard - - self.assertFalse(output, "This source path does not contains wildcard") - def test_get_sftp_tree_behavior(self): operator = SFTPToWasbOperator( task_id=TASK_ID, @@ -112,13 +83,13 @@ def test_get_sftp_tree_behavior(self): ) sftp_complete_path, prefix, delimiter = operator.get_tree_behavior() - self.assertEqual(sftp_complete_path, "main_dir", "not matched at expected complete path") - self.assertEqual(prefix, "main_dir/", "Prefix must be before the wildcard") - self.assertEqual(delimiter, "", "Delimiter must be empty") + self.assertEqual(sftp_complete_path, SOURCE_PATH_NO_WILDCARD, "not matched at expected complete path") + self.assertIsNone(prefix, "Prefix must be empty without wildcard") + self.assertIsNone(delimiter, "Delimiter must be empty") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_get_sftp_files_map(self, sftp_hook, mock_hook): + def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ EXPECTED_FILES, [], @@ -139,7 +110,7 @@ def test_get_sftp_files_map(self, sftp_hook, mock_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_upload_wasb(self, sftp_hook, mock_hook): + def test_copy_files_to_wasb(self, sftp_hook, mock_hook): operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=SOURCE_PATH_NO_WILDCARD, @@ -159,7 +130,7 @@ def test_upload_wasb(self, sftp_hook, mock_hook): self.assertEqual(len(files), 1, "no matched at expected uploaded files") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_sftp_move(self, sftp_hook): + def test_delete_files(self, sftp_hook): sftp_mock = sftp_hook.return_value operator = SFTPToWasbOperator( @@ -179,11 +150,6 @@ def test_sftp_move(self, sftp_hook): @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_execute(self, sftp_hook, mock_hook): - sftp_hook.return_value.get_tree_map.return_value = [ - ["main_dir/test_object.json"], - [], - [], - ] operator = SFTPToWasbOperator( task_id=TASK_ID, @@ -194,39 +160,12 @@ def test_execute(self, sftp_hook, mock_hook): move_object=False, ) - operator.execute(None) - - sftp_hook.return_value.get_tree_map.assert_called_with( - "main_dir", prefix="main_dir/test_object", delimiter=".json" - ) - - sftp_hook.return_value.retrieve_file.assert_has_calls( - [mock.call("main_dir/test_object.json", mock.ANY)] - ) - - mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") - - sftp_hook.delete_file.assert_not_called() - - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') - @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') - def test_execute_moving_files(self, sftp_hook, mock_hook): sftp_hook.return_value.get_tree_map.return_value = [ - ["main_dir/test_object.json"], + [WILDCARD_FILE_NAME], [], [], ] - operator = SFTPToWasbOperator( - task_id=TASK_ID, - sftp_source_path=WILDCARD_FILE_NAME, - sftp_conn_id=SFTP_CONN_ID, - container_name=CONTAINER_NAME, - wasb_conn_id=WASB_CONN_ID, - blob_prefix=BLOB_PREFIX, - move_object=True, - ) - operator.execute(None) sftp_hook.return_value.get_tree_map.assert_called_with( @@ -237,8 +176,6 @@ def test_execute_moving_files(self, sftp_hook, mock_hook): [mock.call("main_dir/test_object.json", mock.ANY)] ) - mock_hook.return_value.load_file.assert_called_once_with( - mock.ANY, CONTAINER_NAME, BLOB_PREFIX + "test_object.json" - ) + mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test.json") - self.assertTrue(sftp_hook.return_value.delete_file.called, "It did perform move of file") + operator.sftp_hook.delete_file.assert_not_called() From 2e6ee82d30b2281269871e17d8cf5aa969fb2f22 Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 16 Feb 2021 01:13:29 -0300 Subject: [PATCH 50/79] Fix tests --- CONTRIBUTING.rst | 2 +- .../microsoft/azure/transfers/sftp_to_wasb.py | 1 - .../microsoft/azure/transfers/test_sftp_to_wasb.py | 10 +++++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/CONTRIBUTING.rst b/CONTRIBUTING.rst index 90be9975b3e90..f63a28b5af7aa 100644 --- a/CONTRIBUTING.rst +++ b/CONTRIBUTING.rst @@ -663,7 +663,7 @@ dingding http discord http google amazon,apache.beam,apache.cassandra,cncf.kubernetes,facebook,microsoft.azure,microsoft.mssql,mysql,oracle,postgres,presto,salesforce,sftp,ssh hashicorp google -microsoft.azure google,oracle +microsoft.azure google,oracle,sftp microsoft.mssql odbc mysql amazon,presto,vertica opsgenie http diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 8a4c86170f9d6..93c36744d6c6f 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -111,7 +111,6 @@ def get_sftp_files_map(self) -> List[SftpFile]: return sftp_files - @staticmethod def get_tree_behavior(self) -> Tuple[str, Optional[str], Optional[str]]: """Extracts from source path the tree behavior to interact with the remote folder""" self.check_wildcards_limit() diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 626ec9b14df33..389173c1e77ce 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -83,9 +83,9 @@ def test_get_sftp_tree_behavior(self): ) sftp_complete_path, prefix, delimiter = operator.get_tree_behavior() - self.assertEqual(sftp_complete_path, SOURCE_PATH_NO_WILDCARD, "not matched at expected complete path") - self.assertIsNone(prefix, "Prefix must be empty without wildcard") - self.assertIsNone(delimiter, "Delimiter must be empty") + self.assertEqual(sftp_complete_path, 'main_dir', "not matched at expected complete path") + self.assertEqual(prefix, 'main_dir/', "Prefix must be EQUAL TO wildcard") + self.assertEqual(delimiter, "", "Delimiter must be empty") @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') @@ -161,7 +161,7 @@ def test_execute(self, sftp_hook, mock_hook): ) sftp_hook.return_value.get_tree_map.return_value = [ - [WILDCARD_FILE_NAME], + ["main_dir/test_object.json"], [], [], ] @@ -176,6 +176,6 @@ def test_execute(self, sftp_hook, mock_hook): [mock.call("main_dir/test_object.json", mock.ANY)] ) - mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test.json") + mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") operator.sftp_hook.delete_file.assert_not_called() From 56743c39b52f238f58875f5fa551bb5c36e3f579 Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 16 Feb 2021 01:26:06 -0300 Subject: [PATCH 51/79] Adds dry-run and new tests --- .../microsoft/azure/transfers/sftp_to_wasb.py | 13 ++++ .../azure/transfers/test_sftp_to_wasb.py | 75 ++++++++++++++++++- 2 files changed, 87 insertions(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 93c36744d6c6f..952b5d2ffb1e3 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -86,6 +86,19 @@ def __init__( self.load_options = load_options if load_options is not None else {} self.move_object = move_object + def dry_run(self) -> None: + super(SFTPToWasbOperator, self).dry_run() + sftp_files: List[SftpFile] = self.get_sftp_files_map() + for file in sftp_files: + self.log.info( + 'Process will upload file from (SFTP) %s to wasb://%s as %s', + file.sftp_file_path, + self.container_name, + file.blob_name, + ) + if self.move_object: + self.log.info("Executing delete of %s", file) + def execute(self, context: Optional[Dict[Any, Any]]) -> None: """Upload a file from SFTP to Azure Blob Storage.""" sftp_files: List[SftpFile] = self.get_sftp_files_map() diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 389173c1e77ce..e88d7ad76382e 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -87,6 +87,45 @@ def test_get_sftp_tree_behavior(self): self.assertEqual(prefix, 'main_dir/', "Prefix must be EQUAL TO wildcard") self.assertEqual(delimiter, "", "Delimiter must be empty") + def test_get_sftp_tree_behavior_without_wildcard(self): + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False, + ) + sftp_complete_path, prefix, delimiter = operator.get_tree_behavior() + + self.assertEqual(sftp_complete_path, 'main_dir/', "not matched at expected complete path") + self.assertIsNone(prefix, "Prefix must be NONE when no wildcard") + self.assertIsNone(delimiter, "Delimiter must be none") + + def test_source_path_contains_wildcard(self): + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_PATH, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False, + ) + output = operator.source_path_contains_wildcard + self.assertTrue(output, "This path contains a wildpath") + + def test_source_path_not_contains_wildcard(self): + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=SOURCE_PATH_NO_WILDCARD, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=False, + ) + output = operator.source_path_contains_wildcard + self.assertFalse(output, "This path does not contains a wildpath") + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): @@ -178,4 +217,38 @@ def test_execute(self, sftp_hook, mock_hook): mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") - operator.sftp_hook.delete_file.assert_not_called() + sftp_hook.return_value.delete_file.assert_not_called() + + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') + @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') + def test_execute_moved_files(self, sftp_hook, mock_hook): + + operator = SFTPToWasbOperator( + task_id=TASK_ID, + sftp_source_path=WILDCARD_FILE_NAME, + sftp_conn_id=SFTP_CONN_ID, + container_name=CONTAINER_NAME, + wasb_conn_id=WASB_CONN_ID, + move_object=True, + blob_prefix=BLOB_PREFIX + ) + + sftp_hook.return_value.get_tree_map.return_value = [ + ["main_dir/test_object.json"], + [], + [], + ] + + operator.execute(None) + + sftp_hook.return_value.get_tree_map.assert_called_with( + "main_dir", prefix="main_dir/test_object", delimiter=".json" + ) + + sftp_hook.return_value.retrieve_file.assert_has_calls( + [mock.call("main_dir/test_object.json", mock.ANY)] + ) + + mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, BLOB_PREFIX+"test_object" + ".json") + self.assertTrue(sftp_hook.return_value.delete_file.called, "File must be moved") From 8f40c9d2374cc01a739a04b5889658699e78da64 Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 16 Feb 2021 01:42:10 -0300 Subject: [PATCH 52/79] Fix files --- .../providers/microsoft/azure/transfers/sftp_to_wasb.py | 2 +- .../microsoft/azure/transfers/test_sftp_to_wasb.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 952b5d2ffb1e3..61083025783eb 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -87,7 +87,7 @@ def __init__( self.move_object = move_object def dry_run(self) -> None: - super(SFTPToWasbOperator, self).dry_run() + super().dry_run() sftp_files: List[SftpFile] = self.get_sftp_files_map() for file in sftp_files: self.log.info( diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index e88d7ad76382e..eba4c4350d530 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -230,7 +230,7 @@ def test_execute_moved_files(self, sftp_hook, mock_hook): container_name=CONTAINER_NAME, wasb_conn_id=WASB_CONN_ID, move_object=True, - blob_prefix=BLOB_PREFIX + blob_prefix=BLOB_PREFIX, ) sftp_hook.return_value.get_tree_map.return_value = [ @@ -249,6 +249,7 @@ def test_execute_moved_files(self, sftp_hook, mock_hook): [mock.call("main_dir/test_object.json", mock.ANY)] ) - mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, BLOB_PREFIX+"test_object" - ".json") + mock_hook.return_value.load_file.assert_called_once_with( + mock.ANY, CONTAINER_NAME, BLOB_PREFIX + "test_object" ".json" + ) self.assertTrue(sftp_hook.return_value.delete_file.called, "File must be moved") From 8e7caddcc27b1ed5d5fb2fd6a09dc59d680b0618 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Tue, 16 Feb 2021 06:29:12 -0300 Subject: [PATCH 53/79] Update airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py Co-authored-by: Ephraim Anierobi --- airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 61083025783eb..82b922eefe069 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -72,7 +72,6 @@ def __init__( wasb_conn_id: str = 'wasb_default', load_options: Optional[dict] = None, move_object: bool = False, - *args, **kwargs, ) -> None: super().__init__(*args, **kwargs) From e3d9bf7b8630baf0de5f1d1d70b0cbac35f21dad Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Tue, 16 Feb 2021 06:29:19 -0300 Subject: [PATCH 54/79] Update airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py Co-authored-by: Ephraim Anierobi --- airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 82b922eefe069..c7ea05a1d629b 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -74,7 +74,7 @@ def __init__( move_object: bool = False, **kwargs, ) -> None: - super().__init__(*args, **kwargs) + super().__init__(**kwargs) self.sftp_source_path = sftp_source_path self.blob_prefix = blob_prefix From 1bf756e7e1f81797857d70e915e7697d5a672c76 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Tue, 16 Feb 2021 06:29:32 -0300 Subject: [PATCH 55/79] Update airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py Co-authored-by: Ephraim Anierobi --- airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index c7ea05a1d629b..2bed67875d4ef 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -82,7 +82,7 @@ def __init__( self.wasb_conn_id = wasb_conn_id self.container_name = container_name self.wasb_conn_id = wasb_conn_id - self.load_options = load_options if load_options is not None else {} + self.load_options = load_options or {} self.move_object = move_object def dry_run(self) -> None: From abeb394e9e9440fb87a82eb33a08ebf03b0a0627 Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 16 Feb 2021 11:05:07 -0300 Subject: [PATCH 56/79] All parameters only be called by keywords --- airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 2bed67875d4ef..15a72f7c8cc98 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -65,6 +65,7 @@ class SFTPToWasbOperator(BaseOperator): @apply_defaults def __init__( self, + *, sftp_source_path: str, container_name: str, blob_prefix: str = "", From e355da7d7cc2c5b70ca054e00c5f9dd4dea3e8a7 Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 16 Feb 2021 12:41:37 -0300 Subject: [PATCH 57/79] Changes rst file --- .../microsoft/azure/example_dags/example_sftp_to_wasb.py | 6 +++--- airflow/providers/microsoft/azure/provider.yaml | 2 +- airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py | 4 ++++ .../operators/{sftp_to_azure_blob.rst => sftp_to_wasb.rst} | 6 ++++-- 4 files changed, 12 insertions(+), 6 deletions(-) rename docs/apache-airflow-providers-microsoft-azure/operators/{sftp_to_azure_blob.rst => sftp_to_wasb.rst} (96%) diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index 959b131b870a4..2679167315aba 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -26,12 +26,12 @@ SFTP_SRC_PATH = os.environ.get("AZURE_SFTP_SRC_PATH", "test-sftp-azure") with DAG( - "example_sftp_to_azure_blob", + "example_sftp_to_wasb", schedule_interval=None, start_date=days_ago(1), # Override to match your needs ) as dag: - # [START how_to_sftp_to_azure_blob] + # [START how_to_sftp_to_wasb] transfer_files_to_azure = SFTPToWasbOperator( task_id="transfer_files_to_gcs", # SFTP args @@ -42,4 +42,4 @@ container_name=AZURE_CONTAINER_NAME, blob_prefix=BLOB_PREFIX, ) - # [END how_to_sftp_to_azure_blob] + # [END how_to_sftp_to_wasb] diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index 963cd73d9c335..815d2ffed55c7 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -147,7 +147,7 @@ transfers: python-module: airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs - source-integration-name: SSH File Transfer Protocol (SFTP) target-integration-name: Microsoft Azure Data Lake Storage - how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_azure_blob.rst + how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst python-module: airflow.providers.microsoft.azure.transfers.sftp_to_wasb hook-class-names: diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 15a72f7c8cc98..606332b48c327 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -37,6 +37,10 @@ class SFTPToWasbOperator(BaseOperator): """ Transfer files to Azure Blob Storage from SFTP server. + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:SFTPToWasbOperator` + :param sftp_source_path: The sftp remote path. This is the specified file path for downloading the single file or multiple files from the SFTP server. You can use only one wildcard within your path. The wildcard can appear diff --git a/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_azure_blob.rst b/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst similarity index 96% rename from docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_azure_blob.rst rename to docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst index 42ca7d792ceff..d0d6486676760 100644 --- a/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_azure_blob.rst +++ b/docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst @@ -16,6 +16,8 @@ specific language governing permissions and limitations under the License. + + Azure Blob Storage Transfer Operator ==================================== The Blob service stores text and binary data as objects in the cloud. @@ -61,5 +63,5 @@ Example usage: .. exampleinclude:: /../../airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py :language: python :dedent: 4 - :start-after: [START how_to_sftp_to_azure_blob] - :end-before: [END how_to_sftp_to_azure_blob] + :start-after: [START how_to_sftp_to_wasb] + :end-before: [END how_to_sftp_to_wasb] From e517eb34322c232af676d275c87293785ec18048 Mon Sep 17 00:00:00 2001 From: guilherme Date: Fri, 19 Feb 2021 22:28:46 -0300 Subject: [PATCH 58/79] New test system --- .../example_dags/example_sftp_to_wasb.py | 2 +- .../providers/microsoft/azure/provider.yaml | 2 +- .../transfers/test_sftp_to_wasb_system.py | 48 +++++++++++++++++++ 3 files changed, 50 insertions(+), 2 deletions(-) create mode 100644 tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index 2679167315aba..1fb6f22c2b6c6 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -33,7 +33,7 @@ # [START how_to_sftp_to_wasb] transfer_files_to_azure = SFTPToWasbOperator( - task_id="transfer_files_to_gcs", + task_id="transfer_files_from_sftp_to_wasb", # SFTP args sftp_conn_id="sftp_default", sftp_source_path=SFTP_SRC_PATH, diff --git a/airflow/providers/microsoft/azure/provider.yaml b/airflow/providers/microsoft/azure/provider.yaml index 815d2ffed55c7..69cfba9de3c93 100644 --- a/airflow/providers/microsoft/azure/provider.yaml +++ b/airflow/providers/microsoft/azure/provider.yaml @@ -146,7 +146,7 @@ transfers: how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/azure_blob_to_gcs.rst python-module: airflow.providers.microsoft.azure.transfers.azure_blob_to_gcs - source-integration-name: SSH File Transfer Protocol (SFTP) - target-integration-name: Microsoft Azure Data Lake Storage + target-integration-name: Microsoft Azure Blob Storage how-to-guide: /docs/apache-airflow-providers-microsoft-azure/operators/sftp_to_wasb.rst python-module: airflow.providers.microsoft.azure.transfers.sftp_to_wasb diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py new file mode 100644 index 0000000000000..a68cb7ca0e47f --- /dev/null +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import os + +import pytest + +from airflow.providers.microsoft.azure.example_dags.example_sftp_to_wasb import SFTP_SRC_PATH +from tests.test_utils.azure_system_helpers import ( + AZURE_DAG_FOLDER, + AzureSystemTest, + provide_wasb_default_connection, +) +CREDENTIALS_DIR = os.environ.get('CREDENTIALS_DIR', '/files/airflow-breeze-config/keys') +WASB_DEFAULT_KEY = 'wasb_key.json' +CREDENTIALS_PATH = os.path.join(CREDENTIALS_DIR, WASB_DEFAULT_KEY) +FILENAME = "TEST.TXT" + +@pytest.mark.backend('postgres', 'mysql') +@pytest.mark.credential_file(WASB_DEFAULT_KEY) +class TestSFTPToWasbSystem(AzureSystemTest): + def setUp(self): + super().setUp() + self.create_dummy_file(FILENAME, SFTP_SRC_PATH) + + def tearDown(self): + os.remove(SFTP_SRC_PATH) + super().tearDown() + + @provide_wasb_default_connection(CREDENTIALS_PATH) + def test_run_example_file_to_wasb(self): + print("ok") + self.run_dag('example_sftp_to_wasb', AZURE_DAG_FOLDER) From debd10b3877cb24426498615d85819750c826bdb Mon Sep 17 00:00:00 2001 From: guilherme Date: Sat, 20 Feb 2021 00:30:22 -0300 Subject: [PATCH 59/79] Static fix --- .../microsoft/azure/transfers/test_sftp_to_wasb_system.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py index a68cb7ca0e47f..4d93beeac53af 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py @@ -26,11 +26,13 @@ AzureSystemTest, provide_wasb_default_connection, ) + CREDENTIALS_DIR = os.environ.get('CREDENTIALS_DIR', '/files/airflow-breeze-config/keys') WASB_DEFAULT_KEY = 'wasb_key.json' CREDENTIALS_PATH = os.path.join(CREDENTIALS_DIR, WASB_DEFAULT_KEY) FILENAME = "TEST.TXT" + @pytest.mark.backend('postgres', 'mysql') @pytest.mark.credential_file(WASB_DEFAULT_KEY) class TestSFTPToWasbSystem(AzureSystemTest): From e3fcd629bb18715c3771152b34f640a19322119d Mon Sep 17 00:00:00 2001 From: guilherme Date: Sat, 20 Feb 2021 16:55:01 -0300 Subject: [PATCH 60/79] Start CI --- .../microsoft/azure/transfers/test_sftp_to_wasb_system.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py index 4d93beeac53af..6c691bb0497e5 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py @@ -17,7 +17,6 @@ import os - import pytest from airflow.providers.microsoft.azure.example_dags.example_sftp_to_wasb import SFTP_SRC_PATH From 893dfd2d4ae5ff042714aefb9fa1b183a17a1aad Mon Sep 17 00:00:00 2001 From: guilherme Date: Sat, 20 Feb 2021 18:11:54 -0300 Subject: [PATCH 61/79] Fix error from pylint --- .../microsoft/azure/transfers/test_sftp_to_wasb_system.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py index 6c691bb0497e5..4d93beeac53af 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py @@ -17,6 +17,7 @@ import os + import pytest from airflow.providers.microsoft.azure.example_dags.example_sftp_to_wasb import SFTP_SRC_PATH From 9da5f919f3857227abbb1631b49069399a19257f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Sat, 20 Feb 2021 20:03:41 -0300 Subject: [PATCH 62/79] Fix template fields at airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py Co-authored-by: Ephraim Anierobi --- airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 606332b48c327..b1f745d6accb6 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -64,7 +64,7 @@ class SFTPToWasbOperator(BaseOperator): :type move_object: bool """ - template_fields = ("sftp_source_path", "container_name", "blob_name") + template_fields = ("sftp_source_path", "container_name", "blob_prefix") @apply_defaults def __init__( From 85e6c2c3cc0e904a64dd8b3889b71f59babffd51 Mon Sep 17 00:00:00 2001 From: guilherme Date: Mon, 22 Feb 2021 02:11:01 -0300 Subject: [PATCH 63/79] Loads sftp connection from files and deletes files from azure and sftp --- .../example_dags/example_sftp_to_wasb.py | 16 +++++- .../transfers/test_sftp_to_wasb_system.py | 31 ++++++++--- tests/test_utils/sftp_system_helpers.py | 51 +++++++++++++++++++ 3 files changed, 90 insertions(+), 8 deletions(-) create mode 100644 tests/test_utils/sftp_system_helpers.py diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index 1fb6f22c2b6c6..a86d396825662 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -19,11 +19,16 @@ from airflow import DAG from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator +from airflow.providers.sftp.operators.sftp import SFTPOperator from airflow.utils.dates import days_ago AZURE_CONTAINER_NAME = os.environ.get("AZURE_CONTAINER_NAME", "airflow") BLOB_PREFIX = os.environ.get("AZURE_BLOB_PREFIX", "airflow") -SFTP_SRC_PATH = os.environ.get("AZURE_SFTP_SRC_PATH", "test-sftp-azure") +SFTP_SRC_PATH = os.environ.get("AZURE_WASB_SFTP_SRC_PATH", "/sftp") +LOCAL_FILE_PATH = os.environ.get("FILE_TO_SFTPWASB_LOCAL_SRC_PATH", "/tmp") +SAMPLE_FILE_NAME = os.environ.get("FILE_TO_SFTPWASB", "sftp_to_wasb_test.txt") +FILE_COMPLETE_PATH = os.path.join(LOCAL_FILE_PATH, SAMPLE_FILE_NAME) +SFTP_FILE_COMPLETE_PATH = os.path.join(SFTP_SRC_PATH, SAMPLE_FILE_NAME) with DAG( "example_sftp_to_wasb", @@ -31,6 +36,13 @@ start_date=days_ago(1), # Override to match your needs ) as dag: + transfer_files_to_sftp = SFTPOperator( + task_id="transfer_files_from_local_to_sftp", + ssh_conn_id="sftp_default", + local_filepath=FILE_COMPLETE_PATH, + remote_filepath=SFTP_FILE_COMPLETE_PATH, + ) + # [START how_to_sftp_to_wasb] transfer_files_to_azure = SFTPToWasbOperator( task_id="transfer_files_from_sftp_to_wasb", @@ -43,3 +55,5 @@ blob_prefix=BLOB_PREFIX, ) # [END how_to_sftp_to_wasb] + + transfer_files_to_sftp >> transfer_files_to_azure diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py index 4d93beeac53af..fac5abb0cd820 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py @@ -20,31 +20,48 @@ import pytest -from airflow.providers.microsoft.azure.example_dags.example_sftp_to_wasb import SFTP_SRC_PATH +from airflow.providers.microsoft.azure.example_dags.example_sftp_to_wasb import ( + AZURE_CONTAINER_NAME, + BLOB_PREFIX, + FILE_COMPLETE_PATH, + LOCAL_FILE_PATH, + SAMPLE_FILE_NAME, + SFTP_FILE_COMPLETE_PATH, + SFTP_SRC_PATH, +) +from airflow.providers.microsoft.azure.hooks.wasb import WasbHook +from airflow.providers.sftp.hooks.sftp import SFTPHook from tests.test_utils.azure_system_helpers import ( AZURE_DAG_FOLDER, AzureSystemTest, provide_wasb_default_connection, ) +from tests.test_utils.sftp_system_helpers import provide_sftp_default_connection CREDENTIALS_DIR = os.environ.get('CREDENTIALS_DIR', '/files/airflow-breeze-config/keys') +SFTP_DEFAULT_KEY = 'sftp_key.json' WASB_DEFAULT_KEY = 'wasb_key.json' -CREDENTIALS_PATH = os.path.join(CREDENTIALS_DIR, WASB_DEFAULT_KEY) -FILENAME = "TEST.TXT" +CREDENTIALS_SFTP_PATH = os.path.join(CREDENTIALS_DIR, SFTP_DEFAULT_KEY) +CREDENTIALS_WASB_PATH = os.path.join(CREDENTIALS_DIR, WASB_DEFAULT_KEY) @pytest.mark.backend('postgres', 'mysql') @pytest.mark.credential_file(WASB_DEFAULT_KEY) +@pytest.mark.credential_file(SFTP_DEFAULT_KEY) class TestSFTPToWasbSystem(AzureSystemTest): def setUp(self): super().setUp() - self.create_dummy_file(FILENAME, SFTP_SRC_PATH) + self.create_dummy_file(SAMPLE_FILE_NAME, LOCAL_FILE_PATH) def tearDown(self): - os.remove(SFTP_SRC_PATH) + os.remove(FILE_COMPLETE_PATH) super().tearDown() - @provide_wasb_default_connection(CREDENTIALS_PATH) + @provide_wasb_default_connection(CREDENTIALS_WASB_PATH) + @provide_sftp_default_connection(CREDENTIALS_SFTP_PATH) def test_run_example_file_to_wasb(self): - print("ok") self.run_dag('example_sftp_to_wasb', AZURE_DAG_FOLDER) + WasbHook(wasb_conn_id="wasb_default").delete_file( + AZURE_CONTAINER_NAME, BLOB_PREFIX + SAMPLE_FILE_NAME + ) + SFTPHook(ssh_conn_id="sftp_default").delete_file(SFTP_FILE_COMPLETE_PATH) diff --git a/tests/test_utils/sftp_system_helpers.py b/tests/test_utils/sftp_system_helpers.py new file mode 100644 index 0000000000000..accf3c9504591 --- /dev/null +++ b/tests/test_utils/sftp_system_helpers.py @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import json +import os +from contextlib import contextmanager + +from airflow.exceptions import AirflowException +from airflow.models import Connection +from airflow.utils.process_utils import patch_environ + +SFTP_CONNECTION_ID = os.environ.get("SFTP_CONNECTION_ID", "sftp_default") + + +@contextmanager +def provide_sftp_default_connection(key_file_path: str): + """ + Context manager to provide a temporary value for sftp_default connection + + :param key_file_path: Path to file with sftp_default credentials .json file. + :type key_file_path: str + """ + if not key_file_path.endswith(".json"): + raise AirflowException("Use a JSON key file.") + with open(key_file_path) as credentials: + creds = json.load(credentials) + conn = Connection( + conn_id=SFTP_CONNECTION_ID, + conn_type="ssh", + port=creds.get("port", None), + host=creds.get("host", None), + login=creds.get("login", None), + password=creds.get("password", None), + extra=json.dumps(creds.get('extra', None)), + ) + with patch_environ({f"AIRFLOW_CONN_{conn.conn_id.upper()}": conn.get_uri()}): + yield From a32af79a966350653bcc844046818496834ce849 Mon Sep 17 00:00:00 2001 From: guilherme Date: Mon, 22 Feb 2021 09:02:19 -0300 Subject: [PATCH 64/79] Removes unnecessary import --- .../microsoft/azure/transfers/test_sftp_to_wasb_system.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py index fac5abb0cd820..e63d1c1e74978 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py @@ -27,7 +27,6 @@ LOCAL_FILE_PATH, SAMPLE_FILE_NAME, SFTP_FILE_COMPLETE_PATH, - SFTP_SRC_PATH, ) from airflow.providers.microsoft.azure.hooks.wasb import WasbHook from airflow.providers.sftp.hooks.sftp import SFTPHook From 68cf8e9766184882835ec338c70d90b74c3a6fc7 Mon Sep 17 00:00:00 2001 From: guilherme Date: Thu, 25 Feb 2021 23:51:38 -0300 Subject: [PATCH 65/79] Load all hooks as operators in example dag to sftp_to_Wasb --- .../example_dags/example_sftp_to_wasb.py | 19 ++++++++++++++++++- .../transfers/test_sftp_to_wasb_system.py | 9 --------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index a86d396825662..20a680e5eef04 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -18,7 +18,10 @@ import os from airflow import DAG +from airflow.operators.python import PythonOperator +from airflow.providers.microsoft.azure.operators.wasb_delete_blob import WasbDeleteBlobOperator from airflow.providers.microsoft.azure.transfers.sftp_to_wasb import SFTPToWasbOperator +from airflow.providers.sftp.hooks.sftp import SFTPHook from airflow.providers.sftp.operators.sftp import SFTPOperator from airflow.utils.dates import days_ago @@ -30,6 +33,12 @@ FILE_COMPLETE_PATH = os.path.join(LOCAL_FILE_PATH, SAMPLE_FILE_NAME) SFTP_FILE_COMPLETE_PATH = os.path.join(SFTP_SRC_PATH, SAMPLE_FILE_NAME) + +def delete_sftp_file(): + """Delete a file at SFTP SERVER""" + SFTPHook(ssh_conn_id="sftp_default").delete_file(SFTP_FILE_COMPLETE_PATH) + + with DAG( "example_sftp_to_wasb", schedule_interval=None, @@ -56,4 +65,12 @@ ) # [END how_to_sftp_to_wasb] - transfer_files_to_sftp >> transfer_files_to_azure + delete_blob_files = WasbDeleteBlobOperator( + wasb_conn_id="wasb_default", + container_name=AZURE_CONTAINER_NAME, + blob_name=BLOB_PREFIX + SAMPLE_FILE_NAME, + ) + + delete_sftp_step = PythonOperator(task_id="delete_sftp_file", python_callable=delete_sftp_file) + + transfer_files_to_sftp >> transfer_files_to_azure >> delete_blob_files >> delete_sftp_step diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py index e63d1c1e74978..fd2c1ab1e5334 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py @@ -21,15 +21,10 @@ import pytest from airflow.providers.microsoft.azure.example_dags.example_sftp_to_wasb import ( - AZURE_CONTAINER_NAME, - BLOB_PREFIX, FILE_COMPLETE_PATH, LOCAL_FILE_PATH, SAMPLE_FILE_NAME, - SFTP_FILE_COMPLETE_PATH, ) -from airflow.providers.microsoft.azure.hooks.wasb import WasbHook -from airflow.providers.sftp.hooks.sftp import SFTPHook from tests.test_utils.azure_system_helpers import ( AZURE_DAG_FOLDER, AzureSystemTest, @@ -60,7 +55,3 @@ def tearDown(self): @provide_sftp_default_connection(CREDENTIALS_SFTP_PATH) def test_run_example_file_to_wasb(self): self.run_dag('example_sftp_to_wasb', AZURE_DAG_FOLDER) - WasbHook(wasb_conn_id="wasb_default").delete_file( - AZURE_CONTAINER_NAME, BLOB_PREFIX + SAMPLE_FILE_NAME - ) - SFTPHook(ssh_conn_id="sftp_default").delete_file(SFTP_FILE_COMPLETE_PATH) From 236b13337f3e03486a54a5a2238e796484b04b96 Mon Sep 17 00:00:00 2001 From: guilherme Date: Fri, 26 Feb 2021 01:46:56 -0300 Subject: [PATCH 66/79] Missing task id --- .../microsoft/azure/example_dags/example_sftp_to_wasb.py | 1 + 1 file changed, 1 insertion(+) diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index 20a680e5eef04..12b2c5d854034 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -66,6 +66,7 @@ def delete_sftp_file(): # [END how_to_sftp_to_wasb] delete_blob_files = WasbDeleteBlobOperator( + task_id="delete_blob_files", wasb_conn_id="wasb_default", container_name=AZURE_CONTAINER_NAME, blob_name=BLOB_PREFIX + SAMPLE_FILE_NAME, From c2da2372a1b88d3ab50398c5d89a5fdf7819acad Mon Sep 17 00:00:00 2001 From: guilherme Date: Fri, 26 Feb 2021 07:46:05 -0300 Subject: [PATCH 67/79] Changes variable name --- .../microsoft/azure/example_dags/example_sftp_to_wasb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index 12b2c5d854034..68cdf402f04fa 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -65,7 +65,7 @@ def delete_sftp_file(): ) # [END how_to_sftp_to_wasb] - delete_blob_files = WasbDeleteBlobOperator( + delete_blob_files_step = WasbDeleteBlobOperator( task_id="delete_blob_files", wasb_conn_id="wasb_default", container_name=AZURE_CONTAINER_NAME, @@ -74,4 +74,4 @@ def delete_sftp_file(): delete_sftp_step = PythonOperator(task_id="delete_sftp_file", python_callable=delete_sftp_file) - transfer_files_to_sftp >> transfer_files_to_azure >> delete_blob_files >> delete_sftp_step + transfer_files_to_sftp >> transfer_files_to_azure >> delete_blob_files_step >> delete_sftp_step From c1032f005635e7fc22394a0bbeafeb69afb216de Mon Sep 17 00:00:00 2001 From: guilherme Date: Fri, 26 Feb 2021 17:43:50 -0300 Subject: [PATCH 68/79] Changes variable name --- .../microsoft/azure/example_dags/example_sftp_to_wasb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index 68cdf402f04fa..e4a3d2a713bd0 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -65,7 +65,7 @@ def delete_sftp_file(): ) # [END how_to_sftp_to_wasb] - delete_blob_files_step = WasbDeleteBlobOperator( + delete_blob_file_step = WasbDeleteBlobOperator( task_id="delete_blob_files", wasb_conn_id="wasb_default", container_name=AZURE_CONTAINER_NAME, @@ -74,4 +74,4 @@ def delete_sftp_file(): delete_sftp_step = PythonOperator(task_id="delete_sftp_file", python_callable=delete_sftp_file) - transfer_files_to_sftp >> transfer_files_to_azure >> delete_blob_files_step >> delete_sftp_step + transfer_files_to_sftp >> transfer_files_to_azure >> delete_blob_file_step >> delete_sftp_step From 9390f02f544660e0de90929d973d8852d89e7220 Mon Sep 17 00:00:00 2001 From: guilherme Date: Sun, 28 Feb 2021 00:00:35 -0300 Subject: [PATCH 69/79] Renames variable name --- .../microsoft/azure/example_dags/example_sftp_to_wasb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index e4a3d2a713bd0..f75d792184624 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -45,7 +45,7 @@ def delete_sftp_file(): start_date=days_ago(1), # Override to match your needs ) as dag: - transfer_files_to_sftp = SFTPOperator( + transfer_files_to_sftp_step = SFTPOperator( task_id="transfer_files_from_local_to_sftp", ssh_conn_id="sftp_default", local_filepath=FILE_COMPLETE_PATH, @@ -74,4 +74,4 @@ def delete_sftp_file(): delete_sftp_step = PythonOperator(task_id="delete_sftp_file", python_callable=delete_sftp_file) - transfer_files_to_sftp >> transfer_files_to_azure >> delete_blob_file_step >> delete_sftp_step + transfer_files_to_sftp_step >> transfer_files_to_azure >> delete_blob_file_step >> delete_sftp_step From d35cb993b209a281f195bbcad43e2855e4523a0a Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 2 Mar 2021 00:21:37 -0300 Subject: [PATCH 70/79] Changes to assert use in tests --- .../azure/transfers/test_sftp_to_wasb.py | 44 +++++++++---------- 1 file changed, 21 insertions(+), 23 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index eba4c4350d530..0abb3c841d3bb 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -49,11 +49,11 @@ def test_init(self): wasb_conn_id=WASB_CONN_ID, move_object=False, ) - self.assertEqual(operator.sftp_source_path, SOURCE_PATH_NO_WILDCARD) - self.assertEqual(operator.sftp_conn_id, SFTP_CONN_ID) - self.assertEqual(operator.container_name, CONTAINER_NAME) - self.assertEqual(operator.wasb_conn_id, WASB_CONN_ID) - self.assertEqual(operator.blob_prefix, BLOB_PREFIX) + assert operator.sftp_source_path == SOURCE_PATH_NO_WILDCARD + assert operator.sftp_conn_id == SFTP_CONN_ID + assert operator.container_name == CONTAINER_NAME + assert operator.wasb_conn_id == WASB_CONN_ID + assert operator.blob_prefix == BLOB_PREFIX @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook', autospec=True) def test_execute_more_than_one_wildcard_exception(self, mock_hook): @@ -70,7 +70,7 @@ def test_execute_more_than_one_wildcard_exception(self, mock_hook): operator.check_wildcards_limit() err = cm.exception - self.assertIn("Only one wildcard '*' is allowed", str(err)) + assert "Only one wildcard '*' is allowed" in str(err) def test_get_sftp_tree_behavior(self): operator = SFTPToWasbOperator( @@ -83,9 +83,9 @@ def test_get_sftp_tree_behavior(self): ) sftp_complete_path, prefix, delimiter = operator.get_tree_behavior() - self.assertEqual(sftp_complete_path, 'main_dir', "not matched at expected complete path") - self.assertEqual(prefix, 'main_dir/', "Prefix must be EQUAL TO wildcard") - self.assertEqual(delimiter, "", "Delimiter must be empty") + assert sftp_complete_path == 'main_dir', "not matched at expected complete path" + assert prefix == 'main_dir/', "Prefix must be EQUAL TO wildcard" + assert delimiter == "", "Delimiter must be empty" def test_get_sftp_tree_behavior_without_wildcard(self): operator = SFTPToWasbOperator( @@ -98,9 +98,9 @@ def test_get_sftp_tree_behavior_without_wildcard(self): ) sftp_complete_path, prefix, delimiter = operator.get_tree_behavior() - self.assertEqual(sftp_complete_path, 'main_dir/', "not matched at expected complete path") - self.assertIsNone(prefix, "Prefix must be NONE when no wildcard") - self.assertIsNone(delimiter, "Delimiter must be none") + assert sftp_complete_path == 'main_dir/', "not matched at expected complete path" + assert prefix is None, "Prefix must be NONE when no wildcard" + assert delimiter is None, "Delimiter must be none" def test_source_path_contains_wildcard(self): operator = SFTPToWasbOperator( @@ -112,7 +112,7 @@ def test_source_path_contains_wildcard(self): move_object=False, ) output = operator.source_path_contains_wildcard - self.assertTrue(output, "This path contains a wildpath") + assert output is True, "This path contains a wildpath" def test_source_path_not_contains_wildcard(self): operator = SFTPToWasbOperator( @@ -124,7 +124,7 @@ def test_source_path_not_contains_wildcard(self): move_object=False, ) output = operator.source_path_contains_wildcard - self.assertFalse(output, "This path does not contains a wildpath") + assert output is False, "This path does not contains a wildpath" @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') @@ -144,8 +144,8 @@ def test_get_sftp_files_map_no_wildcard(self, sftp_hook, mock_hook): ) files = operator.get_sftp_files_map() - self.assertEqual(len(files), 1, "no matched at expected found files") - self.assertEqual(files[0].blob_name, EXPECTED_BLOB_NAME, "expected blob name not matched") + assert len(files) == 1, "no matched at expected found files" + assert files[0].blob_name == EXPECTED_BLOB_NAME, "expected blob name not matched" @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') @@ -166,7 +166,7 @@ def test_copy_files_to_wasb(self, sftp_hook, mock_hook): mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, EXPECTED_BLOB_NAME) - self.assertEqual(len(files), 1, "no matched at expected uploaded files") + assert len(files) == 1, "no matched at expected uploaded files" @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_delete_files(self, sftp_hook): @@ -184,12 +184,11 @@ def test_delete_files(self, sftp_hook): sftp_file_paths = EXPECTED_FILES operator.delete_files(sftp_file_paths) - sftp_mock.delete_file.assert_has_calls([mock.call(EXPECTED_FILES[0])]) + assert sftp_mock.delete_file.assert_has_calls([mock.call(EXPECTED_FILES[0])]) @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_execute(self, sftp_hook, mock_hook): - operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_FILE_NAME, @@ -215,14 +214,13 @@ def test_execute(self, sftp_hook, mock_hook): [mock.call("main_dir/test_object.json", mock.ANY)] ) - mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") + assert mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") - sftp_hook.return_value.delete_file.assert_not_called() + assert sftp_hook.return_value.delete_file.assert_not_called() @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') def test_execute_moved_files(self, sftp_hook, mock_hook): - operator = SFTPToWasbOperator( task_id=TASK_ID, sftp_source_path=WILDCARD_FILE_NAME, @@ -252,4 +250,4 @@ def test_execute_moved_files(self, sftp_hook, mock_hook): mock_hook.return_value.load_file.assert_called_once_with( mock.ANY, CONTAINER_NAME, BLOB_PREFIX + "test_object" ".json" ) - self.assertTrue(sftp_hook.return_value.delete_file.called, "File must be moved") + assert sftp_hook.return_value.delete_file.called is True, "File must be moved" From d23e4ca1db4805952a7bab47224dfa5216cf39cd Mon Sep 17 00:00:00 2001 From: guilherme Date: Tue, 2 Mar 2021 22:52:17 -0300 Subject: [PATCH 71/79] Fix imports --- .../providers/microsoft/azure/transfers/test_sftp_to_wasb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index 0abb3c841d3bb..a2df2535c7679 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -214,7 +214,9 @@ def test_execute(self, sftp_hook, mock_hook): [mock.call("main_dir/test_object.json", mock.ANY)] ) - assert mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") + assert mock_hook.return_value.load_file.assert_called_once_with( + mock.ANY, CONTAINER_NAME, "test_object.json" + ) assert sftp_hook.return_value.delete_file.assert_not_called() From 1f407fa289063c3d564253aed4ed9ba8d95b65dd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Tue, 2 Mar 2021 22:53:16 -0300 Subject: [PATCH 72/79] Update airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py Co-authored-by: Ephraim Anierobi --- .../microsoft/azure/example_dags/example_sftp_to_wasb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index f75d792184624..7ce6e095a9249 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -27,9 +27,9 @@ AZURE_CONTAINER_NAME = os.environ.get("AZURE_CONTAINER_NAME", "airflow") BLOB_PREFIX = os.environ.get("AZURE_BLOB_PREFIX", "airflow") -SFTP_SRC_PATH = os.environ.get("AZURE_WASB_SFTP_SRC_PATH", "/sftp") -LOCAL_FILE_PATH = os.environ.get("FILE_TO_SFTPWASB_LOCAL_SRC_PATH", "/tmp") -SAMPLE_FILE_NAME = os.environ.get("FILE_TO_SFTPWASB", "sftp_to_wasb_test.txt") +SFTP_SRC_PATH = os.environ.get("SFTP_SRC_PATH", "/sftp") +LOCAL_FILE_PATH = os.environ.get("LOCAL_SRC_PATH", "/tmp") +SAMPLE_FILENAME = os.environ.get("SFTP_SAMPLE_FILENAME", "sftp_to_wasb_test.txt") FILE_COMPLETE_PATH = os.path.join(LOCAL_FILE_PATH, SAMPLE_FILE_NAME) SFTP_FILE_COMPLETE_PATH = os.path.join(SFTP_SRC_PATH, SAMPLE_FILE_NAME) From e0e577c98e7cd62a750875a8273098de8263e843 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Tue, 2 Mar 2021 22:53:27 -0300 Subject: [PATCH 73/79] Update airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py Co-authored-by: Ephraim Anierobi --- .../microsoft/azure/example_dags/example_sftp_to_wasb.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index 7ce6e095a9249..f98f392b73c07 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -30,8 +30,8 @@ SFTP_SRC_PATH = os.environ.get("SFTP_SRC_PATH", "/sftp") LOCAL_FILE_PATH = os.environ.get("LOCAL_SRC_PATH", "/tmp") SAMPLE_FILENAME = os.environ.get("SFTP_SAMPLE_FILENAME", "sftp_to_wasb_test.txt") -FILE_COMPLETE_PATH = os.path.join(LOCAL_FILE_PATH, SAMPLE_FILE_NAME) -SFTP_FILE_COMPLETE_PATH = os.path.join(SFTP_SRC_PATH, SAMPLE_FILE_NAME) +FILE_COMPLETE_PATH = os.path.join(LOCAL_FILE_PATH, SAMPLE_FILENAME) +SFTP_FILE_COMPLETE_PATH = os.path.join(SFTP_SRC_PATH, SAMPLE_FILENAME) def delete_sftp_file(): From 57ebd24335a09895c043e14906fb4c42585dee2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Thu, 4 Mar 2021 00:03:52 -0300 Subject: [PATCH 74/79] Fix issue to change of sample file name in test system --- .../microsoft/azure/example_dags/example_sftp_to_wasb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py index f98f392b73c07..32d1b96abd2cc 100644 --- a/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/example_dags/example_sftp_to_wasb.py @@ -69,7 +69,7 @@ def delete_sftp_file(): task_id="delete_blob_files", wasb_conn_id="wasb_default", container_name=AZURE_CONTAINER_NAME, - blob_name=BLOB_PREFIX + SAMPLE_FILE_NAME, + blob_name=BLOB_PREFIX + SAMPLE_FILENAME, ) delete_sftp_step = PythonOperator(task_id="delete_sftp_file", python_callable=delete_sftp_file) From b3bd462911abe95566d382f66c8c862e0098df93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Thu, 4 Mar 2021 00:04:53 -0300 Subject: [PATCH 75/79] Fix change of SAMPLE_FILENAME in test system --- .../microsoft/azure/transfers/test_sftp_to_wasb_system.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py index fd2c1ab1e5334..600f84fc33585 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb_system.py @@ -23,7 +23,7 @@ from airflow.providers.microsoft.azure.example_dags.example_sftp_to_wasb import ( FILE_COMPLETE_PATH, LOCAL_FILE_PATH, - SAMPLE_FILE_NAME, + SAMPLE_FILENAME, ) from tests.test_utils.azure_system_helpers import ( AZURE_DAG_FOLDER, @@ -45,7 +45,7 @@ class TestSFTPToWasbSystem(AzureSystemTest): def setUp(self): super().setUp() - self.create_dummy_file(SAMPLE_FILE_NAME, LOCAL_FILE_PATH) + self.create_dummy_file(SAMPLE_FILENAME, LOCAL_FILE_PATH) def tearDown(self): os.remove(FILE_COMPLETE_PATH) From f00d7c4516abaa01fed0ba4c5337163295050c73 Mon Sep 17 00:00:00 2001 From: "guilhg2@gmail.com" Date: Sat, 6 Mar 2021 20:21:06 -0300 Subject: [PATCH 76/79] Removes assert from mocking elements --- .../microsoft/azure/transfers/test_sftp_to_wasb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index a2df2535c7679..b50211b600b6c 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -184,7 +184,7 @@ def test_delete_files(self, sftp_hook): sftp_file_paths = EXPECTED_FILES operator.delete_files(sftp_file_paths) - assert sftp_mock.delete_file.assert_has_calls([mock.call(EXPECTED_FILES[0])]) + sftp_mock.delete_file.assert_has_calls([mock.call(EXPECTED_FILES[0])]) @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') @@ -214,11 +214,11 @@ def test_execute(self, sftp_hook, mock_hook): [mock.call("main_dir/test_object.json", mock.ANY)] ) - assert mock_hook.return_value.load_file.assert_called_once_with( + mock_hook.return_value.load_file.assert_called_once_with( mock.ANY, CONTAINER_NAME, "test_object.json" ) - assert sftp_hook.return_value.delete_file.assert_not_called() + sftp_hook.return_value.delete_file.assert_not_called() @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.WasbHook') @mock.patch('airflow.providers.microsoft.azure.transfers.sftp_to_wasb.SFTPHook') From ade3c6c5c0e89fe527020de34e7e370130e4bbf3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Sat, 6 Mar 2021 21:38:59 -0300 Subject: [PATCH 77/79] Update test_sftp_to_wasb.py --- .../microsoft/azure/transfers/test_sftp_to_wasb.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py index b50211b600b6c..dfd5f7f01ccb3 100644 --- a/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py +++ b/tests/providers/microsoft/azure/transfers/test_sftp_to_wasb.py @@ -214,9 +214,7 @@ def test_execute(self, sftp_hook, mock_hook): [mock.call("main_dir/test_object.json", mock.ANY)] ) - mock_hook.return_value.load_file.assert_called_once_with( - mock.ANY, CONTAINER_NAME, "test_object.json" - ) + mock_hook.return_value.load_file.assert_called_once_with(mock.ANY, CONTAINER_NAME, "test_object.json") sftp_hook.return_value.delete_file.assert_not_called() @@ -250,6 +248,6 @@ def test_execute_moved_files(self, sftp_hook, mock_hook): ) mock_hook.return_value.load_file.assert_called_once_with( - mock.ANY, CONTAINER_NAME, BLOB_PREFIX + "test_object" ".json" + mock.ANY, CONTAINER_NAME, BLOB_PREFIX + "test_object.json" ) assert sftp_hook.return_value.delete_file.called is True, "File must be moved" From e298ea5990db90cb5c3b65d3a4a3562f44e84aed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Thu, 11 Mar 2021 08:59:51 -0300 Subject: [PATCH 78/79] Update airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py Co-authored-by: Ash Berlin-Taylor --- airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index b1f745d6accb6..01d35408726ae 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -21,7 +21,10 @@ from tempfile import NamedTemporaryFile from typing import Any, Dict, List, Optional, Tuple -from cached_property import cached_property +try: + from functools import cached_property +except ImportError: + from cached_property import cached_property from airflow.exceptions import AirflowException from airflow.models import BaseOperator From 45b658e24878689f6d8181bfa6975199abeebb89 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Guilherme=20Da=20Silva=20Gon=C3=A7alves?= Date: Thu, 11 Mar 2021 09:00:01 -0300 Subject: [PATCH 79/79] Update airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py Co-authored-by: Ash Berlin-Taylor --- airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py index 01d35408726ae..78917e57af372 100644 --- a/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py +++ b/airflow/providers/microsoft/azure/transfers/sftp_to_wasb.py @@ -59,7 +59,7 @@ class SFTPToWasbOperator(BaseOperator): :param wasb_conn_id: Reference to the wasb connection. :type wasb_conn_id: str :param load_options: Optional keyword arguments that - `WasbHook.load_file()` takes. + ``WasbHook.load_file()`` takes. :type load_options: dict :param move_object: When move object is True, the object is moved instead of copied to the new location. This is the equivalent of a mv command