diff --git a/airflow/providers/amazon/aws/example_dags/example_sql_to_s3.py b/airflow/providers/amazon/aws/example_dags/example_sql_to_s3.py new file mode 100644 index 0000000000000..df2abee0f3052 --- /dev/null +++ b/airflow/providers/amazon/aws/example_dags/example_sql_to_s3.py @@ -0,0 +1,44 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + + +import os +from datetime import datetime + +from airflow import models +from airflow.providers.amazon.aws.transfers.sql_to_s3 import SqlToS3Operator + +S3_BUCKET = os.environ.get("S3_BUCKET", "test-bucket") +S3_KEY = os.environ.get("S3_KEY", "key") +SQL_QUERY = os.environ.get("SQL_QUERY", "SHOW tables") + +with models.DAG( + "example_sql_to_s3", + schedule_interval=None, + start_date=datetime(2021, 1, 1), + catchup=False, +) as dag: + # [START howto_transfer_sql_to_s3] + sql_to_s3_task = SqlToS3Operator( + task_id="sql_to_s3_task", + sql_conn_id="mysql_default", + query=SQL_QUERY, + s3_bucket=S3_BUCKET, + s3_key=S3_KEY, + replace=True, + ) + # [END howto_transfer_sql_to_s3] diff --git a/airflow/providers/amazon/aws/transfers/sql_to_s3.py b/airflow/providers/amazon/aws/transfers/sql_to_s3.py index b82263be3420c..f399c271416e4 100644 --- a/airflow/providers/amazon/aws/transfers/sql_to_s3.py +++ b/airflow/providers/amazon/aws/transfers/sql_to_s3.py @@ -51,7 +51,11 @@ class SqlToS3Operator(BaseOperator): """ - Saves data from an specific SQL query into a file in S3. + Saves data from a specific SQL query into a file in S3. + + .. seealso:: + For more information on how to use this operator, take a look at the guide: + :ref:`howto/operator:SqlToS3Operator` :param query: the sql query to be executed. If you want to execute a file, place the absolute path of it, ending with .sql extension. (templated) diff --git a/airflow/providers/amazon/provider.yaml b/airflow/providers/amazon/provider.yaml index aeaf630105ce7..aea6f484df852 100644 --- a/airflow/providers/amazon/provider.yaml +++ b/airflow/providers/amazon/provider.yaml @@ -475,6 +475,7 @@ transfers: python-module: airflow.providers.amazon.aws.transfers.mongo_to_s3 - source-integration-name: MySQL target-integration-name: Amazon Simple Storage Service (S3) + how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.mysql_to_s3 - source-integration-name: Amazon Redshift target-integration-name: Amazon Simple Storage Service (S3) @@ -512,6 +513,7 @@ transfers: python-module: airflow.providers.amazon.aws.transfers.local_to_s3 - source-integration-name: SQL target-integration-name: Amazon Simple Storage Service (S3) + how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst python-module: airflow.providers.amazon.aws.transfers.sql_to_s3 hook-class-names: # deprecated - to be removed after providers add dependency on Airflow 2.2.0+ diff --git a/docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst b/docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst new file mode 100644 index 0000000000000..20c8a1fe232c8 --- /dev/null +++ b/docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst @@ -0,0 +1,52 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +SQL to Amazon S3 Transfer Operator +================================== + +Use SqlToS3Operator to copy data from a SQL server to an Amazon Simple Storage Service (S3) file. +SqlToS3Operator is compatible with any SQL connection as long as the SQL hook has function that +converts the SQL result to `pandas dataframe `__ +(e.g. MySQL, Hive, ...). + +Prerequisite Tasks +^^^^^^^^^^^^^^^^^^ + +.. include:: ../_partials/prerequisite_tasks.rst + +.. _howto/operator:SqlToS3Operator: + +MySQL to Amazon S3 +^^^^^^^^^^^^^^^^^^ + +This example sends the response of a MySQL query to an Amazon S3 file. + +To get more information about this operator visit: +:class:`~airflow.providers.amazon.aws.transfers.sql_to_s3.SqlToS3Operator` + +Example usage: + +.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_sql_to_s3.py + :language: python + :dedent: 4 + :start-after: [START howto_transfer_sql_to_s3] + :end-before: [END howto_transfer_sql_to_s3] + +Reference +^^^^^^^^^ + +* `AWS boto3 Library Documentation for Amazon S3 `__