Add doc and sample dag for SqlToS3Operator

airflow/providers/amazon/aws/example_dags/example_sql_to_s3.py

@@ -0,0 +1,44 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+
+import os
+from datetime import datetime
+
+from airflow import models
+from airflow.providers.amazon.aws.transfers.sql_to_s3 import SqlToS3Operator
+
+S3_BUCKET = os.environ.get("S3_BUCKET", "test-bucket")
+S3_KEY = os.environ.get("S3_KEY", "key")
+SQL_QUERY = os.environ.get("SQL_QUERY", "SHOW tables")
+
+with models.DAG(
+    "example_sql_to_s3",
+    schedule_interval=None,
+    start_date=datetime(2021, 1, 1),
+    catchup=False,
+) as dag:
+    # [START howto_transfer_sql_to_s3]
+    sql_to_s3_task = SqlToS3Operator(
+        task_id="sql_to_s3_task",
+        sql_conn_id="mysql_default",
+        query=SQL_QUERY,
+        s3_bucket=S3_BUCKET,
+        s3_key=S3_KEY,
+        replace=True,
+    )
+    # [END howto_transfer_sql_to_s3]

airflow/providers/amazon/aws/transfers/sql_to_s3.py

@@ -51,7 +51,11 @@
 
 class SqlToS3Operator(BaseOperator):
     """
-    Saves data from an specific SQL query into a file in S3.
+    Saves data from a specific SQL query into a file in S3.
+
+    .. seealso::
+        For more information on how to use this operator, take a look at the guide:
+        :ref:`howto/operator:SqlToS3Operator`
 
     :param query: the sql query to be executed. If you want to execute a file, place the absolute path of it,
         ending with .sql extension. (templated)

airflow/providers/amazon/provider.yaml

@@ -475,6 +475,7 @@ transfers:
     python-module: airflow.providers.amazon.aws.transfers.mongo_to_s3
   - source-integration-name: MySQL
     target-integration-name: Amazon Simple Storage Service (S3)
+    how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst
     python-module: airflow.providers.amazon.aws.transfers.mysql_to_s3
   - source-integration-name: Amazon Redshift
     target-integration-name: Amazon Simple Storage Service (S3)
@@ -512,6 +513,7 @@ transfers:
     python-module: airflow.providers.amazon.aws.transfers.local_to_s3
   - source-integration-name: SQL
     target-integration-name: Amazon Simple Storage Service (S3)
+    how-to-guide: /docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst
     python-module: airflow.providers.amazon.aws.transfers.sql_to_s3
 
 hook-class-names:  # deprecated - to be removed after providers add dependency on Airflow 2.2.0+

docs/apache-airflow-providers-amazon/operators/transfer/sql_to_s3.rst

@@ -0,0 +1,52 @@
+ .. Licensed to the Apache Software Foundation (ASF) under one
+    or more contributor license agreements.  See the NOTICE file
+    distributed with this work for additional information
+    regarding copyright ownership.  The ASF licenses this file
+    to you under the Apache License, Version 2.0 (the
+    "License"); you may not use this file except in compliance
+    with the License.  You may obtain a copy of the License at
+
+ ..   http://www.apache.org/licenses/LICENSE-2.0
+
+ .. Unless required by applicable law or agreed to in writing,
+    software distributed under the License is distributed on an
+    "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+    KIND, either express or implied.  See the License for the
+    specific language governing permissions and limitations
+    under the License.
+
+SQL to Amazon S3 Transfer Operator
+==================================
+
+Use SqlToS3Operator to copy data from a SQL server to an Amazon Simple Storage Service (S3) file.
+SqlToS3Operator is compatible with any SQL connection as long as the SQL hook has function that
+converts the SQL result to `pandas dataframe <https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html>`__
+(e.g. MySQL, Hive, ...).
+
+Prerequisite Tasks
+^^^^^^^^^^^^^^^^^^
+
+.. include:: ../_partials/prerequisite_tasks.rst
+
+.. _howto/operator:SqlToS3Operator:
+
+MySQL to Amazon S3
+^^^^^^^^^^^^^^^^^^
+
+This example sends the response of a MySQL query to an Amazon S3 file.
+
+To get more information about this operator visit:
+:class:`~airflow.providers.amazon.aws.transfers.sql_to_s3.SqlToS3Operator`
+
+Example usage:
+
+.. exampleinclude:: /../../airflow/providers/amazon/aws/example_dags/example_sql_to_s3.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_transfer_sql_to_s3]
+    :end-before: [END howto_transfer_sql_to_s3]
+
+Reference
+^^^^^^^^^
+
+* `AWS boto3 Library Documentation for Amazon S3 <https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/s3.html>`__