From 72dd72d80395210fc7e089ea01a6cd901562716f Mon Sep 17 00:00:00 2001 From: hankehly Date: Thu, 31 Mar 2022 09:24:05 +0900 Subject: [PATCH 1/3] Add autodetect parameter --- airflow/providers/google/cloud/operators/bigquery.py | 5 +++++ tests/providers/google/cloud/operators/test_bigquery.py | 1 + 2 files changed, 6 insertions(+) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 0b826be038329..e4a6d9fe482ee 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -964,6 +964,8 @@ class BigQueryCreateExternalTableOperator(BaseOperator): :param schema_object: If set, a GCS object path pointing to a .json file that contains the schema for the table. (templated) :param source_format: File format of the data. + :param autodetect: Try to detect schema and format options automatically. + Any option specified explicitly will be honored. :param compression: [Optional] The compression type of the data source. Possible values include GZIP and NONE. The default value is NONE. @@ -1028,6 +1030,7 @@ def __init__( schema_fields: Optional[List] = None, schema_object: Optional[str] = None, source_format: Optional[str] = None, + autodetect: bool = False, compression: Optional[str] = None, skip_leading_rows: Optional[int] = None, field_delimiter: Optional[str] = None, @@ -1116,6 +1119,7 @@ def __init__( self.bigquery_conn_id = bigquery_conn_id self.google_cloud_storage_conn_id = google_cloud_storage_conn_id self.delegate_to = delegate_to + self.autodetect = autodetect self.src_fmt_configs = src_fmt_configs or {} self.labels = labels @@ -1153,6 +1157,7 @@ def execute(self, context: 'Context') -> None: schema_fields=schema_fields, source_uris=source_uris, source_format=self.source_format, + autodetect=self.autodetect, compression=self.compression, skip_leading_rows=self.skip_leading_rows, field_delimiter=self.field_delimiter, diff --git a/tests/providers/google/cloud/operators/test_bigquery.py b/tests/providers/google/cloud/operators/test_bigquery.py index 8584e12b1665e..f64e7dc27484f 100644 --- a/tests/providers/google/cloud/operators/test_bigquery.py +++ b/tests/providers/google/cloud/operators/test_bigquery.py @@ -205,6 +205,7 @@ def test_execute(self, mock_hook): schema_fields=[], source_uris=[f'gs://{TEST_GCS_BUCKET}/{source_object}' for source_object in TEST_GCS_DATA], source_format=TEST_SOURCE_FORMAT, + autodetect=False, compression='NONE', skip_leading_rows=0, field_delimiter=',', From 6994d601fc391b17bd67a85b82b832220549c4b0 Mon Sep 17 00:00:00 2001 From: hankehly Date: Sun, 3 Apr 2022 17:39:52 +0900 Subject: [PATCH 2/3] Update docstring --- airflow/providers/google/cloud/operators/bigquery.py | 4 +++- tests/providers/google/cloud/operators/test_bigquery.py | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index e4a6d9fe482ee..e8711c3129a18 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -965,7 +965,8 @@ class BigQueryCreateExternalTableOperator(BaseOperator): contains the schema for the table. (templated) :param source_format: File format of the data. :param autodetect: Try to detect schema and format options automatically. - Any option specified explicitly will be honored. + The schema_fields and schema_object options will be honored when specified explicitly. + https://cloud.google.com/bigquery/docs/schema-detect#schema_auto-detection_for_external_data_sources :param compression: [Optional] The compression type of the data source. Possible values include GZIP and NONE. The default value is NONE. @@ -1060,6 +1061,7 @@ def __init__( skip_leading_rows, field_delimiter, max_bad_records, + autodetect, quote_character, allow_quoted_newlines, allow_jagged_rows, diff --git a/tests/providers/google/cloud/operators/test_bigquery.py b/tests/providers/google/cloud/operators/test_bigquery.py index f64e7dc27484f..880be0978f171 100644 --- a/tests/providers/google/cloud/operators/test_bigquery.py +++ b/tests/providers/google/cloud/operators/test_bigquery.py @@ -197,6 +197,7 @@ def test_execute(self, mock_hook): bucket=TEST_GCS_BUCKET, source_objects=TEST_GCS_DATA, source_format=TEST_SOURCE_FORMAT, + autodetect=True, ) operator.execute(None) @@ -205,7 +206,7 @@ def test_execute(self, mock_hook): schema_fields=[], source_uris=[f'gs://{TEST_GCS_BUCKET}/{source_object}' for source_object in TEST_GCS_DATA], source_format=TEST_SOURCE_FORMAT, - autodetect=False, + autodetect=True, compression='NONE', skip_leading_rows=0, field_delimiter=',', From 055d01e448f6a49b566933e1720645ce1dc14fa7 Mon Sep 17 00:00:00 2001 From: hankehly Date: Mon, 4 Apr 2022 13:44:33 +0900 Subject: [PATCH 3/3] Update google provider documentation --- .../operators/cloud/bigquery.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst index 5cab46e37b6d6..96038ce8c191a 100644 --- a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst +++ b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst @@ -193,6 +193,9 @@ Or you may point the operator to a Google Cloud Storage object name where the sc :start-after: [START howto_operator_bigquery_create_table_schema_json] :end-before: [END howto_operator_bigquery_create_table_schema_json] +To use BigQuery `schema auto-detection `__, +set the ``autodetect`` flag instead of providing explicit schema information. + .. _howto/operator:BigQueryGetDataOperator: Fetch data from table