From b7d34b57fdd5957baf034cd6a689dc412768751a Mon Sep 17 00:00:00 2001 From: shahkshitij15 Date: Sat, 12 Mar 2022 17:05:23 +0530 Subject: [PATCH 1/3] Added autodetect arg to Create External Table Operator --- airflow/providers/google/cloud/operators/bigquery.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index f7d6e7d1206f8..29647b4d8c8fa 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -956,6 +956,7 @@ class BigQueryCreateExternalTableOperator(BaseOperator): {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"}] Should not be set when source_format is 'DATASTORE_BACKUP'. + :param autodetect: (Optional) If True will automatically detect schema of file. :param table_resource: Table resource as described in documentation: https://cloud.google.com/bigquery/docs/reference/rest/v2/tables#Table If provided all other parameters are ignored. External schema from object will be resolved. @@ -1024,6 +1025,7 @@ def __init__( destination_project_dataset_table: Optional[str] = None, table_resource: Optional[Dict[str, Any]] = None, schema_fields: Optional[List] = None, + autodetect: bool = True, schema_object: Optional[str] = None, source_format: Optional[str] = None, compression: Optional[str] = None, @@ -1051,6 +1053,7 @@ def __init__( destination_project_dataset_table, schema_fields, source_format, + autodetect, compression, skip_leading_rows, field_delimiter, @@ -1107,6 +1110,7 @@ def __init__( if table_resource and kwargs_passed: raise ValueError("You provided both `table_resource` and exclusive keywords arguments.") + self.autodetect = autodetect self.max_bad_records = max_bad_records self.quote_character = quote_character self.allow_quoted_newlines = allow_quoted_newlines @@ -1161,6 +1165,7 @@ def execute(self, context: 'Context') -> None: src_fmt_configs=self.src_fmt_configs, labels=self.labels, encryption_configuration=self.encryption_configuration, + autodetect=self.autodetect ) From 9acd3894ee97ed4e0dd3103c138bde205168d9ba Mon Sep 17 00:00:00 2001 From: shahkshitij15 Date: Sat, 12 Mar 2022 19:04:23 +0530 Subject: [PATCH 2/3] Add autodetect arg --- airflow/providers/google/cloud/operators/bigquery.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/google/cloud/operators/bigquery.py b/airflow/providers/google/cloud/operators/bigquery.py index 29647b4d8c8fa..bf1fed295339f 100644 --- a/airflow/providers/google/cloud/operators/bigquery.py +++ b/airflow/providers/google/cloud/operators/bigquery.py @@ -1165,7 +1165,7 @@ def execute(self, context: 'Context') -> None: src_fmt_configs=self.src_fmt_configs, labels=self.labels, encryption_configuration=self.encryption_configuration, - autodetect=self.autodetect + autodetect=self.autodetect, ) From be3898c0880928addf35bf945f03d8302afa89c8 Mon Sep 17 00:00:00 2001 From: shahkshitij15 Date: Sat, 19 Mar 2022 22:04:37 +0530 Subject: [PATCH 3/3] Added test cases --- tests/providers/google/cloud/operators/test_bigquery.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/providers/google/cloud/operators/test_bigquery.py b/tests/providers/google/cloud/operators/test_bigquery.py index 7917ca13c695e..a060337e767c1 100644 --- a/tests/providers/google/cloud/operators/test_bigquery.py +++ b/tests/providers/google/cloud/operators/test_bigquery.py @@ -196,6 +196,7 @@ def test_execute(self, mock_hook): bucket=TEST_GCS_BUCKET, source_objects=TEST_GCS_DATA, source_format=TEST_SOURCE_FORMAT, + autodetect = False, ) operator.execute(None) @@ -214,6 +215,7 @@ def test_execute(self, mock_hook): src_fmt_configs={}, labels=None, encryption_configuration=None, + autodetect = False, )