From f9be8adb1a0c3deaf9d23c591216be053b8e5132 Mon Sep 17 00:00:00 2001 From: Chuan Qiu <35706996+c-nuro@users.noreply.github.com> Date: Fri, 10 Sep 2021 10:08:21 -0700 Subject: [PATCH 1/2] Remove check for at least one schema For the case when updating an existing table or insert data to a particular partition, no schema is needed. Autodetect doesn't always work, e.g. cannot distinguish partition correctly. Other options requires forking the schema to airflow. --- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index 5a9edc827918e..7cd7aedb920e4 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -274,10 +274,6 @@ def execute(self, context): object_name=self.schema_object, ) schema_fields = json.loads(blob.decode("utf-8")) - elif self.schema_object is None and self.autodetect is False: - raise AirflowException( - 'At least one of `schema_fields`, `schema_object`, or `autodetect` must be passed.' - ) else: schema_fields = None From 3e111255edfe8ecb05a1f2ce1bbac34b91c2125d Mon Sep 17 00:00:00 2001 From: Chuan Qiu <35706996+c-nuro@users.noreply.github.com> Date: Mon, 13 Sep 2021 09:51:21 -0700 Subject: [PATCH 2/2] Update gcs_to_bigquery.py --- airflow/providers/google/cloud/transfers/gcs_to_bigquery.py | 1 - 1 file changed, 1 deletion(-) diff --git a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py index 7cd7aedb920e4..065b473b747b2 100644 --- a/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py +++ b/airflow/providers/google/cloud/transfers/gcs_to_bigquery.py @@ -20,7 +20,6 @@ import json from typing import Optional, Sequence, Union -from airflow.exceptions import AirflowException from airflow.models import BaseOperator from airflow.providers.google.cloud.hooks.bigquery import BigQueryHook from airflow.providers.google.cloud.hooks.gcs import GCSHook