From 0d01a7c385aac1c7e1fc9ad4a8de1ef42c224aba Mon Sep 17 00:00:00 2001 From: atikulmunna Date: Thu, 30 Jul 2026 14:26:14 +0600 Subject: [PATCH 1/3] Check GCSToS3Operator match_glob support after template rendering `match_glob` is a template field, so it is rendered after the constructor runs. The support check in `__init__` therefore tested the un-rendered Jinja expression for truthiness: a templated `match_glob` was rejected at Dag-parse time regardless of what it rendered to, and the error surfaced as a Dag import error rather than on the task. Move the check to `execute()`, onto the `elif` branch of the existing `__is_match_glob_supported` test, which is where the rendered value would otherwise be forwarded to `GCSHook.list()`. This mirrors the same change made for the sibling `GCSToAzureBlobStorageOperator` in gcs_to_wasb.py. Add a test for the error, which was previously uncovered, and remove the class from the validate-operators-init exemption list. related: #70296 Signed-off-by: atikulmunna --- .../amazon/aws/transfers/gcs_to_s3.py | 8 ++++---- .../amazon/aws/transfers/test_gcs_to_s3.py | 18 ++++++++++++++++++ .../validate_operators_init_exemptions.txt | 1 - 3 files changed, 22 insertions(+), 5 deletions(-) diff --git a/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py b/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py index 867f2d3d8cbec..6378a1361375f 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py @@ -146,10 +146,6 @@ def __init__( self.__is_match_glob_supported = False except ImportError: # __version__ was added in 10.1.0, so this means it's < 10.3.0 self.__is_match_glob_supported = False - if not self.__is_match_glob_supported and match_glob: - raise AirflowException( - "The 'match_glob' parameter requires 'apache-airflow-providers-google>=10.3.0'." - ) self.match_glob = match_glob self.gcp_user_project = gcp_user_project @@ -184,6 +180,10 @@ def execute(self, context: Context) -> list[str]: } if self.__is_match_glob_supported: list_kwargs["match_glob"] = self.match_glob + elif self.match_glob: + raise AirflowException( + "The 'match_glob' parameter requires 'apache-airflow-providers-google>=10.3.0'." + ) gcs_files = gcs_hook.list(**list_kwargs) # type: ignore diff --git a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py index 014f26f494348..4844cecc50d9c 100644 --- a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py +++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py @@ -23,6 +23,7 @@ import pytest from moto import mock_aws +from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.amazon.aws.transfers.gcs_to_s3 import GCSToS3Operator @@ -75,6 +76,23 @@ def test_execute__match_glob(self, mock_hook): user_project=None, ) + @mock.patch("airflow.providers.google.__version__", "10.2.0") + @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") + def test_execute__match_glob_requires_recent_google_provider(self, mock_hook): + operator = GCSToS3Operator( + task_id=TASK_ID, + gcs_bucket=GCS_BUCKET, + prefix=PREFIX, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + match_glob=f"**/*{DELIMITER}", + ) + + with pytest.raises(AirflowException, match="match_glob"): + operator.execute(None) + + mock_hook.return_value.list.assert_not_called() + @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") def test_execute_incremental(self, mock_hook): mock_hook.return_value.list.return_value = MOCK_FILES diff --git a/scripts/ci/prek/validate_operators_init_exemptions.txt b/scripts/ci/prek/validate_operators_init_exemptions.txt index c993ca2a16354..ed69a0cd15f4c 100644 --- a/scripts/ci/prek/validate_operators_init_exemptions.txt +++ b/scripts/ci/prek/validate_operators_init_exemptions.txt @@ -9,7 +9,6 @@ providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneS providers/amazon/src/airflow/providers/amazon/aws/operators/neptune.py::NeptuneStopDbClusterOperator providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py::SageMakerCreateNotebookOperator providers/amazon/src/airflow/providers/amazon/aws/operators/sagemaker.py::SageMakerProcessingOperator -providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py::GCSToS3Operator providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/operators/pod.py::KubernetesPodOperator providers/google/src/airflow/providers/google/cloud/operators/cloud_batch.py::CloudBatchSubmitJobOperator providers/google/src/airflow/providers/google/cloud/operators/cloud_build.py::CloudBuildCreateBuildOperator From 454597cc6d4a8a9e89d5f4779bbea89e29ec9cfc Mon Sep 17 00:00:00 2001 From: atikulmunna Date: Thu, 30 Jul 2026 19:59:25 +0600 Subject: [PATCH 2/3] Keep the match_glob check in __init__ and narrow it to ValueError Addresses review feedback on #70723. The check asks whether `match_glob` was passed rather than inspecting its rendered value, so per the false positives section of #70296 it is a provision check and belongs in the constructor. Moving it to `execute()` was wrong on two counts: with `render_template_as_native_obj=True` a provided field can render to None, and raising per task instance hides a static authoring mistake that a Dag import error would surface immediately. Rewrite it in place instead, using `is not None` polarity rather than a truthiness test on the un-rendered string, and narrow `AirflowException` to `ValueError` to match #70359. Since #70505 narrowed the hook to sanction provision checks written this way, the exemption entry still goes. Narrowing removes the file's only `raise AirflowException`, so drop its entry from generated/known_airflow_exceptions.txt to keep the check-no-new-airflow-exceptions allowlist in sync. related: #70296 Signed-off-by: atikulmunna --- generated/known_airflow_exceptions.txt | 1 - .../amazon/aws/transfers/gcs_to_s3.py | 8 +++--- .../amazon/aws/transfers/test_gcs_to_s3.py | 26 +++++++------------ 3 files changed, 13 insertions(+), 22 deletions(-) diff --git a/generated/known_airflow_exceptions.txt b/generated/known_airflow_exceptions.txt index 66ee6590bc84e..d5ba56e459185 100644 --- a/generated/known_airflow_exceptions.txt +++ b/generated/known_airflow_exceptions.txt @@ -114,7 +114,6 @@ providers/amazon/src/airflow/providers/amazon/aws/sensors/sagemaker.py::8 providers/amazon/src/airflow/providers/amazon/aws/sensors/sagemaker_unified_studio.py::1 providers/amazon/src/airflow/providers/amazon/aws/sensors/sqs.py::2 providers/amazon/src/airflow/providers/amazon/aws/sensors/step_function.py::1 -providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py::1 providers/amazon/src/airflow/providers/amazon/aws/transfers/redshift_to_s3.py::1 providers/amazon/src/airflow/providers/amazon/aws/transfers/s3_to_dynamodb.py::3 providers/amazon/src/airflow/providers/amazon/aws/transfers/s3_to_redshift.py::3 diff --git a/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py b/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py index 6378a1361375f..f4c83a50e1d91 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/transfers/gcs_to_s3.py @@ -26,7 +26,7 @@ from packaging.version import Version from airflow.providers.amazon.aws.hooks.s3 import S3Hook -from airflow.providers.common.compat.sdk import AirflowException, BaseOperator +from airflow.providers.common.compat.sdk import BaseOperator from airflow.providers.google.cloud.hooks.gcs import GCSHook if TYPE_CHECKING: @@ -146,6 +146,8 @@ def __init__( self.__is_match_glob_supported = False except ImportError: # __version__ was added in 10.1.0, so this means it's < 10.3.0 self.__is_match_glob_supported = False + if not self.__is_match_glob_supported and match_glob is not None: + raise ValueError("The 'match_glob' parameter requires 'apache-airflow-providers-google>=10.3.0'.") self.match_glob = match_glob self.gcp_user_project = gcp_user_project @@ -180,10 +182,6 @@ def execute(self, context: Context) -> list[str]: } if self.__is_match_glob_supported: list_kwargs["match_glob"] = self.match_glob - elif self.match_glob: - raise AirflowException( - "The 'match_glob' parameter requires 'apache-airflow-providers-google>=10.3.0'." - ) gcs_files = gcs_hook.list(**list_kwargs) # type: ignore diff --git a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py index 4844cecc50d9c..c328abcea099f 100644 --- a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py +++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py @@ -23,7 +23,6 @@ import pytest from moto import mock_aws -from airflow.exceptions import AirflowException from airflow.providers.amazon.aws.hooks.s3 import S3Hook from airflow.providers.amazon.aws.transfers.gcs_to_s3 import GCSToS3Operator @@ -77,21 +76,16 @@ def test_execute__match_glob(self, mock_hook): ) @mock.patch("airflow.providers.google.__version__", "10.2.0") - @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") - def test_execute__match_glob_requires_recent_google_provider(self, mock_hook): - operator = GCSToS3Operator( - task_id=TASK_ID, - gcs_bucket=GCS_BUCKET, - prefix=PREFIX, - dest_aws_conn_id="aws_default", - dest_s3_key=S3_BUCKET, - match_glob=f"**/*{DELIMITER}", - ) - - with pytest.raises(AirflowException, match="match_glob"): - operator.execute(None) - - mock_hook.return_value.list.assert_not_called() + def test_match_glob_requires_recent_google_provider(self): + with pytest.raises(ValueError, match="match_glob"): + GCSToS3Operator( + task_id=TASK_ID, + gcs_bucket=GCS_BUCKET, + prefix=PREFIX, + dest_aws_conn_id="aws_default", + dest_s3_key=S3_BUCKET, + match_glob=f"**/*{DELIMITER}", + ) @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook") def test_execute_incremental(self, mock_hook): From 3c8d638003588475a854749702f9007a8b3927ee Mon Sep 17 00:00:00 2001 From: atikulmunna Date: Sat, 1 Aug 2026 11:54:54 +0600 Subject: [PATCH 3/3] Parametrise the match_glob test over the empty-string case Addresses review feedback on #70723. The test only passed a truthy `match_glob`, so it would also have passed against the old `if not ... and match_glob:` condition; the only thing failing it on main was the `AirflowException` to `ValueError` swap, not the polarity change that was the substance of the review. Parametrise over a glob and the empty string so `is not None` is pinned: `match_glob=""` is rejected under the new condition and accepted under the old one, which is exactly the behaviour change this PR makes. related: #70296 Signed-off-by: atikulmunna --- .../amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py index c328abcea099f..880dc3538a51a 100644 --- a/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py +++ b/providers/amazon/tests/unit/amazon/aws/transfers/test_gcs_to_s3.py @@ -76,7 +76,8 @@ def test_execute__match_glob(self, mock_hook): ) @mock.patch("airflow.providers.google.__version__", "10.2.0") - def test_match_glob_requires_recent_google_provider(self): + @pytest.mark.parametrize("match_glob", [f"**/*{DELIMITER}", ""]) + def test_match_glob_requires_recent_google_provider(self, match_glob): with pytest.raises(ValueError, match="match_glob"): GCSToS3Operator( task_id=TASK_ID, @@ -84,7 +85,7 @@ def test_match_glob_requires_recent_google_provider(self): prefix=PREFIX, dest_aws_conn_id="aws_default", dest_s3_key=S3_BUCKET, - match_glob=f"**/*{DELIMITER}", + match_glob=match_glob, ) @mock.patch("airflow.providers.amazon.aws.transfers.gcs_to_s3.GCSHook")