From d1360e2f261690386fa23c8c6b1d659d636c2138 Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Thu, 10 Mar 2022 09:08:47 +0000
Subject: [PATCH 01/23] New design of Airflow System Tests

Migrate BigQuery system tests to new design (See AIP-47 for details)

Change-Id: Ia0579e22b495a81c5b4bc9fe37dd2067406c934d
---
 .../operators/cloud/bigquery.rst              |  61 +++--
 tests/always/test_example_dags.py             |  82 +++---
 tests/always/test_project_structure.py        | 113 +++------
 .../cloud/operators/test_bigquery_system.py   |  63 -----
 tests/system/README.md                        |  96 +++++++
 tests/system/conftest.py                      |  47 ++++
 tests/system/providers/google/README.md       |  99 ++++++++
 .../resources/example_bigquery_query.sql      |  20 ++
 .../resources/update_table_schema.json        |  12 +
 .../google/bigquery/resources/us-states.csv   |  51 ++++
 .../google/bigquery/test_bigquery_dataset.py  |  94 +++++++
 .../bigquery/test_bigquery_operations.py      | 106 ++++++++
 .../test_bigquery_operations_location.py      |  86 +++++++
 .../google/bigquery/test_bigquery_queries.py  | 237 ++++++++++++++++++
 .../google/bigquery/test_bigquery_sensors.py  | 127 ++++++++++
 .../google/bigquery/test_bigquery_tables.py   | 230 +++++++++++++++++
 .../bigquery/test_bigquery_to_bigquery.py     | 102 ++++++++
 .../google/bigquery/test_bigquery_to_gcs.py   | 103 ++++++++
 tests/system/utils/__init__.py                |  16 ++
 tests/system/utils/watcher.py                 |  25 ++
 20 files changed, 1571 insertions(+), 199 deletions(-)
 delete mode 100644 tests/providers/google/cloud/operators/test_bigquery_system.py
 create mode 100644 tests/system/README.md
 create mode 100644 tests/system/conftest.py
 create mode 100644 tests/system/providers/google/README.md
 create mode 100644 tests/system/providers/google/bigquery/resources/example_bigquery_query.sql
 create mode 100644 tests/system/providers/google/bigquery/resources/update_table_schema.json
 create mode 100644 tests/system/providers/google/bigquery/resources/us-states.csv
 create mode 100644 tests/system/providers/google/bigquery/test_bigquery_dataset.py
 create mode 100644 tests/system/providers/google/bigquery/test_bigquery_operations.py
 create mode 100644 tests/system/providers/google/bigquery/test_bigquery_operations_location.py
 create mode 100644 tests/system/providers/google/bigquery/test_bigquery_queries.py
 create mode 100644 tests/system/providers/google/bigquery/test_bigquery_sensors.py
 create mode 100644 tests/system/providers/google/bigquery/test_bigquery_tables.py
 create mode 100644 tests/system/providers/google/bigquery/test_bigquery_to_bigquery.py
 create mode 100644 tests/system/providers/google/bigquery/test_bigquery_to_gcs.py
 create mode 100644 tests/system/utils/__init__.py
 create mode 100644 tests/system/utils/watcher.py

diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
index bcf947f4164a5..371cdca7a85ca 100644
--- a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
+++ b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
@@ -29,7 +29,7 @@ data.
 Prerequisite Tasks
 ^^^^^^^^^^^^^^^^^^
 
-.. include::/operators/_partials/prerequisite_tasks.rst
+.. include:: ../_partials/prerequisite_tasks.rst
 
 Manage datasets
 ^^^^^^^^^^^^^^^
@@ -42,7 +42,7 @@ Create dataset
 To create an empty dataset in a BigQuery database you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyDatasetOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_dataset]
@@ -58,7 +58,7 @@ To get the details of an existing dataset you can use
 
 This operator returns a `Dataset Resource <https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource>`__.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_get_dataset]
@@ -72,7 +72,7 @@ List tables in dataset
 To retrieve the list of tables in a given dataset use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryGetDatasetTablesOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_get_dataset_tables]
@@ -89,7 +89,7 @@ To update a table in BigQuery you can use
 The update method replaces the entire Table resource, whereas the patch
 method only replaces fields that are provided in the submitted Table resource.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_update_table]
@@ -106,7 +106,7 @@ To update a dataset in BigQuery you can use
 The update method replaces the entire dataset resource, whereas the patch
 method only replaces fields that are provided in the submitted dataset resource.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_update_dataset]
@@ -120,7 +120,7 @@ Delete dataset
 To delete an existing dataset from a BigQuery database you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteDatasetOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_dataset]
@@ -143,7 +143,7 @@ ways. You may either directly pass the schema fields in, or you may point the
 operator to a Google Cloud Storage object name. The object in Google Cloud
 Storage must be a JSON file with the schema fields in it.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_table]
@@ -151,7 +151,7 @@ Storage must be a JSON file with the schema fields in it.
 
 You can use this operator to create a view on top of an existing table.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_view]
@@ -160,7 +160,7 @@ You can use this operator to create a view on top of an existing table.
 You can also use this operator to create a materialized view that periodically
 cache results of a query for increased performance and efficiency.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_materialized_view]
@@ -177,15 +177,22 @@ you can use
 
 Similarly to
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyTableOperator`
-you may either directly pass the schema fields in, or you may point the operator
-to a Google Cloud Storage object name.
+you can directly pass the schema fields in.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_operations.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_external_table]
     :end-before: [END howto_operator_bigquery_create_external_table]
 
+Or you may point the operator to a Google Cloud Storage object name where the schema is stored.
+
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+    :language: python
+    :dedent: 4
+    :start-after: [START howto_operator_bigquery_create_table_schema_json]
+    :end-before: [END howto_operator_bigquery_create_table_schema_json]
+
 .. _howto/operator:BigQueryGetDataOperator:
 
 Fetch data from table
@@ -201,7 +208,7 @@ returned list will be equal to the number of rows fetched. Each element in the
 list will again be a list where elements would represent the column values for
 that row.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_get_data]
@@ -218,7 +225,7 @@ To upsert a table you can use
 This operator either updates the existing table or creates a new, empty table
 in the given dataset.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_upsert_table]
@@ -235,7 +242,7 @@ To update the schema of a table you can use
 This operator updates the schema field values supplied, while leaving the rest unchanged. This is useful
 for instance to set new field descriptions on an existing table schema.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_update_table_schema]
@@ -249,7 +256,7 @@ Delete table
 To delete an existing table you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_table]
@@ -257,7 +264,7 @@ To delete an existing table you can use
 
 You can also use this operator to delete a view.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_view]
@@ -265,7 +272,7 @@ You can also use this operator to delete a view.
 
 You can also use this operator to delete a materialized view.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_materialized_view]
@@ -278,7 +285,7 @@ Execute BigQuery jobs
 
 Let's say you would like to execute the following query.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
     :language: python
     :dedent: 0
     :start-after: [START howto_operator_bigquery_query]
@@ -288,7 +295,7 @@ To execute the SQL query in a specific BigQuery database you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryInsertJobOperator` with
 proper query job configuration that can be Jinja templated.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_insert_job]
@@ -300,7 +307,7 @@ For more information on types of BigQuery job please check
 If you want to include some files in your configuration you can use ``include`` clause of Jinja template
 language as follow:
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_select_job]
@@ -329,7 +336,7 @@ This operator expects a sql query that will return a single row. Each value on
 that first row is evaluated using python ``bool`` casting. If any of the values
 return ``False`` the check is failed and errors out.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_check]
@@ -347,7 +354,7 @@ This operator expects a sql query that will return a single row. Each value on
 that first row is evaluated against ``pass_value`` which can be either a string
 or numeric value. If numeric, you can also specify ``tolerance``.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_value_check]
@@ -362,7 +369,7 @@ To check that the values of metrics given as SQL expressions are within a certai
 tolerance of the ones from ``days_back`` before you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryIntervalCheckOperator`.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_interval_check]
@@ -380,7 +387,7 @@ use the ``{{ ds_nodash }}`` macro as the table name suffix.
 
 :class:`~airflow.providers.google.cloud.sensors.bigquery.BigQueryTableExistenceSensor`.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_sensors.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_sensors.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_bigquery_table]
@@ -392,7 +399,7 @@ Check that a Table Partition exists
 To check that a table exists and has a partition you can use.
 :class:`~airflow.providers.google.cloud.sensors.bigquery.BigQueryTablePartitionExistenceSensor`.
 
-.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_bigquery_sensors.py
+.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_sensors.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_bigquery_table_partition]
diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py
index a21056dcfc087..fc20ab48c935b 100644
--- a/tests/always/test_example_dags.py
+++ b/tests/always/test_example_dags.py
@@ -16,9 +16,10 @@
 # under the License.
 
 import os
-import unittest
 from glob import glob
 
+import pytest
+
 from airflow.models import DagBag
 from tests.test_utils.asserts import assert_queries_count
 
@@ -29,33 +30,52 @@
 NO_DB_QUERY_EXCEPTION = ["/airflow/example_dags/example_subdag_operator.py"]
 
 
-class TestExampleDags(unittest.TestCase):
-    def test_should_be_importable(self):
-        example_dags = list(glob(f"{ROOT_FOLDER}/airflow/**/example_dags/example_*.py", recursive=True))
-        assert 0 != len(example_dags)
-        for filepath in example_dags:
-            relative_filepath = os.path.relpath(filepath, ROOT_FOLDER)
-            with self.subTest(f"File {relative_filepath} should contain dags"):
-                dagbag = DagBag(
-                    dag_folder=filepath,
-                    include_examples=False,
-                )
-                assert 0 == len(dagbag.import_errors), f"import_errors={str(dagbag.import_errors)}"
-                assert len(dagbag.dag_ids) >= 1
-
-    def test_should_not_do_database_queries(self):
-        example_dags = glob(f"{ROOT_FOLDER}/airflow/**/example_dags/example_*.py", recursive=True)
-        example_dags = [
-            dag_file
-            for dag_file in example_dags
-            if any(not dag_file.endswith(e) for e in NO_DB_QUERY_EXCEPTION)
-        ]
-        assert 0 != len(example_dags)
-        for filepath in example_dags:
-            relative_filepath = os.path.relpath(filepath, ROOT_FOLDER)
-            with self.subTest(f"File {relative_filepath} shouldn't do database queries"):
-                with assert_queries_count(0):
-                    DagBag(
-                        dag_folder=filepath,
-                        include_examples=False,
-                    )
+def example_dags():
+    example_dirs = ["airflow/**/example_dags/example_*.py", "tests/system/providers/**/test_*.py"]
+    for example_dir in example_dirs:
+        yield from glob(f"{ROOT_FOLDER}/{example_dir}", recursive=True)
+
+
+def example_dags_except_db_exception():
+    return [
+        dag_file
+        for dag_file in example_dags()
+        if any(not dag_file.endswith(e) for e in NO_DB_QUERY_EXCEPTION)
+    ]
+
+
+def set_env_vars(*args):
+    """Mocks required ENV variables so it's possible to import example dags"""
+    for arg in args:
+        if arg not in os.environ:
+            os.environ[arg] = "PLACEHOLDER"
+
+
+def relative_path(path):
+    return os.path.relpath(path, ROOT_FOLDER)
+
+
+class TestExampleDags:
+    @pytest.fixture(autouse=True, scope="session")
+    def required_env_vars(self):
+        # Common
+        set_env_vars("SYSTEM_TESTS_ENV_ID")
+        # Google
+        set_env_vars("SYSTEM_TESTS_GCP_PROJECT", "SYSTEM_TESTS_GCP_SERVICE_ACCOUNT")
+
+    @pytest.mark.parametrize("example", list(example_dags()), ids=relative_path)
+    def test_should_be_importable(self, example):
+        dagbag = DagBag(
+            dag_folder=example,
+            include_examples=False,
+        )
+        assert 0 == len(dagbag.import_errors), f"import_errors={str(dagbag.import_errors)}"
+        assert len(dagbag.dag_ids) >= 1
+
+    @pytest.mark.parametrize("example", example_dags_except_db_exception(), ids=relative_path)
+    def test_should_not_do_database_queries(self, example):
+        with assert_queries_count(0):
+            DagBag(
+                dag_folder=example,
+                include_examples=False,
+            )
diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py
index 9c019f418744f..89bfe9521ca1c 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -153,12 +153,12 @@ def get_classes_from_file(filepath: str):
 
 class TestGoogleProviderProjectStructure(unittest.TestCase):
     MISSING_EXAMPLE_DAGS = {
-        ('cloud', 'adls_to_gcs'),
-        ('cloud', 'sql_to_gcs'),
-        ('cloud', 'bigquery_to_mysql'),
-        ('cloud', 'cassandra_to_gcs'),
-        ('suite', 'drive'),
-        ('ads', 'ads_to_gcs'),
+        'adls_to_gcs',
+        'sql_to_gcs',
+        'bigquery_to_mysql',
+        'cassandra_to_gcs',
+        'drive',
+        'ads_to_gcs',
     }
 
     # Those operators are deprecated and we do not need examples for them
@@ -192,6 +192,7 @@ class TestGoogleProviderProjectStructure(unittest.TestCase):
 
     # Please at the examples to those operators at the earliest convenience :)
     MISSING_EXAMPLES_FOR_OPERATORS = {
+        'airflow.providers.google.cloud.operators.dataproc.DataprocInstantiateInlineWorkflowTemplateOperator',
         'airflow.providers.google.cloud.operators.mlengine.MLEngineTrainingCancelJobOperator',
         'airflow.providers.google.cloud.operators.dlp.CloudDLPGetStoredInfoTypeOperator',
         'airflow.providers.google.cloud.operators.dlp.CloudDLPReidentifyContentOperator',
@@ -217,87 +218,36 @@ class TestGoogleProviderProjectStructure(unittest.TestCase):
         'airflow.providers.google.cloud.sensors.gcs.GCSUploadSessionCompleteSensor',
     }
 
-    def test_example_dags(self):
-        operators_modules = itertools.chain(
-            *(self.find_resource_files(resource_type=d) for d in ["operators", "sensors", "transfers"])
-        )
-        example_dags_files = self.find_resource_files(resource_type="example_dags")
-        # Generate tuple of department and service e.g. ('marketing_platform', 'display_video')
-        operator_sets = [(f.split("/")[-3], f.split("/")[-1].rsplit(".")[0]) for f in operators_modules]
-        example_sets = [
-            (f.split("/")[-3], f.split("/")[-1].rsplit(".")[0].replace("example_", "", 1))
-            for f in example_dags_files
-        ]
-
-        def has_example_dag(operator_set):
-            for e in example_sets:
-                if e[0] != operator_set[0]:
-                    continue
-                if e[1].startswith(operator_set[1]):
-                    return True
-
-            return False
-
-        with self.subTest("Detect missing example dags"):
-            missing_example = {s for s in operator_sets if not has_example_dag(s)}
-            missing_example -= self.MISSING_EXAMPLE_DAGS
-            assert set() == missing_example
-
-        with self.subTest("Keep update missing example dags list"):
-            new_example_dag = set(example_sets).intersection(set(self.MISSING_EXAMPLE_DAGS))
-            if new_example_dag:
-                new_example_dag_text = '\n'.join(str(f) for f in new_example_dag)
-                self.fail(
-                    "You've added a example dag currently listed as missing:\n"
-                    f"{new_example_dag_text}"
-                    "\n"
-                    "Thank you very much.\n"
-                    "Can you remove it from the list of missing example, please?"
-                )
-
-        with self.subTest("Remove extra elements"):
-            extra_example_dags = set(self.MISSING_EXAMPLE_DAGS) - set(operator_sets)
-            if extra_example_dags:
-                new_example_dag_text = '\n'.join(str(f) for f in extra_example_dags)
-                self.fail(
-                    "You've added a example dag currently listed as missing:\n"
-                    f"{new_example_dag_text}"
-                    "\n"
-                    "Thank you very much.\n"
-                    "Can you remove it from the list of missing example, please?"
-                )
-
     def test_missing_example_for_operator(self):
-        missing_operators = []
-
+        """
+        Assert that all operators defined under operators, sensors and transfers directories
+        are used in any of the example dags
+        """
+        all_operators = set()
+        services = set()
         for resource_type in ["operators", "sensors", "transfers"]:
             operator_files = set(
                 self.find_resource_files(top_level_directory="airflow", resource_type=resource_type)
             )
             for filepath in operator_files:
                 service_name = os.path.basename(filepath)[: -(len(".py"))]
-                example_dags = list(
-                    glob.glob(
-                        f"{ROOT_FOLDER}/airflow/providers/google/*/example_dags/example_{service_name}*.py"
-                    )
-                )
-                if not example_dags:
-                    # Ignore. We have separate tests that detect this.
+                if service_name in self.MISSING_EXAMPLE_DAGS:
                     continue
-                example_paths = {
-                    path for example_dag in example_dags for path in get_imports_from_file(example_dag)
-                }
-                example_paths = {
-                    path for path in example_paths if f'.{resource_type}.{service_name}.' in path
-                }
-                print("example_paths=", example_paths)
+                services.add(service_name)
                 operators_paths = set(get_classes_from_file(f"{ROOT_FOLDER}/{filepath}"))
-                missing_operators.extend(operators_paths - example_paths)
-        full_set = set()
-        full_set.update(self.MISSING_EXAMPLES_FOR_OPERATORS)
-        full_set.update(self.DEPRECATED_OPERATORS)
-        full_set.update(self.BASE_OPERATORS)
-        assert set(missing_operators) == full_set
+                all_operators.update(operators_paths)
+
+        for service in services:
+            example_dags = list(self.examples_for_service(service))
+            example_paths = {
+                path for example_dag in example_dags for path in get_imports_from_file(example_dag)
+            }
+            all_operators -= example_paths
+
+        all_operators -= self.MISSING_EXAMPLES_FOR_OPERATORS
+        all_operators -= self.DEPRECATED_OPERATORS
+        all_operators -= self.BASE_OPERATORS
+        assert set() == all_operators
 
     @parameterized.expand(
         itertools.product(["_system.py", "_system_helper.py"], ["operators", "sensors", "transfers"])
@@ -330,6 +280,13 @@ def find_resource_files(
         resource_files = (f for f in resource_files if not f.endswith("__init__.py"))
         return resource_files
 
+    @staticmethod
+    def examples_for_service(service_name):
+        yield from glob.glob(
+            f"{ROOT_FOLDER}/airflow/providers/google/*/example_dags/example_{service_name}*.py"
+        )
+        yield from glob.glob(f"{ROOT_FOLDER}/tests/system/providers/google/{service_name}/test_*.py")
+
 
 class TestOperatorsHooks(unittest.TestCase):
     def test_no_illegal_suffixes(self):
diff --git a/tests/providers/google/cloud/operators/test_bigquery_system.py b/tests/providers/google/cloud/operators/test_bigquery_system.py
deleted file mode 100644
index adb0cc71dfbf3..0000000000000
--- a/tests/providers/google/cloud/operators/test_bigquery_system.py
+++ /dev/null
@@ -1,63 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""System tests for Google Cloud Build operators"""
-import pytest
-
-from airflow.providers.google.cloud.example_dags.example_bigquery_transfer import DATA_EXPORT_BUCKET_NAME
-from tests.providers.google.cloud.utils.gcp_authenticator import GCP_BIGQUERY_KEY
-from tests.test_utils.gcp_system_helpers import CLOUD_DAG_FOLDER, GoogleSystemTest, provide_gcp_context
-
-
-@pytest.mark.backend("mysql", "postgres")
-@pytest.mark.system("google.cloud")
-@pytest.mark.credential_file(GCP_BIGQUERY_KEY)
-class BigQueryExampleDagsSystemTest(GoogleSystemTest):
-    """
-    System tests for Google BigQuery operators
-    It use a real service.
-    """
-
-    @provide_gcp_context(GCP_BIGQUERY_KEY)
-    def setUp(self):
-        super().setUp()
-        self.create_gcs_bucket(DATA_EXPORT_BUCKET_NAME)
-
-    @provide_gcp_context(GCP_BIGQUERY_KEY)
-    def test_run_example_dag_operations(self):
-        self.run_dag('example_bigquery_operations', CLOUD_DAG_FOLDER)
-
-    @provide_gcp_context(GCP_BIGQUERY_KEY)
-    def test_run_example_dag_operations_location(self):
-        self.run_dag('example_bigquery_operations_location', CLOUD_DAG_FOLDER)
-
-    @provide_gcp_context(GCP_BIGQUERY_KEY)
-    def test_run_example_dag_queries(self):
-        self.run_dag('example_bigquery_queries', CLOUD_DAG_FOLDER)
-
-    @provide_gcp_context(GCP_BIGQUERY_KEY)
-    def test_run_example_dag_sensors(self):
-        self.run_dag('example_bigquery_sensors', CLOUD_DAG_FOLDER)
-
-    @provide_gcp_context(GCP_BIGQUERY_KEY)
-    def test_run_example_dag_queries_location(self):
-        self.run_dag('example_bigquery_queries_location', CLOUD_DAG_FOLDER)
-
-    @provide_gcp_context(GCP_BIGQUERY_KEY)
-    def tearDown(self):
-        self.delete_gcs_bucket(DATA_EXPORT_BUCKET_NAME)
-        super().tearDown()
diff --git a/tests/system/README.md b/tests/system/README.md
new file mode 100644
index 0000000000000..0da4c260a0d96
--- /dev/null
+++ b/tests/system/README.md
@@ -0,0 +1,96 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Airflow System Tests
+
+System tests verify the correctness of Airflow Operators by running them in DAGs and allowing to communicate with
+external services. A system test tries to look as close to a regular DAG as possible, and it generally checks the
+"happy path" (a scenario featuring no errors) ensuring that the Operator works as expected.
+
+The purpose of these tests is to:
+
+- assure high quality of providers and their integration with Airflow core,
+- avoid regression in providers when doing changes to the Airflow,
+- autogenerate documentation for Operators from code,
+- provide runnable example DAGs with use cases for different Operators,
+- serve both as examples and test files.
+
+> This is the new design of system tests which temporarily exists along with the old one documented at
+> [TESTING.rst](../../TESTING.rst) and soon will completely replace it. The new design is based on the
+> [AIP-47](https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-47+New+design+of+Airflow+System+Tests).
+> Please use it and write any new system tests according to this documentation.
+
+## How to run system tests
+
+There are multiple ways of running system tests. Each system test is a self-contained DAG, so it can be run as any
+other DAG. Some tests may require access to external services, enabled APIs or specific permissions. Make sure to
+prepare your  environment correctly, depending on the system tests you want to run - some may require additional
+configuration which should be documented by the relevant providers in their subdirectory
+`tests/system/providers/<provider_name>/README.md`.
+
+### Running via Airflow
+
+If you have a working Airflow environment with a scheduler and a webserver, you can import system test files into
+your Airflow instance and they will be automatically triggered. If the setup of the environment is correct
+(depending on the type of tests you want to run), they should be executed without any issues. The instructions on
+how to set up the environment is documented in each provider's system tests directory. Make sure that all resource
+required by the tests are also imported.
+
+### Running via Pytest
+
+System tests can be executed using pytest. You can either run them using your IDE (if you have installed
+plugin/widget supporting pytest) or using following example command:
+
+```commandline
+# pytest --system [provider_name] [path_to_test(s)]
+pytest --system google tests/system/providers/google/bigquery/test_bigquery_queries.py
+```
+
+You can specify several --system flags if you want to execute tests for several providers:
+
+```commandline
+pytest --system google --system aws tests/system
+```
+
+### Running via Airflow CLI
+
+It is possible to run system tests using Airflow CLI. To execute a specific system test, you need to provide
+`dag_id` of the test to be run, `execution_date` (preferably the one from the past) and a `-S/--subdir` option
+followed by the path where the tests are stored (the command by default looks into `$AIRFLOW_HOME/dags`):
+
+```commandline
+# airflow dags test -S [path_to_tests] [dag_id] [execution date]
+airflow dags test -S tests/system bigquery_dataset 2022-01-01
+```
+
+> Some additional setup may be required to use Airflow CLI. Please refer
+> [here](https://airflow.apache.org/docs/apache-airflow/stable/usage-cli.html) for a documentation.
+
+
+## How to write system tests
+
+If you are going to implement new system tests, it is recommended to familiarize with the content of the
+[AIP-47](https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-47+New+design+of+Airflow+System+Tests). There are
+many changes in comparison to the old design documented at [TESTING.rst](../../TESTING.rst), so you need to be
+aware of them and be compliant with the new design.
+
+To make it easier to migrate old system tests or write new ones, we
+documented the whole **process of migration in details** (which can be found
+[here](https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-47+New+design+of+Airflow+System+Tests#AIP47NewdesignofAirflowSystemTests-Processofmigrationindetails))
+and also prepared an example of a test (located just below the migration details).
diff --git a/tests/system/conftest.py b/tests/system/conftest.py
new file mode 100644
index 0000000000000..3f9ccce5e6bf5
--- /dev/null
+++ b/tests/system/conftest.py
@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import re
+from pathlib import Path
+
+import pytest
+
+OLD_EXECUTOR = os.environ.get("AIRFLOW__CORE__EXECUTOR", default=None)
+
+
+def pytest_configure(config):
+    os.environ["AIRFLOW__CORE__EXECUTOR"] = "DebugExecutor"
+
+
+def pytest_unconfigure(config):
+    if OLD_EXECUTOR is not None:
+        os.environ["AIRFLOW__CORE__EXECUTOR"] = OLD_EXECUTOR
+
+
+def pytest_collection_modifyitems(config, items):
+    """Add @pytest.mark.system(provider_name) for every system test"""
+    rootdir = Path(config.rootdir)
+    for item in items:
+        if item.get_closest_marker("system"):  # already marked
+            continue
+        rel_path = Path(item.fspath).relative_to(rootdir)
+        match = re.match(".+/providers/([^/]+)", str(rel_path))
+        if not match:
+            continue
+        provider = match.group(1)
+        item.add_marker(pytest.mark.system(provider))
diff --git a/tests/system/providers/google/README.md b/tests/system/providers/google/README.md
new file mode 100644
index 0000000000000..402d74783b5a9
--- /dev/null
+++ b/tests/system/providers/google/README.md
@@ -0,0 +1,99 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+
+# Google provider system tests
+
+## Tests structure
+
+All Google-related system tests are located inside this subdirectory of system tests which is
+`tests/system/providers/google/`. They are grouped in directories by the related service name, e.g. all BigQuery
+tests are stored inside `tests/system/providers/google/bigquery/` directory. In each directory you will find test files
+as self-contained DAGs (one DAG per file). Each test may require some additional resources which should be placed in
+`resources` directory found on the same level as tests. Each test file should start with prefix `test_*`. If there
+is anything more needed for the test to be executed, it should be documented in the docstrings.
+
+Example files structure:
+
+```─├│└
+tests/system/providers/google
+├── bigquery
+│   ├── resources
+│   │   ├── example_bigquery_query.sql
+│   │   └── us-states.csv
+│   ├── test_bigquery_queries.py
+│   ├── test_bigquery_operations.py
+.   .
+│   └── test_bigquery_*.py
+├── dataflow
+├── gcs
+.
+└── *
+```
+
+## Initial configuration
+
+Each test requires some environment variables. Check how to set them up on your operating system, but on UNIX-based
+OSes this should work:
+
+```commandline
+export NAME_OF_ENV_VAR=value
+```
+
+To confirm that it is set up correctly, run `echo $NAME_OF_ENV_VAR` which will display its value.
+
+### Required environment variables
+
+- `SYSTEM_TESTS_GCP_PROJECT` - GCP project name that will be used to run system tests (this can be checked on the UI
+  dashboard of the GCP or by running `gcloud config list`).
+
+- `SYSTEM_TESTS_ENV_ID` - environment ID that is unique across between different executions of system tests (if they
+  are run in parallel). The need for this variable comes from the possibility, that the tests may be run on various
+  versions of Airflow at the same time using CI environment. If this is the case, the value of this variable ensures
+  that all resources that are created during the tests, will not interfere with other resources in the same project,
+  that can be created at the same time.
+
+  If you run your tests in parallel, make sure to set this variable with randomized generated value, preferably no
+  longer than 8 characters (which should be fine to avoid collisions). Otherwise, please put whatever value you want,
+  but use only lowercase letters A-Z and digits 0-9.
+
+  The value of this environment variable is commonly attached to other variables like bucket names so it needs to
+  follow the same guidelines as bucket names, which can be found
+  [here](https://cloud.google.com/storage/docs/naming-buckets#requirements).
+
+### GCP Project setup
+
+System tests are executed in a working environment, in this case a GCP project that is deposited with credits. Such
+project is prepared and dedicated for the exclusive use of the Airflow Community, especially for the CI integrated
+system tests. If you want to run the tests manually, you need to have your own GCP project and configure it properly.
+
+Tests need to know which GCP project to use to execute them. If your environment is not already configured to use
+particular GCP project (like Airflow running in GCP Composer) you need to configure it using
+[gcloud CLI](https://cloud.google.com/sdk/gcloud). Use your project ID to execute this command:
+
+```commandline
+gcloud config set project <project-id>
+```
+
+Keep in mind that some additional commands may be required.
+
+
+## Settings for specific tests
+
+Some tests may require extra setup. If this is the case, the steps should be documented inside the docstring of
+related test file.
diff --git a/tests/system/providers/google/bigquery/resources/example_bigquery_query.sql b/tests/system/providers/google/bigquery/resources/example_bigquery_query.sql
new file mode 100644
index 0000000000000..b629f273e4dc3
--- /dev/null
+++ b/tests/system/providers/google/bigquery/resources/example_bigquery_query.sql
@@ -0,0 +1,20 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+   http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied.  See the License for the
+ specific language governing permissions and limitations
+ under the License.
+*/
+
+SELECT * FROM {{ DATASET }}.{{ TABLE }}
diff --git a/tests/system/providers/google/bigquery/resources/update_table_schema.json b/tests/system/providers/google/bigquery/resources/update_table_schema.json
new file mode 100644
index 0000000000000..f28d55f7996a7
--- /dev/null
+++ b/tests/system/providers/google/bigquery/resources/update_table_schema.json
@@ -0,0 +1,12 @@
+[
+   {
+      "mode": "NULLABLE",
+      "name": "friendlyName",
+      "type": "STRING"
+   },
+   {
+      "mode": "NULLABLE",
+      "name": "description",
+      "type": "STRING"
+   }
+]
diff --git a/tests/system/providers/google/bigquery/resources/us-states.csv b/tests/system/providers/google/bigquery/resources/us-states.csv
new file mode 100644
index 0000000000000..52cc001884b5a
--- /dev/null
+++ b/tests/system/providers/google/bigquery/resources/us-states.csv
@@ -0,0 +1,51 @@
+name,post_abbr
+Alabama,AL
+Alaska,AK
+Arizona,AZ
+Arkansas,AR
+California,CA
+Colorado,CO
+Connecticut,CT
+Delaware,DE
+Florida,FL
+Georgia,GA
+Hawaii,HI
+Idaho,ID
+Illinois,IL
+Indiana,IN
+Iowa,IA
+Kansas,KS
+Kentucky,KY
+Louisiana,LA
+Maine,ME
+Maryland,MD
+Massachusetts,MA
+Michigan,MI
+Minnesota,MN
+Mississippi,MS
+Missouri,MO
+Montana,MT
+Nebraska,NE
+Nevada,NV
+New Hampshire,NH
+New Jersey,NJ
+New Mexico,NM
+New York,NY
+North Carolina,NC
+North Dakota,ND
+Ohio,OH
+Oklahoma,OK
+Oregon,OR
+Pennsylvania,PA
+Rhode Island,RI
+South Carolina,SC
+South Dakota,SD
+Tennessee,TN
+Texas,TX
+Utah,UT
+Vermont,VT
+Virginia,VA
+Washington,WA
+West Virginia,WV
+Wisconsin,WI
+Wyoming,WY
diff --git a/tests/system/providers/google/bigquery/test_bigquery_dataset.py b/tests/system/providers/google/bigquery/test_bigquery_dataset.py
new file mode 100644
index 0000000000000..70edcca65d475
--- /dev/null
+++ b/tests/system/providers/google/bigquery/test_bigquery_dataset.py
@@ -0,0 +1,94 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG for Google BigQuery service testing dataset operations.
+"""
+import os
+from datetime import datetime
+
+from airflow import models
+from airflow.operators.bash import BashOperator
+from airflow.providers.google.cloud.operators.bigquery import (
+    BigQueryCreateEmptyDatasetOperator,
+    BigQueryDeleteDatasetOperator,
+    BigQueryGetDatasetOperator,
+    BigQueryUpdateDatasetOperator,
+)
+from airflow.utils.trigger_rule import TriggerRule
+from tests.system.utils.watcher import watcher
+
+ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+DAG_ID = "bigquery_dataset"
+
+DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
+
+
+with models.DAG(
+    DAG_ID,
+    schedule_interval="@once",
+    start_date=datetime(2021, 1, 1),
+    catchup=False,
+    tags=["example", "bigquery"],
+) as dag:
+    # [START howto_operator_bigquery_create_dataset]
+    create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create_dataset", dataset_id=DATASET_NAME)
+    # [END howto_operator_bigquery_create_dataset]
+
+    # [START howto_operator_bigquery_update_dataset]
+    update_dataset = BigQueryUpdateDatasetOperator(
+        task_id="update_dataset",
+        dataset_id=DATASET_NAME,
+        dataset_resource={"description": "Updated dataset"},
+    )
+    # [END howto_operator_bigquery_update_dataset]
+
+    # [START howto_operator_bigquery_get_dataset]
+    get_dataset = BigQueryGetDatasetOperator(task_id="get-dataset", dataset_id=DATASET_NAME)
+    # [END howto_operator_bigquery_get_dataset]
+
+    get_dataset_result = BashOperator(
+        task_id="get_dataset_result",
+        bash_command="echo \"{{ task_instance.xcom_pull('get-dataset')['id'] }}\"",
+    )
+
+    # [START howto_operator_bigquery_delete_dataset]
+    delete_dataset = BigQueryDeleteDatasetOperator(
+        task_id="delete_dataset", dataset_id=DATASET_NAME, delete_contents=True
+    )
+    # [END howto_operator_bigquery_delete_dataset]
+    delete_dataset.trigger_rule = TriggerRule.ALL_DONE
+
+    (
+        # TEST BODY
+        create_dataset
+        >> update_dataset
+        >> get_dataset
+        >> get_dataset_result
+        # TEST TEARDOWN
+        >> delete_dataset
+    )
+
+    list(dag.tasks) >> watcher()
+
+
+def test_run():
+    from airflow.utils.state import State
+
+    dag.clear(dag_run_state=State.NONE)
+    dag.run()
diff --git a/tests/system/providers/google/bigquery/test_bigquery_operations.py b/tests/system/providers/google/bigquery/test_bigquery_operations.py
new file mode 100644
index 0000000000000..54534bd591782
--- /dev/null
+++ b/tests/system/providers/google/bigquery/test_bigquery_operations.py
@@ -0,0 +1,106 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG for Google BigQuery service local file upload and external table creation.
+"""
+import os
+from datetime import datetime
+from pathlib import Path
+
+from airflow import models
+from airflow.providers.google.cloud.operators.bigquery import (
+    BigQueryCreateEmptyDatasetOperator,
+    BigQueryCreateExternalTableOperator,
+    BigQueryDeleteDatasetOperator,
+)
+from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
+from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator
+from airflow.utils.trigger_rule import TriggerRule
+from tests.system.utils.watcher import watcher
+
+ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+DAG_ID = "bigquery_operations"
+
+DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
+DATA_SAMPLE_GCS_BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
+DATA_SAMPLE_GCS_OBJECT_NAME = "bigquery/us-states/us-states.csv"
+CSV_FILE_LOCAL_PATH = str(Path(__file__).parent / "resources" / "us-states.csv")
+
+
+with models.DAG(
+    DAG_ID,
+    schedule_interval="@once",
+    start_date=datetime(2021, 1, 1),
+    catchup=False,
+    tags=["example", "bigquery"],
+) as dag:
+    create_bucket = GCSCreateBucketOperator(task_id="create_bucket", bucket_name=DATA_SAMPLE_GCS_BUCKET_NAME)
+
+    create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create_dataset", dataset_id=DATASET_NAME)
+
+    upload_file = LocalFilesystemToGCSOperator(
+        task_id="upload_file_to_bucket",
+        src=CSV_FILE_LOCAL_PATH,
+        dst=DATA_SAMPLE_GCS_OBJECT_NAME,
+        bucket=DATA_SAMPLE_GCS_BUCKET_NAME,
+    )
+
+    # [START howto_operator_bigquery_create_external_table]
+    create_external_table = BigQueryCreateExternalTableOperator(
+        task_id="create_external_table",
+        destination_project_dataset_table=f"{DATASET_NAME}.external_table",
+        bucket=DATA_SAMPLE_GCS_BUCKET_NAME,
+        source_objects=[DATA_SAMPLE_GCS_OBJECT_NAME],
+        schema_fields=[
+            {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
+            {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
+        ],
+    )
+    # [END howto_operator_bigquery_create_external_table]
+
+    delete_dataset = BigQueryDeleteDatasetOperator(
+        task_id="delete_dataset",
+        dataset_id=DATASET_NAME,
+        delete_contents=True,
+        trigger_rule=TriggerRule.ALL_DONE,
+    )
+
+    delete_bucket = GCSDeleteBucketOperator(
+        task_id="delete_bucket", bucket_name=DATA_SAMPLE_GCS_BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE
+    )
+
+    (
+        # TEST SETUP
+        [create_bucket, create_dataset]
+        # TEST BODY
+        >> upload_file
+        >> create_external_table
+        # TEST TEARDOWN
+        >> delete_dataset
+        >> delete_bucket
+    )
+
+    list(dag.tasks) >> watcher()
+
+
+def test_run():
+    from airflow.utils.state import State
+
+    dag.clear(dag_run_state=State.NONE)
+    dag.run()
diff --git a/tests/system/providers/google/bigquery/test_bigquery_operations_location.py b/tests/system/providers/google/bigquery/test_bigquery_operations_location.py
new file mode 100644
index 0000000000000..09ba3e349be00
--- /dev/null
+++ b/tests/system/providers/google/bigquery/test_bigquery_operations_location.py
@@ -0,0 +1,86 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG for Google BigQuery service testing data structures with location.
+"""
+import os
+from datetime import datetime
+
+from airflow import models
+from airflow.providers.google.cloud.operators.bigquery import (
+    BigQueryCreateEmptyDatasetOperator,
+    BigQueryCreateEmptyTableOperator,
+    BigQueryDeleteDatasetOperator,
+)
+from airflow.utils.trigger_rule import TriggerRule
+from tests.system.utils.watcher import watcher
+
+ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+DAG_ID = "bigquery_operations_location"
+
+BQ_LOCATION = "europe-north1"
+DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
+
+
+with models.DAG(
+    DAG_ID,
+    schedule_interval="@once",
+    start_date=datetime(2021, 1, 1),
+    catchup=False,
+    tags=["example", "bigquery"],
+) as dag:
+    create_dataset_with_location = BigQueryCreateEmptyDatasetOperator(
+        task_id="create_dataset_with_location",
+        dataset_id=DATASET_NAME,
+        location=BQ_LOCATION,
+    )
+
+    create_table_with_location = BigQueryCreateEmptyTableOperator(
+        task_id="create_table_with_location",
+        dataset_id=DATASET_NAME,
+        table_id="test_table",
+        schema_fields=[
+            {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
+            {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
+        ],
+    )
+
+    delete_dataset_with_location = BigQueryDeleteDatasetOperator(
+        task_id="delete_dataset_with_location",
+        dataset_id=DATASET_NAME,
+        delete_contents=True,
+        trigger_rule=TriggerRule.ALL_DONE,
+    )
+
+    (
+        # TEST BODY
+        create_dataset_with_location
+        >> create_table_with_location
+        # TEST TEARDOWN
+        >> delete_dataset_with_location
+    )
+
+    list(dag.tasks) >> watcher()
+
+
+def test_run():
+    from airflow.utils.state import State
+
+    dag.clear(dag_run_state=State.NONE)
+    dag.run()
diff --git a/tests/system/providers/google/bigquery/test_bigquery_queries.py b/tests/system/providers/google/bigquery/test_bigquery_queries.py
new file mode 100644
index 0000000000000..9d84976c2ddff
--- /dev/null
+++ b/tests/system/providers/google/bigquery/test_bigquery_queries.py
@@ -0,0 +1,237 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG for Google BigQuery service.
+"""
+import os
+from datetime import datetime
+from pathlib import Path
+
+from airflow import models
+from airflow.operators.bash import BashOperator
+from airflow.providers.google.cloud.operators.bigquery import (
+    BigQueryCheckOperator,
+    BigQueryCreateEmptyDatasetOperator,
+    BigQueryCreateEmptyTableOperator,
+    BigQueryDeleteDatasetOperator,
+    BigQueryGetDataOperator,
+    BigQueryInsertJobOperator,
+    BigQueryIntervalCheckOperator,
+    BigQueryValueCheckOperator,
+)
+from airflow.utils.trigger_rule import TriggerRule
+from tests.system.utils.watcher import watcher
+
+ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+PROJECT_ID = os.environ["SYSTEM_TESTS_GCP_PROJECT"]
+LOCATION = "us-east1"
+QUERY_SQL_PATH = str(Path(__file__).parent / "resources" / "example_bigquery_query.sql")
+
+TABLE_1 = "table1"
+TABLE_2 = "table2"
+
+SCHEMA = [
+    {"name": "value", "type": "INTEGER", "mode": "REQUIRED"},
+    {"name": "name", "type": "STRING", "mode": "NULLABLE"},
+    {"name": "ds", "type": "DATE", "mode": "NULLABLE"},
+]
+
+
+locations = [None, LOCATION]
+for index, location in enumerate(locations, 1):
+    DAG_ID = "bigquery_queries_location" if location else "bigquery_queries"
+    DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
+    DATASET = f"{DATASET_NAME}{index}"
+    INSERT_DATE = datetime.now().strftime("%Y-%m-%d")
+    # [START howto_operator_bigquery_query]
+    INSERT_ROWS_QUERY = (
+        f"INSERT {DATASET}.{TABLE_1} VALUES "
+        f"(42, 'monty python', '{INSERT_DATE}'), "
+        f"(42, 'fishy fish', '{INSERT_DATE}');"
+    )
+    # [END howto_operator_bigquery_query]
+
+    with models.DAG(
+        DAG_ID,
+        schedule_interval="@once",
+        start_date=datetime(2021, 1, 1),
+        catchup=False,
+        tags=["example", "bigquery"],
+        user_defined_macros={"DATASET": DATASET, "TABLE": TABLE_1, "QUERY_SQL_PATH": QUERY_SQL_PATH},
+    ) as dag:
+        create_dataset = BigQueryCreateEmptyDatasetOperator(
+            task_id="create_dataset",
+            dataset_id=DATASET,
+            location=location,
+        )
+
+        create_table_1 = BigQueryCreateEmptyTableOperator(
+            task_id="create_table_1",
+            dataset_id=DATASET,
+            table_id=TABLE_1,
+            schema_fields=SCHEMA,
+            location=location,
+        )
+
+        create_table_2 = BigQueryCreateEmptyTableOperator(
+            task_id="create_table_2",
+            dataset_id=DATASET,
+            table_id=TABLE_2,
+            schema_fields=SCHEMA,
+            location=location,
+        )
+
+        # [START howto_operator_bigquery_insert_job]
+        insert_query_job = BigQueryInsertJobOperator(
+            task_id="insert_query_job",
+            configuration={
+                "query": {
+                    "query": INSERT_ROWS_QUERY,
+                    "useLegacySql": False,
+                }
+            },
+            location=location,
+        )
+        # [END howto_operator_bigquery_insert_job]
+
+        # [START howto_operator_bigquery_select_job]
+        select_query_job = BigQueryInsertJobOperator(
+            task_id="select_query_job",
+            configuration={
+                "query": {
+                    "query": "{% include QUERY_SQL_PATH %}",
+                    "useLegacySql": False,
+                }
+            },
+            location=location,
+        )
+        # [END howto_operator_bigquery_select_job]
+
+        execute_insert_query = BigQueryInsertJobOperator(
+            task_id="execute_insert_query",
+            configuration={
+                "query": {
+                    "query": INSERT_ROWS_QUERY,
+                    "useLegacySql": False,
+                }
+            },
+            location=location,
+        )
+
+        execute_query_save = BigQueryInsertJobOperator(
+            task_id="execute_query_save",
+            configuration={
+                "query": {
+                    "query": f"SELECT * FROM {DATASET}.{TABLE_1}",
+                    "useLegacySql": False,
+                    "destinationTable": {
+                        'projectId': PROJECT_ID,
+                        'datasetId': DATASET,
+                        'tableId': TABLE_2,
+                    },
+                }
+            },
+            location=location,
+        )
+
+        bigquery_execute_multi_query = BigQueryInsertJobOperator(
+            task_id="execute_multi_query",
+            configuration={
+                "query": {
+                    "query": [
+                        f"SELECT * FROM {DATASET}.{TABLE_2}",
+                        f"SELECT COUNT(*) FROM {DATASET}.{TABLE_2}",
+                    ],
+                    "useLegacySql": False,
+                }
+            },
+            location=location,
+        )
+
+        # [START howto_operator_bigquery_get_data]
+        get_data = BigQueryGetDataOperator(
+            task_id="get_data",
+            dataset_id=DATASET,
+            table_id=TABLE_1,
+            max_results=10,
+            selected_fields="value,name",
+            location=location,
+        )
+        # [END howto_operator_bigquery_get_data]
+
+        get_data_result = BashOperator(
+            task_id="get_data_result",
+            bash_command=f"echo {get_data.output}",
+        )
+
+        # [START howto_operator_bigquery_check]
+        check_count = BigQueryCheckOperator(
+            task_id="check_count",
+            sql=f"SELECT COUNT(*) FROM {DATASET}.{TABLE_1}",
+            use_legacy_sql=False,
+            location=location,
+        )
+        # [END howto_operator_bigquery_check]
+
+        # [START howto_operator_bigquery_value_check]
+        check_value = BigQueryValueCheckOperator(
+            task_id="check_value",
+            sql=f"SELECT COUNT(*) FROM {DATASET}.{TABLE_1}",
+            pass_value=4,
+            use_legacy_sql=False,
+            location=location,
+        )
+        # [END howto_operator_bigquery_value_check]
+
+        # [START howto_operator_bigquery_interval_check]
+        check_interval = BigQueryIntervalCheckOperator(
+            task_id="check_interval",
+            table=f"{DATASET}.{TABLE_1}",
+            days_back=1,
+            metrics_thresholds={"COUNT(*)": 1.5},
+            use_legacy_sql=False,
+            location=location,
+        )
+        # [END howto_operator_bigquery_interval_check]
+
+        delete_dataset = BigQueryDeleteDatasetOperator(
+            task_id="delete_dataset",
+            dataset_id=DATASET,
+            delete_contents=True,
+            trigger_rule=TriggerRule.ALL_DONE,
+        )
+
+        # TEST SETUP
+        create_dataset >> [create_table_1, create_table_2]
+        # TEST BODY
+        [create_table_1, create_table_2] >> insert_query_job >> [select_query_job, execute_insert_query]
+        execute_insert_query >> get_data >> get_data_result >> delete_dataset
+        execute_insert_query >> execute_query_save >> bigquery_execute_multi_query >> delete_dataset
+        execute_insert_query >> [check_count, check_value, check_interval] >> delete_dataset
+
+        list(dag.tasks) >> watcher()
+
+    globals()[DAG_ID] = dag
+
+
+def test_run():
+    from airflow.utils.state import State
+
+    dag.clear(dag_run_state=State.NONE)
+    dag.run()
diff --git a/tests/system/providers/google/bigquery/test_bigquery_sensors.py b/tests/system/providers/google/bigquery/test_bigquery_sensors.py
new file mode 100644
index 0000000000000..efe2c5390bc02
--- /dev/null
+++ b/tests/system/providers/google/bigquery/test_bigquery_sensors.py
@@ -0,0 +1,127 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG for Google BigQuery Sensors.
+"""
+import os
+from datetime import datetime
+
+from airflow import models
+from airflow.providers.google.cloud.operators.bigquery import (
+    BigQueryCreateEmptyDatasetOperator,
+    BigQueryCreateEmptyTableOperator,
+    BigQueryDeleteDatasetOperator,
+    BigQueryInsertJobOperator,
+)
+from airflow.providers.google.cloud.sensors.bigquery import (
+    BigQueryTableExistenceSensor,
+    BigQueryTablePartitionExistenceSensor,
+)
+from airflow.utils.trigger_rule import TriggerRule
+from tests.system.utils.watcher import watcher
+
+ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+DAG_ID = "bigquery_sensors"
+
+DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
+PROJECT_ID = os.environ["SYSTEM_TESTS_GCP_PROJECT"]
+
+TABLE_NAME = "partitioned_table"
+INSERT_DATE = datetime.now().strftime("%Y-%m-%d")
+
+PARTITION_NAME = "{{ ds_nodash }}"
+
+INSERT_ROWS_QUERY = f"INSERT {DATASET_NAME}.{TABLE_NAME} VALUES " "(42, '{{ ds }}')"
+
+SCHEMA = [
+    {"name": "value", "type": "INTEGER", "mode": "REQUIRED"},
+    {"name": "ds", "type": "DATE", "mode": "NULLABLE"},
+]
+
+
+with models.DAG(
+    DAG_ID,
+    schedule_interval="@once",
+    start_date=datetime(2021, 1, 1),
+    catchup=False,
+    tags=["example", "bigquery"],
+    user_defined_macros={"DATASET": DATASET_NAME, "TABLE": TABLE_NAME},
+    default_args={"project_id": PROJECT_ID},
+) as dag:
+
+    create_dataset = BigQueryCreateEmptyDatasetOperator(
+        task_id="create_dataset", dataset_id=DATASET_NAME, project_id=PROJECT_ID
+    )
+
+    create_table = BigQueryCreateEmptyTableOperator(
+        task_id="create_table",
+        dataset_id=DATASET_NAME,
+        table_id=TABLE_NAME,
+        schema_fields=SCHEMA,
+        time_partitioning={
+            "type": "DAY",
+            "field": "ds",
+        },
+    )
+    # [START howto_sensor_bigquery_table]
+    check_table_exists = BigQueryTableExistenceSensor(
+        task_id="check_table_exists", project_id=PROJECT_ID, dataset_id=DATASET_NAME, table_id=TABLE_NAME
+    )
+    # [END howto_sensor_bigquery_table]
+
+    execute_insert_query = BigQueryInsertJobOperator(
+        task_id="execute_insert_query",
+        configuration={
+            "query": {
+                "query": INSERT_ROWS_QUERY,
+                "useLegacySql": False,
+            }
+        },
+    )
+
+    # [START howto_sensor_bigquery_table_partition]
+    check_table_partition_exists = BigQueryTablePartitionExistenceSensor(
+        task_id="check_table_partition_exists",
+        project_id=PROJECT_ID,
+        dataset_id=DATASET_NAME,
+        table_id=TABLE_NAME,
+        partition_id=PARTITION_NAME,
+    )
+    # [END howto_sensor_bigquery_table_partition]
+
+    delete_dataset = BigQueryDeleteDatasetOperator(
+        task_id="delete_dataset",
+        dataset_id=DATASET_NAME,
+        delete_contents=True,
+        trigger_rule=TriggerRule.ALL_DONE,
+    )
+
+    create_dataset >> create_table
+    create_table >> [check_table_exists, execute_insert_query]
+    execute_insert_query >> check_table_partition_exists
+    [check_table_exists, check_table_partition_exists] >> delete_dataset
+
+    list(dag.tasks) >> watcher()
+
+
+def test_run():
+    from airflow.utils.state import State
+
+    dag.clear(dag_run_state=State.NONE)
+    dag.run()
diff --git a/tests/system/providers/google/bigquery/test_bigquery_tables.py b/tests/system/providers/google/bigquery/test_bigquery_tables.py
new file mode 100644
index 0000000000000..3338f08f45593
--- /dev/null
+++ b/tests/system/providers/google/bigquery/test_bigquery_tables.py
@@ -0,0 +1,230 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Example Airflow DAG for Google BigQuery service testing tables.
+"""
+import os
+import time
+from datetime import datetime
+from pathlib import Path
+
+from airflow import models
+from airflow.providers.google.cloud.operators.bigquery import (
+    BigQueryCreateEmptyDatasetOperator,
+    BigQueryCreateEmptyTableOperator,
+    BigQueryDeleteDatasetOperator,
+    BigQueryDeleteTableOperator,
+    BigQueryGetDatasetTablesOperator,
+    BigQueryUpdateDatasetOperator,
+    BigQueryUpdateTableOperator,
+    BigQueryUpdateTableSchemaOperator,
+    BigQueryUpsertTableOperator,
+)
+from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
+from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator
+from airflow.utils.trigger_rule import TriggerRule
+from tests.system.utils.watcher import watcher
+
+ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+DAG_ID = "bigquery_tables"
+
+BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
+DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
+PROJECT_ID = os.environ["SYSTEM_TESTS_GCP_PROJECT"]
+SCHEMA_JSON_LOCAL_SRC = str(Path(__file__).parent / "resources" / "update_table_schema.json")
+SCHEMA_JSON_DESTINATION = "update_table_schema.json"
+GCS_PATH_TO_SCHEMA_JSON = f"gs://{BUCKET_NAME}/{SCHEMA_JSON_DESTINATION}"
+
+
+with models.DAG(
+    DAG_ID,
+    schedule_interval="@once",
+    start_date=datetime(2021, 1, 1),
+    catchup=False,
+    tags=["example", "bigquery"],
+) as dag:
+    create_bucket = GCSCreateBucketOperator(
+        task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID
+    )
+
+    upload_schema_json = LocalFilesystemToGCSOperator(
+        task_id="upload_schema_json",
+        src=SCHEMA_JSON_LOCAL_SRC,
+        dst=SCHEMA_JSON_DESTINATION,
+        bucket=BUCKET_NAME,
+    )
+
+    create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create_dataset", dataset_id=DATASET_NAME)
+
+    # [START howto_operator_bigquery_create_table]
+    create_table = BigQueryCreateEmptyTableOperator(
+        task_id="create_table",
+        dataset_id=DATASET_NAME,
+        table_id="test_table",
+        schema_fields=[
+            {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
+            {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
+        ],
+    )
+    # [END howto_operator_bigquery_create_table]
+
+    # [START howto_operator_bigquery_create_view]
+    create_view = BigQueryCreateEmptyTableOperator(
+        task_id="create_view",
+        dataset_id=DATASET_NAME,
+        table_id="test_view",
+        view={
+            "query": f"SELECT * FROM `{PROJECT_ID}.{DATASET_NAME}.test_table`",
+            "useLegacySql": False,
+        },
+    )
+    # [END howto_operator_bigquery_create_view]
+
+    # [START howto_operator_bigquery_create_materialized_view]
+    create_materialized_view = BigQueryCreateEmptyTableOperator(
+        task_id="create_materialized_view",
+        dataset_id=DATASET_NAME,
+        table_id="test_materialized_view",
+        materialized_view={
+            "query": f"SELECT SUM(salary) AS sum_salary " f"FROM `{PROJECT_ID}.{DATASET_NAME}.test_table`",
+            "enableRefresh": True,
+            "refreshIntervalMs": 2000000,
+        },
+    )
+    # [END howto_operator_bigquery_create_materialized_view]
+
+    # [START howto_operator_bigquery_delete_view]
+    delete_view = BigQueryDeleteTableOperator(
+        task_id="delete_view",
+        deletion_dataset_table=f"{PROJECT_ID}.{DATASET_NAME}.test_view",
+    )
+    # [END howto_operator_bigquery_delete_view]
+
+    # [START howto_operator_bigquery_update_table]
+    update_table = BigQueryUpdateTableOperator(
+        task_id="update_table",
+        dataset_id=DATASET_NAME,
+        table_id="test_table",
+        fields=["friendlyName", "description"],
+        table_resource={
+            "friendlyName": "Updated Table",
+            "description": "Updated Table",
+        },
+    )
+    # [END howto_operator_bigquery_update_table]
+
+    # [START howto_operator_bigquery_upsert_table]
+    upsert_table = BigQueryUpsertTableOperator(
+        task_id="upsert_table",
+        dataset_id=DATASET_NAME,
+        table_resource={
+            "tableReference": {"tableId": "test_table_id"},
+            "expirationTime": (int(time.time()) + 300) * 1000,
+        },
+    )
+    # [END howto_operator_bigquery_upsert_table]
+
+    # [START howto_operator_bigquery_update_table_schema]
+    update_table_schema = BigQueryUpdateTableSchemaOperator(
+        task_id="update_table_schema",
+        dataset_id=DATASET_NAME,
+        table_id="test_table",
+        schema_fields_updates=[
+            {"name": "emp_name", "description": "Name of employee"},
+            {"name": "salary", "description": "Monthly salary in USD"},
+        ],
+    )
+    # [END howto_operator_bigquery_update_table_schema]
+
+    # [START howto_operator_bigquery_create_table_schema_json]
+    update_table_schema_json = BigQueryCreateEmptyTableOperator(
+        task_id="update_table_schema_json",
+        dataset_id=DATASET_NAME,
+        table_id="test_table",
+        gcs_schema_object=GCS_PATH_TO_SCHEMA_JSON,
+    )
+    # [END howto_operator_bigquery_create_table_schema_json]
+
+    # [START howto_operator_bigquery_delete_materialized_view]
+    delete_materialized_view = BigQueryDeleteTableOperator(
+        task_id="delete_materialized_view",
+        deletion_dataset_table=f"{PROJECT_ID}.{DATASET_NAME}.test_materialized_view",
+    )
+    # [END howto_operator_bigquery_delete_materialized_view]
+
+    # [START howto_operator_bigquery_get_dataset_tables]
+    get_dataset_tables = BigQueryGetDatasetTablesOperator(
+        task_id="get_dataset_tables", dataset_id=DATASET_NAME
+    )
+
+    update_dataset = BigQueryUpdateDatasetOperator(
+        task_id="update_dataset",
+        dataset_id=DATASET_NAME,
+        dataset_resource={"description": "Updated dataset"},
+    )
+
+    # [START howto_operator_bigquery_delete_table]
+    delete_table = BigQueryDeleteTableOperator(
+        task_id="delete_table",
+        deletion_dataset_table=f"{PROJECT_ID}.{DATASET_NAME}.test_table",
+    )
+    # [END howto_operator_bigquery_delete_table]
+
+    delete_dataset = BigQueryDeleteDatasetOperator(
+        task_id="delete_dataset", dataset_id=DATASET_NAME, delete_contents=True
+    )
+    delete_dataset.trigger_rule = TriggerRule.ALL_DONE
+
+    delete_bucket = GCSDeleteBucketOperator(
+        task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE
+    )
+
+    (
+        # TEST SETUP
+        create_bucket
+        >> create_dataset
+        >> upload_schema_json
+        # TEST BODY
+        >> update_dataset
+        >> create_table
+        >> create_view
+        >> create_materialized_view
+        >> [
+            get_dataset_tables,
+            delete_view,
+        ]
+        >> update_table
+        >> upsert_table
+        >> update_table_schema
+        >> update_table_schema_json
+        >> delete_materialized_view
+        >> delete_table
+        # TEST TEARDOWN
+        >> delete_bucket
+        >> delete_dataset
+    )
+
+    list(dag.tasks) >> watcher()
+
+
+def test_run():
+    from airflow.utils.state import State
+
+    dag.clear(dag_run_state=State.NONE)
+    dag.run()
diff --git a/tests/system/providers/google/bigquery/test_bigquery_to_bigquery.py b/tests/system/providers/google/bigquery/test_bigquery_to_bigquery.py
new file mode 100644
index 0000000000000..1084ba0e2d77a
--- /dev/null
+++ b/tests/system/providers/google/bigquery/test_bigquery_to_bigquery.py
@@ -0,0 +1,102 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Airflow System Test DAG that verifies BigQueryToBigQueryOperator.
+"""
+import os
+from datetime import datetime
+
+from airflow import models
+from airflow.providers.google.cloud.operators.bigquery import (
+    BigQueryCreateEmptyDatasetOperator,
+    BigQueryCreateEmptyTableOperator,
+    BigQueryDeleteDatasetOperator,
+)
+from airflow.providers.google.cloud.transfers.bigquery_to_bigquery import BigQueryToBigQueryOperator
+from airflow.utils.trigger_rule import TriggerRule
+from tests.system.utils.watcher import watcher
+
+ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+DAG_ID = "bigquery_to_bigquery"
+
+DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
+ORIGIN = "origin"
+TARGET = "target"
+
+
+with models.DAG(
+    DAG_ID,
+    schedule_interval="@once",
+    start_date=datetime(2021, 1, 1),
+    catchup=False,
+    tags=["example", "bigquery"],
+) as dag:
+    create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create_dataset", dataset_id=DATASET_NAME)
+
+    create_origin_table = BigQueryCreateEmptyTableOperator(
+        task_id="create_origin_table",
+        dataset_id=DATASET_NAME,
+        table_id="origin",
+        schema_fields=[
+            {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
+            {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
+        ],
+    )
+
+    create_target_table = BigQueryCreateEmptyTableOperator(
+        task_id="create_target_table",
+        dataset_id=DATASET_NAME,
+        table_id="target",
+        schema_fields=[
+            {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
+            {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
+        ],
+    )
+
+    copy_selected_data = BigQueryToBigQueryOperator(
+        task_id="copy_selected_data",
+        source_project_dataset_tables=f"{DATASET_NAME}.{ORIGIN}",
+        destination_project_dataset_table=f"{DATASET_NAME}.{TARGET}",
+    )
+
+    delete_dataset = BigQueryDeleteDatasetOperator(
+        task_id="delete_dataset",
+        dataset_id=DATASET_NAME,
+        delete_contents=True,
+        trigger_rule=TriggerRule.ALL_DONE,
+    )
+
+    (
+        # TEST SETUP
+        create_dataset
+        >> [create_origin_table, create_target_table]
+        # TEST BODY
+        >> copy_selected_data
+        # TEST TEARDOWN
+        >> delete_dataset
+    )
+
+    list(dag.tasks) >> watcher()
+
+
+def test_run():
+    from airflow.utils.state import State
+
+    dag.clear(dag_run_state=State.NONE)
+    dag.run()
diff --git a/tests/system/providers/google/bigquery/test_bigquery_to_gcs.py b/tests/system/providers/google/bigquery/test_bigquery_to_gcs.py
new file mode 100644
index 0000000000000..9c99580cebd39
--- /dev/null
+++ b/tests/system/providers/google/bigquery/test_bigquery_to_gcs.py
@@ -0,0 +1,103 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""
+Airflow System Test DAG that verifies BigQueryToGCSOperator.
+"""
+import os
+from datetime import datetime
+
+from airflow import models
+from airflow.providers.google.cloud.operators.bigquery import (
+    BigQueryCreateEmptyDatasetOperator,
+    BigQueryCreateEmptyTableOperator,
+    BigQueryDeleteDatasetOperator,
+)
+from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
+from airflow.providers.google.cloud.transfers.bigquery_to_gcs import BigQueryToGCSOperator
+from airflow.utils.trigger_rule import TriggerRule
+from tests.system.utils.watcher import watcher
+
+ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+DAG_ID = "bigquery_to_gcs"
+
+DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
+BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
+PROJECT_ID = os.environ["SYSTEM_TESTS_GCP_PROJECT"]
+BUCKET_FILE = "test.csv"
+TABLE = "test"
+
+
+with models.DAG(
+    DAG_ID,
+    schedule_interval="@once",
+    start_date=datetime(2021, 1, 1),
+    catchup=False,
+    tags=["example", "bigquery"],
+) as dag:
+    create_bucket = GCSCreateBucketOperator(
+        task_id="create_bucket", bucket_name=BUCKET_NAME, project_id=PROJECT_ID
+    )
+
+    create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create_dataset", dataset_id=DATASET_NAME)
+
+    create_table = BigQueryCreateEmptyTableOperator(
+        task_id="create_table",
+        dataset_id=DATASET_NAME,
+        table_id=TABLE,
+        schema_fields=[
+            {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
+            {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
+        ],
+    )
+
+    bigquery_to_gcs = BigQueryToGCSOperator(
+        task_id="bigquery_to_gcs",
+        source_project_dataset_table=f"{DATASET_NAME}.{TABLE}",
+        destination_cloud_storage_uris=[f"gs://{BUCKET_NAME}/{BUCKET_FILE}"],
+    )
+
+    delete_bucket = GCSDeleteBucketOperator(
+        task_id="delete_bucket", bucket_name=BUCKET_NAME, trigger_rule=TriggerRule.ALL_DONE
+    )
+
+    delete_dataset = BigQueryDeleteDatasetOperator(
+        task_id="delete_dataset",
+        dataset_id=DATASET_NAME,
+        delete_contents=True,
+        trigger_rule=TriggerRule.ALL_DONE,
+    )
+
+    (
+        # TEST SETUP
+        [create_bucket, create_dataset]
+        >> create_table
+        # TEST BODY
+        >> bigquery_to_gcs
+        # TEST TEARDOWN
+        >> [delete_bucket, delete_dataset]
+    )
+
+    list(dag.tasks) >> watcher()
+
+
+def test_run():
+    from airflow.utils.state import State
+
+    dag.clear(dag_run_state=State.NONE)
+    dag.run()
diff --git a/tests/system/utils/__init__.py b/tests/system/utils/__init__.py
new file mode 100644
index 0000000000000..13a83393a9124
--- /dev/null
+++ b/tests/system/utils/__init__.py
@@ -0,0 +1,16 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
diff --git a/tests/system/utils/watcher.py b/tests/system/utils/watcher.py
new file mode 100644
index 0000000000000..2c0d37c17afc2
--- /dev/null
+++ b/tests/system/utils/watcher.py
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+from airflow.decorators import task
+from airflow.exceptions import AirflowException
+
+
+@task(trigger_rule="one_failed", retries=0)
+def watcher():
+    """Watcher task raises an AirflowException and is used to 'watch' tasks for failures
+    and propagates fail status to the whole DAG Run"""
+    raise AirflowException("Failing task because one or more upstream tasks failed.")

From ccb13345bf3325c983e3947f6dd9fa8c7d3cd3ab Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Thu, 10 Mar 2022 12:31:45 +0000
Subject: [PATCH 02/23] Remove Bigquery example dags already migrated to new
 system tests

Change-Id: Id9594771fe12e22f052f1929c7afc7fe43aec166
---
 .../example_bigquery_operations.py            | 266 ------------------
 .../example_dags/example_bigquery_queries.py  | 217 --------------
 .../example_dags/example_bigquery_query.sql   |  20 --
 .../example_dags/example_bigquery_sensors.py  | 112 --------
 4 files changed, 615 deletions(-)
 delete mode 100644 airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
 delete mode 100644 airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
 delete mode 100644 airflow/providers/google/cloud/example_dags/example_bigquery_query.sql
 delete mode 100644 airflow/providers/google/cloud/example_dags/example_bigquery_sensors.py

diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py b/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
deleted file mode 100644
index d12a749474fe3..0000000000000
--- a/airflow/providers/google/cloud/example_dags/example_bigquery_operations.py
+++ /dev/null
@@ -1,266 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-Example Airflow DAG for Google BigQuery service.
-"""
-import os
-import time
-from datetime import datetime
-from urllib.parse import urlparse
-
-from airflow import models
-from airflow.operators.bash import BashOperator
-from airflow.providers.google.cloud.operators.bigquery import (
-    BigQueryCreateEmptyDatasetOperator,
-    BigQueryCreateEmptyTableOperator,
-    BigQueryCreateExternalTableOperator,
-    BigQueryDeleteDatasetOperator,
-    BigQueryDeleteTableOperator,
-    BigQueryGetDatasetOperator,
-    BigQueryGetDatasetTablesOperator,
-    BigQueryUpdateDatasetOperator,
-    BigQueryUpdateTableOperator,
-    BigQueryUpdateTableSchemaOperator,
-    BigQueryUpsertTableOperator,
-)
-
-START_DATE = datetime(2021, 1, 1)
-
-PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
-BQ_LOCATION = "europe-north1"
-
-DATASET_NAME = os.environ.get("GCP_BIGQUERY_DATASET_NAME", "test_dataset_operations")
-LOCATION_DATASET_NAME = f"{DATASET_NAME}_location"
-DATA_SAMPLE_GCS_URL = os.environ.get(
-    "GCP_BIGQUERY_DATA_GCS_URL",
-    "gs://INVALID BUCKET NAME/bigquery/us-states/us-states.csv",
-)
-
-DATA_SAMPLE_GCS_URL_PARTS = urlparse(DATA_SAMPLE_GCS_URL)
-DATA_SAMPLE_GCS_BUCKET_NAME = DATA_SAMPLE_GCS_URL_PARTS.netloc
-DATA_SAMPLE_GCS_OBJECT_NAME = DATA_SAMPLE_GCS_URL_PARTS.path[1:]
-
-
-with models.DAG(
-    "example_bigquery_operations",
-    schedule_interval='@once',  # Override to match your needs
-    start_date=START_DATE,
-    catchup=False,
-    tags=["example"],
-) as dag:
-    # [START howto_operator_bigquery_create_table]
-    create_table = BigQueryCreateEmptyTableOperator(
-        task_id="create_table",
-        dataset_id=DATASET_NAME,
-        table_id="test_table",
-        schema_fields=[
-            {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
-            {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
-        ],
-    )
-    # [END howto_operator_bigquery_create_table]
-
-    # [START howto_operator_bigquery_update_table_schema]
-    update_table_schema = BigQueryUpdateTableSchemaOperator(
-        task_id="update_table_schema",
-        dataset_id=DATASET_NAME,
-        table_id="test_table",
-        schema_fields_updates=[
-            {"name": "emp_name", "description": "Name of employee"},
-            {"name": "salary", "description": "Monthly salary in USD"},
-        ],
-    )
-    # [END howto_operator_bigquery_update_table_schema]
-
-    # [START howto_operator_bigquery_delete_table]
-    delete_table = BigQueryDeleteTableOperator(
-        task_id="delete_table",
-        deletion_dataset_table=f"{PROJECT_ID}.{DATASET_NAME}.test_table",
-    )
-    # [END howto_operator_bigquery_delete_table]
-
-    # [START howto_operator_bigquery_create_view]
-    create_view = BigQueryCreateEmptyTableOperator(
-        task_id="create_view",
-        dataset_id=DATASET_NAME,
-        table_id="test_view",
-        view={
-            "query": f"SELECT * FROM `{PROJECT_ID}.{DATASET_NAME}.test_table`",
-            "useLegacySql": False,
-        },
-    )
-    # [END howto_operator_bigquery_create_view]
-
-    # [START howto_operator_bigquery_delete_view]
-    delete_view = BigQueryDeleteTableOperator(
-        task_id="delete_view", deletion_dataset_table=f"{PROJECT_ID}.{DATASET_NAME}.test_view"
-    )
-    # [END howto_operator_bigquery_delete_view]
-
-    # [START howto_operator_bigquery_create_materialized_view]
-    create_materialized_view = BigQueryCreateEmptyTableOperator(
-        task_id="create_materialized_view",
-        dataset_id=DATASET_NAME,
-        table_id="test_materialized_view",
-        materialized_view={
-            "query": f"SELECT SUM(salary)  AS sum_salary FROM `{PROJECT_ID}.{DATASET_NAME}.test_table`",
-            "enableRefresh": True,
-            "refreshIntervalMs": 2000000,
-        },
-    )
-    # [END howto_operator_bigquery_create_materialized_view]
-
-    # [START howto_operator_bigquery_delete_materialized_view]
-    delete_materialized_view = BigQueryDeleteTableOperator(
-        task_id="delete_materialized_view",
-        deletion_dataset_table=f"{PROJECT_ID}.{DATASET_NAME}.test_materialized_view",
-    )
-    # [END howto_operator_bigquery_delete_materialized_view]
-
-    # [START howto_operator_bigquery_create_external_table]
-    create_external_table = BigQueryCreateExternalTableOperator(
-        task_id="create_external_table",
-        table_resource={
-            "tableReference": {
-                "projectId": PROJECT_ID,
-                "datasetId": DATASET_NAME,
-                "tableId": "external_table",
-            },
-            "schema": {
-                "fields": [
-                    {"name": "name", "type": "STRING"},
-                    {"name": "post_abbr", "type": "STRING"},
-                ]
-            },
-            "externalDataConfiguration": {
-                "sourceFormat": "CSV",
-                "compression": "NONE",
-                "csvOptions": {"skipLeadingRows": 1},
-                "sourceUris": [DATA_SAMPLE_GCS_URL],
-            },
-        },
-    )
-    # [END howto_operator_bigquery_create_external_table]
-
-    # [START howto_operator_bigquery_upsert_table]
-    upsert_table = BigQueryUpsertTableOperator(
-        task_id="upsert_table",
-        dataset_id=DATASET_NAME,
-        table_resource={
-            "tableReference": {"tableId": "test_table_id"},
-            "expirationTime": (int(time.time()) + 300) * 1000,
-        },
-    )
-    # [END howto_operator_bigquery_upsert_table]
-
-    # [START howto_operator_bigquery_create_dataset]
-    create_dataset = BigQueryCreateEmptyDatasetOperator(task_id="create-dataset", dataset_id=DATASET_NAME)
-    # [END howto_operator_bigquery_create_dataset]
-
-    # [START howto_operator_bigquery_get_dataset_tables]
-    get_dataset_tables = BigQueryGetDatasetTablesOperator(
-        task_id="get_dataset_tables", dataset_id=DATASET_NAME
-    )
-    # [END howto_operator_bigquery_get_dataset_tables]
-
-    # [START howto_operator_bigquery_get_dataset]
-    get_dataset = BigQueryGetDatasetOperator(task_id="get-dataset", dataset_id=DATASET_NAME)
-    # [END howto_operator_bigquery_get_dataset]
-
-    get_dataset_result = BashOperator(
-        task_id="get_dataset_result",
-        bash_command="echo \"{{ task_instance.xcom_pull('get-dataset')['id'] }}\"",
-    )
-
-    # [START howto_operator_bigquery_update_table]
-    update_table = BigQueryUpdateTableOperator(
-        task_id="update_table",
-        dataset_id=DATASET_NAME,
-        table_id="test_table",
-        fields=["friendlyName", "description"],
-        table_resource={
-            "friendlyName": "Updated Table",
-            "description": "Updated Table",
-        },
-    )
-    # [END howto_operator_bigquery_update_table]
-
-    # [START howto_operator_bigquery_update_dataset]
-    update_dataset = BigQueryUpdateDatasetOperator(
-        task_id="update_dataset",
-        dataset_id=DATASET_NAME,
-        dataset_resource={"description": "Updated dataset"},
-    )
-    # [END howto_operator_bigquery_update_dataset]
-
-    # [START howto_operator_bigquery_delete_dataset]
-    delete_dataset = BigQueryDeleteDatasetOperator(
-        task_id="delete_dataset", dataset_id=DATASET_NAME, delete_contents=True
-    )
-    # [END howto_operator_bigquery_delete_dataset]
-
-    create_dataset >> update_dataset >> get_dataset >> get_dataset_result >> delete_dataset
-
-    (
-        update_dataset
-        >> create_table
-        >> create_view
-        >> create_materialized_view
-        >> update_table
-        >> [
-            get_dataset_tables,
-            delete_view,
-        ]
-        >> upsert_table
-        >> update_table_schema
-        >> delete_materialized_view
-        >> delete_table
-        >> delete_dataset
-    )
-    update_dataset >> create_external_table >> delete_dataset
-
-with models.DAG(
-    "example_bigquery_operations_location",
-    schedule_interval='@once',  # Override to match your needs
-    start_date=START_DATE,
-    catchup=False,
-    tags=["example"],
-) as dag_with_location:
-    create_dataset_with_location = BigQueryCreateEmptyDatasetOperator(
-        task_id="create_dataset_with_location",
-        dataset_id=LOCATION_DATASET_NAME,
-        location=BQ_LOCATION,
-    )
-
-    create_table_with_location = BigQueryCreateEmptyTableOperator(
-        task_id="create_table_with_location",
-        dataset_id=LOCATION_DATASET_NAME,
-        table_id="test_table",
-        schema_fields=[
-            {"name": "emp_name", "type": "STRING", "mode": "REQUIRED"},
-            {"name": "salary", "type": "INTEGER", "mode": "NULLABLE"},
-        ],
-    )
-
-    delete_dataset_with_location = BigQueryDeleteDatasetOperator(
-        task_id="delete_dataset_with_location",
-        dataset_id=LOCATION_DATASET_NAME,
-        delete_contents=True,
-    )
-    create_dataset_with_location >> create_table_with_location >> delete_dataset_with_location
diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_queries.py b/airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
deleted file mode 100644
index f4e435518dc2a..0000000000000
--- a/airflow/providers/google/cloud/example_dags/example_bigquery_queries.py
+++ /dev/null
@@ -1,217 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-Example Airflow DAG for Google BigQuery service.
-"""
-import os
-from datetime import datetime
-
-from airflow import models
-from airflow.operators.bash import BashOperator
-from airflow.providers.google.cloud.operators.bigquery import (
-    BigQueryCheckOperator,
-    BigQueryCreateEmptyDatasetOperator,
-    BigQueryCreateEmptyTableOperator,
-    BigQueryDeleteDatasetOperator,
-    BigQueryGetDataOperator,
-    BigQueryInsertJobOperator,
-    BigQueryIntervalCheckOperator,
-    BigQueryValueCheckOperator,
-)
-
-PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
-DATASET_NAME = os.environ.get("GCP_BIGQUERY_DATASET_NAME", "test_dataset")
-LOCATION = "southamerica-east1"
-
-TABLE_1 = "table1"
-TABLE_2 = "table2"
-
-SCHEMA = [
-    {"name": "value", "type": "INTEGER", "mode": "REQUIRED"},
-    {"name": "name", "type": "STRING", "mode": "NULLABLE"},
-    {"name": "ds", "type": "DATE", "mode": "NULLABLE"},
-]
-
-locations = [None, LOCATION]
-for index, location in enumerate(locations, 1):
-    dag_id = "example_bigquery_queries_location" if location else "example_bigquery_queries"
-    DATASET = DATASET_NAME + str(index)
-    INSERT_DATE = datetime.now().strftime("%Y-%m-%d")
-    # [START howto_operator_bigquery_query]
-    INSERT_ROWS_QUERY = (
-        f"INSERT {DATASET}.{TABLE_1} VALUES "
-        f"(42, 'monthy python', '{INSERT_DATE}'), "
-        f"(42, 'fishy fish', '{INSERT_DATE}');"
-    )
-    # [END howto_operator_bigquery_query]
-
-    with models.DAG(
-        dag_id,
-        schedule_interval='@once',  # Override to match your needs
-        start_date=datetime(2021, 1, 1),
-        catchup=False,
-        tags=["example"],
-        user_defined_macros={"DATASET": DATASET, "TABLE": TABLE_1},
-    ) as dag_with_locations:
-        create_dataset = BigQueryCreateEmptyDatasetOperator(
-            task_id="create-dataset",
-            dataset_id=DATASET,
-            location=location,
-        )
-
-        create_table_1 = BigQueryCreateEmptyTableOperator(
-            task_id="create_table_1",
-            dataset_id=DATASET,
-            table_id=TABLE_1,
-            schema_fields=SCHEMA,
-            location=location,
-        )
-
-        create_table_2 = BigQueryCreateEmptyTableOperator(
-            task_id="create_table_2",
-            dataset_id=DATASET,
-            table_id=TABLE_2,
-            schema_fields=SCHEMA,
-            location=location,
-        )
-
-        create_dataset >> [create_table_1, create_table_2]
-
-        delete_dataset = BigQueryDeleteDatasetOperator(
-            task_id="delete_dataset", dataset_id=DATASET, delete_contents=True
-        )
-
-        # [START howto_operator_bigquery_insert_job]
-        insert_query_job = BigQueryInsertJobOperator(
-            task_id="insert_query_job",
-            configuration={
-                "query": {
-                    "query": INSERT_ROWS_QUERY,
-                    "useLegacySql": False,
-                }
-            },
-            location=location,
-        )
-        # [END howto_operator_bigquery_insert_job]
-
-        # [START howto_operator_bigquery_select_job]
-        select_query_job = BigQueryInsertJobOperator(
-            task_id="select_query_job",
-            configuration={
-                "query": {
-                    "query": "{% include 'example_bigquery_query.sql' %}",
-                    "useLegacySql": False,
-                }
-            },
-            location=location,
-        )
-        # [END howto_operator_bigquery_select_job]
-
-        execute_insert_query = BigQueryInsertJobOperator(
-            task_id="execute_insert_query",
-            configuration={
-                "query": {
-                    "query": INSERT_ROWS_QUERY,
-                    "useLegacySql": False,
-                }
-            },
-            location=location,
-        )
-
-        bigquery_execute_multi_query = BigQueryInsertJobOperator(
-            task_id="execute_multi_query",
-            configuration={
-                "query": {
-                    "query": f"SELECT * FROM {DATASET}.{TABLE_2};SELECT COUNT(*) FROM {DATASET}.{TABLE_2}",
-                    "useLegacySql": False,
-                }
-            },
-            location=location,
-        )
-
-        execute_query_save = BigQueryInsertJobOperator(
-            task_id="execute_query_save",
-            configuration={
-                "query": {
-                    "query": f"SELECT * FROM {DATASET}.{TABLE_1}",
-                    "useLegacySql": False,
-                    "destinationTable": {
-                        'projectId': PROJECT_ID,
-                        'datasetId': DATASET,
-                        'tableId': TABLE_2,
-                    },
-                }
-            },
-            location=location,
-        )
-
-        # [START howto_operator_bigquery_get_data]
-        get_data = BigQueryGetDataOperator(
-            task_id="get_data",
-            dataset_id=DATASET,
-            table_id=TABLE_1,
-            max_results=10,
-            selected_fields="value,name",
-            location=location,
-        )
-        # [END howto_operator_bigquery_get_data]
-
-        get_data_result = BashOperator(
-            task_id="get_data_result",
-            bash_command=f"echo {get_data.output}",
-        )
-
-        # [START howto_operator_bigquery_check]
-        check_count = BigQueryCheckOperator(
-            task_id="check_count",
-            sql=f"SELECT COUNT(*) FROM {DATASET}.{TABLE_1}",
-            use_legacy_sql=False,
-            location=location,
-        )
-        # [END howto_operator_bigquery_check]
-
-        # [START howto_operator_bigquery_value_check]
-        check_value = BigQueryValueCheckOperator(
-            task_id="check_value",
-            sql=f"SELECT COUNT(*) FROM {DATASET}.{TABLE_1}",
-            pass_value=4,
-            use_legacy_sql=False,
-            location=location,
-        )
-        # [END howto_operator_bigquery_value_check]
-
-        # [START howto_operator_bigquery_interval_check]
-        check_interval = BigQueryIntervalCheckOperator(
-            task_id="check_interval",
-            table=f"{DATASET}.{TABLE_1}",
-            days_back=1,
-            metrics_thresholds={"COUNT(*)": 1.5},
-            use_legacy_sql=False,
-            location=location,
-        )
-        # [END howto_operator_bigquery_interval_check]
-
-        [create_table_1, create_table_2] >> insert_query_job >> select_query_job
-
-        insert_query_job >> execute_insert_query
-        execute_insert_query >> get_data >> get_data_result >> delete_dataset
-        execute_insert_query >> execute_query_save >> bigquery_execute_multi_query >> delete_dataset
-        execute_insert_query >> [check_count, check_value, check_interval] >> delete_dataset
-
-    globals()[dag_id] = dag_with_locations
diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_query.sql b/airflow/providers/google/cloud/example_dags/example_bigquery_query.sql
deleted file mode 100644
index b629f273e4dc3..0000000000000
--- a/airflow/providers/google/cloud/example_dags/example_bigquery_query.sql
+++ /dev/null
@@ -1,20 +0,0 @@
-/*
- Licensed to the Apache Software Foundation (ASF) under one
- or more contributor license agreements.  See the NOTICE file
- distributed with this work for additional information
- regarding copyright ownership.  The ASF licenses this file
- to you under the Apache License, Version 2.0 (the
- "License"); you may not use this file except in compliance
- with the License.  You may obtain a copy of the License at
-
-   http://www.apache.org/licenses/LICENSE-2.0
-
- Unless required by applicable law or agreed to in writing,
- software distributed under the License is distributed on an
- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- KIND, either express or implied.  See the License for the
- specific language governing permissions and limitations
- under the License.
-*/
-
-SELECT * FROM {{ DATASET }}.{{ TABLE }}
diff --git a/airflow/providers/google/cloud/example_dags/example_bigquery_sensors.py b/airflow/providers/google/cloud/example_dags/example_bigquery_sensors.py
deleted file mode 100644
index 28a86fc4017c1..0000000000000
--- a/airflow/providers/google/cloud/example_dags/example_bigquery_sensors.py
+++ /dev/null
@@ -1,112 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-"""
-Example Airflow DAG for Google BigQuery Sensors.
-"""
-import os
-from datetime import datetime
-
-from airflow import models
-from airflow.providers.google.cloud.operators.bigquery import (
-    BigQueryCreateEmptyDatasetOperator,
-    BigQueryCreateEmptyTableOperator,
-    BigQueryDeleteDatasetOperator,
-    BigQueryInsertJobOperator,
-)
-from airflow.providers.google.cloud.sensors.bigquery import (
-    BigQueryTableExistenceSensor,
-    BigQueryTablePartitionExistenceSensor,
-)
-
-PROJECT_ID = os.environ.get("GCP_PROJECT_ID", "example-project")
-DATASET_NAME = os.environ.get("GCP_BIGQUERY_DATASET_NAME", "test_sensors_dataset")
-
-TABLE_NAME = "partitioned_table"
-INSERT_DATE = datetime.now().strftime("%Y-%m-%d")
-
-PARTITION_NAME = "{{ ds_nodash }}"
-
-INSERT_ROWS_QUERY = f"INSERT {DATASET_NAME}.{TABLE_NAME} VALUES " "(42, '{{ ds }}')"
-
-SCHEMA = [
-    {"name": "value", "type": "INTEGER", "mode": "REQUIRED"},
-    {"name": "ds", "type": "DATE", "mode": "NULLABLE"},
-]
-
-dag_id = "example_bigquery_sensors"
-
-with models.DAG(
-    dag_id,
-    schedule_interval='@once',  # Override to match your needs
-    start_date=datetime(2021, 1, 1),
-    catchup=False,
-    tags=["example"],
-    user_defined_macros={"DATASET": DATASET_NAME, "TABLE": TABLE_NAME},
-    default_args={"project_id": PROJECT_ID},
-) as dag_with_locations:
-    create_dataset = BigQueryCreateEmptyDatasetOperator(
-        task_id="create-dataset", dataset_id=DATASET_NAME, project_id=PROJECT_ID
-    )
-
-    create_table = BigQueryCreateEmptyTableOperator(
-        task_id="create_table",
-        dataset_id=DATASET_NAME,
-        table_id=TABLE_NAME,
-        schema_fields=SCHEMA,
-        time_partitioning={
-            "type": "DAY",
-            "field": "ds",
-        },
-    )
-    # [START howto_sensor_bigquery_table]
-    check_table_exists = BigQueryTableExistenceSensor(
-        task_id="check_table_exists", project_id=PROJECT_ID, dataset_id=DATASET_NAME, table_id=TABLE_NAME
-    )
-    # [END howto_sensor_bigquery_table]
-
-    execute_insert_query = BigQueryInsertJobOperator(
-        task_id="execute_insert_query",
-        configuration={
-            "query": {
-                "query": INSERT_ROWS_QUERY,
-                "useLegacySql": False,
-            }
-        },
-    )
-
-    # [START howto_sensor_bigquery_table_partition]
-    check_table_partition_exists = BigQueryTablePartitionExistenceSensor(
-        task_id="check_table_partition_exists",
-        project_id=PROJECT_ID,
-        dataset_id=DATASET_NAME,
-        table_id=TABLE_NAME,
-        partition_id=PARTITION_NAME,
-    )
-    # [END howto_sensor_bigquery_table_partition]
-
-    delete_dataset = BigQueryDeleteDatasetOperator(
-        task_id="delete_dataset", dataset_id=DATASET_NAME, delete_contents=True
-    )
-
-    create_dataset >> create_table
-    create_table >> check_table_exists
-    create_table >> execute_insert_query
-    execute_insert_query >> check_table_partition_exists
-    check_table_exists >> delete_dataset
-    check_table_partition_exists >> delete_dataset

From fc16df615bc02d5f29714b58bd64a4a6a23c3b89 Mon Sep 17 00:00:00 2001
From: Mateusz Nojek <matnojek@gmail.com>
Date: Mon, 14 Mar 2022 17:48:25 +0000
Subject: [PATCH 03/23] Rename test files with 'example_*' as a prefix

---
 .../operators/cloud/bigquery.rst              | 50 +++++++++----------
 pytest.ini                                    |  1 +
 tests/system/README.md                        |  2 +-
 tests/system/providers/google/README.md       | 10 ++--
 ...dataset.py => example_bigquery_dataset.py} |  0
 ...ions.py => example_bigquery_operations.py} |  0
 ...> example_bigquery_operations_location.py} |  0
 ...queries.py => example_bigquery_queries.py} |  0
 ...sensors.py => example_bigquery_sensors.py} |  0
 ...y_tables.py => example_bigquery_tables.py} |  0
 ...ery.py => example_bigquery_to_bigquery.py} |  0
 ...y_to_gcs.py => example_bigquery_to_gcs.py} |  0
 12 files changed, 32 insertions(+), 31 deletions(-)
 rename tests/system/providers/google/bigquery/{test_bigquery_dataset.py => example_bigquery_dataset.py} (100%)
 rename tests/system/providers/google/bigquery/{test_bigquery_operations.py => example_bigquery_operations.py} (100%)
 rename tests/system/providers/google/bigquery/{test_bigquery_operations_location.py => example_bigquery_operations_location.py} (100%)
 rename tests/system/providers/google/bigquery/{test_bigquery_queries.py => example_bigquery_queries.py} (100%)
 rename tests/system/providers/google/bigquery/{test_bigquery_sensors.py => example_bigquery_sensors.py} (100%)
 rename tests/system/providers/google/bigquery/{test_bigquery_tables.py => example_bigquery_tables.py} (100%)
 rename tests/system/providers/google/bigquery/{test_bigquery_to_bigquery.py => example_bigquery_to_bigquery.py} (100%)
 rename tests/system/providers/google/bigquery/{test_bigquery_to_gcs.py => example_bigquery_to_gcs.py} (100%)

diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
index 371cdca7a85ca..383c3ac0bb413 100644
--- a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
+++ b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
@@ -42,7 +42,7 @@ Create dataset
 To create an empty dataset in a BigQuery database you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyDatasetOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_dataset.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_dataset]
@@ -58,7 +58,7 @@ To get the details of an existing dataset you can use
 
 This operator returns a `Dataset Resource <https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource>`__.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_dataset.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_get_dataset]
@@ -72,7 +72,7 @@ List tables in dataset
 To retrieve the list of tables in a given dataset use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryGetDatasetTablesOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_get_dataset_tables]
@@ -89,7 +89,7 @@ To update a table in BigQuery you can use
 The update method replaces the entire Table resource, whereas the patch
 method only replaces fields that are provided in the submitted Table resource.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_update_table]
@@ -106,7 +106,7 @@ To update a dataset in BigQuery you can use
 The update method replaces the entire dataset resource, whereas the patch
 method only replaces fields that are provided in the submitted dataset resource.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_dataset.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_update_dataset]
@@ -120,7 +120,7 @@ Delete dataset
 To delete an existing dataset from a BigQuery database you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteDatasetOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_dataset.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_dataset]
@@ -143,7 +143,7 @@ ways. You may either directly pass the schema fields in, or you may point the
 operator to a Google Cloud Storage object name. The object in Google Cloud
 Storage must be a JSON file with the schema fields in it.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_table]
@@ -151,7 +151,7 @@ Storage must be a JSON file with the schema fields in it.
 
 You can use this operator to create a view on top of an existing table.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_view]
@@ -160,7 +160,7 @@ You can use this operator to create a view on top of an existing table.
 You can also use this operator to create a materialized view that periodically
 cache results of a query for increased performance and efficiency.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_materialized_view]
@@ -179,7 +179,7 @@ Similarly to
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyTableOperator`
 you can directly pass the schema fields in.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_operations.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_external_table]
@@ -187,7 +187,7 @@ you can directly pass the schema fields in.
 
 Or you may point the operator to a Google Cloud Storage object name where the schema is stored.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_table_schema_json]
@@ -208,7 +208,7 @@ returned list will be equal to the number of rows fetched. Each element in the
 list will again be a list where elements would represent the column values for
 that row.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_get_data]
@@ -225,7 +225,7 @@ To upsert a table you can use
 This operator either updates the existing table or creates a new, empty table
 in the given dataset.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_upsert_table]
@@ -242,7 +242,7 @@ To update the schema of a table you can use
 This operator updates the schema field values supplied, while leaving the rest unchanged. This is useful
 for instance to set new field descriptions on an existing table schema.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_update_table_schema]
@@ -256,7 +256,7 @@ Delete table
 To delete an existing table you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_table]
@@ -264,7 +264,7 @@ To delete an existing table you can use
 
 You can also use this operator to delete a view.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_view]
@@ -272,7 +272,7 @@ You can also use this operator to delete a view.
 
 You can also use this operator to delete a materialized view.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_materialized_view]
@@ -285,7 +285,7 @@ Execute BigQuery jobs
 
 Let's say you would like to execute the following query.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 0
     :start-after: [START howto_operator_bigquery_query]
@@ -295,7 +295,7 @@ To execute the SQL query in a specific BigQuery database you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryInsertJobOperator` with
 proper query job configuration that can be Jinja templated.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_insert_job]
@@ -307,7 +307,7 @@ For more information on types of BigQuery job please check
 If you want to include some files in your configuration you can use ``include`` clause of Jinja template
 language as follow:
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_select_job]
@@ -336,7 +336,7 @@ This operator expects a sql query that will return a single row. Each value on
 that first row is evaluated using python ``bool`` casting. If any of the values
 return ``False`` the check is failed and errors out.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_check]
@@ -354,7 +354,7 @@ This operator expects a sql query that will return a single row. Each value on
 that first row is evaluated against ``pass_value`` which can be either a string
 or numeric value. If numeric, you can also specify ``tolerance``.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_value_check]
@@ -369,7 +369,7 @@ To check that the values of metrics given as SQL expressions are within a certai
 tolerance of the ones from ``days_back`` before you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryIntervalCheckOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_interval_check]
@@ -387,7 +387,7 @@ use the ``{{ ds_nodash }}`` macro as the table name suffix.
 
 :class:`~airflow.providers.google.cloud.sensors.bigquery.BigQueryTableExistenceSensor`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_sensors.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_sensors.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_bigquery_table]
@@ -399,7 +399,7 @@ Check that a Table Partition exists
 To check that a table exists and has a partition you can use.
 :class:`~airflow.providers.google.cloud.sensors.bigquery.BigQueryTablePartitionExistenceSensor`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/test_bigquery_sensors.py
+.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_sensors.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_bigquery_table_partition]
diff --git a/pytest.ini b/pytest.ini
index 26e748291ed39..53004a28436f1 100644
--- a/pytest.ini
+++ b/pytest.ini
@@ -35,3 +35,4 @@ filterwarnings =
 markers =
     need_serialized_dag
 asyncio_mode = strict
+python_files = test_*.py example_*.py
diff --git a/tests/system/README.md b/tests/system/README.md
index 0da4c260a0d96..4c15e5a9e4e06 100644
--- a/tests/system/README.md
+++ b/tests/system/README.md
@@ -59,7 +59,7 @@ plugin/widget supporting pytest) or using following example command:
 
 ```commandline
 # pytest --system [provider_name] [path_to_test(s)]
-pytest --system google tests/system/providers/google/bigquery/test_bigquery_queries.py
+pytest --system google tests/system/providers/google/bigquery/example_bigquery_queries.py
 ```
 
 You can specify several --system flags if you want to execute tests for several providers:
diff --git a/tests/system/providers/google/README.md b/tests/system/providers/google/README.md
index 402d74783b5a9..eb2558087336f 100644
--- a/tests/system/providers/google/README.md
+++ b/tests/system/providers/google/README.md
@@ -25,21 +25,21 @@ All Google-related system tests are located inside this subdirectory of system t
 `tests/system/providers/google/`. They are grouped in directories by the related service name, e.g. all BigQuery
 tests are stored inside `tests/system/providers/google/bigquery/` directory. In each directory you will find test files
 as self-contained DAGs (one DAG per file). Each test may require some additional resources which should be placed in
-`resources` directory found on the same level as tests. Each test file should start with prefix `test_*`. If there
+`resources` directory found on the same level as tests. Each test file should start with prefix `example_*`. If there
 is anything more needed for the test to be executed, it should be documented in the docstrings.
 
 Example files structure:
 
-```─├│└
+```
 tests/system/providers/google
 ├── bigquery
 │   ├── resources
 │   │   ├── example_bigquery_query.sql
 │   │   └── us-states.csv
-│   ├── test_bigquery_queries.py
-│   ├── test_bigquery_operations.py
+│   ├── example_bigquery_queries.py
+│   ├── example_bigquery_operations.py
 .   .
-│   └── test_bigquery_*.py
+│   └── example_bigquery_*.py
 ├── dataflow
 ├── gcs
 .
diff --git a/tests/system/providers/google/bigquery/test_bigquery_dataset.py b/tests/system/providers/google/bigquery/example_bigquery_dataset.py
similarity index 100%
rename from tests/system/providers/google/bigquery/test_bigquery_dataset.py
rename to tests/system/providers/google/bigquery/example_bigquery_dataset.py
diff --git a/tests/system/providers/google/bigquery/test_bigquery_operations.py b/tests/system/providers/google/bigquery/example_bigquery_operations.py
similarity index 100%
rename from tests/system/providers/google/bigquery/test_bigquery_operations.py
rename to tests/system/providers/google/bigquery/example_bigquery_operations.py
diff --git a/tests/system/providers/google/bigquery/test_bigquery_operations_location.py b/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
similarity index 100%
rename from tests/system/providers/google/bigquery/test_bigquery_operations_location.py
rename to tests/system/providers/google/bigquery/example_bigquery_operations_location.py
diff --git a/tests/system/providers/google/bigquery/test_bigquery_queries.py b/tests/system/providers/google/bigquery/example_bigquery_queries.py
similarity index 100%
rename from tests/system/providers/google/bigquery/test_bigquery_queries.py
rename to tests/system/providers/google/bigquery/example_bigquery_queries.py
diff --git a/tests/system/providers/google/bigquery/test_bigquery_sensors.py b/tests/system/providers/google/bigquery/example_bigquery_sensors.py
similarity index 100%
rename from tests/system/providers/google/bigquery/test_bigquery_sensors.py
rename to tests/system/providers/google/bigquery/example_bigquery_sensors.py
diff --git a/tests/system/providers/google/bigquery/test_bigquery_tables.py b/tests/system/providers/google/bigquery/example_bigquery_tables.py
similarity index 100%
rename from tests/system/providers/google/bigquery/test_bigquery_tables.py
rename to tests/system/providers/google/bigquery/example_bigquery_tables.py
diff --git a/tests/system/providers/google/bigquery/test_bigquery_to_bigquery.py b/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
similarity index 100%
rename from tests/system/providers/google/bigquery/test_bigquery_to_bigquery.py
rename to tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
diff --git a/tests/system/providers/google/bigquery/test_bigquery_to_gcs.py b/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
similarity index 100%
rename from tests/system/providers/google/bigquery/test_bigquery_to_gcs.py
rename to tests/system/providers/google/bigquery/example_bigquery_to_gcs.py

From b97f75a05e26eaf2a304273fde691e39521aa5c4 Mon Sep 17 00:00:00 2001
From: Jarek Potiuk <jarek@potiuk.com>
Date: Mon, 14 Mar 2022 20:21:12 +0100
Subject: [PATCH 04/23] fixup! Rename test files with 'example_*' as a prefix

---
 .pre-commit-config.yaml                       |  7 ++
 BREEZE.rst                                    | 10 +--
 STATIC_CODE_CHECKS.rst                        |  2 +
 breeze-complete                               |  1 +
 .../src/airflow_breeze/pre_commit_ids.py      |  1 +
 .../pre_commit_check_watcher_in_examples.py   | 71 +++++++++++++++++++
 6 files changed, 87 insertions(+), 5 deletions(-)
 create mode 100755 scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 9bc797137185c..edcee32a87abe 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -575,6 +575,13 @@ repos:
         pass_filenames: false
         require_serial: true
         additional_dependencies: ['rich']
+      - id: check-watcher-in-examples
+        name: Check if watcher is properly used in examples
+        entry: ./scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
+        language: python
+        files: ^tests/system/.*/example_[^/]*.py$
+        pass_filenames: true
+        additional_dependencies: ['rich']
       - id: markdownlint
         name: Run markdownlint
         description: Checks the style of Markdown files.
diff --git a/BREEZE.rst b/BREEZE.rst
index 3ce3068a12589..e2edab3aee55b 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -2292,11 +2292,11 @@ This is the current syntax for  `./breeze <./breeze>`_:
                  build-providers-dependencies chart-schema-lint capitalized-breeze
                  changelog-duplicates check-apache-license check-builtin-literals
                  check-executables-have-shebangs check-extras-order check-hooks-apply
-                 check-integrations check-merge-conflict check-xml daysago-import-check
-                 debug-statements detect-private-key docstring-params doctoc dont-use-safe-filter
-                 end-of-file-fixer fix-encoding-pragma flake8 flynt codespell forbid-tabs helm-lint
-                 identity incorrect-use-of-LoggingMixin insert-license isort json-schema
-                 language-matters lint-dockerfile lint-openapi markdownlint mermaid
+                 check-integrations check-merge-conflict check-xml check-watcher-in-examples
+                 daysago-import-check debug-statements detect-private-key docstring-params doctoc
+                 dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 flynt codespell
+                 forbid-tabs helm-lint identity incorrect-use-of-LoggingMixin insert-license isort
+                 json-schema language-matters lint-dockerfile lint-openapi markdownlint mermaid
                  migration-reference mixed-line-ending mypy mypy-helm no-providers-in-core-examples
                  no-relative-imports persist-credentials-disabled pre-commit-descriptions
                  pre-commit-hook-names pretty-format-json provide-create-sessions
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 087b0716a5f6a..8ea4fdbf469f6 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -166,6 +166,8 @@ require Breeze Docker images to be installed locally.
 ------------------------------------ ---------------------------------------------------------------- ------------
 ``check-xml``                          Checks XML files with xmllint
 ------------------------------------ ---------------------------------------------------------------- ------------
+``check-watcher-in-examples``          Check if watcher is properly used in examples
+------------------------------------ ---------------------------------------------------------------- ------------
 ``daysago-import-check``               Checks if daysago is properly imported
 ------------------------------------ ---------------------------------------------------------------- ------------
 ``debug-statements``                   Detects accidentally committed debug statements
diff --git a/breeze-complete b/breeze-complete
index a3b8f1bcc66d4..b16fb3512cbfd 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -98,6 +98,7 @@ check-hooks-apply
 check-integrations
 check-merge-conflict
 check-xml
+check-watcher-in-examples
 daysago-import-check
 debug-statements
 detect-private-key
diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
index 4786fce4a83ad..fc020a6e3229b 100644
--- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
+++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
@@ -40,6 +40,7 @@
     'check-hooks-apply',
     'check-integrations',
     'check-merge-conflict',
+    'check-watcher-in-examples',
     'check-xml',
     'codespell',
     'daysago-import-check',
diff --git a/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py b/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
new file mode 100755
index 0000000000000..cb7d1458d66a8
--- /dev/null
+++ b/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
@@ -0,0 +1,71 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import sys
+from pathlib import Path
+from typing import List
+
+from rich.console import Console
+
+if __name__ not in ("__main__", "__mp_main__"):
+    raise SystemExit(
+        "This file is intended to be executed as an executable program. You cannot use it as a module."
+        f"To run this script, run the ./{__file__} command [FILE] ..."
+    )
+
+
+console = Console(color_system="standard", width=200)
+
+errors: List[str] = []
+
+WATCHER_APPEND_INSTRUCTION = "list(dag.tasks) >> watcher()"
+
+
+def _check_file(file: Path):
+    content = file.read_text()
+    if "from tests.system.utils.watcher import watcher" in content:
+        index = content.find(WATCHER_APPEND_INSTRUCTION)
+        if index == -1:
+            errors.append(
+                f"[red]The example {file} imports tests.system.utils.watcher "
+                f"but does not use it properly![/]\n\n"
+                "[yellow]Make sure you have:[/]\n\n"
+                f"        {WATCHER_APPEND_INSTRUCTION}\n\n"
+                "[yellow]as last instruction in your example DAG.[/]\n"
+            )
+        else:
+            operator_leftshift_index = content.find("<<", index + len(WATCHER_APPEND_INSTRUCTION))
+            operator_rightshift_index = content.find(">>", index + len(WATCHER_APPEND_INSTRUCTION))
+            if operator_leftshift_index != -1 or operator_rightshift_index != -1:
+                errors.append(
+                    f"[red]In the  example {file} "
+                    f"watcher is not last instruction in your DAG (there are << "
+                    f"or >> operators after it)![/]\n\n"
+                    "[yellow]Make sure you have:[/]\n"
+                    f"        {WATCHER_APPEND_INSTRUCTION}\n\n"
+                    "[yellow]as last instruction in your example DAG.[/]\n"
+                )
+
+
+if __name__ == '__main__':
+    for file in sys.argv[1:]:
+        _check_file(Path(file))
+    if errors:
+        console.print("[red]There were some errors in the example files[/]\n")
+        for error in errors:
+            console.print(error)
+        sys.exit(1)

From ba9a3894a97cbedb99cc4ae7360387f10adf3163 Mon Sep 17 00:00:00 2001
From: Jarek Potiuk <jarek@potiuk.com>
Date: Mon, 14 Mar 2022 20:44:24 +0100
Subject: [PATCH 05/23] fixup! fixup! Rename test files with 'example_*' as a
 prefix

---
 .../pre_commit_check_watcher_in_examples.py       | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py b/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
index cb7d1458d66a8..34330dd6cc560 100755
--- a/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
+++ b/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
@@ -34,6 +34,14 @@
 
 WATCHER_APPEND_INSTRUCTION = "list(dag.tasks) >> watcher()"
 
+PYTEST_FUNCTION = """
+def test_run():
+    from airflow.utils.state import State
+
+    dag.clear(dag_run_state=State.NONE)
+    dag.run()
+"""
+
 
 def _check_file(file: Path):
     content = file.read_text()
@@ -59,6 +67,13 @@ def _check_file(file: Path):
                     f"        {WATCHER_APPEND_INSTRUCTION}\n\n"
                     "[yellow]as last instruction in your example DAG.[/]\n"
                 )
+        if PYTEST_FUNCTION not in content:
+            errors.append(
+                f"[yellow]The  example {file} missed the pytest function at the end.[/]\n\n"
+                "All example tests should have this function added:\n\n" + PYTEST_FUNCTION + "\n\n"
+                "[yellow]Automatically adding it now!\n"
+            )
+            file.write_text(content + "\n" + PYTEST_FUNCTION)
 
 
 if __name__ == '__main__':

From abfef7df467ca0e1170f2bd2cd21e1c78b0ac346 Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Tue, 15 Mar 2022 10:08:49 +0000
Subject: [PATCH 06/23] Fix always tests to work with example_* prefix

Change-Id: I7a9c93d0ecc7a12edca933ea693c9219e9e3a5b5
---
 tests/always/test_example_dags.py      | 43 +++++++++++++-------------
 tests/always/test_project_structure.py |  2 +-
 2 files changed, 23 insertions(+), 22 deletions(-)

diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py
index fc20ab48c935b..81208d0bd0718 100644
--- a/tests/always/test_example_dags.py
+++ b/tests/always/test_example_dags.py
@@ -31,7 +31,7 @@
 
 
 def example_dags():
-    example_dirs = ["airflow/**/example_dags/example_*.py", "tests/system/providers/**/test_*.py"]
+    example_dirs = ["airflow/**/example_dags/example_*.py", "tests/system/providers/**/example_*.py"]
     for example_dir in example_dirs:
         yield from glob(f"{ROOT_FOLDER}/{example_dir}", recursive=True)
 
@@ -55,27 +55,28 @@ def relative_path(path):
     return os.path.relpath(path, ROOT_FOLDER)
 
 
-class TestExampleDags:
-    @pytest.fixture(autouse=True, scope="session")
-    def required_env_vars(self):
-        # Common
-        set_env_vars("SYSTEM_TESTS_ENV_ID")
-        # Google
-        set_env_vars("SYSTEM_TESTS_GCP_PROJECT", "SYSTEM_TESTS_GCP_SERVICE_ACCOUNT")
+@pytest.fixture(autouse=True, scope="session")
+def required_env_vars():
+    # Common
+    set_env_vars("SYSTEM_TESTS_ENV_ID")
+    # Google
+    set_env_vars("SYSTEM_TESTS_GCP_PROJECT", "SYSTEM_TESTS_GCP_SERVICE_ACCOUNT")
 
-    @pytest.mark.parametrize("example", list(example_dags()), ids=relative_path)
-    def test_should_be_importable(self, example):
-        dagbag = DagBag(
+
+@pytest.mark.parametrize("example", list(example_dags()), ids=relative_path)
+def test_should_be_importable(example):
+    dagbag = DagBag(
+        dag_folder=example,
+        include_examples=False,
+    )
+    assert 0 == len(dagbag.import_errors), f"import_errors={str(dagbag.import_errors)}"
+    assert len(dagbag.dag_ids) >= 1
+
+
+@pytest.mark.parametrize("example", example_dags_except_db_exception(), ids=relative_path)
+def test_should_not_do_database_queries(example):
+    with assert_queries_count(0):
+        DagBag(
             dag_folder=example,
             include_examples=False,
         )
-        assert 0 == len(dagbag.import_errors), f"import_errors={str(dagbag.import_errors)}"
-        assert len(dagbag.dag_ids) >= 1
-
-    @pytest.mark.parametrize("example", example_dags_except_db_exception(), ids=relative_path)
-    def test_should_not_do_database_queries(self, example):
-        with assert_queries_count(0):
-            DagBag(
-                dag_folder=example,
-                include_examples=False,
-            )
diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py
index 89bfe9521ca1c..2c839fad017ef 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -285,7 +285,7 @@ def examples_for_service(service_name):
         yield from glob.glob(
             f"{ROOT_FOLDER}/airflow/providers/google/*/example_dags/example_{service_name}*.py"
         )
-        yield from glob.glob(f"{ROOT_FOLDER}/tests/system/providers/google/{service_name}/test_*.py")
+        yield from glob.glob(f"{ROOT_FOLDER}/tests/system/providers/google/{service_name}/example_*.py")
 
 
 class TestOperatorsHooks(unittest.TestCase):

From d9106a145653627b8bcbcfcae49c158d53dfbd96 Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Tue, 15 Mar 2022 13:26:23 +0000
Subject: [PATCH 07/23] Skip tests if required env vars are not defined

Change-Id: Iee9cecaf2f1991c48ead28e91e384c55af6271d1
---
 tests/always/test_example_dags.py             | 15 ------
 tests/system/conftest.py                      | 26 ++++++++--
 .../bigquery/example_bigquery_dataset.py      |  2 +-
 .../bigquery/example_bigquery_operations.py   |  2 +-
 .../example_bigquery_operations_location.py   |  2 +-
 .../bigquery/example_bigquery_queries.py      |  4 +-
 .../bigquery/example_bigquery_sensors.py      |  4 +-
 .../bigquery/example_bigquery_tables.py       |  4 +-
 .../bigquery/example_bigquery_to_bigquery.py  |  2 +-
 .../bigquery/example_bigquery_to_gcs.py       |  4 +-
 tests/system/providers/google/conftest.py     | 52 +++++++++++++++++++
 11 files changed, 87 insertions(+), 30 deletions(-)
 create mode 100644 tests/system/providers/google/conftest.py

diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py
index 81208d0bd0718..4d05207f37082 100644
--- a/tests/always/test_example_dags.py
+++ b/tests/always/test_example_dags.py
@@ -44,25 +44,10 @@ def example_dags_except_db_exception():
     ]
 
 
-def set_env_vars(*args):
-    """Mocks required ENV variables so it's possible to import example dags"""
-    for arg in args:
-        if arg not in os.environ:
-            os.environ[arg] = "PLACEHOLDER"
-
-
 def relative_path(path):
     return os.path.relpath(path, ROOT_FOLDER)
 
 
-@pytest.fixture(autouse=True, scope="session")
-def required_env_vars():
-    # Common
-    set_env_vars("SYSTEM_TESTS_ENV_ID")
-    # Google
-    set_env_vars("SYSTEM_TESTS_GCP_PROJECT", "SYSTEM_TESTS_GCP_SERVICE_ACCOUNT")
-
-
 @pytest.mark.parametrize("example", list(example_dags()), ids=relative_path)
 def test_should_be_importable(example):
     dagbag = DagBag(
diff --git a/tests/system/conftest.py b/tests/system/conftest.py
index 3f9ccce5e6bf5..0ee3a5342843e 100644
--- a/tests/system/conftest.py
+++ b/tests/system/conftest.py
@@ -22,6 +22,7 @@
 import pytest
 
 OLD_EXECUTOR = os.environ.get("AIRFLOW__CORE__EXECUTOR", default=None)
+REQUIRED_ENV = ("SYSTEM_TESTS_ENV_ID",)
 
 
 def pytest_configure(config):
@@ -33,15 +34,34 @@ def pytest_unconfigure(config):
         os.environ["AIRFLOW__CORE__EXECUTOR"] = OLD_EXECUTOR
 
 
+def is_env_not_ready():
+    """Check if all required environments are present. If  return error message to be used in skip marker"""
+    for env in REQUIRED_ENV:
+        if env not in os.environ:
+            return f"Missing required environment variable {env}"
+    return ""
+
+
+def is_system_mark_present(item, provider):
+    """Don't mark with missing env variables if system mark is missing"""
+    selected_systems = item.config.getoption("--system")
+    return selected_systems and (provider in selected_systems or "all" in selected_systems)
+
+
 def pytest_collection_modifyitems(config, items):
-    """Add @pytest.mark.system(provider_name) for every system test"""
+    """
+    Add @pytest.mark.system(provider_name) for every system test.
+    Skip tests if environment is not configured
+    """
+    not_ready = is_env_not_ready()
+    skip = pytest.mark.skip(reason=not_ready)
     rootdir = Path(config.rootdir)
     for item in items:
-        if item.get_closest_marker("system"):  # already marked
-            continue
         rel_path = Path(item.fspath).relative_to(rootdir)
         match = re.match(".+/providers/([^/]+)", str(rel_path))
         if not match:
             continue
         provider = match.group(1)
         item.add_marker(pytest.mark.system(provider))
+        if not_ready and is_system_mark_present(item, provider):
+            item.add_marker(skip)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_dataset.py b/tests/system/providers/google/bigquery/example_bigquery_dataset.py
index 70edcca65d475..26ed04b34fb5f 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_dataset.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_dataset.py
@@ -33,7 +33,7 @@
 from airflow.utils.trigger_rule import TriggerRule
 from tests.system.utils.watcher import watcher
 
-ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_dataset"
 
 DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
diff --git a/tests/system/providers/google/bigquery/example_bigquery_operations.py b/tests/system/providers/google/bigquery/example_bigquery_operations.py
index 54534bd591782..84203f2039e3f 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_operations.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_operations.py
@@ -34,7 +34,7 @@
 from airflow.utils.trigger_rule import TriggerRule
 from tests.system.utils.watcher import watcher
 
-ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_operations"
 
 DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
diff --git a/tests/system/providers/google/bigquery/example_bigquery_operations_location.py b/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
index 09ba3e349be00..c0f7c93c2f221 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
@@ -31,7 +31,7 @@
 from airflow.utils.trigger_rule import TriggerRule
 from tests.system.utils.watcher import watcher
 
-ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_operations_location"
 
 BQ_LOCATION = "europe-north1"
diff --git a/tests/system/providers/google/bigquery/example_bigquery_queries.py b/tests/system/providers/google/bigquery/example_bigquery_queries.py
index 9d84976c2ddff..685197c0758a6 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_queries.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_queries.py
@@ -38,8 +38,8 @@
 from airflow.utils.trigger_rule import TriggerRule
 from tests.system.utils.watcher import watcher
 
-ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
-PROJECT_ID = os.environ["SYSTEM_TESTS_GCP_PROJECT"]
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
+PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
 LOCATION = "us-east1"
 QUERY_SQL_PATH = str(Path(__file__).parent / "resources" / "example_bigquery_query.sql")
 
diff --git a/tests/system/providers/google/bigquery/example_bigquery_sensors.py b/tests/system/providers/google/bigquery/example_bigquery_sensors.py
index efe2c5390bc02..0200aec616214 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_sensors.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_sensors.py
@@ -36,11 +36,11 @@
 from airflow.utils.trigger_rule import TriggerRule
 from tests.system.utils.watcher import watcher
 
-ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_sensors"
 
 DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
-PROJECT_ID = os.environ["SYSTEM_TESTS_GCP_PROJECT"]
+PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT", "")
 
 TABLE_NAME = "partitioned_table"
 INSERT_DATE = datetime.now().strftime("%Y-%m-%d")
diff --git a/tests/system/providers/google/bigquery/example_bigquery_tables.py b/tests/system/providers/google/bigquery/example_bigquery_tables.py
index 3338f08f45593..048766efb58ef 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_tables.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_tables.py
@@ -41,12 +41,12 @@
 from airflow.utils.trigger_rule import TriggerRule
 from tests.system.utils.watcher import watcher
 
-ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_tables"
 
 BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
 DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
-PROJECT_ID = os.environ["SYSTEM_TESTS_GCP_PROJECT"]
+PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
 SCHEMA_JSON_LOCAL_SRC = str(Path(__file__).parent / "resources" / "update_table_schema.json")
 SCHEMA_JSON_DESTINATION = "update_table_schema.json"
 GCS_PATH_TO_SCHEMA_JSON = f"gs://{BUCKET_NAME}/{SCHEMA_JSON_DESTINATION}"
diff --git a/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py b/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
index 1084ba0e2d77a..991c7413bc9e6 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
@@ -32,7 +32,7 @@
 from airflow.utils.trigger_rule import TriggerRule
 from tests.system.utils.watcher import watcher
 
-ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_to_bigquery"
 
 DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
diff --git a/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py b/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
index 9c99580cebd39..57a437c001fe3 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
@@ -33,12 +33,12 @@
 from airflow.utils.trigger_rule import TriggerRule
 from tests.system.utils.watcher import watcher
 
-ENV_ID = os.environ["SYSTEM_TESTS_ENV_ID"]
+ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_to_gcs"
 
 DATASET_NAME = f"dataset_{DAG_ID}_{ENV_ID}"
 BUCKET_NAME = f"bucket_{DAG_ID}_{ENV_ID}"
-PROJECT_ID = os.environ["SYSTEM_TESTS_GCP_PROJECT"]
+PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
 BUCKET_FILE = "test.csv"
 TABLE = "test"
 
diff --git a/tests/system/providers/google/conftest.py b/tests/system/providers/google/conftest.py
new file mode 100644
index 0000000000000..a0d85d1b09bed
--- /dev/null
+++ b/tests/system/providers/google/conftest.py
@@ -0,0 +1,52 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+from pathlib import Path
+
+import pytest
+
+PROVIDER = Path(__file__).parent.name
+REQUIRED_ENV = ("SYSTEM_TESTS_GCP_PROJECT",)
+
+
+def is_env_not_ready():
+    """Check if all required environments are present. If  return error message to be used in skip marker"""
+    for env in REQUIRED_ENV:
+        if env not in os.environ:
+            return f"Missing required environment variable {env}"
+    return ""
+
+
+def is_system_mark_present(item):
+    """Don't mark with missing env variables if system mark is missing"""
+    selected_systems = item.config.getoption("--system")
+    return selected_systems and (PROVIDER in selected_systems or "all" in selected_systems)
+
+
+def pytest_collection_modifyitems(config, items):
+    """
+    Skip tests if environment is not configured
+    """
+    not_ready = is_env_not_ready()
+    if not not_ready:
+        return
+    skip = pytest.mark.skip(reason=not_ready)
+    for item in items:
+        system_mark_present = is_system_mark_present(item)
+        if system_mark_present and not_ready:
+            item.add_marker(skip)

From 8342b978db07ee2d04481daaea973fbfcfc4177e Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Tue, 15 Mar 2022 13:41:18 +0000
Subject: [PATCH 08/23] Fix paths to example dags in docs

Change-Id: I1479b3cad103a710c7a6ff5d218f68fed98fe20a
---
 .../operators/cloud/bigquery.rst              | 50 +++++++++----------
 tests/system/conftest.py                      |  4 +-
 tests/system/providers/google/conftest.py     |  8 ++-
 3 files changed, 30 insertions(+), 32 deletions(-)

diff --git a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
index 383c3ac0bb413..5cab46e37b6d6 100644
--- a/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
+++ b/docs/apache-airflow-providers-google/operators/cloud/bigquery.rst
@@ -42,7 +42,7 @@ Create dataset
 To create an empty dataset in a BigQuery database you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyDatasetOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_dataset.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_dataset]
@@ -58,7 +58,7 @@ To get the details of an existing dataset you can use
 
 This operator returns a `Dataset Resource <https://cloud.google.com/bigquery/docs/reference/rest/v2/datasets#resource>`__.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_dataset.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_get_dataset]
@@ -72,7 +72,7 @@ List tables in dataset
 To retrieve the list of tables in a given dataset use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryGetDatasetTablesOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_get_dataset_tables]
@@ -89,7 +89,7 @@ To update a table in BigQuery you can use
 The update method replaces the entire Table resource, whereas the patch
 method only replaces fields that are provided in the submitted Table resource.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_update_table]
@@ -106,7 +106,7 @@ To update a dataset in BigQuery you can use
 The update method replaces the entire dataset resource, whereas the patch
 method only replaces fields that are provided in the submitted dataset resource.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_dataset.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_update_dataset]
@@ -120,7 +120,7 @@ Delete dataset
 To delete an existing dataset from a BigQuery database you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteDatasetOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_dataset.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_dataset.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_dataset]
@@ -143,7 +143,7 @@ ways. You may either directly pass the schema fields in, or you may point the
 operator to a Google Cloud Storage object name. The object in Google Cloud
 Storage must be a JSON file with the schema fields in it.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_table]
@@ -151,7 +151,7 @@ Storage must be a JSON file with the schema fields in it.
 
 You can use this operator to create a view on top of an existing table.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_view]
@@ -160,7 +160,7 @@ You can use this operator to create a view on top of an existing table.
 You can also use this operator to create a materialized view that periodically
 cache results of a query for increased performance and efficiency.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_materialized_view]
@@ -179,7 +179,7 @@ Similarly to
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryCreateEmptyTableOperator`
 you can directly pass the schema fields in.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_operations.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_operations.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_external_table]
@@ -187,7 +187,7 @@ you can directly pass the schema fields in.
 
 Or you may point the operator to a Google Cloud Storage object name where the schema is stored.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_create_table_schema_json]
@@ -208,7 +208,7 @@ returned list will be equal to the number of rows fetched. Each element in the
 list will again be a list where elements would represent the column values for
 that row.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_get_data]
@@ -225,7 +225,7 @@ To upsert a table you can use
 This operator either updates the existing table or creates a new, empty table
 in the given dataset.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_upsert_table]
@@ -242,7 +242,7 @@ To update the schema of a table you can use
 This operator updates the schema field values supplied, while leaving the rest unchanged. This is useful
 for instance to set new field descriptions on an existing table schema.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_update_table_schema]
@@ -256,7 +256,7 @@ Delete table
 To delete an existing table you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryDeleteTableOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_table]
@@ -264,7 +264,7 @@ To delete an existing table you can use
 
 You can also use this operator to delete a view.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_view]
@@ -272,7 +272,7 @@ You can also use this operator to delete a view.
 
 You can also use this operator to delete a materialized view.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_tables.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_tables.py
     :language: python
     :dedent: 4
     :start-after: [START howto_operator_bigquery_delete_materialized_view]
@@ -285,7 +285,7 @@ Execute BigQuery jobs
 
 Let's say you would like to execute the following query.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 0
     :start-after: [START howto_operator_bigquery_query]
@@ -295,7 +295,7 @@ To execute the SQL query in a specific BigQuery database you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryInsertJobOperator` with
 proper query job configuration that can be Jinja templated.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_insert_job]
@@ -307,7 +307,7 @@ For more information on types of BigQuery job please check
 If you want to include some files in your configuration you can use ``include`` clause of Jinja template
 language as follow:
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_select_job]
@@ -336,7 +336,7 @@ This operator expects a sql query that will return a single row. Each value on
 that first row is evaluated using python ``bool`` casting. If any of the values
 return ``False`` the check is failed and errors out.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_check]
@@ -354,7 +354,7 @@ This operator expects a sql query that will return a single row. Each value on
 that first row is evaluated against ``pass_value`` which can be either a string
 or numeric value. If numeric, you can also specify ``tolerance``.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_value_check]
@@ -369,7 +369,7 @@ To check that the values of metrics given as SQL expressions are within a certai
 tolerance of the ones from ``days_back`` before you can use
 :class:`~airflow.providers.google.cloud.operators.bigquery.BigQueryIntervalCheckOperator`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_queries.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_queries.py
     :language: python
     :dedent: 8
     :start-after: [START howto_operator_bigquery_interval_check]
@@ -387,7 +387,7 @@ use the ``{{ ds_nodash }}`` macro as the table name suffix.
 
 :class:`~airflow.providers.google.cloud.sensors.bigquery.BigQueryTableExistenceSensor`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_sensors.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_sensors.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_bigquery_table]
@@ -399,7 +399,7 @@ Check that a Table Partition exists
 To check that a table exists and has a partition you can use.
 :class:`~airflow.providers.google.cloud.sensors.bigquery.BigQueryTablePartitionExistenceSensor`.
 
-.. exampleinclude:: /../../tests/system/google/bigquery/example_bigquery_sensors.py
+.. exampleinclude:: /../../tests/system/providers/google/bigquery/example_bigquery_sensors.py
     :language: python
     :dedent: 4
     :start-after: [START howto_sensor_bigquery_table_partition]
diff --git a/tests/system/conftest.py b/tests/system/conftest.py
index 0ee3a5342843e..e6066acb526ee 100644
--- a/tests/system/conftest.py
+++ b/tests/system/conftest.py
@@ -22,7 +22,7 @@
 import pytest
 
 OLD_EXECUTOR = os.environ.get("AIRFLOW__CORE__EXECUTOR", default=None)
-REQUIRED_ENV = ("SYSTEM_TESTS_ENV_ID",)
+REQUIRED_ENV_VARS = ("SYSTEM_TESTS_ENV_ID",)
 
 
 def pytest_configure(config):
@@ -36,7 +36,7 @@ def pytest_unconfigure(config):
 
 def is_env_not_ready():
     """Check if all required environments are present. If  return error message to be used in skip marker"""
-    for env in REQUIRED_ENV:
+    for env in REQUIRED_ENV_VARS:
         if env not in os.environ:
             return f"Missing required environment variable {env}"
     return ""
diff --git a/tests/system/providers/google/conftest.py b/tests/system/providers/google/conftest.py
index a0d85d1b09bed..fcc637b969444 100644
--- a/tests/system/providers/google/conftest.py
+++ b/tests/system/providers/google/conftest.py
@@ -21,12 +21,12 @@
 import pytest
 
 PROVIDER = Path(__file__).parent.name
-REQUIRED_ENV = ("SYSTEM_TESTS_GCP_PROJECT",)
+REQUIRED_ENV_VARS = ("SYSTEM_TESTS_GCP_PROJECT",)
 
 
 def is_env_not_ready():
     """Check if all required environments are present. If  return error message to be used in skip marker"""
-    for env in REQUIRED_ENV:
+    for env in REQUIRED_ENV_VARS:
         if env not in os.environ:
             return f"Missing required environment variable {env}"
     return ""
@@ -39,9 +39,7 @@ def is_system_mark_present(item):
 
 
 def pytest_collection_modifyitems(config, items):
-    """
-    Skip tests if environment is not configured
-    """
+    """Skip tests if environment is not configured"""
     not_ready = is_env_not_ready()
     if not not_ready:
         return

From 35689e4515fccb0653679a09c88de70ea05a51f7 Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Tue, 15 Mar 2022 13:58:21 +0000
Subject: [PATCH 09/23] Use autofixture to skip tests with missing env vars

Change-Id: I67005b1bc5f5d0170a1436fba8b324d419912303
---
 tests/system/conftest.py                  | 25 ++++++++----------
 tests/system/providers/google/conftest.py | 31 +++--------------------
 2 files changed, 14 insertions(+), 42 deletions(-)

diff --git a/tests/system/conftest.py b/tests/system/conftest.py
index e6066acb526ee..ac475e761a41b 100644
--- a/tests/system/conftest.py
+++ b/tests/system/conftest.py
@@ -17,6 +17,7 @@
 
 import os
 import re
+from itertools import chain
 from pathlib import Path
 
 import pytest
@@ -34,18 +35,18 @@ def pytest_unconfigure(config):
         os.environ["AIRFLOW__CORE__EXECUTOR"] = OLD_EXECUTOR
 
 
-def is_env_not_ready():
-    """Check if all required environments are present. If  return error message to be used in skip marker"""
-    for env in REQUIRED_ENV_VARS:
-        if env not in os.environ:
-            return f"Missing required environment variable {env}"
-    return ""
+@pytest.fixture()
+def provider_env_vars():
+    """Override this fixture in provider's conftest.py"""
+    return ()
 
 
-def is_system_mark_present(item, provider):
-    """Don't mark with missing env variables if system mark is missing"""
-    selected_systems = item.config.getoption("--system")
-    return selected_systems and (provider in selected_systems or "all" in selected_systems)
+@pytest.fixture(autouse=True)
+def skip_if_env_var_not_set(provider_env_vars):
+    for env in chain(REQUIRED_ENV_VARS, provider_env_vars):
+        if env not in os.environ:
+            pytest.skip(f"Missing required environment variable {env}")
+            return
 
 
 def pytest_collection_modifyitems(config, items):
@@ -53,8 +54,6 @@ def pytest_collection_modifyitems(config, items):
     Add @pytest.mark.system(provider_name) for every system test.
     Skip tests if environment is not configured
     """
-    not_ready = is_env_not_ready()
-    skip = pytest.mark.skip(reason=not_ready)
     rootdir = Path(config.rootdir)
     for item in items:
         rel_path = Path(item.fspath).relative_to(rootdir)
@@ -63,5 +62,3 @@ def pytest_collection_modifyitems(config, items):
             continue
         provider = match.group(1)
         item.add_marker(pytest.mark.system(provider))
-        if not_ready and is_system_mark_present(item, provider):
-            item.add_marker(skip)
diff --git a/tests/system/providers/google/conftest.py b/tests/system/providers/google/conftest.py
index fcc637b969444..bb3aa4f981f38 100644
--- a/tests/system/providers/google/conftest.py
+++ b/tests/system/providers/google/conftest.py
@@ -15,36 +15,11 @@
 # specific language governing permissions and limitations
 # under the License.
 
-import os
-from pathlib import Path
-
 import pytest
 
-PROVIDER = Path(__file__).parent.name
 REQUIRED_ENV_VARS = ("SYSTEM_TESTS_GCP_PROJECT",)
 
 
-def is_env_not_ready():
-    """Check if all required environments are present. If  return error message to be used in skip marker"""
-    for env in REQUIRED_ENV_VARS:
-        if env not in os.environ:
-            return f"Missing required environment variable {env}"
-    return ""
-
-
-def is_system_mark_present(item):
-    """Don't mark with missing env variables if system mark is missing"""
-    selected_systems = item.config.getoption("--system")
-    return selected_systems and (PROVIDER in selected_systems or "all" in selected_systems)
-
-
-def pytest_collection_modifyitems(config, items):
-    """Skip tests if environment is not configured"""
-    not_ready = is_env_not_ready()
-    if not not_ready:
-        return
-    skip = pytest.mark.skip(reason=not_ready)
-    for item in items:
-        system_mark_present = is_system_mark_present(item)
-        if system_mark_present and not_ready:
-            item.add_marker(skip)
+@pytest.fixture()
+def provider_env_vars():
+    return REQUIRED_ENV_VARS

From ec25fea94aa2ee8f90e49bc7dd06c185d366732b Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Tue, 15 Mar 2022 14:05:57 +0000
Subject: [PATCH 10/23] Remove deprecated doc string

Change-Id: I935c4de2ca5d36c2b9b8ddbab80af0b50e4226d6
---
 tests/system/conftest.py | 5 +----
 1 file changed, 1 insertion(+), 4 deletions(-)

diff --git a/tests/system/conftest.py b/tests/system/conftest.py
index ac475e761a41b..15dfec49477f9 100644
--- a/tests/system/conftest.py
+++ b/tests/system/conftest.py
@@ -50,10 +50,7 @@ def skip_if_env_var_not_set(provider_env_vars):
 
 
 def pytest_collection_modifyitems(config, items):
-    """
-    Add @pytest.mark.system(provider_name) for every system test.
-    Skip tests if environment is not configured
-    """
+    """Add @pytest.mark.system(provider_name) for every system test."""
     rootdir = Path(config.rootdir)
     for item in items:
         rel_path = Path(item.fspath).relative_to(rootdir)

From 7acc96e542ead837954e9ae07374fdd7c33c9356 Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Wed, 16 Mar 2022 09:12:24 +0000
Subject: [PATCH 11/23] Auto-format BREEZE.rst

Change-Id: Ice6fa8f33530d842f78df95aba1e3d6d72ec9697
---
 BREEZE.rst | 18 +++++++++---------
 1 file changed, 9 insertions(+), 9 deletions(-)

diff --git a/BREEZE.rst b/BREEZE.rst
index c9eb100f40e71..17e3c55b1647c 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -2292,15 +2292,15 @@ This is the current syntax for  `./breeze <./breeze>`_:
                  build-providers-dependencies chart-schema-lint capitalized-breeze
                  changelog-duplicates check-apache-license check-builtin-literals
                  check-executables-have-shebangs check-extras-order check-hooks-apply
-                 check-integrations check-merge-conflict check-xml check-watcher-in-examples daysago-import-check
-                 debug-statements detect-private-key docstring-params doctoc dont-use-safe-filter
-                 end-of-file-fixer fix-encoding-pragma flake8 flynt forbidden-xcom-get-value
-                 codespell forbid-tabs helm-lint identity incorrect-use-of-LoggingMixin
-                 insert-license isort json-schema language-matters lint-dockerfile lint-openapi
-                 markdownlint mermaid migration-reference mixed-line-ending mypy mypy-helm
-                 no-providers-in-core-examples no-relative-imports persist-credentials-disabled
-                 pre-commit-descriptions pre-commit-hook-names pretty-format-json
-                 provide-create-sessions providers-changelogs providers-init-file
+                 check-integrations check-merge-conflict check-xml check-watcher-in-examples
+                 daysago-import-check debug-statements detect-private-key docstring-params doctoc
+                 dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 flynt
+                 forbidden-xcom-get-value codespell forbid-tabs helm-lint identity
+                 incorrect-use-of-LoggingMixin insert-license isort json-schema language-matters
+                 lint-dockerfile lint-openapi markdownlint mermaid migration-reference
+                 mixed-line-ending mypy mypy-helm no-providers-in-core-examples no-relative-imports
+                 persist-credentials-disabled pre-commit-descriptions pre-commit-hook-names
+                 pretty-format-json provide-create-sessions providers-changelogs providers-init-file
                  providers-subpackages-init-file provider-yamls pydevd pydocstyle python-no-log-warn
                  pyupgrade restrict-start_date rst-backticks setup-order setup-extra-packages
                  shellcheck sort-in-the-wild sort-spelling-wordlist stylelint trailing-whitespace

From 28c07c347112d66cb6460d646ee1dfb7ca40a5cd Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Wed, 16 Mar 2022 09:13:41 +0000
Subject: [PATCH 12/23] Fix missing documentation tag end

Change-Id: Ic7c5ffce6e67a1d5f8b32817be24e51864ab6abf
---
 .../system/providers/google/bigquery/example_bigquery_tables.py  | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/system/providers/google/bigquery/example_bigquery_tables.py b/tests/system/providers/google/bigquery/example_bigquery_tables.py
index 048766efb58ef..c561286452cd0 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_tables.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_tables.py
@@ -172,6 +172,7 @@
     get_dataset_tables = BigQueryGetDatasetTablesOperator(
         task_id="get_dataset_tables", dataset_id=DATASET_NAME
     )
+    # [END howto_operator_bigquery_get_dataset_tables]
 
     update_dataset = BigQueryUpdateDatasetOperator(
         task_id="update_dataset",

From 5c54739d0c22cce2be1efd5c29403ec20c6ba485 Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Fri, 18 Mar 2022 12:18:56 +0000
Subject: [PATCH 13/23] Remove unnecessary conversion to list

Change-Id: Ib15e66bd4fd49fd3db586c8b01753e4a19299bf5
---
 tests/always/test_example_dags.py      | 4 ++--
 tests/always/test_project_structure.py | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/tests/always/test_example_dags.py b/tests/always/test_example_dags.py
index 4d05207f37082..058e342e30aea 100644
--- a/tests/always/test_example_dags.py
+++ b/tests/always/test_example_dags.py
@@ -48,13 +48,13 @@ def relative_path(path):
     return os.path.relpath(path, ROOT_FOLDER)
 
 
-@pytest.mark.parametrize("example", list(example_dags()), ids=relative_path)
+@pytest.mark.parametrize("example", example_dags(), ids=relative_path)
 def test_should_be_importable(example):
     dagbag = DagBag(
         dag_folder=example,
         include_examples=False,
     )
-    assert 0 == len(dagbag.import_errors), f"import_errors={str(dagbag.import_errors)}"
+    assert len(dagbag.import_errors) == 0, f"import_errors={str(dagbag.import_errors)}"
     assert len(dagbag.dag_ids) >= 1
 
 
diff --git a/tests/always/test_project_structure.py b/tests/always/test_project_structure.py
index 2c839fad017ef..6cf994ebe8bbe 100644
--- a/tests/always/test_project_structure.py
+++ b/tests/always/test_project_structure.py
@@ -238,7 +238,7 @@ def test_missing_example_for_operator(self):
                 all_operators.update(operators_paths)
 
         for service in services:
-            example_dags = list(self.examples_for_service(service))
+            example_dags = self.examples_for_service(service)
             example_paths = {
                 path for example_dag in example_dags for path in get_imports_from_file(example_dag)
             }

From f083a080b96b5f53177025ed0fe2a81bc7b254be Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Fri, 18 Mar 2022 12:25:58 +0000
Subject: [PATCH 14/23] Remove redundant between word

Change-Id: Id305440af10f2b2e01fb5d2713c5e5f7db6028f7
---
 tests/system/providers/google/README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/system/providers/google/README.md b/tests/system/providers/google/README.md
index eb2558087336f..35d07c8e1e72d 100644
--- a/tests/system/providers/google/README.md
+++ b/tests/system/providers/google/README.md
@@ -62,7 +62,7 @@ To confirm that it is set up correctly, run `echo $NAME_OF_ENV_VAR` which will d
 - `SYSTEM_TESTS_GCP_PROJECT` - GCP project name that will be used to run system tests (this can be checked on the UI
   dashboard of the GCP or by running `gcloud config list`).
 
-- `SYSTEM_TESTS_ENV_ID` - environment ID that is unique across between different executions of system tests (if they
+- `SYSTEM_TESTS_ENV_ID` - environment ID that is unique across different executions of system tests (if they
   are run in parallel). The need for this variable comes from the possibility, that the tests may be run on various
   versions of Airflow at the same time using CI environment. If this is the case, the value of this variable ensures
   that all resources that are created during the tests, will not interfere with other resources in the same project,

From efb9fc7b49897c79681dc15aac4cb9eb0d4f3deb Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Mon, 21 Mar 2022 07:13:28 +0000
Subject: [PATCH 15/23] pytest_configure -> auto-fixture

Change-Id: I7abfaf201f7c18d62d77d0c1f5191998b6f76006
---
 tests/system/conftest.py | 13 ++++++-------
 1 file changed, 6 insertions(+), 7 deletions(-)

diff --git a/tests/system/conftest.py b/tests/system/conftest.py
index 15dfec49477f9..633125d95ac5d 100644
--- a/tests/system/conftest.py
+++ b/tests/system/conftest.py
@@ -22,17 +22,16 @@
 
 import pytest
 
-OLD_EXECUTOR = os.environ.get("AIRFLOW__CORE__EXECUTOR", default=None)
 REQUIRED_ENV_VARS = ("SYSTEM_TESTS_ENV_ID",)
 
 
-def pytest_configure(config):
+@pytest.fixture(scope="package")
+def use_debug_executor():
+    prev_executor = os.environ.get("AIRFLOW__CORE__EXECUTOR", default=None)
     os.environ["AIRFLOW__CORE__EXECUTOR"] = "DebugExecutor"
-
-
-def pytest_unconfigure(config):
-    if OLD_EXECUTOR is not None:
-        os.environ["AIRFLOW__CORE__EXECUTOR"] = OLD_EXECUTOR
+    yield
+    if prev_executor is not None:
+        os.environ["AIRFLOW__CORE__EXECUTOR"] = prev_executor
 
 
 @pytest.fixture()

From b5b0d7def4ecd58ef146f5272790bd5b8d5506b4 Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Mon, 21 Mar 2022 07:24:07 +0000
Subject: [PATCH 16/23] Replace test_run method in system tests by common
 method

Change-Id: I8a439d8d4198778fe7a724db19fe905fde5a3041
---
 .../pre_commit/pre_commit_check_watcher_in_examples.py   | 6 ++----
 .../google/bigquery/example_bigquery_dataset.py          | 6 ++----
 .../google/bigquery/example_bigquery_operations.py       | 6 ++----
 .../bigquery/example_bigquery_operations_location.py     | 6 ++----
 .../google/bigquery/example_bigquery_queries.py          | 6 ++----
 .../google/bigquery/example_bigquery_sensors.py          | 6 ++----
 .../providers/google/bigquery/example_bigquery_tables.py | 6 ++----
 .../google/bigquery/example_bigquery_to_bigquery.py      | 6 ++----
 .../providers/google/bigquery/example_bigquery_to_gcs.py | 6 ++----
 tests/system/utils/__init__.py                           | 9 +++++++++
 10 files changed, 27 insertions(+), 36 deletions(-)

diff --git a/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py b/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
index 34330dd6cc560..4c8cf0eb33834 100755
--- a/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
+++ b/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
@@ -35,11 +35,9 @@
 WATCHER_APPEND_INSTRUCTION = "list(dag.tasks) >> watcher()"
 
 PYTEST_FUNCTION = """
-def test_run():
-    from airflow.utils.state import State
+from tests.system.utils import get_test_run  # noqa: E402
 
-    dag.clear(dag_run_state=State.NONE)
-    dag.run()
+test_run = get_test_run(dag)
 """
 
 
diff --git a/tests/system/providers/google/bigquery/example_bigquery_dataset.py b/tests/system/providers/google/bigquery/example_bigquery_dataset.py
index 26ed04b34fb5f..152725a1460cd 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_dataset.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_dataset.py
@@ -87,8 +87,6 @@
     list(dag.tasks) >> watcher()
 
 
-def test_run():
-    from airflow.utils.state import State
+from tests.system.utils import get_test_run  # noqa: E402
 
-    dag.clear(dag_run_state=State.NONE)
-    dag.run()
+test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_operations.py b/tests/system/providers/google/bigquery/example_bigquery_operations.py
index 84203f2039e3f..6e316a86bea33 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_operations.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_operations.py
@@ -99,8 +99,6 @@
     list(dag.tasks) >> watcher()
 
 
-def test_run():
-    from airflow.utils.state import State
+from tests.system.utils import get_test_run  # noqa: E402
 
-    dag.clear(dag_run_state=State.NONE)
-    dag.run()
+test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_operations_location.py b/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
index c0f7c93c2f221..926af06c868d6 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
@@ -79,8 +79,6 @@
     list(dag.tasks) >> watcher()
 
 
-def test_run():
-    from airflow.utils.state import State
+from tests.system.utils import get_test_run  # noqa: E402
 
-    dag.clear(dag_run_state=State.NONE)
-    dag.run()
+test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_queries.py b/tests/system/providers/google/bigquery/example_bigquery_queries.py
index 685197c0758a6..b54536249980b 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_queries.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_queries.py
@@ -230,8 +230,6 @@
     globals()[DAG_ID] = dag
 
 
-def test_run():
-    from airflow.utils.state import State
+from tests.system.utils import get_test_run  # noqa: E402
 
-    dag.clear(dag_run_state=State.NONE)
-    dag.run()
+test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_sensors.py b/tests/system/providers/google/bigquery/example_bigquery_sensors.py
index 0200aec616214..486ed6d18adf4 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_sensors.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_sensors.py
@@ -120,8 +120,6 @@
     list(dag.tasks) >> watcher()
 
 
-def test_run():
-    from airflow.utils.state import State
+from tests.system.utils import get_test_run  # noqa: E402
 
-    dag.clear(dag_run_state=State.NONE)
-    dag.run()
+test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_tables.py b/tests/system/providers/google/bigquery/example_bigquery_tables.py
index c561286452cd0..3017dfc036b71 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_tables.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_tables.py
@@ -224,8 +224,6 @@
     list(dag.tasks) >> watcher()
 
 
-def test_run():
-    from airflow.utils.state import State
+from tests.system.utils import get_test_run  # noqa: E402
 
-    dag.clear(dag_run_state=State.NONE)
-    dag.run()
+test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py b/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
index 991c7413bc9e6..48b661a41f1a3 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
@@ -95,8 +95,6 @@
     list(dag.tasks) >> watcher()
 
 
-def test_run():
-    from airflow.utils.state import State
+from tests.system.utils import get_test_run  # noqa: E402
 
-    dag.clear(dag_run_state=State.NONE)
-    dag.run()
+test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py b/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
index 57a437c001fe3..cb79ace6fb4d8 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
@@ -96,8 +96,6 @@
     list(dag.tasks) >> watcher()
 
 
-def test_run():
-    from airflow.utils.state import State
+from tests.system.utils import get_test_run  # noqa: E402
 
-    dag.clear(dag_run_state=State.NONE)
-    dag.run()
+test_run = get_test_run(dag)
diff --git a/tests/system/utils/__init__.py b/tests/system/utils/__init__.py
index 13a83393a9124..31b0672b907d9 100644
--- a/tests/system/utils/__init__.py
+++ b/tests/system/utils/__init__.py
@@ -14,3 +14,12 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+from airflow.utils.state import State
+
+
+def get_test_run(dag):
+    def test_run():
+        dag.clear(dag_run_state=State.NONE)
+        dag.run()
+
+    return test_run

From b5c05c74a0132a4df27490354206334ca9bac529 Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Mon, 21 Mar 2022 12:09:51 +0000
Subject: [PATCH 17/23] Mock os.environ with AIRFLOW__CORE__EXECUTOR env

Change-Id: I04e3133cba03f5a9d2daec4052df542bab835ee8
---
 tests/system/conftest.py | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/tests/system/conftest.py b/tests/system/conftest.py
index 633125d95ac5d..d0e82187e1c50 100644
--- a/tests/system/conftest.py
+++ b/tests/system/conftest.py
@@ -19,19 +19,17 @@
 import re
 from itertools import chain
 from pathlib import Path
+from unittest import mock
 
 import pytest
 
 REQUIRED_ENV_VARS = ("SYSTEM_TESTS_ENV_ID",)
 
 
-@pytest.fixture(scope="package")
+@pytest.fixture(scope="package", autouse=True)
 def use_debug_executor():
-    prev_executor = os.environ.get("AIRFLOW__CORE__EXECUTOR", default=None)
-    os.environ["AIRFLOW__CORE__EXECUTOR"] = "DebugExecutor"
-    yield
-    if prev_executor is not None:
-        os.environ["AIRFLOW__CORE__EXECUTOR"] = prev_executor
+    with mock.patch.dict("os.environ", AIRFLOW__CORE__EXECUTOR="DebugExecutor"):
+        yield
 
 
 @pytest.fixture()

From e44a778e7fe5f028c457fb05f017ec7b0f94da33 Mon Sep 17 00:00:00 2001
From: Mateusz Nojek <matnojek@gmail.com>
Date: Mon, 21 Mar 2022 13:28:57 +0000
Subject: [PATCH 18/23] Update pre-commit hook, move watcher import, small
 refactor

---
 .pre-commit-config.yaml                       |  6 ++--
 BREEZE.rst                                    |  2 +-
 STATIC_CODE_CHECKS.rst                        |  2 +-
 breeze-complete                               |  2 +-
 .../src/airflow_breeze/pre_commit_ids.py      |  2 +-
 docs/apache-airflow/best-practices.rst        | 29 +++++++++++++------
 ...es.py => pre_commit_check_system_tests.py} | 26 ++++++++---------
 .../bigquery/example_bigquery_dataset.py      |  5 +++-
 .../bigquery/example_bigquery_operations.py   |  5 +++-
 .../example_bigquery_operations_location.py   |  5 +++-
 .../bigquery/example_bigquery_queries.py      |  5 +++-
 .../bigquery/example_bigquery_sensors.py      |  5 +++-
 .../bigquery/example_bigquery_tables.py       |  5 +++-
 .../bigquery/example_bigquery_to_bigquery.py  |  5 +++-
 .../bigquery/example_bigquery_to_gcs.py       |  5 +++-
 tests/system/utils/watcher.py                 |  3 +-
 16 files changed, 74 insertions(+), 38 deletions(-)
 rename scripts/ci/pre_commit/{pre_commit_check_watcher_in_examples.py => pre_commit_check_system_tests.py} (76%)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index c3b75d38455d5..6d4ac980dc6ca 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -582,9 +582,9 @@ repos:
         pass_filenames: false
         require_serial: true
         additional_dependencies: ['rich']
-      - id: check-watcher-in-examples
-        name: Check if watcher is properly used in examples
-        entry: ./scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
+      - id: check-system-tests
+        name: Check if system tests have required segments of code
+        entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests.py
         language: python
         files: ^tests/system/.*/example_[^/]*.py$
         pass_filenames: true
diff --git a/BREEZE.rst b/BREEZE.rst
index 17e3c55b1647c..6b503917533fa 100644
--- a/BREEZE.rst
+++ b/BREEZE.rst
@@ -2292,7 +2292,7 @@ This is the current syntax for  `./breeze <./breeze>`_:
                  build-providers-dependencies chart-schema-lint capitalized-breeze
                  changelog-duplicates check-apache-license check-builtin-literals
                  check-executables-have-shebangs check-extras-order check-hooks-apply
-                 check-integrations check-merge-conflict check-xml check-watcher-in-examples
+                 check-integrations check-merge-conflict check-xml check-system-tests
                  daysago-import-check debug-statements detect-private-key docstring-params doctoc
                  dont-use-safe-filter end-of-file-fixer fix-encoding-pragma flake8 flynt
                  forbidden-xcom-get-value codespell forbid-tabs helm-lint identity
diff --git a/STATIC_CODE_CHECKS.rst b/STATIC_CODE_CHECKS.rst
index 2ec6cc04d6293..821d02a1cd91a 100644
--- a/STATIC_CODE_CHECKS.rst
+++ b/STATIC_CODE_CHECKS.rst
@@ -166,7 +166,7 @@ require Breeze Docker images to be installed locally.
 ------------------------------------ ---------------------------------------------------------------- ------------
 ``check-xml``                          Checks XML files with xmllint
 ------------------------------------ ---------------------------------------------------------------- ------------
-``check-watcher-in-examples``          Check if watcher is properly used in examples
+``check-system-tests``                 Check if system tests have required segments of code
 ------------------------------------ ---------------------------------------------------------------- ------------
 ``daysago-import-check``               Checks if daysago is properly imported
 ------------------------------------ ---------------------------------------------------------------- ------------
diff --git a/breeze-complete b/breeze-complete
index f03200ba7a4d6..6f91c9f124ff2 100644
--- a/breeze-complete
+++ b/breeze-complete
@@ -98,7 +98,7 @@ check-hooks-apply
 check-integrations
 check-merge-conflict
 check-xml
-check-watcher-in-examples
+check-system-tests
 daysago-import-check
 debug-statements
 detect-private-key
diff --git a/dev/breeze/src/airflow_breeze/pre_commit_ids.py b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
index 270c9aa25e5ca..2686d120ace45 100644
--- a/dev/breeze/src/airflow_breeze/pre_commit_ids.py
+++ b/dev/breeze/src/airflow_breeze/pre_commit_ids.py
@@ -40,7 +40,7 @@
     'check-hooks-apply',
     'check-integrations',
     'check-merge-conflict',
-    'check-watcher-in-examples',
+    'check-system-tests',
     'check-xml',
     'codespell',
     'daysago-import-check',
diff --git a/docs/apache-airflow/best-practices.rst b/docs/apache-airflow/best-practices.rst
index d62020319704a..d4606548d3dd4 100644
--- a/docs/apache-airflow/best-practices.rst
+++ b/docs/apache-airflow/best-practices.rst
@@ -247,18 +247,26 @@ each parameter by following the links):
 Example of watcher pattern with trigger rules
 ---------------------------------------------
 
-The watcher pattern is how we call a DAG with a task that is "watching" the states of the other tasks. It's primary purpose is to fail a DAG Run when any other task fail.
+The watcher pattern is how we call a DAG with a task that is "watching" the states of the other tasks.
+It's primary purpose is to fail a DAG Run when any other task fail.
 The need came from the Airflow system tests that are DAGs with different tasks (similarly like a test containing steps).
 
-Normally, when any task fails, all other tasks are not executed and the whole DAG Run gets failed status too. But when we use trigger rules, we can disrupt the normal flow of running tasks and the whole DAG may represent different status that we expect.
-For example, we can have a teardown task (with trigger rule set to ``"all_done"``) that will be executed regardless of the state of the other tasks (e.g. to clean up the resources). In such situation, the DAG would always run this task and the DAG Run will get the status of this particular task, so we can potentially lose the information about failing tasks.
-If we want to ensure that the DAG with teardown task would fail if any task fails, we need to use the watcher pattern.
-The watcher task is a task that will always fail if triggered, but it needs to be triggered only if any other task fails. It needs to have a trigger rule set to ``"one_failed"`` and it needs also to be a downstream task for all other tasks in the DAG.
-Thanks to this, if every other task will pass, the watcher will be skipped, but when something fails, the watcher task will be executed and fail making the DAG Run fail too.
+Normally, when any task fails, all other tasks are not executed and the whole DAG Run gets failed status too. But
+when we use trigger rules, we can disrupt the normal flow of running tasks and the whole DAG may represent different
+status that we expect. For example, we can have a teardown task (with trigger rule set to ``TriggerRule.ALL_DONE``)
+that will be executed regardless of the state of the other tasks (e.g. to clean up the resources). In such
+situation, the DAG would always run this task and the DAG Run will get the status of this particular task, so we can
+potentially lose the information about failing tasks. If we want to ensure that the DAG with teardown task would fail
+if any task fails, we need to  use the watcher pattern. The watcher task is a task that will always fail if
+triggered, but it needs to be triggered only if any other task fails. It needs to have a trigger rule set to
+``TriggerRule.ONE_FAILED`` and it needs also to be a  downstream task for all other tasks in the DAG. Thanks to
+this, if every other task will pass, the watcher will be skipped, but when something fails, the watcher task will be
+executed and fail making the DAG Run fail too.
 
 .. note::
 
-    Be aware that trigger rules only rely on the direct upstream (parent) tasks, e.g. ``one_failed`` will ignore any failed (or ``upstream_failed``) tasks that are not a direct parent of the parameterized task.
+    Be aware that trigger rules only rely on the direct upstream (parent) tasks, e.g. ``TriggerRule.ONE_FAILED``
+    will ignore any failed (or ``upstream_failed``) tasks that are not a direct parent of the parameterized task.
 
 It's easier to grab the concept with an example. Let's say that we have the following DAG:
 
@@ -270,9 +278,10 @@ It's easier to grab the concept with an example. Let's say that we have the foll
     from airflow.exceptions import AirflowException
     from airflow.operators.bash import BashOperator
     from airflow.operators.python import PythonOperator
+    from airflow.utils.trigger_rule import TriggerRule
 
 
-    @task(trigger_rule="one_failed", retries=0)
+    @task(trigger_rule=TriggerRule.ONE_FAILED, retries=0)
     def watcher():
         raise AirflowException("Failing task because one or more upstream tasks failed.")
 
@@ -290,7 +299,9 @@ It's easier to grab the concept with an example. Let's say that we have the foll
             task_id="passing_task", bash_command="echo passing_task"
         )
         teardown = BashOperator(
-            task_id="teardown", bash_command="echo teardown", trigger_rule="all_done"
+            task_id="teardown",
+            bash_command="echo teardown",
+            trigger_rule=TriggerRule.ALL_DONE,
         )
 
         failing_task >> passing_task >> teardown
diff --git a/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py b/scripts/ci/pre_commit/pre_commit_check_system_tests.py
similarity index 76%
rename from scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
rename to scripts/ci/pre_commit/pre_commit_check_system_tests.py
index 4c8cf0eb33834..e0032bc1dbc82 100755
--- a/scripts/ci/pre_commit/pre_commit_check_watcher_in_examples.py
+++ b/scripts/ci/pre_commit/pre_commit_check_system_tests.py
@@ -24,7 +24,7 @@
 if __name__ not in ("__main__", "__mp_main__"):
     raise SystemExit(
         "This file is intended to be executed as an executable program. You cannot use it as a module."
-        f"To run this script, run the ./{__file__} command [FILE] ..."
+        f"To execute this script, run ./{__file__} [FILE] ..."
     )
 
 
@@ -51,27 +51,27 @@ def _check_file(file: Path):
                 f"but does not use it properly![/]\n\n"
                 "[yellow]Make sure you have:[/]\n\n"
                 f"        {WATCHER_APPEND_INSTRUCTION}\n\n"
-                "[yellow]as last instruction in your example DAG.[/]\n"
+                "[yellow]as the last instruction in your example DAG.[/]\n"
             )
         else:
             operator_leftshift_index = content.find("<<", index + len(WATCHER_APPEND_INSTRUCTION))
             operator_rightshift_index = content.find(">>", index + len(WATCHER_APPEND_INSTRUCTION))
             if operator_leftshift_index != -1 or operator_rightshift_index != -1:
                 errors.append(
-                    f"[red]In the  example {file} "
-                    f"watcher is not last instruction in your DAG (there are << "
-                    f"or >> operators after it)![/]\n\n"
+                    f"[red]In the example {file} "
+                    f"watcher is not the last instruction in your DAG "
+                    f"(there are << or >> operators after it)![/]\n\n"
                     "[yellow]Make sure you have:[/]\n"
                     f"        {WATCHER_APPEND_INSTRUCTION}\n\n"
-                    "[yellow]as last instruction in your example DAG.[/]\n"
+                    "[yellow]as the last instruction in your example DAG.[/]\n"
                 )
-        if PYTEST_FUNCTION not in content:
-            errors.append(
-                f"[yellow]The  example {file} missed the pytest function at the end.[/]\n\n"
-                "All example tests should have this function added:\n\n" + PYTEST_FUNCTION + "\n\n"
-                "[yellow]Automatically adding it now!\n"
-            )
-            file.write_text(content + "\n" + PYTEST_FUNCTION)
+    if PYTEST_FUNCTION not in content:
+        errors.append(
+            f"[yellow]The example {file} missed the pytest function at the end.[/]\n\n"
+            "All example tests should have this function added:\n\n" + PYTEST_FUNCTION + "\n\n"
+            "[yellow]Automatically adding it now![/]\n"
+        )
+        file.write_text(content + "\n" + PYTEST_FUNCTION)
 
 
 if __name__ == '__main__':
diff --git a/tests/system/providers/google/bigquery/example_bigquery_dataset.py b/tests/system/providers/google/bigquery/example_bigquery_dataset.py
index 152725a1460cd..7a45ae2fe761a 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_dataset.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_dataset.py
@@ -31,7 +31,6 @@
     BigQueryUpdateDatasetOperator,
 )
 from airflow.utils.trigger_rule import TriggerRule
-from tests.system.utils.watcher import watcher
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_dataset"
@@ -84,6 +83,10 @@
         >> delete_dataset
     )
 
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
     list(dag.tasks) >> watcher()
 
 
diff --git a/tests/system/providers/google/bigquery/example_bigquery_operations.py b/tests/system/providers/google/bigquery/example_bigquery_operations.py
index 6e316a86bea33..2ebbb13aff4ce 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_operations.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_operations.py
@@ -32,7 +32,6 @@
 from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
 from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator
 from airflow.utils.trigger_rule import TriggerRule
-from tests.system.utils.watcher import watcher
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_operations"
@@ -96,6 +95,10 @@
         >> delete_bucket
     )
 
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
     list(dag.tasks) >> watcher()
 
 
diff --git a/tests/system/providers/google/bigquery/example_bigquery_operations_location.py b/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
index 926af06c868d6..f28e0fe847aeb 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
@@ -29,7 +29,6 @@
     BigQueryDeleteDatasetOperator,
 )
 from airflow.utils.trigger_rule import TriggerRule
-from tests.system.utils.watcher import watcher
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_operations_location"
@@ -76,6 +75,10 @@
         >> delete_dataset_with_location
     )
 
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
     list(dag.tasks) >> watcher()
 
 
diff --git a/tests/system/providers/google/bigquery/example_bigquery_queries.py b/tests/system/providers/google/bigquery/example_bigquery_queries.py
index b54536249980b..0389c951456f5 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_queries.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_queries.py
@@ -36,7 +36,6 @@
     BigQueryValueCheckOperator,
 )
 from airflow.utils.trigger_rule import TriggerRule
-from tests.system.utils.watcher import watcher
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 PROJECT_ID = os.environ.get("SYSTEM_TESTS_GCP_PROJECT")
@@ -225,6 +224,10 @@
         execute_insert_query >> execute_query_save >> bigquery_execute_multi_query >> delete_dataset
         execute_insert_query >> [check_count, check_value, check_interval] >> delete_dataset
 
+        from tests.system.utils.watcher import watcher
+
+        # This test needs watcher in order to properly mark success/failure
+        # when "tearDown" task with trigger rule is part of the DAG
         list(dag.tasks) >> watcher()
 
     globals()[DAG_ID] = dag
diff --git a/tests/system/providers/google/bigquery/example_bigquery_sensors.py b/tests/system/providers/google/bigquery/example_bigquery_sensors.py
index 486ed6d18adf4..8dadcdc509bc4 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_sensors.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_sensors.py
@@ -34,7 +34,6 @@
     BigQueryTablePartitionExistenceSensor,
 )
 from airflow.utils.trigger_rule import TriggerRule
-from tests.system.utils.watcher import watcher
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_sensors"
@@ -117,6 +116,10 @@
     execute_insert_query >> check_table_partition_exists
     [check_table_exists, check_table_partition_exists] >> delete_dataset
 
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
     list(dag.tasks) >> watcher()
 
 
diff --git a/tests/system/providers/google/bigquery/example_bigquery_tables.py b/tests/system/providers/google/bigquery/example_bigquery_tables.py
index 3017dfc036b71..9be271ca2674c 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_tables.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_tables.py
@@ -39,7 +39,6 @@
 from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
 from airflow.providers.google.cloud.transfers.local_to_gcs import LocalFilesystemToGCSOperator
 from airflow.utils.trigger_rule import TriggerRule
-from tests.system.utils.watcher import watcher
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_tables"
@@ -221,6 +220,10 @@
         >> delete_dataset
     )
 
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
     list(dag.tasks) >> watcher()
 
 
diff --git a/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py b/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
index 48b661a41f1a3..8a8ad2cfe7bb4 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
@@ -30,7 +30,6 @@
 )
 from airflow.providers.google.cloud.transfers.bigquery_to_bigquery import BigQueryToBigQueryOperator
 from airflow.utils.trigger_rule import TriggerRule
-from tests.system.utils.watcher import watcher
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_to_bigquery"
@@ -92,6 +91,10 @@
         >> delete_dataset
     )
 
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
     list(dag.tasks) >> watcher()
 
 
diff --git a/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py b/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
index cb79ace6fb4d8..cdded3a76e9fa 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
@@ -31,7 +31,6 @@
 from airflow.providers.google.cloud.operators.gcs import GCSCreateBucketOperator, GCSDeleteBucketOperator
 from airflow.providers.google.cloud.transfers.bigquery_to_gcs import BigQueryToGCSOperator
 from airflow.utils.trigger_rule import TriggerRule
-from tests.system.utils.watcher import watcher
 
 ENV_ID = os.environ.get("SYSTEM_TESTS_ENV_ID")
 DAG_ID = "bigquery_to_gcs"
@@ -93,6 +92,10 @@
         >> [delete_bucket, delete_dataset]
     )
 
+    from tests.system.utils.watcher import watcher
+
+    # This test needs watcher in order to properly mark success/failure
+    # when "tearDown" task with trigger rule is part of the DAG
     list(dag.tasks) >> watcher()
 
 
diff --git a/tests/system/utils/watcher.py b/tests/system/utils/watcher.py
index 2c0d37c17afc2..2fa6d14bd85cc 100644
--- a/tests/system/utils/watcher.py
+++ b/tests/system/utils/watcher.py
@@ -16,9 +16,10 @@
 # under the License.
 from airflow.decorators import task
 from airflow.exceptions import AirflowException
+from airflow.utils.trigger_rule import TriggerRule
 
 
-@task(trigger_rule="one_failed", retries=0)
+@task(trigger_rule=TriggerRule.ONE_FAILED, retries=0)
 def watcher():
     """Watcher task raises an AirflowException and is used to 'watch' tasks for failures
     and propagates fail status to the whole DAG Run"""

From 11be7c043dfd721ddecdcaa5b624cbc5041d3319 Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Wed, 23 Mar 2022 08:12:33 +0000
Subject: [PATCH 19/23] Update docs lint checks with new system tests design

Change-Id: Icd0575ab1789dd6a0ca8d812f28725447382e0d5
---
 .../operators/cloud/data_loss_prevention.rst         |  2 +-
 docs/exts/docs_build/lint_checks.py                  | 12 +++++++++---
 2 files changed, 10 insertions(+), 4 deletions(-)

diff --git a/docs/apache-airflow-providers-google/operators/cloud/data_loss_prevention.rst b/docs/apache-airflow-providers-google/operators/cloud/data_loss_prevention.rst
index d0092d1984664..520b45b7987a0 100644
--- a/docs/apache-airflow-providers-google/operators/cloud/data_loss_prevention.rst
+++ b/docs/apache-airflow-providers-google/operators/cloud/data_loss_prevention.rst
@@ -297,7 +297,7 @@ Configuration information defines how you want the sensitive data de-identified.
 
 This config can either be saved and persisted in de-identification templates or defined in a :class:`~google.cloud.dlp_v2.types.DeidentifyConfig` object:
 
-.. literalinclude:: /../../airflow/providers/google/cloud/example_dags/example_dlp.py
+.. exampleinclude:: /../../airflow/providers/google/cloud/example_dags/example_dlp.py
     :language: python
     :start-after: [START dlp_deidentify_config_example]
     :end-before: [END dlp_deidentify_config_example]
diff --git a/docs/exts/docs_build/lint_checks.py b/docs/exts/docs_build/lint_checks.py
index a1af22ef5fe8e..ca35694926b67 100644
--- a/docs/exts/docs_build/lint_checks.py
+++ b/docs/exts/docs_build/lint_checks.py
@@ -231,12 +231,12 @@ def find_modules(deprecated_only: bool = False) -> Set[str]:
 
 def check_exampleinclude_for_example_dags() -> List[DocBuildError]:
     """Checks all exampleincludes for  example dags."""
-    all_docs_files = glob(f"${DOCS_DIR}/**/*.rst", recursive=True)
+    all_docs_files = glob(f"{DOCS_DIR}/**/*.rst", recursive=True)
     build_errors = []
     for doc_file in all_docs_files:
         build_error = assert_file_not_contains(
             file_path=doc_file,
-            pattern=r"literalinclude::.+example_dags",
+            pattern=r"literalinclude::.+(?:example_dags|tests/system/)",
             message=(
                 "literalinclude directive is prohibited for example DAGs. \n"
                 "You should use the exampleinclude directive to include example DAGs."
@@ -265,12 +265,18 @@ def check_enforce_code_block() -> List[DocBuildError]:
     return build_errors
 
 
+def find_example_dags(provider_dir):
+    system_tests_dir = provider_dir.replace(f"{ROOT_PACKAGE_DIR}/", "")
+    yield from glob(f"{provider_dir}/**/*example_dags", recursive=True)
+    yield from glob(f"{ROOT_PROJECT_DIR}/tests/system/{system_tests_dir}/*/", recursive=True)
+
+
 def check_example_dags_in_provider_tocs() -> List[DocBuildError]:
     """Checks that each documentation for provider packages has a link to example DAGs in the TOC."""
     build_errors = []
 
     for provider in ALL_PROVIDER_YAMLS:
-        example_dags_dirs = list(glob(f"{provider['package-dir']}/**/example_dags", recursive=True))
+        example_dags_dirs = list(find_example_dags(provider['package-dir']))
         if not example_dags_dirs:
             continue
         doc_file_path = f"{DOCS_DIR}/{provider['package-name']}/index.rst"

From a9c7bc1ea27c1ed075e8534eff4cc63c90d37b64 Mon Sep 17 00:00:00 2001
From: Mateusz Nojek <matnojek@gmail.com>
Date: Wed, 23 Mar 2022 09:30:19 +0100
Subject: [PATCH 20/23] Remove whitespace in
 docs/exts/docs_build/lint_checks.py

---
 docs/exts/docs_build/lint_checks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/docs/exts/docs_build/lint_checks.py b/docs/exts/docs_build/lint_checks.py
index ca35694926b67..d80419d7ebd62 100644
--- a/docs/exts/docs_build/lint_checks.py
+++ b/docs/exts/docs_build/lint_checks.py
@@ -230,7 +230,7 @@ def find_modules(deprecated_only: bool = False) -> Set[str]:
 
 
 def check_exampleinclude_for_example_dags() -> List[DocBuildError]:
-    """Checks all exampleincludes for  example dags."""
+    """Checks all exampleincludes for example dags."""
     all_docs_files = glob(f"{DOCS_DIR}/**/*.rst", recursive=True)
     build_errors = []
     for doc_file in all_docs_files:

From fa1e1a3256cac9310e7c45816fee386453869490 Mon Sep 17 00:00:00 2001
From: Mateusz Nojek <matnojek@gmail.com>
Date: Wed, 23 Mar 2022 15:23:46 +0000
Subject: [PATCH 21/23] Update pytest function and pre-commit hook, improve
 docs

---
 .../pre_commit_check_system_tests.py          |  9 ++++++-
 tests/system/README.md                        | 24 ++++++++++++-------
 .../bigquery/example_bigquery_dataset.py      |  1 +
 .../bigquery/example_bigquery_operations.py   |  1 +
 .../example_bigquery_operations_location.py   |  1 +
 .../bigquery/example_bigquery_queries.py      | 10 ++++----
 .../bigquery/example_bigquery_sensors.py      |  1 +
 .../bigquery/example_bigquery_tables.py       |  1 +
 .../bigquery/example_bigquery_to_bigquery.py  |  1 +
 .../bigquery/example_bigquery_to_gcs.py       |  1 +
 10 files changed, 37 insertions(+), 13 deletions(-)

diff --git a/scripts/ci/pre_commit/pre_commit_check_system_tests.py b/scripts/ci/pre_commit/pre_commit_check_system_tests.py
index e0032bc1dbc82..fcb4beefb29fc 100755
--- a/scripts/ci/pre_commit/pre_commit_check_system_tests.py
+++ b/scripts/ci/pre_commit/pre_commit_check_system_tests.py
@@ -15,6 +15,7 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
+import re
 import sys
 from pathlib import Path
 from typing import List
@@ -37,8 +38,14 @@
 PYTEST_FUNCTION = """
 from tests.system.utils import get_test_run  # noqa: E402
 
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
 test_run = get_test_run(dag)
 """
+PYTEST_FUNCTION_PATTERN = re.compile(
+    r"from tests\.system\.utils import get_test_run  # noqa: E402\s+"
+    r"(?:# .+\))?\s+"
+    r"test_run = get_test_run\(dag\)"
+)
 
 
 def _check_file(file: Path):
@@ -65,7 +72,7 @@ def _check_file(file: Path):
                     f"        {WATCHER_APPEND_INSTRUCTION}\n\n"
                     "[yellow]as the last instruction in your example DAG.[/]\n"
                 )
-    if PYTEST_FUNCTION not in content:
+    if not PYTEST_FUNCTION_PATTERN.search(content):
         errors.append(
             f"[yellow]The example {file} missed the pytest function at the end.[/]\n\n"
             "All example tests should have this function added:\n\n" + PYTEST_FUNCTION + "\n\n"
diff --git a/tests/system/README.md b/tests/system/README.md
index 4c15e5a9e4e06..912bfc248ce0d 100644
--- a/tests/system/README.md
+++ b/tests/system/README.md
@@ -19,6 +19,12 @@
 
 # Airflow System Tests
 
+- [How to run system tests](#how_to_run)
+  - [Running via Airflow](#run_via_airflow)
+  - [Running via Pytest](#run_via_pytest)
+  - [Running via Airflow CLI](#run_via_airflow_cli)
+- [How to write system tests](#how_to_write)
+
 System tests verify the correctness of Airflow Operators by running them in DAGs and allowing to communicate with
 external services. A system test tries to look as close to a regular DAG as possible, and it generally checks the
 "happy path" (a scenario featuring no errors) ensuring that the Operator works as expected.
@@ -36,7 +42,7 @@ The purpose of these tests is to:
 > [AIP-47](https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-47+New+design+of+Airflow+System+Tests).
 > Please use it and write any new system tests according to this documentation.
 
-## How to run system tests
+## How to run system tests <a name="how_to_run"></a>
 
 There are multiple ways of running system tests. Each system test is a self-contained DAG, so it can be run as any
 other DAG. Some tests may require access to external services, enabled APIs or specific permissions. Make sure to
@@ -44,7 +50,7 @@ prepare your  environment correctly, depending on the system tests you want to r
 configuration which should be documented by the relevant providers in their subdirectory
 `tests/system/providers/<provider_name>/README.md`.
 
-### Running via Airflow
+### Running via Airflow <a name="run_via_airflow"></a>
 
 If you have a working Airflow environment with a scheduler and a webserver, you can import system test files into
 your Airflow instance and they will be automatically triggered. If the setup of the environment is correct
@@ -52,23 +58,25 @@ your Airflow instance and they will be automatically triggered. If the setup of
 how to set up the environment is documented in each provider's system tests directory. Make sure that all resource
 required by the tests are also imported.
 
-### Running via Pytest
+### Running via Pytest <a name="run_via_pytest"></a>
 
-System tests can be executed using pytest. You can either run them using your IDE (if you have installed
-plugin/widget supporting pytest) or using following example command:
+Running system tests with pytest is the easiest with Breeze. Thanks to it, you don't need to bother about setting up
+the correct environment, that is able to execute the tests.
+You can either run them using your IDE (if you have installed plugin/widget supporting pytest) or using the following
+example of command:
 
 ```commandline
 # pytest --system [provider_name] [path_to_test(s)]
 pytest --system google tests/system/providers/google/bigquery/example_bigquery_queries.py
 ```
 
-You can specify several --system flags if you want to execute tests for several providers:
+You can specify several `--system` flags if you want to execute tests for several providers:
 
 ```commandline
 pytest --system google --system aws tests/system
 ```
 
-### Running via Airflow CLI
+### Running via Airflow CLI <a name="run_via_airflow_cli"></a>
 
 It is possible to run system tests using Airflow CLI. To execute a specific system test, you need to provide
 `dag_id` of the test to be run, `execution_date` (preferably the one from the past) and a `-S/--subdir` option
@@ -83,7 +91,7 @@ airflow dags test -S tests/system bigquery_dataset 2022-01-01
 > [here](https://airflow.apache.org/docs/apache-airflow/stable/usage-cli.html) for a documentation.
 
 
-## How to write system tests
+## How to write system tests <a name="how_to_write"></a>
 
 If you are going to implement new system tests, it is recommended to familiarize with the content of the
 [AIP-47](https://cwiki.apache.org/confluence/display/AIRFLOW/AIP-47+New+design+of+Airflow+System+Tests). There are
diff --git a/tests/system/providers/google/bigquery/example_bigquery_dataset.py b/tests/system/providers/google/bigquery/example_bigquery_dataset.py
index 7a45ae2fe761a..f5e6d7f044020 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_dataset.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_dataset.py
@@ -92,4 +92,5 @@
 
 from tests.system.utils import get_test_run  # noqa: E402
 
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
 test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_operations.py b/tests/system/providers/google/bigquery/example_bigquery_operations.py
index 2ebbb13aff4ce..c2320f96e6278 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_operations.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_operations.py
@@ -104,4 +104,5 @@
 
 from tests.system.utils import get_test_run  # noqa: E402
 
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
 test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_operations_location.py b/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
index f28e0fe847aeb..391a336bfdfbd 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_operations_location.py
@@ -84,4 +84,5 @@
 
 from tests.system.utils import get_test_run  # noqa: E402
 
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
 test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_queries.py b/tests/system/providers/google/bigquery/example_bigquery_queries.py
index 0389c951456f5..b0eb8eaffa767 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_queries.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_queries.py
@@ -51,7 +51,7 @@
     {"name": "ds", "type": "DATE", "mode": "NULLABLE"},
 ]
 
-
+DAGS_LIST = []
 locations = [None, LOCATION]
 for index, location in enumerate(locations, 1):
     DAG_ID = "bigquery_queries_location" if location else "bigquery_queries"
@@ -230,9 +230,11 @@
         # when "tearDown" task with trigger rule is part of the DAG
         list(dag.tasks) >> watcher()
 
+    DAGS_LIST.append(dag)
     globals()[DAG_ID] = dag
 
+for dag in DAGS_LIST:
+    from tests.system.utils import get_test_run  # noqa: E402
 
-from tests.system.utils import get_test_run  # noqa: E402
-
-test_run = get_test_run(dag)
+    # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
+    test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_sensors.py b/tests/system/providers/google/bigquery/example_bigquery_sensors.py
index 8dadcdc509bc4..c51e7c905d73f 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_sensors.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_sensors.py
@@ -125,4 +125,5 @@
 
 from tests.system.utils import get_test_run  # noqa: E402
 
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
 test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_tables.py b/tests/system/providers/google/bigquery/example_bigquery_tables.py
index 9be271ca2674c..be7e909d55155 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_tables.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_tables.py
@@ -229,4 +229,5 @@
 
 from tests.system.utils import get_test_run  # noqa: E402
 
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
 test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py b/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
index 8a8ad2cfe7bb4..b0358046c59df 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_to_bigquery.py
@@ -100,4 +100,5 @@
 
 from tests.system.utils import get_test_run  # noqa: E402
 
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
 test_run = get_test_run(dag)
diff --git a/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py b/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
index cdded3a76e9fa..e5275d91ba281 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_to_gcs.py
@@ -101,4 +101,5 @@
 
 from tests.system.utils import get_test_run  # noqa: E402
 
+# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
 test_run = get_test_run(dag)

From d321fab2723641e3bfe7458a4405617e238a6931 Mon Sep 17 00:00:00 2001
From: Jarek Potiuk <jarek@potiuk.com>
Date: Thu, 24 Mar 2022 21:23:03 +0100
Subject: [PATCH 22/23] fixup! Merge branch 'main' into new_system_tests

---
 .pre-commit-config.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 269bfb0281785..e93abeb063754 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -595,6 +595,7 @@ repos:
         entry: ./scripts/ci/pre_commit/pre_commit_check_system_tests.py
         language: python
         files: ^tests/system/.*/example_[^/]*.py$
+        exclude: ^tests/system/providers/google/bigquery/example_bigquery_queries\.py$
         pass_filenames: true
         additional_dependencies: ['rich']
       - id: markdownlint

From 3832e475b6031448f88807da014b0c946899445b Mon Sep 17 00:00:00 2001
From: Bartlomiej Hirsz <bartomiejh@google.com>
Date: Fri, 25 Mar 2022 10:24:12 +0000
Subject: [PATCH 23/23] Update check_system_tests precommit pattern

Change-Id: Ib8a714c795989c371d456f51513f1be615f2be82
---
 scripts/ci/pre_commit/pre_commit_check_system_tests.py          | 2 +-
 .../providers/google/bigquery/example_bigquery_queries.py       | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/pre_commit/pre_commit_check_system_tests.py b/scripts/ci/pre_commit/pre_commit_check_system_tests.py
index fcb4beefb29fc..8fc162a421105 100755
--- a/scripts/ci/pre_commit/pre_commit_check_system_tests.py
+++ b/scripts/ci/pre_commit/pre_commit_check_system_tests.py
@@ -42,7 +42,7 @@
 test_run = get_test_run(dag)
 """
 PYTEST_FUNCTION_PATTERN = re.compile(
-    r"from tests\.system\.utils import get_test_run  # noqa: E402\s+"
+    r"from tests\.system\.utils import get_test_run(?:  # noqa: E402)?\s+"
     r"(?:# .+\))?\s+"
     r"test_run = get_test_run\(dag\)"
 )
diff --git a/tests/system/providers/google/bigquery/example_bigquery_queries.py b/tests/system/providers/google/bigquery/example_bigquery_queries.py
index b0eb8eaffa767..be34df79ef79e 100644
--- a/tests/system/providers/google/bigquery/example_bigquery_queries.py
+++ b/tests/system/providers/google/bigquery/example_bigquery_queries.py
@@ -234,7 +234,7 @@
     globals()[DAG_ID] = dag
 
 for dag in DAGS_LIST:
-    from tests.system.utils import get_test_run  # noqa: E402
+    from tests.system.utils import get_test_run
 
     # Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest)
     test_run = get_test_run(dag)
