Add providers E2E tests framework and OpenLineage tests

.dockerignore

@@ -51,6 +51,7 @@
 !airflow-ctl-tests
 !shared/
 !airflow-e2e-tests
+!providers-e2e-tests
 
 # Add scripts so that we can use them inside the container
 !scripts

.github/workflows/additional-prod-image-tests.yml

@@ -60,6 +60,14 @@ on:  # yamllint disable-line rule:truthy
         description: "Whether to run Go SDK e2e tests (true/false)"
         required: true
         type: string
+      run-providers-e2e-tests-openlineage:
+        description: "Whether to run OpenLineage e2e tests (true/false)"
+        required: true
+        type: string
+      providers-compatibility-tests-matrix:
+        description: "JSON matrix of released Airflow versions for provider e2e compat tests."
+        required: true
+        type: string
       constraints-branch:
         description: "Branch used to construct constraints URL from."
         required: true
@@ -321,6 +329,35 @@ jobs:
       e2e_test_mode: "go_sdk"
     if: inputs.canary-run == 'true' || inputs.run-go-sdk-e2e-tests == 'true'
 
+  test-providers-e2e-tests-openlineage:
+    name: "Provider E2E tests openlineage"
+    uses: ./.github/workflows/providers-e2e-tests.yml
+    with:
+      provider: "openlineage"
+      provider-display-name: "OpenLineage"
+      runners: ${{ inputs.runners }}
+      platform: ${{ inputs.platform }}
+      default-python-version: "${{ inputs.default-python-version }}"
+      use-uv: ${{ inputs.use-uv }}
+    if: inputs.canary-run == 'true' || inputs.run-providers-e2e-tests-openlineage == 'true'
+
+  test-providers-e2e-tests-openlineage-compat:
+    name: "Provider E2E tests openlineage compat"
+    strategy:
+      fail-fast: false
+      matrix:
+        compat: ${{ fromJSON(inputs.providers-compatibility-tests-matrix) }}
+    uses: ./.github/workflows/providers-e2e-tests.yml
+    with:
+      provider: "openlineage"
+      provider-display-name: "OpenLineage"
+      runners: ${{ inputs.runners }}
+      platform: ${{ inputs.platform }}
+      default-python-version: "${{ inputs.default-python-version }}"
+      use-uv: ${{ inputs.use-uv }}
+      airflow-version: ${{ matrix.compat.airflow-version }}
+    if: inputs.canary-run == 'true' || inputs.run-providers-e2e-tests-openlineage == 'true'
+
   test-ui-e2e-chromium:
     name: "Chromium UI e2e tests with PROD image"
     uses: ./.github/workflows/ui-e2e-tests.yml

.github/workflows/ci-amd.yml

@@ -141,6 +141,7 @@ jobs:
       run-event-driven-e2e-tests: ${{ steps.selective-checks.outputs.run-event-driven-e2e-tests }}
       run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }}
       run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }}
+      run-providers-e2e-tests-openlineage: ${{ steps.selective-checks.outputs.run-providers-e2e-tests-openlineage }}
       run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
       run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }}
       run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }}
@@ -435,7 +436,7 @@ jobs:
           DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
 
   providers:
-    name: "provider distributions tests"
+    name: "Provider distributions tests"
     uses: ./.github/workflows/test-providers.yml
     needs: [build-info, build-ci-images]
     permissions:
@@ -913,6 +914,9 @@ jobs:
       run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }}
       run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }}
       run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }}
+      run-providers-e2e-tests-openlineage: ${{ needs.build-info.outputs.run-providers-e2e-tests-openlineage }}
+      providers-compatibility-tests-matrix: >
+        ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
       run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }}
       run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }}

.github/workflows/ci-arm.yml

@@ -134,6 +134,7 @@ jobs:
       run-event-driven-e2e-tests: ${{ steps.selective-checks.outputs.run-event-driven-e2e-tests }}
       run-java-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-java-sdk-e2e-tests }}
       run-go-sdk-e2e-tests: ${{ steps.selective-checks.outputs.run-go-sdk-e2e-tests }}
+      run-providers-e2e-tests-openlineage: ${{ steps.selective-checks.outputs.run-providers-e2e-tests-openlineage }}
       run-system-tests: ${{ steps.selective-checks.outputs.run-system-tests }}
       run-task-sdk-tests: ${{ steps.selective-checks.outputs.run-task-sdk-tests }}
       run-task-sdk-integration-tests: ${{ steps.selective-checks.outputs.run-task-sdk-integration-tests }}
@@ -428,7 +429,7 @@ jobs:
           DEFAULT_BRANCH: ${{ needs.build-info.outputs.default-branch }}
 
   providers:
-    name: "provider distributions tests"
+    name: "Provider distributions tests"
     uses: ./.github/workflows/test-providers.yml
     needs: [build-info, build-ci-images]
     permissions:
@@ -906,6 +907,9 @@ jobs:
       run-event-driven-e2e-tests: ${{ needs.build-info.outputs.run-event-driven-e2e-tests }}
       run-java-sdk-e2e-tests: ${{ needs.build-info.outputs.run-java-sdk-e2e-tests }}
       run-go-sdk-e2e-tests: ${{ needs.build-info.outputs.run-go-sdk-e2e-tests }}
+      run-providers-e2e-tests-openlineage: ${{ needs.build-info.outputs.run-providers-e2e-tests-openlineage }}
+      providers-compatibility-tests-matrix: >
+        ${{ needs.build-info.outputs.providers-compatibility-tests-matrix }}
       use-uv: ${{ needs.build-info.outputs.use-uv }}
       run-ui-e2e-tests: ${{ needs.build-info.outputs.run-ui-e2e-tests }}
       run-airflow-ctl-integration-tests: ${{ needs.build-info.outputs.run-airflow-ctl-integration-tests }}

.github/workflows/providers-e2e-tests.yml

@@ -0,0 +1,157 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+
+name: Providers E2E Tests
+
+permissions:
+  contents: read
+on:  # yamllint disable-line rule:truthy
+  workflow_dispatch:
+    inputs:
+      provider:
+        description: "The provider whose e2e suite to run (a directory under providers-e2e-tests/)."
+        type: string
+        default: 'openlineage'
+      provider-display-name:
+        description: "Display name for the provider, used only in the job name."
+        type: string
+        default: 'OpenLineage'
+      runners:
+        description: "The array of labels (in json form) determining runners."
+        type: string
+        default: '["ubuntu-24.04"]'
+      platform:
+        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
+        type: string
+        default: 'linux/amd64'
+      default-python-version:
+        description: "Which version of python should be used by default"
+        type: string
+        default: '3.10'
+      use-uv:
+        description: "Whether to use uv to build the image (true/false)"
+        type: string
+        default: 'true'
+      docker-image-tag:
+        description: "Tag of the Docker image to test"
+        type: string
+        required: true
+      airflow-version:
+        description: "Released Airflow version to test with current providers (empty = default PROD image)"
+        type: string
+        default: ""
+
+  workflow_call:
+    inputs:
+      provider:
+        description: "The provider whose e2e suite to run (a directory under providers-e2e-tests/)."
+        type: string
+        default: 'openlineage'
+      provider-display-name:
+        description: "Display name for the provider, used only in the job name."
+        type: string
+        default: 'OpenLineage'
+      runners:
+        description: "The array of labels (in json form) determining runners."
+        required: true
+        type: string
+      platform:
+        description: "Platform for the build - 'linux/amd64' or 'linux/arm64'"
+        required: true
+        type: string
+      default-python-version:
+        description: "Which version of python should be used by default"
+        required: true
+        type: string
+      use-uv:
+        description: "Whether to use uv to build the image (true/false)"
+        required: true
+        type: string
+      docker-image-tag:
+        description: "Tag of the Docker image to test"
+        type: string
+        default: ""
+      airflow-version:
+        description: "Released Airflow version to test with current providers (empty = default PROD image)"
+        type: string
+        default: ""
+
+jobs:
+  test-providers-e2e-tests:
+    timeout-minutes: 60
+    name: "Provider E2E tests \
+      ${{ inputs.provider-display-name }}\
+      ${{ inputs.airflow-version != '' && format(' Compat {0}', inputs.airflow-version) || '' }}"
+    runs-on: ${{ fromJSON(inputs.runners) }}
+    env:
+      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
+      PROVIDER: ${{ inputs.provider }}
+      GITHUB_REPOSITORY: ${{ github.repository }}
+      GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      GITHUB_USERNAME: ${{ github.actor }}
+      VERBOSE: "true"
+    steps:
+      # Gating on job-level `if:` instead of per-step would skip the whole job — and GitHub Actions
+      # does not evaluate a dynamic `name:` (the one above, using `inputs.*`) for a skipped job, so it
+      # renders the raw unevaluated template instead. Gating each step keeps the job itself "running"
+      # (and its name properly resolved), ending in success with every real step skipped for AF2.
+      - name: "Skip notice for Airflow 2.x"
+        if: ${{ startsWith(inputs.airflow-version, '2.') }}
+        run: echo "Skipping $PROVIDER e2e tests — not supported against Airflow 2.x."
+      - name: "Cleanup repo"
+        if: ${{ !startsWith(inputs.airflow-version, '2.') }}
+        shell: bash
+        run: sudo rm -rf ${GITHUB_WORKSPACE}/*
+      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
+        if: ${{ !startsWith(inputs.airflow-version, '2.') }}
+        uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10  # v6.0.3
+        with:
+          fetch-depth: 2
+          persist-credentials: false
+      - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"
+        if: ${{ !startsWith(inputs.airflow-version, '2.') }}
+        uses: ./.github/actions/prepare_breeze_and_image
+        with:
+          platform: ${{ inputs.platform }}
+          image-type: "prod"
+          python: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}
+          use-uv: ${{ inputs.use-uv }}
+          make-mnt-writeable-and-cleanup: true
+        id: breeze
+      - name: "Run ${{ inputs.provider }} e2e tests"
+        if: ${{ !startsWith(inputs.airflow-version, '2.') }}
+        run: >
+          breeze testing providers-e2e-tests "$PROVIDER"
+          ${AIRFLOW_VERSION:+--airflow-version ${AIRFLOW_VERSION}}
+          --skip-mounting-local-volumes
+        env:
+          DOCKER_IMAGE: "${{ inputs.docker-image-tag }}"
+          AIRFLOW_VERSION: "${{ inputs.airflow-version }}"
+      - name: Zip logs
+        if: ${{ always() && !startsWith(inputs.airflow-version, '2.') }}
+        run: |
+          cd "./providers-e2e-tests/$PROVIDER" && zip -r logs.zip logs
+      - name: "Upload logs"
+        if: ${{ always() && !startsWith(inputs.airflow-version, '2.') }}
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a  # v7.0.1
+        with:
+          name: "${{ inputs.provider }}-e2e-test-logs-${{ inputs.airflow-version || 'prod' }}"
+          path: './providers-e2e-tests/${{ inputs.provider }}/logs.zip'
+          retention-days: '7'
+          if-no-files-found: 'error'

.rat-excludes

@@ -45,6 +45,7 @@ venv
 clients/*
 files/*
 dags/*
+providers-e2e-tests/openlineage/dags/dag_doc.md
 generated/*
 .gitmodules
 prod_image_installed_providers.txt

dev/breeze/doc/05_test_commands.rst

@@ -373,6 +373,27 @@ You can override the ``DOCKER_IMAGE`` environment variable to point to the image
 
 The Airflow E2E tests are in ``airflow-e2e-tests/`` folder in the main repo.
 
+Running provider E2E tests
+..........................
+
+You can use Breeze to run provider end-to-end tests, run against a real built Airflow image
+(``$DOCKER_IMAGE``) rather than the in-process ``dag.test()`` path used by provider system tests.
+What "e2e" means, and how the stack under test gets stood up, is entirely up to the provider — the
+OpenLineage suite, for example, deploys a real Airflow via docker-compose and runs its system-test
+DAGs against it, but a different provider could drive it a completely different way. Those tests run
+using the Production image by default; pass ``--airflow-version`` to run against an older released
+Airflow version with the current providers installed from main.
+
+.. image:: ./images/output_testing_providers-e2e-tests.svg
+  :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/images/output_testing_providers-e2e-tests.svg
+  :width: 100%
+  :alt: Breeze testing providers-e2e-tests
+
+The provider E2E tests are in the ``providers-e2e-tests/`` folder in the main repo; see
+``providers-e2e-tests/README.md`` for how to run any provider's suite, how it's wired into CI, and
+what's involved in adding a new provider (``providers-e2e-tests/openlineage/`` is one
+example implementation).
+
 Running Airflow UI E2E tests
 .............................
 

dev/breeze/doc/ci/04_selective_checks.md

@@ -427,6 +427,11 @@ together using `pytest-xdist` (pytest-xdist distributes the tests among parallel
     of affected providers (but not recursively - only direct dependencies are added)
   * if there are any changes to "common" provider code not belonging to any provider (usually system tests
     or tests), then tests for all Providers are run
+* `OpenLineage E2E tests` (the deployed-stack tests under `providers-e2e-tests/openlineage`, exposed as
+  the `run-providers-e2e-tests-openlineage` output) run when the `openlineage` or `common` providers or the
+  `providers-e2e-tests` harness change — and always on `canary` runs (where `full tests needed` also
+  covers core/task-sdk changes). Like the other deployed e2e suites, enabling them forces
+  `PROD Image building`.
 * The specific unit test type is enabled only if changed files match the expected patterns for each type
   (`API`, `CLI`, `WWW`, `Providers` etc.). The `Always` test type is added always if any unit
   tests are run. `Providers` tests are removed if current branch is different than `main`