Interactive LLM example based on the Airflow survey data

.asf.yaml

@@ -151,6 +151,11 @@ github:
         required_approving_review_count: 1
       required_linear_history: true
       required_signatures: false
+    v3-2-stable:
+      required_pull_request_reviews:
+        required_approving_review_count: 1
+      required_linear_history: true
+      required_signatures: false
     providers-fab/v1-5:
       required_pull_request_reviews:
         required_approving_review_count: 1

.github/CODEOWNERS

@@ -105,6 +105,7 @@ airflow-core/src/airflow/ui/public/i18n/locales/zh-TW/ @Lee-W @jason810496 @guan
 /providers/openlineage/ @mobuchowski
 /providers/smtp/ @hussein-awala
 /providers/snowflake/ @potiuk
+/providers/vespa/ @potiuk # + @radu-gheorghe @thomasht86
 
 
 # Generated metadata

.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml

@@ -206,6 +206,7 @@ body:
         - teradata
         - trino
         - vertica
+        - vespa
         - weaviate
         - yandex
         - ydb

.github/actions/prepare_breeze_and_image/action.yml

@@ -45,6 +45,9 @@ runs:
       shell: bash
       run: ./scripts/ci/make_mnt_writeable.sh
       if: inputs.make-mnt-writeable-and-cleanup == 'true'
+    - name: "Free up disk space"
+      shell: bash
+      run: ./scripts/tools/free_up_disk_space.sh
     - name: "Install Breeze"
       uses: ./.github/actions/breeze
       id: breeze

.github/boring-cyborg.yml

@@ -303,6 +303,9 @@ labelPRBasedOnFilePath:
   provider:vertica:
     - providers/vertica/**
 
+  provider:vespa:
+    - providers/vespa/**
+
   provider:weaviate:
     - providers/weaviate/**
 
@@ -575,7 +578,7 @@ labelerFlags:
 firstPRWelcomeComment: >
   Congratulations on your first Pull Request and welcome to the Apache Airflow community!
   If you have any issues or are unsure about any anything please check our
-  Contributors' Guide (https://github.com/apache/airflow/blob/main/contributing-docs/README.rst)
+  [Contributors' Guide](https://github.com/apache/airflow/blob/main/contributing-docs/README.rst)
 
   Here are some useful points:
 

.github/instructions/code-review.instructions.md

@@ -11,7 +11,7 @@ Use these rules when reviewing pull requests to the Apache Airflow repository.
 
 - **Scheduler must never run user code.** It only processes serialized Dags. Flag any scheduler-path code that deserializes or executes Dag/task code.
 - **Flag any task execution code that accesses the metadata DB directly** instead of through the Execution API (`/execution` endpoints).
-- **Flag any code in Dag Processor or Triggerer that breaks process isolation** — these components run user code in isolated processes.
+- **Flag any code in Dag Processor or Triggerer that breaks process isolation** — these components run user code in separate processes from the Scheduler and API Server, but note that they potentially have direct metadata database access and potentially bypass JWT authentication via in-process Execution API transport. This is an intentional design choice documented in the security model, not a security vulnerability.
 - **Flag any provider importing core internals** like `SUPERVISOR_COMMS` or task-runner plumbing. Providers interact through the public SDK and execution API only.
 
 ## Database and Query Correctness

.github/workflows/basic-tests.yml

@@ -301,89 +301,6 @@ jobs:
           fetch-depth: 2
           persist-credentials: false
 
-  upgrade-check:
-    timeout-minutes: 45
-    name: "Upgrade checks"
-    runs-on: ${{ fromJSON(inputs.runners) }}
-    env:
-      PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}"
-    if: inputs.canary-run == 'true'
-    steps:
-      - name: "Cleanup repo"
-        shell: bash
-        run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*"
-      - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
-        with:
-          persist-credentials: false
-      - name: "Install Breeze"
-        uses: ./.github/actions/breeze
-        id: breeze
-      - name: "Install prek"
-        uses: ./.github/actions/install-prek
-        id: prek
-        with:
-          python-version: ${{ steps.breeze.outputs.host-python-version }}
-          platform: ${{ inputs.platform }}
-          save-cache: false
-      - name: "Autoupdate all prek hooks"
-        run: prek autoupdate --cooldown-days 4 --freeze
-      - name: "Check if there are any changes in prek hooks"
-        run: |
-          if ! git diff --exit-code; then
-            echo -e "\n\033[0;31mThere are changes in prek hooks after upgrade check.\033[0m"
-            echo -e "\n\033[0;33mHow to fix:\033[0m Run \`breeze ci upgrade\` locally to fix it!.\n"
-            exit 1
-          fi
-      - name: "Run automated upgrade for chart dependencies"
-        run: >
-          prek
-          --all-files --show-diff-on-failure --color always --verbose
-          --stage manual
-          update-chart-dependencies
-        if: always()
-      # For UV we are not failing the upgrade installers check if it is updated because
-      # it is upgraded very frequently, so we want to manually upgrade it rather than
-      # get notified about it - until it stabilizes in 1.* version
-      - name: "Run automated upgrade for uv,  prek (not failing - just informational)"
-        run: >
-          prek
-          --all-files --show-diff-on-failure --color always --verbose
-          --stage manual upgrade-important-versions || true
-        if: always()
-        env:
-          UPGRADE_FLIT: "false"
-          UPGRADE_GITPYTHON: "false"
-          UPGRADE_GOLANG: "false"
-          UPGRADE_HATCH: "false"
-          UPGRADE_HATCHLING: "false"
-          UPGRADE_MYPY: "false"
-          UPGRADE_NODE_LTS: "false"
-          UPGRADE_PIP: "false"
-          UPGRADE_PYTHON: "false"
-          UPGRADE_PYYAML: "false"
-          UPGRADE_RICH: "false"
-          UPGRADE_RUFF: "false"
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-      - name: "Run automated upgrade for important versions minus uv (failing if needed)"
-        run: |
-          if ! prek \
-            --all-files --show-diff-on-failure --color always --verbose \
-            --stage manual upgrade-important-versions; then
-              echo -e "\n\033[0;31mThere are changes in prek hooks after upgrade check.\033[0m"
-              echo -e "\n\033[0;33mHow to fix:\033[0m Run \`breeze ci upgrade\` locally to fix it!.\n"
-              exit 1
-          fi
-        if: always()
-        env:
-          UPGRADE_UV: "false"
-          UPGRADE_PREK: "false"
-          UPGRADE_MPROCS: "false"
-          UPGRADE_PROTOC: "false"
-          UPGRADE_OPENAPI_GENERATOR: "false"
-          UPGRADE_COOLDOWN_DAYS: "4"
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-
   test-airflow-release-commands:
     timeout-minutes: 80
     name: "Test Airflow release commands"

.github/workflows/check-newsfragment-pr-number.yml

@@ -21,7 +21,7 @@ on:  # yamllint disable-line rule:truthy
   pull_request:
     branches:
       - main
-    types: [opened, reopened, synchronize, labeled, unlabeled]
+    types: [opened, reopened, synchronize]
 
 permissions:
   contents: read

.github/workflows/ci-amd-arm.yml

@@ -347,9 +347,6 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           persist-credentials: false
-      - name: "Free up disk space"
-        shell: bash
-        run: ./scripts/tools/free_up_disk_space.sh
       - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}"
         uses: ./.github/actions/prepare_breeze_and_image
         with:

.github/workflows/k8s-tests.yml

@@ -84,9 +84,6 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           persist-credentials: false
-      - name: "Free up disk space"
-        shell: bash
-        run: ./scripts/tools/free_up_disk_space.sh
       # env.PYTHON_MAJOR_MINOR_VERSION, env.KUBERNETES_VERSION are set in the previous
       # step id: prepare-versions
       - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}"

.github/workflows/scheduled-upgrade-check-main.yml

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "[main] Scheduled CI upgrade check"
+on:  # yamllint disable-line rule:truthy
+  schedule:
+    # Mon, Wed, Fri at 06:00 UTC
+    - cron: '0 6 * * 1,3,5'
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  upgrade-main:
+    name: "[main] Upgrade"
+    uses: ./.github/workflows/upgrade-check.yml
+    with:
+      target-branch: main
+    secrets:
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/scheduled-upgrade-check-v3-2-test.yml

@@ -0,0 +1,34 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+---
+name: "[v3-2-test] Scheduled CI upgrade check"
+on:  # yamllint disable-line rule:truthy
+  schedule:
+    # Tue, Thu at 06:00 UTC
+    - cron: '0 6 * * 2,4'
+  workflow_dispatch:
+permissions:
+  contents: read
+jobs:
+  upgrade-v3-2-test:
+    name: "[v3-2-test] Upgrade"
+    uses: ./.github/workflows/upgrade-check.yml
+    with:
+      target-branch: v3-2-test
+    secrets:
+      SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}

.github/workflows/test-providers.yml

@@ -92,9 +92,6 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           persist-credentials: false
-      - name: "Free up disk space"
-        shell: bash
-        run: ./scripts/tools/free_up_disk_space.sh
       - name: "Install prek"
         uses: ./.github/actions/install-prek
         id: prek
@@ -201,9 +198,6 @@ jobs:
         uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
         with:
           persist-credentials: false
-      - name: "Free up disk space"
-        shell: bash
-        run: ./scripts/tools/free_up_disk_space.sh
       - name: "Install prek"
         uses: ./.github/actions/install-prek
         id: prek