diff --git a/.asf.yaml b/.asf.yaml index b8fb1be32036f..10947cbc7ffac 100644 --- a/.asf.yaml +++ b/.asf.yaml @@ -151,6 +151,11 @@ github: required_approving_review_count: 1 required_linear_history: true required_signatures: false + v3-2-stable: + required_pull_request_reviews: + required_approving_review_count: 1 + required_linear_history: true + required_signatures: false providers-fab/v1-5: required_pull_request_reviews: required_approving_review_count: 1 diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e23ac33fa872c..2da9dbc9c27c9 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -105,6 +105,7 @@ airflow-core/src/airflow/ui/public/i18n/locales/zh-TW/ @Lee-W @jason810496 @guan /providers/openlineage/ @mobuchowski /providers/smtp/ @hussein-awala /providers/snowflake/ @potiuk +/providers/vespa/ @potiuk # + @radu-gheorghe @thomasht86 # Generated metadata diff --git a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml index 61fb8b4415113..33254b162eb7b 100644 --- a/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml +++ b/.github/ISSUE_TEMPLATE/1-airflow_bug_report.yml @@ -206,6 +206,7 @@ body: - teradata - trino - vertica + - vespa - weaviate - yandex - ydb diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 753cfd0378231..2498caad2b80d 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -45,6 +45,9 @@ runs: shell: bash run: ./scripts/ci/make_mnt_writeable.sh if: inputs.make-mnt-writeable-and-cleanup == 'true' + - name: "Free up disk space" + shell: bash + run: ./scripts/tools/free_up_disk_space.sh - name: "Install Breeze" uses: ./.github/actions/breeze id: breeze diff --git a/.github/boring-cyborg.yml b/.github/boring-cyborg.yml index 1df8a75b82bf4..2888b199edc7c 100644 --- a/.github/boring-cyborg.yml +++ b/.github/boring-cyborg.yml @@ -303,6 +303,9 @@ labelPRBasedOnFilePath: provider:vertica: - providers/vertica/** + provider:vespa: + - providers/vespa/** + provider:weaviate: - providers/weaviate/** @@ -575,7 +578,7 @@ labelerFlags: firstPRWelcomeComment: > Congratulations on your first Pull Request and welcome to the Apache Airflow community! If you have any issues or are unsure about any anything please check our - Contributors' Guide (https://github.com/apache/airflow/blob/main/contributing-docs/README.rst) + [Contributors' Guide](https://github.com/apache/airflow/blob/main/contributing-docs/README.rst) Here are some useful points: diff --git a/.github/instructions/code-review.instructions.md b/.github/instructions/code-review.instructions.md index 0d4ce8a87913a..cd480bdcaf706 100644 --- a/.github/instructions/code-review.instructions.md +++ b/.github/instructions/code-review.instructions.md @@ -11,7 +11,7 @@ Use these rules when reviewing pull requests to the Apache Airflow repository. - **Scheduler must never run user code.** It only processes serialized Dags. Flag any scheduler-path code that deserializes or executes Dag/task code. - **Flag any task execution code that accesses the metadata DB directly** instead of through the Execution API (`/execution` endpoints). -- **Flag any code in Dag Processor or Triggerer that breaks process isolation** — these components run user code in isolated processes. +- **Flag any code in Dag Processor or Triggerer that breaks process isolation** — these components run user code in separate processes from the Scheduler and API Server, but note that they potentially have direct metadata database access and potentially bypass JWT authentication via in-process Execution API transport. This is an intentional design choice documented in the security model, not a security vulnerability. - **Flag any provider importing core internals** like `SUPERVISOR_COMMS` or task-runner plumbing. Providers interact through the public SDK and execution API only. ## Database and Query Correctness diff --git a/.github/workflows/basic-tests.yml b/.github/workflows/basic-tests.yml index 883f432c9ff23..5f449ab2227a9 100644 --- a/.github/workflows/basic-tests.yml +++ b/.github/workflows/basic-tests.yml @@ -301,89 +301,6 @@ jobs: fetch-depth: 2 persist-credentials: false - upgrade-check: - timeout-minutes: 45 - name: "Upgrade checks" - runs-on: ${{ fromJSON(inputs.runners) }} - env: - PYTHON_MAJOR_MINOR_VERSION: "${{ inputs.default-python-version }}" - if: inputs.canary-run == 'true' - steps: - - name: "Cleanup repo" - shell: bash - run: docker run -v "${GITHUB_WORKSPACE}:/workspace" -u 0:0 bash -c "rm -rf /workspace/*" - - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )" - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 - with: - persist-credentials: false - - name: "Install Breeze" - uses: ./.github/actions/breeze - id: breeze - - name: "Install prek" - uses: ./.github/actions/install-prek - id: prek - with: - python-version: ${{ steps.breeze.outputs.host-python-version }} - platform: ${{ inputs.platform }} - save-cache: false - - name: "Autoupdate all prek hooks" - run: prek autoupdate --cooldown-days 4 --freeze - - name: "Check if there are any changes in prek hooks" - run: | - if ! git diff --exit-code; then - echo -e "\n\033[0;31mThere are changes in prek hooks after upgrade check.\033[0m" - echo -e "\n\033[0;33mHow to fix:\033[0m Run \`breeze ci upgrade\` locally to fix it!.\n" - exit 1 - fi - - name: "Run automated upgrade for chart dependencies" - run: > - prek - --all-files --show-diff-on-failure --color always --verbose - --stage manual - update-chart-dependencies - if: always() - # For UV we are not failing the upgrade installers check if it is updated because - # it is upgraded very frequently, so we want to manually upgrade it rather than - # get notified about it - until it stabilizes in 1.* version - - name: "Run automated upgrade for uv, prek (not failing - just informational)" - run: > - prek - --all-files --show-diff-on-failure --color always --verbose - --stage manual upgrade-important-versions || true - if: always() - env: - UPGRADE_FLIT: "false" - UPGRADE_GITPYTHON: "false" - UPGRADE_GOLANG: "false" - UPGRADE_HATCH: "false" - UPGRADE_HATCHLING: "false" - UPGRADE_MYPY: "false" - UPGRADE_NODE_LTS: "false" - UPGRADE_PIP: "false" - UPGRADE_PYTHON: "false" - UPGRADE_PYYAML: "false" - UPGRADE_RICH: "false" - UPGRADE_RUFF: "false" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - - name: "Run automated upgrade for important versions minus uv (failing if needed)" - run: | - if ! prek \ - --all-files --show-diff-on-failure --color always --verbose \ - --stage manual upgrade-important-versions; then - echo -e "\n\033[0;31mThere are changes in prek hooks after upgrade check.\033[0m" - echo -e "\n\033[0;33mHow to fix:\033[0m Run \`breeze ci upgrade\` locally to fix it!.\n" - exit 1 - fi - if: always() - env: - UPGRADE_UV: "false" - UPGRADE_PREK: "false" - UPGRADE_MPROCS: "false" - UPGRADE_PROTOC: "false" - UPGRADE_OPENAPI_GENERATOR: "false" - UPGRADE_COOLDOWN_DAYS: "4" - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - test-airflow-release-commands: timeout-minutes: 80 name: "Test Airflow release commands" diff --git a/.github/workflows/check-newsfragment-pr-number.yml b/.github/workflows/check-newsfragment-pr-number.yml index be31eae726e85..2c9fce37b827f 100644 --- a/.github/workflows/check-newsfragment-pr-number.yml +++ b/.github/workflows/check-newsfragment-pr-number.yml @@ -21,7 +21,7 @@ on: # yamllint disable-line rule:truthy pull_request: branches: - main - types: [opened, reopened, synchronize, labeled, unlabeled] + types: [opened, reopened, synchronize] permissions: contents: read diff --git a/.github/workflows/ci-amd-arm.yml b/.github/workflows/ci-amd-arm.yml index 64e19d1981e37..c094a6ee82565 100644 --- a/.github/workflows/ci-amd-arm.yml +++ b/.github/workflows/ci-amd-arm.yml @@ -347,9 +347,6 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: "Free up disk space" - shell: bash - run: ./scripts/tools/free_up_disk_space.sh - name: "Prepare breeze & CI image: ${{ needs.build-info.outputs.default-python-version }}" uses: ./.github/actions/prepare_breeze_and_image with: diff --git a/.github/workflows/k8s-tests.yml b/.github/workflows/k8s-tests.yml index ac86ac2ee4497..dd45336d5e79f 100644 --- a/.github/workflows/k8s-tests.yml +++ b/.github/workflows/k8s-tests.yml @@ -84,9 +84,6 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: "Free up disk space" - shell: bash - run: ./scripts/tools/free_up_disk_space.sh # env.PYTHON_MAJOR_MINOR_VERSION, env.KUBERNETES_VERSION are set in the previous # step id: prepare-versions - name: "Prepare breeze & PROD image: ${{ env.PYTHON_MAJOR_MINOR_VERSION }}" diff --git a/.github/workflows/scheduled-upgrade-check-main.yml b/.github/workflows/scheduled-upgrade-check-main.yml new file mode 100644 index 0000000000000..5978721b97195 --- /dev/null +++ b/.github/workflows/scheduled-upgrade-check-main.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: "[main] Scheduled CI upgrade check" +on: # yamllint disable-line rule:truthy + schedule: + # Mon, Wed, Fri at 06:00 UTC + - cron: '0 6 * * 1,3,5' + workflow_dispatch: +permissions: + contents: read +jobs: + upgrade-main: + name: "[main] Upgrade" + uses: ./.github/workflows/upgrade-check.yml + with: + target-branch: main + secrets: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/scheduled-upgrade-check-v3-2-test.yml b/.github/workflows/scheduled-upgrade-check-v3-2-test.yml new file mode 100644 index 0000000000000..268dd798f2e7d --- /dev/null +++ b/.github/workflows/scheduled-upgrade-check-v3-2-test.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: "[v3-2-test] Scheduled CI upgrade check" +on: # yamllint disable-line rule:truthy + schedule: + # Tue, Thu at 06:00 UTC + - cron: '0 6 * * 2,4' + workflow_dispatch: +permissions: + contents: read +jobs: + upgrade-v3-2-test: + name: "[v3-2-test] Upgrade" + uses: ./.github/workflows/upgrade-check.yml + with: + target-branch: v3-2-test + secrets: + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} diff --git a/.github/workflows/test-providers.yml b/.github/workflows/test-providers.yml index d6c268db849e0..db1f31f2453aa 100644 --- a/.github/workflows/test-providers.yml +++ b/.github/workflows/test-providers.yml @@ -92,9 +92,6 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: "Free up disk space" - shell: bash - run: ./scripts/tools/free_up_disk_space.sh - name: "Install prek" uses: ./.github/actions/install-prek id: prek @@ -201,9 +198,6 @@ jobs: uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: persist-credentials: false - - name: "Free up disk space" - shell: bash - run: ./scripts/tools/free_up_disk_space.sh - name: "Install prek" uses: ./.github/actions/install-prek id: prek diff --git a/.github/workflows/upgrade-check.yml b/.github/workflows/upgrade-check.yml new file mode 100644 index 0000000000000..55a7383665ece --- /dev/null +++ b/.github/workflows/upgrade-check.yml @@ -0,0 +1,125 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# +--- +name: Upgrade check +on: # yamllint disable-line rule:truthy + workflow_call: + inputs: + target-branch: + description: >- + Branch to upgrade (e.g. 'main' or 'v3-2-test') + required: true + type: string + secrets: + SLACK_BOT_TOKEN: + description: "Slack bot token for notifications" + required: true +permissions: + contents: write + pull-requests: write +env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }} + TARGET_BRANCH: ${{ inputs.target-branch }} +jobs: + createupgrade-check: + timeout-minutes: 45 + name: >- + [${{ inputs.target-branch }}] Upgrade checks and PR + runs-on: ["ubuntu-22.04"] + steps: + - name: >- + [${{ inputs.target-branch }}] Cleanup repo + shell: bash + run: > + docker run -v "${GITHUB_WORKSPACE}:/workspace" + -u 0:0 bash -c "rm -rf /workspace/*" + - name: >- + [${{ inputs.target-branch }}] Checkout + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: ${{ inputs.target-branch }} + fetch-depth: 0 + persist-credentials: false + - name: >- + [${{ inputs.target-branch }}] Configure git credentials + run: | + git remote set-url origin \ + "https://x-access-token:${GITHUB_TOKEN}@github.com/${GITHUB_REPOSITORY}.git" + git config user.name "github-actions[bot]" + git config user.email \ + "41898282+github-actions[bot]@users.noreply.github.com" + - name: >- + [${{ inputs.target-branch }}] Install Breeze + uses: ./.github/actions/breeze + id: breeze + - name: >- + [${{ inputs.target-branch }}] Install prek + uses: ./.github/actions/install-prek + id: prek + with: + python-version: >- + ${{ steps.breeze.outputs.host-python-version }} + platform: "linux/amd64" + save-cache: false + - name: >- + [${{ inputs.target-branch }}] Run breeze ci upgrade + run: > + breeze ci upgrade + --target-branch "${TARGET_BRANCH}" + --create-pr + --draft + --switch-to-base + --no-k8s-schema-sync + --answer yes + - name: >- + [${{ inputs.target-branch }}] Find upgrade PR + id: find-pr + run: | + PR_URL=$(gh pr list \ + --repo "${GITHUB_REPOSITORY}" \ + --head "ci-upgrade-${TARGET_BRANCH}" \ + --base "${TARGET_BRANCH}" \ + --state open \ + --json url \ + --jq '.[0].url' 2>/dev/null || true) + echo "pr-url=${PR_URL}" >> "${GITHUB_OUTPUT}" + - name: >- + [${{ inputs.target-branch }}] Notify Slack + if: steps.find-pr.outputs.pr-url != '' + uses: >- + slackapi/slack-github-action@af78098f536edbc4de71162a307590698245be95 + with: + method: chat.postMessage + token: ${{ env.SLACK_BOT_TOKEN }} + payload: | + channel: "internal-airflow-ci-cd" + text: >- + 🔧 [${{ inputs.target-branch }}] CI upgrade PR + ready for review. Please undraft, review and + merge: ${{ steps.find-pr.outputs.pr-url }} + blocks: + - type: section + text: + type: mrkdwn + text: >- + 🔧 *[${{ inputs.target-branch }}] CI upgrade + PR ready for review* + + Please undraft, review and merge: + <${{ steps.find-pr.outputs.pr-url }}|View PR> diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 11a4b47d9d64b..df62237b6f36f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -448,7 +448,7 @@ repos: types_or: [python, pyi] args: [--fix] require_serial: true - additional_dependencies: ['ruff==0.15.8'] + additional_dependencies: ['ruff==0.15.9'] exclude: ^airflow-core/tests/unit/dags/test_imports\.py$|^performance/tests/test_.*\.py$ - id: ruff-format name: Run 'ruff format' @@ -977,6 +977,12 @@ repos: language: python pass_filenames: true files: \.py$ + - id: check-no-new-airflow-exceptions + name: Check that no new raise AirflowException usages are added + entry: ./scripts/ci/prek/check_new_airflow_exception_usage.py + language: python + pass_filenames: true + files: ^(airflow-core|airflow-ctl|task-sdk|providers|shared)/.*\.py$ - id: bandit name: bandit description: "Bandit is a tool for finding common security issues in Python code" diff --git a/AGENTS.md b/AGENTS.md index b9ef07b381d46..33d1486cbc209 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -63,19 +63,46 @@ UV workspace monorepo. Key paths: - `ci/prek/` — prek (pre-commit) hook scripts; shared utilities in `common_prek_utils.py` - `tests/` — pytest tests for the scripts; run with `uv run --project scripts pytest scripts/tests/` +The `uv.lock` file is generated by `uv lock`, `uv sync` and is committed to the repo - it contains snapshot of +versions of all dependencies used for development of Airflow. If at any point in time you have a conflict +with `uv.lock`, simply delete it and run `uv lock` to regenerate it. + ## Architecture Boundaries 1. Users author Dags with the Task SDK (`airflow.sdk`). -2. Dag Processor parses Dag files in isolated processes and stores serialized Dags in the metadata DB. +2. Dag File Processor parses Dag files in separate processes and stores serialized Dags in the metadata DB. Software guards prevent individual parsing processes from accessing the database directly and enforce use of the Execution API, but these guards do not protect against intentional bypassing by malicious or misconfigured code. 3. Scheduler reads serialized Dags — **never runs user code** — and creates Dag runs / task instances. -4. Workers execute tasks via Task SDK and communicate with the API server through the Execution API — **never access the metadata DB directly**. +4. Workers execute tasks via Task SDK and communicate with the API server through the Execution API — **never access the metadata DB directly**. Each task receives a short-lived JWT token scoped to its task instance ID. 5. API Server serves the React UI and handles all client-database interactions. -6. Triggerer evaluates deferred tasks/sensors in isolated processes. +6. Triggerer evaluates deferred tasks/sensors in separate processes. Like the Dag File Processor, software guards steer it through the Execution API rather than direct database access, but these guards do not protect against intentional bypassing by malicious or misconfigured code. 7. Shared libraries that are symbolically linked to different Python distributions are in `shared` folder. 8. Airflow uses `uv workspace` feature to keep all the distributions sharing dependencies and venv 9. Each of the distributions should declare other needed distributions: `uv --project sync` command acts on the selected project in the monorepo with only dependencies that it has +## Security Model + +When reviewing code, writing security documentation, or performing security research, keep in +mind the following aspects of Airflow's security model. The authoritative reference is +[`airflow-core/docs/security/security_model.rst`](airflow-core/docs/security/security_model.rst) +and [`airflow-core/docs/security/jwt_token_authentication.rst`](airflow-core/docs/security/jwt_token_authentication.rst). + +Particularly, the intentional design choices, that are not security vulnerabilities and should not be +reported as such are described in "What is NOT considered a security vulnerability" chapter of the security model. + +**When flagging security concerns, distinguish between:** + +1. **Actual vulnerabilities** — code that violates the documented security model (e.g., a worker + gaining database access it shouldn't have, a Scheduler executing user code, an unauthenticated + user accessing protected endpoints). +2. **Known limitations** — documented gaps where the current implementation doesn't provide full + isolation (e.g., DFP/Triggerer database access, shared Execution API resources, multi-team + not enforcing task-level isolation). These are tracked for improvement in future versions and + should not be reported as new findings. +3. **Deployment hardening opportunities** — measures a Deployment Manager can take to improve + isolation beyond what Airflow enforces natively (e.g., per-component configuration, asymmetric + JWT keys, network policies). These belong in deployment guidance, not as code-level issues. + # Shared libraries - shared libraries provide implementation of some common utilities like logging, configuration where the code should be reused in different distributions (potentially in different versions) diff --git a/COMMITTERS.rst b/COMMITTERS.rst index 23a52436918c4..cd82d303ae67d 100644 --- a/COMMITTERS.rst +++ b/COMMITTERS.rst @@ -283,7 +283,7 @@ To be able to merge PRs, committers have to integrate their GitHub ID with Apach New PMC Member Onboarding steps ------------------------------- -1. Familiarise yourself with `https://community.apache.org/pmc/responsibilities.html`_ +1. Familiarise yourself with `PMC Responsibilities `_ 2. Subscribe to the private mailing list: ``private@airflow.apache.org``. Do this by sending an empty email to ``private-subscribe@airflow.apache.org`` and following the instructions in the automated response you'll receive. 3. Ask another PMC member to add you to ``#pmc-private`` channel on slack diff --git a/Dockerfile b/Dockerfile index d56e1fb165f8d..b3333db01533e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -48,10 +48,10 @@ ARG AIRFLOW_UID="50000" ARG AIRFLOW_USER_HOME_DIR=/home/airflow # latest released version here -ARG AIRFLOW_VERSION="3.1.8" +ARG AIRFLOW_VERSION="3.2.0" ARG BASE_IMAGE="debian:bookworm-slim" -ARG AIRFLOW_PYTHON_VERSION="3.12.13" +ARG AIRFLOW_PYTHON_VERSION="3.13.13" # PYTHON_LTO: Controls whether Python is built with Link-Time Optimization (LTO). # @@ -1874,7 +1874,7 @@ ENV DEV_APT_DEPS=${DEV_APT_DEPS} \ ARG PYTHON_LTO ENV RUSTUP_HOME="/usr/local/rustup" -ENV CARGO_HOME="/usr/local/cargo" +ENV CARGO_HOME="/home/airflow/.cargo" ENV PATH="${CARGO_HOME}/bin:${PATH}" COPY --from=scripts install_os_dependencies.sh /scripts/docker/ diff --git a/Dockerfile.ci b/Dockerfile.ci index 7aaf2814e3e38..cf9f7437dc4b0 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -1673,9 +1673,9 @@ ENV DEV_APT_COMMAND=${DEV_APT_COMMAND} \ ADDITIONAL_DEV_APT_COMMAND=${ADDITIONAL_DEV_APT_COMMAND} -ARG AIRFLOW_PYTHON_VERSION="3.12.13" +ARG AIRFLOW_PYTHON_VERSION="3.13.13" ENV AIRFLOW_PYTHON_VERSION=${AIRFLOW_PYTHON_VERSION} -ENV GOLANG_MAJOR_MINOR_VERSION="1.26.1" +ENV GOLANG_MAJOR_MINOR_VERSION="1.26.2" ENV RUSTUP_HOME="/usr/local/rustup" ENV CARGO_HOME="/usr/local/cargo" ENV PATH="${CARGO_HOME}/bin:${PATH}" diff --git a/INTHEWILD.md b/INTHEWILD.md index 1feeeae161c9b..07d403be9eca4 100644 --- a/INTHEWILD.md +++ b/INTHEWILD.md @@ -306,6 +306,7 @@ Currently, **officially** using Airflow: 1. [Interia](http://www.interia.pl) 1. [Intuit](https://www.intuit.com/) 1. [Investorise](https://investorise.com/) [[@svenvarkel](https://github.com/svenvarkel)] +1. [Ipregistry](https://ipregistry.co) [[@ipregistry](https://github.com/ipregistry)] 1. [iS2.co](https://www.is2.co) [[@iS2co](https://github.com/iS2co)] 1. [Jagex](https://www.jagex.com) [[@anumsheraz](https://github.com/AnumSheraz), [@trucnguyenlam](https://github.com/trucnguyenlam), [@lumez](https://github.com/lumez)] 1. [Jampp](https://github.com/jampp) diff --git a/README.md b/README.md index 38dacf897e0c7..f8e3e858c9b8a 100644 --- a/README.md +++ b/README.md @@ -98,14 +98,14 @@ Airflow is not a streaming solution, but it is often used to process real-time d Apache Airflow is tested with: -| | Main version (dev) | Stable version (3.1.8) | Stable version (2.11.2) | -|------------|------------------------------------|------------------------|------------------------------| -| Python | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12, 3.13 | 3.10, 3.11, 3.12 | -| Platform | AMD64/ARM64 | AMD64/ARM64 | AMD64/ARM64(\*) | -| Kubernetes | 1.30, 1.31, 1.32, 1.33, 1.34, 1.35 | 1.30, 1.31, 1.32, 1.33 | 1.26, 1.27, 1.28, 1.29, 1.30 | -| PostgreSQL | 14, 15, 16, 17, 18 | 13, 14, 15, 16, 17 | 12, 13, 14, 15, 16 | -| MySQL | 8.0, 8.4, Innovation | 8.0, 8.4, Innovation | 8.0, Innovation | -| SQLite | 3.15.0+ | 3.15.0+ | 3.15.0+ | +| | Main version (dev) | Stable version (3.2.0) | Stable version (2.11.2) | +|------------|------------------------------------|-------------------------------------|------------------------------| +| Python | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12 | +| Platform | AMD64/ARM64 | AMD64/ARM64 | AMD64/ARM64(\*) | +| Kubernetes | 1.30, 1.31, 1.32, 1.33, 1.34, 1.35 | 1.30, 1.31, 1.32, 1.33, 1.34, 1.35 | 1.26, 1.27, 1.28, 1.29, 1.30 | +| PostgreSQL | 14, 15, 16, 17, 18 | 14, 15, 16, 17, 18 | 12, 13, 14, 15, 16 | +| MySQL | 8.0, 8.4, Innovation | 8.0, 8.4, Innovation | 8.0, Innovation | +| SQLite | 3.15.0+ | 3.15.0+ | 3.15.0+ | \* Experimental @@ -171,15 +171,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==3.1.8' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.1.8/constraints-3.10.txt" +pip install 'apache-airflow==3.2.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.2.0/constraints-3.10.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==3.1.8' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.1.8/constraints-3.10.txt" +pip install 'apache-airflow[postgres,google]==3.2.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.2.0/constraints-3.10.txt" ``` For information on installing provider distributions, check @@ -293,7 +293,7 @@ Apache Airflow version life cycle: | Version | Current Patch/Minor | State | First Release | Limited Maintenance | EOL/Terminated | |-----------|-----------------------|---------------------|-----------------|-----------------------|------------------| -| 3 | 3.1.8 | Maintenance | Apr 22, 2025 | TBD | TBD | +| 3 | 3.2.0 | Maintenance | Apr 22, 2025 | TBD | TBD | | 2 | 2.11.2 | Limited maintenance | Dec 17, 2020 | Oct 22, 2025 | Apr 22, 2026 | | 1.10 | 1.10.15 | EOL | Aug 27, 2018 | Dec 17, 2020 | June 17, 2021 | | 1.9 | 1.9.0 | EOL | Jan 03, 2018 | Aug 27, 2018 | Aug 27, 2018 | diff --git a/RELEASE_NOTES.rst b/RELEASE_NOTES.rst index 1accb35dfb98b..46db14f826169 100644 --- a/RELEASE_NOTES.rst +++ b/RELEASE_NOTES.rst @@ -24,6 +24,655 @@ .. towncrier release notes start +Airflow 3.2.0 (2026-04-07) +-------------------------- + +Significant Changes +^^^^^^^^^^^^^^^^^^^ + +Asset Partitioning +"""""""""""""""""" + +The headline feature of Airflow 3.2.0 is asset partitioning — a major evolution of data-aware +scheduling. Instead of triggering Dags based on an entire asset, you can now schedule downstream +processing based on specific partitions of data. Only the relevant slice of data triggers downstream +work, making pipeline orchestration far more efficient and precise. + +This matters when working with partitioned data lakes — date-partitioned S3 paths, Hive table +partitions, BigQuery table partitions, or any other partitioned data store. Previously, any update +to an asset triggered all downstream Dags regardless of which partition changed. Now only the right +work gets triggered at the right time. + +For detailed usage instructions, see :doc:`/authoring-and-scheduling/assets`. + +Multi-Team Deployments +"""""""""""""""""""""" + +Airflow 3.2 introduces multi-team support, allowing organizations to run multiple isolated teams within a single Airflow deployment. +Each team can have its own Dags, connections, variables, pools, and executors— enabling true resource and permission isolation without requiring separate Airflow instances per team. + +This is particularly valuable for platform teams that serve multiple data engineering or data science teams from shared infrastructure, while maintaining strong boundaries between teams' resources and access. + +For detailed usage instructions, see :doc:`/core-concepts/multi-team`. + +.. warning:: + + Multi-Team Deployments are experimental in 3.2.0 and may change in future versions based on + user feedback. + +Synchronous callback support for Deadline Alerts +"""""""""""""""""""""""""""""""""""""""""""""""" + +Deadline Alerts now support synchronous callbacks via ``SyncCallback`` in addition to the existing +asynchronous ``AsyncCallback``. Synchronous callbacks are executed by the executor (rather than +the triggerer), and can optionally target a specific executor via the ``executor`` parameter. + +A Dag can also define multiple Deadline Alerts by passing a list to the ``deadline`` parameter, +and each alert can use either callback type. + +.. warning:: + + Deadline Alerts are experimental in 3.2.0 and may change in future versions based on + user feedback. Synchronous deadline callbacks (``SyncCallback``) do not currently + support Connections stored in the Airflow metadata database. + +For detailed usage instructions, see :doc:`/howto/deadline-alerts`. + + +UI Enhancements & Performance +""""""""""""""""""""""""""""" + +- **Grid View Virtualization**: + The Grid view now uses virtualization -- only visible rows are rendered to the DOM. This dramatically improves performance when viewing Dags with large numbers of task runs, reducing render time and memory usage for complex Dags. (#60241) + +- **XCom Management in the UI**: + You can now add, edit, and delete XCom values directly from the Airflow UI. This makes it much easier to debug and manage XCom state during development and day-to-day operations without needing CLI commands. (#58921) + +- **HITL Detail History**: + The Human-in-the-Loop approval interface now includes a full history view, letting operators and reviewers see the complete audit trail of approvals and rejections for any task. (#56760, #55952) + +- **Gantt Chart Improvements**: + + - All task tries displayed: Gantt chart now shows every attempt, not just the latest + - Task display names in Gantt: task_display_name shown for better readability (#61438) + - ISO dates in Gantt: Cross-browser consistent date format (#61250) + - Fixed null datetime crash: Gantt chart no longer crashes on tasks with null datetime fields + +New ``--only-idle`` flag for the scheduler CLI +""""""""""""""""""""""""""""""""""""""""""""""" + +The ``airflow scheduler`` command has a new ``--only-idle`` flag that only counts runs when the +scheduler is idle. This helps users run the scheduler once and process all triggered Dags and +queued tasks. It requires and complements the ``--num-runs`` flag so one can set a small value +instead of guessing how many iterations the scheduler needs. + +Replace per-run TI summary requests with a single NDJSON stream +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The grid, graph, gantt, and task-detail views now fetch task-instance +summaries through a single streaming HTTP request +(``GET /ui/grid/ti_summaries/{dag_id}?run_ids=...``) instead of one request +per run. The server emits one JSON line per run as soon as that run's task +instances are ready, so columns appear progressively rather than all at once. + +**What changed:** + +- ``GET /ui/grid/ti_summaries/{dag_id}?run_ids=...`` is now the sole endpoint + for TI summaries, returning an ``application/x-ndjson`` stream where each + line is a serialized ``GridTISummaries`` object for one run. +- The old single-run endpoint ``GET /ui/grid/ti_summaries/{dag_id}/{run_id}`` + has been removed. +- The serialized Dag structure is loaded once and shared across all runs that + share the same ``dag_version_id``, avoiding redundant deserialization. +- All UI views (grid, graph, gantt, task instance, mapped task instance, group + task instance) use the stream endpoint, passing one or more ``run_ids``. + +Structured JSON logging for all API server output +""""""""""""""""""""""""""""""""""""""""""""""""" + +The new ``json_logs`` option under the ``[logging]`` section makes Airflow +produce all its output as newline-delimited JSON (structured logs) instead of +human-readable formatted logs. This covers the API server (gunicorn/uvicorn), +including access logs, warnings, and unhandled exceptions. + +Not all components support this yet — notably ``airflow celery worker`` but +any non-JSON output when ``json_logs`` is enabled will be treated as a bug. (#63365) + +Remove legacy OTel Trace metaclass and shared tracer wrappers +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The interfaces and functions located in ``airflow.traces`` were +internal code that provided a standard way to manage spans in +internal Airflow code. They were not intended as user-facing code +and were never documented. They are no longer needed so we +remove them in 3.2. (#63452) + +Move task-level exception imports into the Task SDK +""""""""""""""""""""""""""""""""""""""""""""""""""" + +Airflow now sources task-facing exceptions (``AirflowSkipException``, ``TaskDeferred``, etc.) from +``airflow.sdk.exceptions``. ``airflow.exceptions`` still exposes the same exceptions, but they are +proxies that emit ``DeprecatedImportWarning`` so Dag authors can migrate before the shim is removed. + +**What changed:** + +- Runtime code now consistently raises the SDK versions of task-level exceptions. +- The Task SDK redefines these classes so workers no longer depend on ``airflow-core`` at runtime. +- ``airflow.providers.common.compat.sdk`` centralizes compatibility imports for providers. + +**Behaviour changes:** + +- Sensors and other helpers that validate user input now raise ``ValueError`` (instead of + ``AirflowException``) when ``poke_interval``/ ``timeout`` arguments are invalid. +- Importing deprecated exception names from ``airflow.exceptions`` logs a warning directing users to + the SDK import path. + +**Exceptions now provided by ``airflow.sdk.exceptions``:** + +- ``AirflowException`` and ``AirflowNotFoundException`` +- ``AirflowRescheduleException`` and ``AirflowSensorTimeout`` +- ``AirflowSkipException``, ``AirflowFailException``, ``AirflowTaskTimeout``, ``AirflowTaskTerminated`` +- ``TaskDeferred``, ``TaskDeferralTimeout``, ``TaskDeferralError`` +- ``DagRunTriggerException`` and ``DownstreamTasksSkipped`` +- ``AirflowDagCycleException`` and ``AirflowInactiveAssetInInletOrOutletException`` +- ``ParamValidationError``, ``DuplicateTaskIdFound``, ``TaskAlreadyInTaskGroup``, ``TaskNotFound``, ``XComNotFound`` +- ``AirflowOptionalProviderFeatureException`` + +**Backward compatibility:** + +- Existing Dags/operators that still import from ``airflow.exceptions`` continue to work, though + they log warnings. +- Providers can rely on ``airflow.providers.common.compat.sdk`` to keep one import path that works + across supported Airflow versions. + +**Migration:** + +- Update custom operators, sensors, and extensions to import exception classes from + ``airflow.sdk.exceptions`` (or from the provider compat shim). +- Adjust custom validation code to expect ``ValueError`` for invalid sensor arguments if it + previously caught ``AirflowException``. + +Support numeric multiplier values for retry_exponential_backoff parameter +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The ``retry_exponential_backoff`` parameter now accepts numeric values to specify custom exponential backoff multipliers for task retries. Previously, this parameter only accepted boolean values (``True`` or ``False``), with ``True`` using a hardcoded multiplier of ``2.0``. + +**New behavior:** + +- Numeric values (e.g., ``2.0``, ``3.5``) directly specify the exponential backoff multiplier +- ``retry_exponential_backoff=2.0`` doubles the delay between each retry attempt +- ``retry_exponential_backoff=0`` or ``False`` disables exponential backoff (uses fixed ``retry_delay``) + +**Backwards compatibility:** + +Existing Dags using boolean values continue to work: + +- ``retry_exponential_backoff=True`` → converted to ``2.0`` (maintains original behavior) +- ``retry_exponential_backoff=False`` → converted to ``0.0`` (no exponential backoff) + +**API changes:** + +The REST API schema for ``retry_exponential_backoff`` has changed from ``type: boolean`` to ``type: number``. API clients must use numeric values (boolean values will be rejected). + +**Migration:** + +While boolean values in Python Dags are automatically converted for backwards compatibility, we recommend updating to explicit numeric values for clarity: + +- Change ``retry_exponential_backoff=True`` → ``retry_exponential_backoff=2.0`` +- Change ``retry_exponential_backoff=False`` → ``retry_exponential_backoff=0`` + +Move serialization/deserialization (serde) logic into Task SDK +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Airflow now sources serde logic from ``airflow.sdk.serde`` instead of +``airflow.serialization.serde``. Serializer modules have moved from ``airflow.serialization.serializers.*`` +to ``airflow.sdk.serde.serializers.*``. The old import paths still work but emit ``DeprecatedImportWarning`` +to guide migration. The backward compatibility layer will be removed in Airflow 4. + +**What changed:** + +- Serialization/deserialization code moved from ``airflow-core`` to ``task-sdk`` package +- Serializer modules moved from ``airflow.serialization.serializers.*`` to ``airflow.sdk.serde.serializers.*`` +- New serializers should be added to ``airflow.sdk.serde.serializers.*`` namespace + +**Code interface changes:** + +- Import serializers from ``airflow.sdk.serde.serializers.*`` instead of ``airflow.serialization.serializers.*`` +- Import serialization functions from ``airflow.sdk.serde`` instead of ``airflow.serialization.serde`` + +**Backward compatibility:** + +- Existing serializers importing from ``airflow.serialization.serializers.*`` continue to work with deprecation warnings +- All existing serializers (builtin, datetime, pandas, numpy, etc.) are available at the new location + +**Migration:** + +- **For existing custom serializers**: Update imports to use ``airflow.sdk.serde.serializers.*`` +- **For new serializers**: Add them to ``airflow.sdk.serde.serializers.*`` namespace (e.g., create ``task-sdk/src/airflow/sdk/serde/serializers/your_serializer.py``) + +Methods removed from PriorityWeightStrategy +"""""""""""""""""""""""""""""""""""""""""""" + +On (experimental) class ``PriorityWeightStrategy``, functions ``serialize()`` +and ``deserialize()`` were never used anywhere, and have been removed. They +should not be relied on in user code. (#59780) + +Methods removed from TaskInstance +""""""""""""""""""""""""""""""""" + +On class ``TaskInstance``, functions ``run()``, ``render_templates()``, +``get_template_context()``, and private members related to them have been +removed. The class has been considered internal since 3.0, and should not be +relied on in user code. (#59780, #59835) + +Modify the information returned by ``DagBag`` +""""""""""""""""""""""""""""""""""""""""""""" + +**New behavior:** + +- ``DagBag`` now uses ``Path.relative_to`` for consistent cross-platform behavior. +- ``FileLoadStat`` now has two additional nullable fields: ``bundle_path`` and ``bundle_name``. + +**Backward compatibility:** + +``FileLoadStat`` will no longer produce paths beginning with ``/`` with the meaning of "relative to the dags folder". +This is a breaking change for any custom code that performs string-based path manipulations relying on this behavior. +Users are advised to update such code to use ``pathlib.Path``. (#59785) + +Remove ``--conn-id`` option from ``airflow connections list`` +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The redundant ``--conn-id`` option has been removed from the ``airflow connections list`` CLI command. +Use ``airflow connections get`` instead. (#59855) + +Add operator-level ``render_template_as_native_obj`` override +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +Operators can now override the Dag-level ``render_template_as_native_obj`` setting, +enabling fine-grained control over whether templates are rendered as native Python +types or strings on a per-task basis. Set ``render_template_as_native_obj=True`` or +``False`` on any operator to override the Dag setting, or leave as ``None`` (default) +to inherit from the Dag. + +Add gunicorn support for API server with zero-downtime worker recycling +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The API server now supports gunicorn as an alternative server with rolling worker restarts +to prevent memory accumulation in long-running processes. + +**Key Benefits:** + +* **Rolling worker restarts**: New workers spawn and pass health checks before old workers + are killed, ensuring zero downtime during worker recycling. + +* **Memory sharing**: Gunicorn uses preload + fork, so workers share memory via + copy-on-write. This significantly reduces total memory usage compared to uvicorn's + multiprocess mode where each worker loads everything independently. + +* **Correct FIFO signal handling**: Gunicorn's SIGTTOU kills the oldest worker (FIFO), + not the newest (LIFO), which is correct for rolling restarts. + +**Configuration:** + +.. code-block:: ini + + [api] + # Use gunicorn instead of uvicorn + server_type = gunicorn + + # Enable rolling worker restarts every 12 hours + worker_refresh_interval = 43200 + + # Restart workers one at a time + worker_refresh_batch_size = 1 + +Or via environment variables: + +.. code-block:: bash + + export AIRFLOW__API__SERVER_TYPE=gunicorn + export AIRFLOW__API__WORKER_REFRESH_INTERVAL=43200 + +**Requirements:** + +Install the gunicorn extra: ``pip install 'apache-airflow-core[gunicorn]'`` + +**Note on uvicorn (default):** + +The default uvicorn mode does not support rolling worker restarts because: + +1. With workers=1, there is no master process to send signals to +2. uvicorn's SIGTTOU kills the newest worker (LIFO), defeating rolling restart purposes +3. Each uvicorn worker loads everything independently with no memory sharing + +If you need worker recycling or memory-efficient multi-worker deployment, use gunicorn. (#60921) + +Improved performance of rendered task instance fields cleanup for Dags with many mapped tasks (~42x faster) +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The config ``max_num_rendered_ti_fields_per_task`` is renamed to ``num_dag_runs_to_retain_rendered_fields`` +(old name still works with deprecation warning). + +Retention is now based on the N most recent dag runs rather than N most recent task executions, +which may result in fewer records retained for conditional/sparse tasks. (#60951) + +AuthManager Backfill permissions are now handled by the ``requires_access_dag`` on the ``DagAccessEntity.Run`` +"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +``is_authorized_backfill`` of the ``BaseAuthManager`` interface has been removed. Core will no longer call this method and their +provider counterpart implementation will be marked as deprecated. +Permissions for backfill operations are now checked against the ``DagAccessEntity.Run`` permission using the existing +``requires_access_dag`` decorator. In other words, if a user has permission to run a Dag, they can perform backfill operations on it. + +Please update your security policies to ensure that users who need to perform backfill operations have the appropriate ``DagAccessEntity.Run`` permissions. (Users +having the Backfill permissions without having the DagRun ones will no longer be able to perform backfill operations without any update) + +Python 3.14 support added +""""""""""""""""""""""""" + +Airflow 3.2.0 adds support for Python 3.14. (#63787) + +Reduce API server memory by eliminating ``SerializedDAG`` loads on task start +""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""" + +The API server no longer loads the full ``SerializedDAG`` when starting tasks, +significantly reducing memory usage. (#60803) + +Remove MySQL client from container images +""""""""""""""""""""""""""""""""""""""""" + +MySQL client support has been removed from official Airflow container images. MySQL users +building on official images must install the client themselves. (#57146) + + +Add support for async callables in PythonOperator +""""""""""""""""""""""""""""""""""""""""""""""""" + +The ``PythonOperator`` parameter ``python_callable`` now also supports async callables in Airflow 3.2, +allowing users to run async def functions without manually managing an event loop. (#60268) + +Make start_date optional for @continuous schedule +""""""""""""""""""""""""""""""""""""""""""""""""" +The ``schedule="@continuous"`` parameter now works without requiring a ``start_date``, and any Dags with this schedule will begin running immediately when unpaused. (#61405) + +New Features +^^^^^^^^^^^^ + +- Add FIPS support by making Python LTO configurable via ``PYTHON_LTO`` build argument (#58337) +- Add support for task queue-based Trigger assignment to specific Triggerer hosts via the new ``--queues`` CLI option for the ``trigger`` command (#59239) +- Add ``--show-values`` and ``--hide-sensitive`` flags to CLI ``connections list`` and ``variables list`` to hide sensitive values by default (#62344) +- Add support for setting individual secrets backend kwargs via ``AIRFLOW__SECRETS__BACKEND_KWARG__`` environment variables (#63312) +- Add ``only_new`` parameter to Dag clear to only clear newly added task instances (#59764) +- Add ``log_timestamp_format`` config option for customizing component log timestamps (#63321) +- Add ``--action-on-existing-key`` option to ``pools import`` and ``connections import`` CLI commands (#62702) +- Add back ``--use-migration-files`` flag for ``airflow db init`` (#62234) +- Add ``AllowedKeyMapper`` for partition key validation in asset partitioning (#61931) +- Add ``ChainMapper`` for chaining multiple partition mappers (#64094) +- Add cryptographic signature verification for Python source packages in Docker builds (#63345) +- Add Human-in-the-Loop (HITL) Review system for ``AgenticOperator`` (#63081) +- Add ``@task.stub`` decorator to allow tasks in other languages to be defined in Dags (#56055) +- Add support for creating connections using URI in SDK (#62211) +- Add note support to ``TriggerDagRunOperator`` (#60810) +- Add ``allowed_run_types`` to whitelist specific Dag run types (#61833) +- Add OR operator support in API search parameters (#60008) +- Add API filtering for Dags by timetable type (#58852) +- Add wildcard support for ``dag_id`` and ``dag_run_id`` in bulk task instance endpoint (#57441) +- Add ``operator_name_pattern``, ``pool_pattern``, ``queue_pattern`` as task instance search filters (#57571) +- Add ``update_mask`` support for bulk PATCH APIs (#54597) +- Add asset event emission listener event (#61718) +- Add ``source`` parameter to ``Param`` (#58615) +- Add lazy filtering for inlet events by time range, ordering, and limit (#54891) +- Add ability to get previous ``TaskInstance`` on ``RuntimeTaskInstance`` (#59712) +- Add required context messages to all DagRun state change notifications (#56272) +- Add ``max_trigger_to_select_per_loop`` config for Triggerer HA setup (#58803) +- Add ``uvicorn_logging_level`` config option to control API server access logs (#56062) +- Add correlation-id support to Execution API for request tracing (#57458) +- Add ``executor.running_dags`` gauge metric to expose count of running Dags (#52815) +- Add submodules support to ``GitDagBundle`` (#59911) +- Add HTTP URL authentication support to ``GitHook`` for Dag bundles (#58194) +- Add stream method to ``RemoteIO`` for ``ObjectStorage`` (#54813) +- Add CLI hot-reload support via ``--dev`` flag (#57741) +- Add ``auth list-envs`` command to list CLI environments and auth status (#61426) +- Add Dag bundles to ``airflow info`` command output (#59124) +- Add new arguments to ``db_clean`` to explicitly include or exclude Dags (#56663) +- UI: Add Jobs page to the Airflow UI (#61512) +- UI: Add version change indicators for Dag and bundle versions in Grid view (#53216) +- UI: Add segmented state bar for collapsed task groups and mapped tasks (#61854) +- UI: Add date range filter for Dag executions (#60772) +- UI: Add "Select Recent Configurations" to trigger form, restoring Airflow 2 functionality (#56406) +- UI: Add copy button to logs (#61185) +- UI: Add filename display to Dag Code tab for easier file identification (#60759) +- UI: Add Dag run state filter to grid view options (#55898) +- UI: Add task upstream/downstream filter to Graph and Grid views (#57237) +- UI: Add filters to Task Instances tab (#56920) +- UI: Add display of active Dag runs count in header with auto-refresh (#58332) +- UI: Add Dag ID pattern search to Dag Runs and Task Instances pages (#55691) +- UI: Add delete button for Dag runs in more options menu (#55696) +- UI: Add depth filter to ``TaskStreamFilter`` (#60549) +- UI: Add theme config support (#58411) +- UI: Add support for ``globalCss`` in custom themes (#61161) +- UI: Add display of logged-in user in settings button (#58981) +- UI: Add tooltip for explaining task filter traversal (#61401) +- UI: Add self-service JWT token generation for API and CLI access (#63195) +- UI: Add bulk operations for edge workers page (#64033) +- UI: Add real-time concurrency control for edge workers (#63142) +- UI: Add ``run_after`` date filter on Dag runs page (#62797) +- UI: Add bundle version filter on Dag runs page (#62810) +- UI: Add icon support for theme customization (#62172) +- UI: Add Monaco editor for all JSON editing fields (#62708) +- UI: Add run type legend tooltip to grid view (#62946) +- UI: Allow customizing gray, black, and white color tokens in ``AIRFLOW__API__THEME`` in addition to brand (#64232) + +Bug Fixes +^^^^^^^^^ + +- Fix sensitive configuration values not being masked in public config APIs; treat the deprecated ``non-sensitive-only`` value as ``True`` (#59880) +- Fix ``InvalidStatsNameException`` for pool names with invalid characters by auto-normalizing them when emitting metrics (#59938) +- Fix JWT tokens appearing in task logs by excluding the token field from workload object representations (#62964) +- Fix security iframe navigation when ``AIRFLOW__API__BASE_URL`` basename is configured (#63141) +- Fix grid view URL for dynamic task groups producing 404 by not appending ``/mapped`` to group URLs (#63205) +- Fix ``ti_skip_downstream`` overwriting RUNNING tasks to SKIPPED in HA deployments (#63266) +- Fix duplicate task execution when running multiple schedulers (#60330) +- Fix callback starvation across Dag bundles (#63795) +- Fix ``@task`` decorator failing for tasks that return falsy values like ``0`` or empty string (#63788) +- Fix ``LatestOnlyOperator`` not working when direct upstream of a dynamically mapped task (#62287) +- Fix inconsistent ``XCom`` return type in mapped task groups with dynamic mapping (#59104) +- Fix task group lookup using wrong Dag version for historical runs, causing 404 errors in grid view (#63360) +- Fix import errors when updating Dags in other bundles (#63615) +- Fix ``DagRun`` span emission crash when ``context_carrier`` is ``None`` (#64087) +- Fix false error logs for partitioned timetables when ``next_dagrun`` fields are ``None`` (#63962) +- Fix timetable serialization error when decoding ``relativedelta`` (#61671) +- Fix ``task_instance_mutation_hook`` receiving ``run_id=None`` during ``TaskInstance`` creation (#63049) +- Fix scheduler crash on ``None`` ``dag_version`` access (#62225) +- Fix ``MetastoreBackend.expunge_all()`` corrupting shared session state (#63080) +- Fix triggerer logger file descriptor closed prematurely when trigger is removed (#62103) +- Fix ``airflowignore`` negation pattern handling for directory-only patterns (#62860) +- Fix false warnings for ``TYPE_CHECKING``-only forward references in TaskFlow decorators (#63053) +- Fix ``structlog`` JSON serialization crash on non-serializable objects (#62656) +- Fix backward compatibility for deadline alert serialization (#63701) +- Fix ``queued_tasks`` type mismatch in hybrid executors (``CeleryKubernetesExecutor``, ``LocalKubernetesExecutor``) (#63744) +- Fix Celery tasks not being registered at worker startup (#63110) +- Fix asset partition detection incorrectly identifying Dags as partitioned (#62864) +- Fix ``pathlib.Path`` objects incorrectly resolved by Jinja templater in Task SDK (#63306) +- Fix state mismatch in Kubernetes executor after pod completion (#63061) +- Fix ``make_partial_model`` for API Pydantic models (#63716) +- Fix WTForms validator compatibility in connection form (#63823) +- Fix ``_execution_api_server_url()`` ignoring configured value and falling back to edge config (#63192) +- Fix ``DetachedInstanceError`` for ``airflow tasks render`` command (#63916) +- Fix scheduler isolating per-dag-run failures to prevent a single DagRun crashing all scheduling (#62893) +- Fix task argument order in ``@task`` definition causing Dag parsing errors (#62174) +- Fix ``limit`` parameter not sent in ``execute_list`` server requests (#63048) +- Fix circular import from ``airflow.configuration`` causing ``ImportError`` on Python 3.14 (#63787) +- Fix ``map_index`` range validation in CLI commands (#62626) +- Fix nullable ORM fields by restoring correct defaults and dropping unreleased corrective migration (#63899) +- Fix race condition in auth manager initialization on concurrent requests (#62431) +- Fix ``FabAuthManager`` race condition on startup with multiple workers (#62737) +- Fix ``FabAuthManager`` race condition when workers concurrently create permissions, roles, and resources (#63842) +- Fix ``JWTValidator`` not handling GUESS algorithm with JWKS (#63115) +- Fix ``FabAuthManager`` first idle MySQL disconnect in token auth (#62919) +- Fix ``JWTBearerTIPathDep`` import errors in Human-In-The-Loop routes (#63277) +- Fix 403 from roles endpoint despite admin rights in FAB provider (#64097) +- Fix task log filters not working in full-screen mode (#62747) +- Fix duplicate log reads when resuming from ``log_pos`` (#63531) +- Fix 404 errors from worker log server for historical retry attempts now handled gracefully (#62475) +- Fix Elasticsearch/OpenSearch logging exception details missing in task log tab (#63739) +- Fix task-level audit logs missing success/running events (#61932) +- Fix ``null`` ``dag_run_conf`` causing serialization error in ``BackfillResponse`` (#63259) +- Fix CLI asset materialization using wrong Dag run type (#63815) +- Fix migration 0094 performance: use SQL instead of Python deserialization (#63628) +- Fix migration reliability: replace ``savepoints`` with per-Dag transactions (#63591) +- Fix slow downgrade performance by adding index to ``deadline.callback_id`` (#63612) +- Fix MySQL reserved keyword ``interval`` causing query failures in ``deadline_alert`` (#63494) +- Fix MySQL ``serialize_dag`` query failure during deadline migration (#63804) +- Fix SQLite downgrade failures caused by FK constraints during batch table recreation (#63437) +- Fix migration 0096 downgrade failing when team table has existing rows (#63449) +- Fix missing warning about hardcoded 24h ``visibility_timeout`` that kills long-running Celery tasks (#62869) +- Fix scheduler memory issue by removing eager loading of all task instances (#60956) +- Fix MySQL sort buffer overflow in deadline alert migration (#61806) +- Fix failing to manually trigger a Dag with ``CronPartitionedTimetable`` (#62441) +- Fix race condition in ``AssetModel`` when updating asset partition DagRun — adds mutex lock (#59183) +- Fix FAB ``auth_manager`` ``load_user`` causing ``PendingRollbackError`` (#61943) +- Fix N+1 query: add ``joinedload`` for asset in ``dags_needing_dagruns()`` (#60957) +- Fix Dag Processor health check threshold matching SchedulerJob/TriggererJob pattern (#58704) +- Fix ``NotMapped`` exception when clearing task instances with downstream/upstream (#58922) +- Fix missing asset events for partitioned DagRun (#61433) +- Fix missing ``partition_key`` filter in ``PALK`` when creating DagRun (#61831) +- Fix Dag params API contract broken by earlier change (#56831) +- Fix OAuth session race condition causing false 401 errors during login (#61287) +- Fix ``ObjectStoragePath`` to exclude ``conn_id`` from storage options passed to fsspec (#62701) +- Fix unable to import list value for Variable (#61508) +- Fix plugin registration returning early on duplicate names (#60498) +- Fix circular import when using ``XComObjectStorageBackend`` (#55805) +- Fix deadline alert hashing bug (#61702) +- Fix task SDK to read ``default_email_on_failure``/``default_email_on_retry`` from config (#59912) +- Fix Celery worker crash on macOS due to non-serializable local function (#62655) +- Fix Redis import race condition in Celery executor (#61362) +- Fix incorrect state query parameter for task instances in Dashboard (#59086) +- Fix ``TaskInstance.get_dagrun`` returning None in ``task_instance_mutation_hook`` (#60726) +- Fix Simple Auth Manager login showing cryptic error on failed authentication (#64303) +- Fix ``dag_display_name`` property bypass for ``DagStats`` query (#64256) +- Fix ``TaskAlreadyRunningError`` not raised when starting an already-running task instance (#60855) +- Fix Teardown tasks not waiting for all in-scope tasks to complete (#64181) +- Fix ``enable_swagger_ui`` config not respected in API server (#64376) +- Fix: add check for xcom permission when result is specified for DagRun wait endpoint (#64415) +- Fix ``conf.has_option`` not respects default provider metadata (#64209) +- Fix teardown scope causing unnecessary database writes during task scheduling (#64558) +- Fix live task log output not visible in stdout when using Elasticsearch log forwarding (#64067) +- Fix ``TaskInstance`` crash when refreshing task weight for non-serialized operators (#64557) +- Fix Variables secrets backend conflict check exiting early when multiple backends are configured (#64062) +- UI: Fix Dag run accessor key on clear task instance page (#64072) +- UI: Fix searchable dropdown not working for Dag params enum fields (#63895) +- UI: Fix newline rendering in Dag warning alert (#63588) +- UI: Fix ``XCom`` edit modal value not repopulating on reopen (#62798) +- UI: Fix task duration tooltip not displaying correctly (#63639) +- UI: Fix elapsed time not showing for running tasks (#63619) +- UI: Fix ``RenderedJsonField`` collapse behavior (#63831) +- UI: Fix ``RenderedJsonField`` not displaying in table cells (#63245) +- UI: Fix full-screen log dropdown z-index after Chakra upgrade (#63816) +- UI: Fix asset materialization run type display (#63819) +- UI: Fix pools with unlimited (``-1``) slots not rendering correctly (#62831) +- UI: Fix ``DurationChart`` labels and disable animation flicker during auto-refresh (#62835) +- UI: Fix 403 error not shown when unauthorized user re-parses Dag (#61560) +- UI: Fix logical date filter on ``/dagruns`` page not working (#62848) +- UI: Fix inflated ``total_received`` count in partitioned Dag runs view (#62786) +- UI: Fix edge executor navigation when behind reverse proxy with subpath (#63777) +- UI: Fix queries not invalidated on Dag run add/delete (#64269) +- UI: Fix ``RenderedJsonField`` flickering when collapsed (#64261) +- UI: Fix Docs menu REST API link visibility when API docs are disabled (#64359) +- UI: Fix ``TISummaries`` not refreshing when ``gridRuns`` are invalidated (#64113) +- UI: Fix guard against null/undefined dates in Gantt chart to prevent RangeError (#64031) +- UI: Block polling requests to endpoints that returned 403 Forbidden (#64333) +- UI: Fix Gantt view still visible when time range is outside ``DagRun`` window (#64179) +- UI: Fix Human-in-the-Loop (HITL) operator options not displaying when exactly 4 choices are configured (#64453) + + +Miscellaneous +^^^^^^^^^^^^^ + +- Deprecate ``api.page_size`` config in favor of ``api.fallback_page_limit`` (#61067) +- Improve Dag callback relevancy by passing a context-relevant task instance based on the Dag's final state instead of an arbitrary lexicographical selection (#61274) +- Optimize ``get_dag_runs`` API endpoint performance (#63940) +- Improve historical metrics endpoint performance (#63526) +- Add TTL cache with single-flight deduplication to Keycloak ``filter_authorized_dag_ids`` (#63184) +- Reduce Celery worker memory usage with ``gc.freeze`` (#62212) +- Eliminate duplicate JOINs in ``get_task_instances`` endpoint (#62910) +- Replace large ``IN`` clause in asset queries with CTE and JOIN for better SQL performance (#62114) +- Add row lock to prevent race conditions during asset-triggered DagRun creation (#60773) +- Add ``ConnectionResponse`` serializer safeguard to prevent accidental sensitive field exposure (#63883) +- Add missing ``dag_id`` filter on DagRun task instances API query (#62750) +- Add missing HTTP timeout to FAB JWKS fetching (#63058) +- Add additional permission check in asset materialization endpoint (#63338) +- Filter backfills list by readable Dags (authorization enforcement) (#63003) +- Hide SQL statements in exception details when ``expose_stacktrace`` is disabled (#63028) +- Use default max depth to redact Variable values in API responses (#63480) +- Validate ``update_mask`` fields in PATCH API endpoints against Pydantic models (#62657) +- Align key/id path validation for variables and connections in Execution API (#63897) +- Add ``order_by`` parameter to GET /permissions endpoint for pagination consistency (#63418) +- Implement truncation logic for rendered template values (#61878) +- Add ``BaseXcom`` to ``airflow.sdk`` public exports (#63116) +- Make ``TaskSDK`` conf respect default config from provider metadata (#62696) +- Add OTel trace import shim via ``airflow.sdk.observability.trace`` (#63554) +- Improve 3.2.0 deadline migration performance (#63920) +- Improve 3.2.0 downgrade migration for ``external_executor_id`` on PostgreSQL (#63625) +- Skip backfilling old ``DagRun.created_at`` during migration for faster upgrades (#63825) +- Add INFO-level logging to asset scheduling path (#63958) +- Improve log file template for ``ExecuteCallback`` by including ``dag_id`` and ``run_id`` (#62616) +- Improve Dag processor timeout logging clarity (#62328) +- Deprecate ``get_connection_form_widgets`` and ``get_ui_field_behaviour`` hook methods (#63711) +- Add missing deprecation warnings for ``[workers]`` config section (#63659) +- Expose ``TaskInstance`` API for external task management (#61568) +- Remove deprecated ``airflow.datasets``, ``airflow.timetables.datasets``, and ``airflow.utils.dag_parsing_context`` modules (#62927) +- Remove ``PyOpenSSL`` from core dependencies (#63869) +- Optimize fail-fast check to avoid loading ``SerializedDAG`` (#56694) +- Improve performance of task queue processing by switching from ``pop(0)`` to ``popleft()`` (#61376) +- Optimize K8s API usage for watching pod events, fixing hanging communication (#59080) +- Remove N+1 database queries for team names (#61471) +- Improve XCom value handling in extra links API (#61641) +- Remove ``.git`` folder from versions in ``GitDagBundle`` to reduce storage size (#57069) +- Deprecate subprocess exec utils from ``airflow.utils.process_utils`` (#57193) +- Improve error handling in edge worker on 405 responses (#60425) +- Improve deferrable ``KubernetesPodOperator`` handling of deleted pods between polls (#56976) +- Improve event log entries when a pod fails for K8s executor (#60800) +- Refactor XCom API to use shared serialization constants (#64148) +- Improve temporal mapper to be timezone aware for asset partitioning (#62709) +- Improve dag version inflation checker logic and fix false-positive detection (#61345) +- Rename ``ToXXXMapper`` to ``StartOfXXXMapper`` in partition-mapper for clarity (#64160) +- Run DB check only for core components in prod entrypoint (#63413) +- Fix partitioned asset events incorrectly triggering non-partition-aware Dags (#63848) +- Improve partitioned DagRun sorting by ``partition_date`` (#62866) +- Allow gray, black, and white color tokens in ``AIRFLOW__API__THEME`` config (#64232) +- Add parent task spans and nest worker/trigger spans for improved observability (#63839) +- UI: Enhance code view to support search and diff (#55467) +- UI: Improve UX for adding custom ``DeadlineReferences`` (#57222) +- UI: Enhance ``FilterBar`` with ``DateRangeFilter`` for compact UI (#56173) +- UI: Move deadline alerts into their own table for UI integration (#58248) +- UI: Persist tag filter selection in Dag grid view (#63273) +- UI: Show HITL review tab only for review-enabled task instances (#63477) +- UI: Updated button styles for adding Connections, Variables, and Pools (#62607) +- UI: Add clear permission toast for 403 errors on user actions (#61588) + + +Doc Only Changes +^^^^^^^^^^^^^^^^ + +- Add documentation marking pre/post-execute task hooks as GA (no longer experimental) (#59656) +- Add ``RedisTaskHandler`` configuration example (#63898) +- Add documentation explaining difference between deferred vs async operators (#63500) +- Add auth manager section in multi-team documentation (#63208) +- Add documentation about shared libraries in ``_shared`` folders (#63468) +- Clarify plugin folder module registration in ``modules_management`` docs (#63634) +- Clarify ``max_active_tasks`` Dag parameter documentation (#63217) +- Clarify HLL in extraction precedence docs (#63723) +- Clarify Ubuntu/Debian venv requirement in quick start guide (#63244) +- Fix Git connection docs to match actual ``GitHook`` parameters (#63265) +- Mention Python 3.14 support in docs (#63950) +- Add Dag documentation for ``example_bash_decorator`` (#62948) +- Add Russian translation for UI (#63450) +- Add Hungarian translation (#62925) +- Complete Traditional Chinese translations (#62652) +- Add asset partition documentation (#63262) +- Add guide for dag version inflation and its checker (#64100) + Airflow 3.1.8 (2026-03-11) -------------------------- diff --git a/airflow-core/.pre-commit-config.yaml b/airflow-core/.pre-commit-config.yaml index 0567187dbf686..0ad9de856d930 100644 --- a/airflow-core/.pre-commit-config.yaml +++ b/airflow-core/.pre-commit-config.yaml @@ -263,6 +263,16 @@ repos: require_serial: true pass_filenames: false files: ^src/airflow/config_templates/config\.yml$ + - id: check-security-doc-constants + name: Check security docs match config.yml constants + entry: ../scripts/ci/prek/check_security_doc_constants.py + language: python + pass_filenames: false + files: > + (?x) + ^src/airflow/config_templates/config\.yml$| + ^docs/security/jwt_token_authentication\.rst$| + ^docs/security/security_model\.rst$ - id: check-airflow-version-checks-in-core language: pygrep name: No AIRFLOW_V_* imports in airflow-core diff --git a/airflow-core/docs/administration-and-deployment/production-deployment.rst b/airflow-core/docs/administration-and-deployment/production-deployment.rst index e69d436488713..e88b94d94ba8b 100644 --- a/airflow-core/docs/administration-and-deployment/production-deployment.rst +++ b/airflow-core/docs/administration-and-deployment/production-deployment.rst @@ -62,9 +62,12 @@ the :doc:`Celery executor `. Once you have configured the executor, it is necessary to make sure that every node in the cluster contains -the same configuration and Dags. Airflow sends simple instructions such as "execute task X of Dag Y", but -does not send any Dag files or configuration. You can use a simple cronjob or any other mechanism to sync -Dags and configs across your nodes, e.g., checkout Dags from git repo every 5 minutes on all nodes. +the Dags and configuration appropriate for its role. Airflow sends simple instructions such as +"execute task X of Dag Y", but does not send any Dag files or configuration. For synchronization of Dags +we recommend the Dag Bundle mechanism (including ``GitDagBundle``), which allows you to make use of +DAG versioning. For security-sensitive deployments, restrict sensitive configuration (JWT signing keys, +database credentials, Fernet keys) to only the components that need them rather than sharing all +configuration across all nodes — see :doc:`/security/security_model` for guidance. Logging diff --git a/airflow-core/docs/best-practices.rst b/airflow-core/docs/best-practices.rst index 4c0eb02986699..b0b75b0086aff 100644 --- a/airflow-core/docs/best-practices.rst +++ b/airflow-core/docs/best-practices.rst @@ -319,7 +319,7 @@ Installing and Using ruff .. code-block:: bash - pip install "ruff>=0.15.8" + pip install "ruff>=0.15.9" 2. **Running ruff**: Execute ``ruff`` to check your Dags for potential issues: @@ -1098,8 +1098,10 @@ The benefits of using those operators are: environment is optimized for the case where you have multiple similar, but different environments. * The dependencies can be pre-vetted by the admins and your security team, no unexpected, new code will be added dynamically. This is good for both, security and stability. -* Complete isolation between tasks. They cannot influence one another in other ways than using standard - Airflow XCom mechanisms. +* Strong process-level isolation between tasks. Tasks run in separate containers/pods and cannot + influence one another at the process or filesystem level. They can still interact through standard + Airflow mechanisms (XComs, connections, variables) via the Execution API. See + :doc:`/security/security_model` for the full isolation model. The drawbacks: diff --git a/airflow-core/docs/configurations-ref.rst b/airflow-core/docs/configurations-ref.rst index 83c5d8a8ed51a..1afe00f1e2c1f 100644 --- a/airflow-core/docs/configurations-ref.rst +++ b/airflow-core/docs/configurations-ref.rst @@ -22,15 +22,22 @@ Configuration Reference This page contains the list of all the available Airflow configurations that you can set in ``airflow.cfg`` file or using environment variables. -Use the same configuration across all the Airflow components. While each component -does not require all, some configurations need to be same otherwise they would not -work as expected. A good example for that is :ref:`secret_key` which -should be same on the Webserver and Worker to allow Webserver to fetch logs from Worker. - -The webserver key is also used to authorize requests to Celery workers when logs are retrieved. The token -generated using the secret key has a short expiry time though - make sure that time on ALL the machines -that you run Airflow components on is synchronized (for example using ntpd) otherwise you might get -"forbidden" errors when the logs are accessed. +Different Airflow components may require different configuration parameters, and for +improved security, you should restrict sensitive configuration to only the components that +need it. Some configuration values must be shared across specific components to work +correctly — for example, the JWT signing key (``[api_auth] jwt_secret`` or +``[api_auth] jwt_private_key_path``) must be consistent across all components that generate +or validate JWT tokens (Scheduler, API Server). However, other sensitive parameters such as +database connection strings or Fernet keys should only be provided to components that need them. + +For security-sensitive deployments, pass configuration values via environment variables +scoped to individual components rather than sharing a single configuration file across all +components. See :doc:`/security/security_model` for details on which configuration +parameters should be restricted to which components. + +Make sure that time on ALL the machines that you run Airflow components on is synchronized +(for example using ntpd) otherwise you might get "forbidden" errors when the logs are +accessed or API calls are made. .. note:: For more information see :doc:`/howto/set-config`. diff --git a/airflow-core/docs/core-concepts/dags.rst b/airflow-core/docs/core-concepts/dags.rst index a7e6fdb2b8c64..8b19ba07e4107 100644 --- a/airflow-core/docs/core-concepts/dags.rst +++ b/airflow-core/docs/core-concepts/dags.rst @@ -774,10 +774,7 @@ in which one Dag can depend on another: - waiting - :class:`~airflow.providers.standard.sensors.external_task.ExternalTaskSensor` Additional difficulty is that one Dag could wait for or trigger several runs of the other Dag -with different data intervals. The **Dag Dependencies** view -``Menu -> Browse -> Dag Dependencies`` helps visualize dependencies between Dags. The dependencies -are calculated by the scheduler during Dag serialization and the webserver uses them to build -the dependency graph. +with different data intervals. These dependencies are calculated by the scheduler during Dag serialization. The dependency detector is configurable, so you can implement your own logic different than the defaults in :class:`~airflow.serialization.serialized_objects.DependencyDetector` diff --git a/airflow-core/docs/core-concepts/multi-team.rst b/airflow-core/docs/core-concepts/multi-team.rst index 6beccc249b1cf..609a79cdf1888 100644 --- a/airflow-core/docs/core-concepts/multi-team.rst +++ b/airflow-core/docs/core-concepts/multi-team.rst @@ -38,7 +38,7 @@ Multi-Team mode is designed for medium to large organizations that typically hav **Use Multi-Team mode when:** - You have many teams that need to share Airflow infrastructure -- You need resource isolation (Variables, Connections, Secrets, etc) between teams +- You need resource isolation (Variables, Connections, Secrets, etc) between teams at the UI and API level (see :doc:`/security/security_model` for task-level isolation limitations) - You want separate execution environments per team - You want separate views per team in the Airflow UI - You want to minimize operational overhead or cost by sharing a single Airflow deployment diff --git a/airflow-core/docs/extra-packages-ref.rst b/airflow-core/docs/extra-packages-ref.rst index a3acd03df1e94..19875b959c5d7 100644 --- a/airflow-core/docs/extra-packages-ref.rst +++ b/airflow-core/docs/extra-packages-ref.rst @@ -299,6 +299,8 @@ These are extras that add dependencies needed for integration with external serv +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | vertica | ``pip install 'apache-airflow[vertica]'`` | Vertica hook support as an Airflow backend | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ +| vespa | ``pip install 'apache-airflow[vespa]'`` | Vespa hooks and operators | ++---------------------+-----------------------------------------------------+-----------------------------------------------------+ | weaviate | ``pip install 'apache-airflow[weaviate]'`` | Weaviate hook and operators | +---------------------+-----------------------------------------------------+-----------------------------------------------------+ | yandex | ``pip install 'apache-airflow[yandex]'`` | Yandex.cloud hooks and operators | diff --git a/airflow-core/docs/faq.rst b/airflow-core/docs/faq.rst index fe48c3695dc81..a5064cc3dfab4 100644 --- a/airflow-core/docs/faq.rst +++ b/airflow-core/docs/faq.rst @@ -666,6 +666,43 @@ try pausing the Dag again, or check the console or server logs if the issue recurs. +API Server +^^^^^^^^^^ + +.. _faq:api-server-memory-growth: + +How to prevent API server memory growth? +----------------------------------------- + +The API server caches serialized Dag objects in memory. Over time, as Dag versions accumulate +(see :ref:`faq:dag-version-inflation`), this cache grows and can consume several gigabytes of memory. + +The recommended solution (available since Airflow 3.2.0) is to use **gunicorn** with **rolling worker +restarts**. Gunicorn periodically recycles worker processes, releasing all accumulated memory. It also +uses ``preload`` + ``fork``, so workers share read-only memory pages via copy-on-write, reducing overall +memory usage by 40-50% compared to uvicorn's multiprocess mode. + +To enable gunicorn with worker recycling: + +.. code-block:: ini + + [api] + server_type = gunicorn + # Restart each worker every 12 hours (43200 seconds) + worker_refresh_interval = 43200 + worker_refresh_batch_size = 1 + +This requires the ``apache-airflow-core[gunicorn]`` extra to be installed. + +See :ref:`config:api__server_type`, :ref:`config:api__worker_refresh_interval`, and +:ref:`config:api__worker_refresh_batch_size` for the full configuration reference. + +.. note:: + + Worker recycling handles memory growth from *any* source, not just the Dag cache. It is the + recommended approach for production API server deployments. + + MySQL and MySQL variant Databases ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ diff --git a/airflow-core/docs/howto/customize-ui.rst b/airflow-core/docs/howto/customize-ui.rst index 3d696f52969b7..b9d03cdf94da5 100644 --- a/airflow-core/docs/howto/customize-ui.rst +++ b/airflow-core/docs/howto/customize-ui.rst @@ -71,6 +71,7 @@ We can provide a JSON configuration to customize the UI. .. important:: - You can customize the ``brand``, ``gray``, ``black``, and ``white`` color tokens, ``globalCss``, and the navigation icon via ``icon`` (and ``icon_dark_mode``). + - All top-level fields (``tokens``, ``globalCss``, ``icon``, ``icon_dark_mode``) are **optional** — you can supply any combination, including an empty ``{}`` to restore OSS defaults. - All color tokens are **optional** — you can override any subset without supplying the others. - ``brand`` and ``gray`` each accept an 11-shade scale with keys ``50``–``950``. - ``black`` and ``white`` each accept a single color: ``{ "value": "oklch(...)" }``. diff --git a/airflow-core/docs/howto/set-config.rst b/airflow-core/docs/howto/set-config.rst index 30d29c924c689..c35df0f4c894b 100644 --- a/airflow-core/docs/howto/set-config.rst +++ b/airflow-core/docs/howto/set-config.rst @@ -157,15 +157,20 @@ the example below. See :doc:`/administration-and-deployment/modules_management` for details on how Python and Airflow manage modules. .. note:: - Use the same configuration across all the Airflow components. While each component - does not require all, some configurations need to be same otherwise they would not - work as expected. A good example for that is :ref:`secret_key` which - should be same on the Webserver and Worker to allow Webserver to fetch logs from Worker. - - The webserver key is also used to authorize requests to Celery workers when logs are retrieved. The token - generated using the secret key has a short expiry time though - make sure that time on ALL the machines - that you run Airflow components on is synchronized (for example using ntpd) otherwise you might get - "forbidden" errors when the logs are accessed. + Different Airflow components may require different configuration parameters. For improved + security, restrict sensitive configuration to only the components that need it rather than + sharing all configuration across all components. Some values must be consistent across specific + components — for example, the JWT signing key must match between components that generate and + validate tokens. However, sensitive parameters such as database connection strings, Fernet keys, + and secrets backend credentials should only be provided to components that actually need them. + + For security-sensitive deployments, pass configuration values via environment variables scoped + to individual components. See :doc:`/security/security_model` for detailed guidance on + restricting configuration parameters. + + Make sure that time on ALL the machines that you run Airflow components on is synchronized + (for example using ntpd) otherwise you might get "forbidden" errors when the logs are + accessed or API calls are made. .. _set-config:configuring-local-settings: diff --git a/airflow-core/docs/installation/supported-versions.rst b/airflow-core/docs/installation/supported-versions.rst index 700ff069d1440..1e7e289968804 100644 --- a/airflow-core/docs/installation/supported-versions.rst +++ b/airflow-core/docs/installation/supported-versions.rst @@ -29,7 +29,7 @@ Apache Airflow® version life cycle: ========= ===================== =================== =============== ===================== ================ Version Current Patch/Minor State First Release Limited Maintenance EOL/Terminated ========= ===================== =================== =============== ===================== ================ -3 3.1.8 Maintenance Apr 22, 2025 TBD TBD +3 3.2.0 Maintenance Apr 22, 2025 TBD TBD 2 2.11.2 Limited maintenance Dec 17, 2020 Oct 22, 2025 Apr 22, 2026 1.10 1.10.15 EOL Aug 27, 2018 Dec 17, 2020 June 17, 2021 1.9 1.9.0 EOL Jan 03, 2018 Aug 27, 2018 Aug 27, 2018 diff --git a/airflow-core/docs/installation/upgrading_to_airflow3.rst b/airflow-core/docs/installation/upgrading_to_airflow3.rst index 2d9c878390db8..ad0b5507b629e 100644 --- a/airflow-core/docs/installation/upgrading_to_airflow3.rst +++ b/airflow-core/docs/installation/upgrading_to_airflow3.rst @@ -54,7 +54,7 @@ In Airflow 3, direct metadata database access from task code is now restricted. - **No Direct Database Access**: Task code can no longer directly import and use Airflow database sessions or models. - **API-Based Resource Access**: All runtime interactions (state transitions, heartbeats, XComs, and resource fetching) are handled through a dedicated Task Execution API. -- **Enhanced Security**: This ensures isolation and security by preventing malicious task code from accessing or modifying the Airflow metadata database. +- **Enhanced Security**: This improves isolation and security by preventing worker task code from directly accessing or modifying the Airflow metadata database. Note that Dag author code potentially still executes with direct database access in the Dag File Processor and Triggerer — see :doc:`/security/security_model` for details. - **Stable Interface**: The Task SDK provides a stable, forward-compatible interface for accessing Airflow resources without direct database dependencies. Step 1: Take care of prerequisites diff --git a/airflow-core/docs/public-airflow-interface.rst b/airflow-core/docs/public-airflow-interface.rst index c768c36a7b170..4f4c09d66d173 100644 --- a/airflow-core/docs/public-airflow-interface.rst +++ b/airflow-core/docs/public-airflow-interface.rst @@ -548,9 +548,10 @@ but in Airflow they are not parts of the Public Interface and might change any t internal implementation detail and you should not assume they will be maintained in a backwards-compatible way. -**Direct metadata database access from task code is no longer allowed**. -Task code cannot directly access the metadata database to query Dag state, task history, -or Dag runs. Instead, use one of the following alternatives: +**Direct metadata database access from code authored by Dag Authors is no longer allowed**. +The code authored by Dag Authors cannot directly access the metadata database to query Dag state, task history, +or Dag runs — workers communicate exclusively through the Execution API. Instead, use one +of the following alternatives: * **Task Context**: Use :func:`~airflow.sdk.get_current_context` to access task instance information and methods like :meth:`~airflow.sdk.types.RuntimeTaskInstanceProtocol.get_dr_count`, diff --git a/airflow-core/docs/security/jwt_token_authentication.rst b/airflow-core/docs/security/jwt_token_authentication.rst new file mode 100644 index 0000000000000..7aa85bba9a381 --- /dev/null +++ b/airflow-core/docs/security/jwt_token_authentication.rst @@ -0,0 +1,398 @@ + .. Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + .. http://www.apache.org/licenses/LICENSE-2.0 + + .. Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +JWT Token Authentication +======================== + +This document describes how JWT (JSON Web Token) authentication works in Apache Airflow +for both the public REST API (Core API) and the internal Execution API used by workers. + +.. contents:: + :local: + :depth: 2 + +Overview +-------- + +Airflow uses JWT tokens as the primary authentication mechanism for its APIs. There are two +distinct JWT authentication flows: + +1. **REST API (Core API)** — used by UI users, CLI tools, and external clients to interact + with the Airflow public API. +2. **Execution API** — used internally by workers, the Dag File Processor, and the Triggerer + to communicate task state and retrieve runtime data (connections, variables, XComs). + +Both flows share the same underlying JWT infrastructure (``JWTGenerator`` and ``JWTValidator`` +classes in ``airflow.api_fastapi.auth.tokens``) but differ in audience, token lifetime, subject +claims, and scope semantics. + + +Signing and Cryptography +------------------------ + +Airflow supports two mutually exclusive signing modes: + +**Symmetric (shared secret)** + Uses a pre-shared secret key (``[api_auth] jwt_secret``) with the **HS512** algorithm. + All components that generate or validate tokens must share the same secret. If no secret + is configured, Airflow auto-generates a random 16-byte key at startup — but this key is + ephemeral and different across processes, which will cause authentication failures in + multi-component deployments. Deployment Managers must explicitly configure this value. + +**Asymmetric (public/private key pair)** + Uses a PEM-encoded private key (``[api_auth] jwt_private_key_path``) for signing and + the corresponding public key for validation. Supported algorithms: **RS256** (``RSA``) and + **EdDSA** (``Ed25519``). The algorithm is auto-detected from the key type when + ``[api_auth] jwt_algorithm`` is set to ``GUESS`` (the default). + + Validation can use either: + + - A JWKS (JSON Web Key Set) endpoint configured via ``[api_auth] trusted_jwks_url`` + (local file or remote HTTP/HTTPS URL, polled periodically for updates). + - The public key derived from the configured private key (automatic fallback when + ``trusted_jwks_url`` is not set). + +REST API Authentication Flow +----------------------------- + +Token acquisition +^^^^^^^^^^^^^^^^^ + +1. A client sends a ``POST`` request to ``/auth/token`` with credentials (e.g., username + and password in JSON body). +2. The auth manager validates the credentials and creates a user object. +3. The auth manager serializes the user into JWT claims and calls ``JWTGenerator.generate()``. +4. The generated token is returned in the response as ``access_token``. + +For UI-based authentication, the token is stored in a secure, HTTP-only cookie (``_token``) +with ``SameSite=Lax``. + +The CLI uses a separate endpoint (``/auth/token/cli``) with a different (shorter) expiration +time. + +Token structure (REST API) +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 15 85 + + * - Claim + - Description + * - ``jti`` + - Unique token identifier (UUID4 hex). Used for token revocation. + * - ``iss`` + - Issuer (from ``[api_auth] jwt_issuer``). + * - ``aud`` + - Audience (from ``[api_auth] jwt_audience``). + * - ``sub`` + - User identifier (serialized by the auth manager). + * - ``iat`` + - Issued-at timestamp (Unix epoch seconds). + * - ``nbf`` + - Not-before timestamp (same as ``iat``). + * - ``exp`` + - Expiration timestamp (``iat + jwt_expiration_time``). + +Token validation (REST API) +^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +On each API request, the token is extracted in this order of precedence: + +1. ``Authorization: Bearer `` header. +2. OAuth2 query parameter. +3. ``_token`` cookie. + +The ``JWTValidator`` verifies the signature, expiry (``exp``), not-before (``nbf``), +issued-at (``iat``), audience, and issuer claims. A configurable leeway +(``[api_auth] jwt_leeway``, default 10 seconds) accounts for clock skew. + +Token revocation (REST API only) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Token revocation applies only to REST API and UI tokens — it is **not** used for Execution API +tokens issued to workers. + +Revoked tokens are tracked in the ``revoked_token`` database table by their ``jti`` claim. +On logout or explicit revocation, the token's ``jti`` and ``exp`` are inserted into this +table. Expired entries are automatically cleaned up at a cadence of ``2× jwt_expiration_time``. + +Token refresh (REST API) +^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``JWTRefreshMiddleware`` runs on UI requests. When the middleware detects that the +current token's ``_token`` cookie is approaching expiry, it calls +``auth_manager.refresh_user()`` to generate a new token and sets it as the updated cookie. + +Default timings (REST API) +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 50 50 + + * - Setting + - Default + * - ``[api_auth] jwt_expiration_time`` + - 86400 seconds (24 hours) + * - ``[api_auth] jwt_cli_expiration_time`` + - 3600 seconds (1 hour) + * - ``[api_auth] jwt_leeway`` + - 10 seconds + + +Execution API Authentication Flow +---------------------------------- + +The Execution API is an API used for use by Airflow itself (not third party callers) +to report and set task state transitions, send heartbeats, and to retrieve connections, +variables, and XComs at task runtime, trigger execution and Dag parsing. + +Token generation (Execution API) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +1. The **Scheduler** generates a JWT for each task instance before + dispatching it (via the executor) to a worker. The executor's + ``jwt_generator`` property creates a ``JWTGenerator`` configured with the ``[execution_api]`` settings. +2. The token's ``sub`` (subject) claim is set to the **task instance UUID**. +3. The token is embedded in the workload JSON payload (``BaseWorkloadSchema.token`` field) + that is sent to the worker process. + +Token structure (Execution API) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 15 85 + + * - Claim + - Description + * - ``jti`` + - Unique token identifier (UUID4 hex). + * - ``iss`` + - Issuer (from ``[api_auth] jwt_issuer``). + * - ``aud`` + - Audience (from ``[execution_api] jwt_audience``, default: ``urn:airflow.apache.org:task``). + * - ``sub`` + - Task instance UUID — the identity of the workload. + * - ``scope`` + - Token scope: ``"execution"`` or ``"workload"``. + * - ``iat`` + - Issued-at timestamp. + * - ``nbf`` + - Not-before timestamp. + * - ``exp`` + - Expiration timestamp (``iat + [execution_api] jwt_expiration_time``). + +Token scopes (Execution API) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The Execution API defines two token scopes: + +**workload** + A restricted scope accepted only on endpoints that explicitly opt in via + ``Security(require_auth, scopes=["token:workload"])``. Used for endpoints that + manage task state transitions. + +**execution** + Accepted by all Execution API endpoints. This is the standard scope for worker + communication and allows access + +Tokens without a ``scope`` claim default to ``"execution"`` for backwards compatibility. + +Token delivery to workers +^^^^^^^^^^^^^^^^^^^^^^^^^ + +The token flows through the execution stack as follows: + +1. **Scheduler** generates the token and embeds it in the workload JSON payload that it passes to + **Executor**. +2. The workload JSON is passed to the worker process (via the executor-specific mechanism: + Celery message, Kubernetes Pod spec, local subprocess arguments, etc.). +3. The worker's ``execute_workload()`` function reads the workload JSON and extracts the token. +4. The ``supervise()`` function receives the token and creates an ``httpx.Client`` instance + with ``BearerAuth(token)`` for all Execution API HTTP requests. +5. The token is included in the ``Authorization: Bearer `` header of every request. + +Token validation (Execution API) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``JWTBearer`` security dependency validates the token once per request: + +1. Extracts the token from the ``Authorization: Bearer`` header. +2. Performs cryptographic signature validation via ``JWTValidator``. +3. Verifies standard claims (``exp``, ``iat``, ``aud`` — ``nbf`` and ``iss`` if configured). +4. Defaults the ``scope`` claim to ``"execution"`` if absent. +5. Creates a ``TIToken`` object with the task instance ID and claims. +6. Caches the validated token on the ASGI request scope for the duration of the request. + +Route-level enforcement is handled by ``require_auth``: + +- Checks the token's ``scope`` against the route's ``allowed_token_types`` (precomputed + by ``ExecutionAPIRoute`` from ``token:*`` Security scopes at route registration time). +- Enforces ``ti:self`` scope — verifies that the token's ``sub`` claim matches the + ``{task_instance_id}`` path parameter, preventing a worker from accessing another task's + endpoints. + +Token refresh (Execution API) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +The ``JWTReissueMiddleware`` automatically refreshes valid tokens that are approaching expiry: + +1. After each response, the middleware checks the token's remaining validity. +2. If less than **20%** of the total validity remains (minimum 30 seconds), the server + generates a new token preserving all original claims (including ``scope`` and ``sub``). +3. The refreshed token is returned in the ``Refreshed-API-Token`` response header. +4. The client's ``_update_auth()`` hook detects this header and transparently updates + the ``BearerAuth`` instance for subsequent requests. + +This mechanism ensures long-running tasks do not lose API access due to token expiry, +without requiring the worker to re-authenticate. + +No token revocation (Execution API) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Execution API tokens are not subject to revocation. They are short-lived (default 10 minutes) +and automatically refreshed by the ``JWTReissueMiddleware``, so revocation is not part of the +Execution API security model. Once an Execution API token is issued to a worker, it remains +valid until it expires. + + + +Default timings (Execution API) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. list-table:: + :header-rows: 1 + :widths: 50 50 + + * - Setting + - Default + * - ``[execution_api] jwt_expiration_time`` + - 600 seconds (10 minutes) + * - ``[execution_api] jwt_audience`` + - ``urn:airflow.apache.org:task`` + * - Token refresh threshold + - 20% of validity remaining (minimum 30 seconds, i.e., at ~120 seconds before expiry + with the default 600-second token lifetime) + + +Dag File Processor and Triggerer +--------------------------------- + +The **Dag File Processor** and **Triggerer** are internal Airflow components that also +interact with the Execution API, but they do so via an **in-process** transport +(``InProcessExecutionAPI``) rather than over the network. This in-process API: + +- Runs the Execution API application directly within the same process, using an ASGI/WSGI + bridge. +- **Potentially bypasses JWT authentication** — the JWT bearer dependency is overridden to + always return a synthetic ``TIToken`` with the ``"execution"`` scope, effectively bypassing + token validation. +- Also potentially bypasses per-resource access controls (connection, variable, and XCom access + checks are overridden to always allow). + +Airflow implements software guards that prevent accidental direct database access from Dag +author code in these components. However, because the child processes that parse Dag files and +execute trigger code run as the **same Unix user** as their parent processes, these guards do +not protect against intentional access. A deliberately malicious Dag author can potentially +retrieve the parent process's database credentials (via ``/proc//environ``, configuration +files, or secrets manager access) and gain full read/write access to the metadata database and +all Execution API operations — without needing a valid JWT token. + +This is in contrast to workers/task execution, where the isolation is implemented ad deployment +level - where sensitive configuration of database credentials is not available to Airflow +processes because they are not set in their deployment configuration at all, and communicate +exclusively through the Execution API. + +In the default deployment, a **single Dag File Processor instance** parses Dag files for all +teams and a **single Triggerer instance** handles all triggers across all teams. This means +that Dag author code from different teams executes within the same process, with potentially +shared access to the in-process Execution API and the metadata database. + +For multi-team deployments that require isolation, Deployment Managers must run **separate +Dag File Processor and Triggerer instances per team** as a deployment-level measure — Airflow +does not provide built-in support for per-team DFP or Triggerer instances. Even with separate +instances, each retains the same Unix user as the parent process. To prevent credential +retrieval, Deployment Managers must implement Unix user-level isolation (running child +processes as a different, low-privilege user) or network-level restrictions. + +See :doc:`/security/security_model` for the full security implications, deployment hardening +guidance, and the planned strategic and tactical improvements. + + +Workload Isolation and Current Limitations +------------------------------------------ + +For a detailed discussion of workload isolation protections, current limitations, and planned +improvements, see :ref:`workload-isolation`. + + +Configuration Reference +------------------------ + +All JWT-related configuration parameters: + +.. list-table:: + :header-rows: 1 + :widths: 40 15 45 + + * - Parameter + - Default + - Description + * - ``[api_auth] jwt_secret`` + - Auto-generated if missing + - Symmetric secret key for signing tokens. Must be the same across all components. Mutually exclusive with ``jwt_private_key_path``. + * - ``[api_auth] jwt_private_key_path`` + - None + - Path to PEM-encoded private key (``RSA`` or ``Ed25519``). Mutually exclusive with ``jwt_secret``. + * - ``[api_auth] jwt_algorithm`` + - ``GUESS`` + - Signing algorithm. Auto-detected from key type: ``HS512`` for symmetric, ``RS256`` for ``RSA``, ``EdDSA`` for ``Ed25519``. + * - ``[api_auth] jwt_kid`` + - Auto (``RFC 7638`` thumbprint) + - Key ID placed in token header. Ignored for symmetric keys. + * - ``[api_auth] jwt_issuer`` + - None + - Issuer claim (``iss``). Recommended to be unique per deployment. + * - ``[api_auth] jwt_audience`` + - None + - Audience claim (``aud``) for REST API tokens. + * - ``[api_auth] jwt_expiration_time`` + - 86400 (24h) + - REST API token lifetime in seconds. + * - ``[api_auth] jwt_cli_expiration_time`` + - 3600 (1h) + - CLI token lifetime in seconds. + * - ``[api_auth] jwt_leeway`` + - 10 + - Clock skew tolerance in seconds for token validation. + * - ``[api_auth] trusted_jwks_url`` + - None + - JWKS endpoint URL or local file path for token validation. Mutually exclusive with ``jwt_secret``. + * - ``[execution_api] jwt_expiration_time`` + - 600 (10 min) + - Execution API token lifetime in seconds. + * - ``[execution_api] jwt_audience`` + - ``urn:airflow.apache.org:task`` + - Audience claim for Execution API tokens. + +.. important:: + + Time synchronization across all Airflow components is critical. Use NTP (e.g., ``ntpd`` or + ``chrony``) to keep clocks in sync. Clock skew beyond the configured ``jwt_leeway`` will cause + authentication failures. diff --git a/airflow-core/docs/security/security_model.rst b/airflow-core/docs/security/security_model.rst index 15b59b250904c..96f6f66783b14 100644 --- a/airflow-core/docs/security/security_model.rst +++ b/airflow-core/docs/security/security_model.rst @@ -62,11 +62,24 @@ Dag authors ........... They can create, modify, and delete Dag files. The -code in Dag files is executed on workers and in the Dag Processor. -Therefore, Dag authors can create and change code executed on workers -and the Dag Processor and potentially access the credentials that the Dag -code uses to access external systems. Dag authors have full access -to the metadata database. +code in Dag files is executed on workers, in the Dag File Processor, +and in the Triggerer. +Therefore, Dag authors can create and change code executed on workers, +the Dag File Processor, and the Triggerer, and potentially access the credentials that the Dag +code uses to access external systems. + +In Airflow 3, the level of database isolation depends on the component: + +* **Workers**: Task code on workers communicates with the API server exclusively through the + Execution API. Workers do not receive database credentials and genuinely cannot access the + metadata database directly. +* **Dag File Processor and Triggerer**: Airflow implements software guards that prevent + accidental direct database access from Dag author code. However, because Dag parsing and + trigger execution processes run as the same Unix user as their parent processes (which do + have database credentials), a deliberately malicious Dag author can potentially retrieve + credentials from the parent process and gain direct database access. See + :ref:`jwt-authentication-and-workload-isolation` for details on the specific mechanisms and + deployment hardening measures. Authenticated UI users ....................... @@ -115,6 +128,8 @@ The primary difference between an operator and admin is the ability to manage an to other users, and access audit logs - only admins are able to do this. Otherwise assume they have the same access as an admin. +.. _connection-configuration-users: + Connection configuration users .............................. @@ -170,6 +185,8 @@ Viewers also do not have permission to access audit logs. For more information on the capabilities of authenticated UI users, see :doc:`apache-airflow-providers-fab:auth-manager/access-control`. +.. _capabilities-of-dag-authors: + Capabilities of Dag authors --------------------------- @@ -193,15 +210,21 @@ not open new security vulnerabilities. Limiting Dag Author access to subset of Dags -------------------------------------------- -Airflow does not have multi-tenancy or multi-team features to provide isolation between different groups of users when -it comes to task execution. While, in Airflow 3.0 and later, Dag Authors cannot directly access database and cannot run -arbitrary queries on the database, they still have access to all Dags in the Airflow installation and they can +Airflow does not yet provide full task-level isolation between different groups of users when +it comes to task execution. While, in Airflow 3.0 and later, worker task code cannot directly access the +metadata database (it communicates through the Execution API), Dag author code that runs in the Dag File +Processor and Triggerer potentially still has direct database access. Regardless of execution context, Dag authors +have access to all Dags in the Airflow installation and they can modify any of those Dags - no matter which Dag the task code is executed for. This means that Dag authors can modify state of any task instance of any Dag, and there are no finer-grained access controls to limit that access. -There is a work in progress on multi-team feature in Airflow that will allow to have some isolation between different -groups of users and potentially limit access of Dag authors to only a subset of Dags, but currently there is no -such feature in Airflow and you can assume that all Dag authors have access to all Dags and can modify their state. +There is an **experimental** multi-team feature in Airflow (``[core] multi_team``) that provides UI-level and +REST API-level RBAC isolation between teams. However, this feature **does not yet guarantee task-level isolation**. +At the task execution level, workloads from different teams still share the same Execution API, signing keys, +connections, and variables. A task from one team can access the same shared resources as a task from another team. +The multi-team feature is a work in progress — task-level isolation and Execution API enforcement of team +boundaries will be improved in future versions of Airflow. Until then, you should assume that all Dag authors +have access to all Dags and shared resources, and can modify their state regardless of team assignment. Security contexts for Dag author submitted code @@ -239,8 +262,15 @@ Triggerer In case of Triggerer, Dag authors can execute arbitrary code in Triggerer. Currently there are no enforcement mechanisms that would allow to isolate tasks that are using deferrable functionality from -each other and arbitrary code from various tasks can be executed in the same process/machine. Deployment -Manager must trust that Dag authors will not abuse this capability. +each other and arbitrary code from various tasks can be executed in the same process/machine. The default +deployment runs a single Triggerer instance that handles triggers from all teams — there is no built-in +support for per-team Triggerer instances. Additionally, the Triggerer uses an in-process Execution API +transport that potentially bypasses JWT authentication and potentially has direct access to the metadata +database. For multi-team deployments, Deployment Managers must run separate Triggerer instances per team +as a deployment-level measure, but even then each instance potentially retains direct database access +and a Dag author +whose trigger code runs there can potentially access the database directly — including data belonging +to other teams. Deployment Manager must trust that Dag authors will not abuse this capability. Dag files not needed for Scheduler and API Server ................................................. @@ -282,6 +312,292 @@ Access to all Dags All Dag authors have access to all Dags in the Airflow deployment. This means that they can view, modify, and update any Dag without restrictions at any time. +.. _jwt-authentication-and-workload-isolation: + +JWT authentication and workload isolation +----------------------------------------- + +Airflow uses JWT (JSON Web Token) authentication for both its public REST API and its internal +Execution API. For a detailed description of the JWT authentication flows, token structure, and +configuration, see :doc:`/security/jwt_token_authentication`. For the current state of workload +isolation protections and their limitations, see :ref:`workload-isolation`. + +Current isolation limitations +............................. + +While Airflow 3 significantly improved the security model by preventing worker task code from +directly accessing the metadata database (workers now communicate exclusively through the +Execution API), **perfect isolation between Dag authors is not yet achieved**. Dag author code +potentially still executes with direct database access in the Dag File Processor and Triggerer. + +**Software guards vs. intentional access** + Airflow implements software-level guards that prevent **accidental and unintentional** direct database + access from Dag author code. The Dag File Processor removes the database session and connection + information before forking child processes that parse Dag files, and worker tasks use the Execution + API exclusively. + + However, these software guards **do not protect against intentional, malicious access**. The child + processes that parse Dag files and execute trigger code run as the **same Unix user** as their parent + processes (the Dag File Processor manager and the Triggerer respectively). Because of how POSIX + process isolation works, a child process running as the same user can retrieve the parent's + credentials through several mechanisms: + + * **Environment variables**: By default, on Linux, any process can read ``/proc//environ`` of another + process running as the same user — so database credentials passed via environment variables + (e.g., ``AIRFLOW__DATABASE__SQL_ALCHEMY_CONN``) can be read from the parent process. This can be + prevented by setting dumpable property of the process which is implemented in supervisor of tasks. + * **Configuration files**: If configuration is stored in files, those files must be readable by the + parent process and are therefore also readable by the child process running as the same user. + * **Command-based secrets** (``_CMD`` suffix options): The child process can execute the same + commands to retrieve secrets. + * **Secrets manager access**: If the parent uses a secrets backend, the child can access the same + secrets manager using credentials available in the process environment or filesystem. + + This means that a deliberately malicious Dag author can retrieve database credentials and gain + **full read/write access to the metadata database** — including the ability to modify any Dag, + task instance, connection, or variable. The software guards address accidental access (e.g., a Dag + author importing ``airflow.settings.Session`` out of habit from Airflow 2) but do not prevent a + determined actor from circumventing them. + + On workers, the isolation can be stronger when Deployment Manager configures worker processes to + not receive database credentials at all (neither via environment variables nor configuration). + Workers should communicate exclusively through the Execution API using short-lived JWT tokens. + A task running on a worker genuinely should not access the metadata database directly — + when it is configured to not have any credentials accessible to it. + +**Dag File Processor and Triggerer run user code only have soft protection to bypass JWT authentication** + The Dag File Processor and Triggerer processes that run user code, + use an in-process transport to access the Execution API, which bypasses JWT authentication. + Since these components execute user-submitted code (Dag files and trigger code respectively), + a Dag author whose code runs in these components + has unrestricted access to all Execution API operations if they bypass the soft protections + — including the ability to read any connection, variable, or XCom — without needing a valid JWT token. + + Furthermore, the Dag File Processor has direct access to the metadata database (it needs this to + store serialized Dags). As described above, Dag author code executing in the Dag File Processor + context could potentially retrieve the database credentials from the parent process and access + the database directly, including the JWT signing key configuration if it is available in the + process environment. If a Dag author obtains the JWT signing key, they could forge arbitrary tokens. + +**Dag File Processor and Triggerer are shared across teams** + In the default deployment, a **single Dag File Processor instance** parses all Dag files and a + **single Triggerer instance** handles all triggers — regardless of team assignment. There is no + built-in support for running per-team Dag File Processor or Triggerer instances. This means that + Dag author code from different teams executes within the same process, potentially sharing the + in-process Execution API and direct database access. + + For multi-team deployments that require separation, Deployment Managers must run **separate + Dag File Processor and Triggerer instances per team** as a deployment-level measure (for example, + by configuring each instance to only process bundles belonging to a specific team). However, even + with separate instances, each Dag File Processor and Triggerer potentially retains direct access + to the metadata database — a Dag author whose code runs in these components can potentially + retrieve credentials from the parent process and access the database directly, including reading + or modifying data belonging to other teams, unless the Deployment Manager implements Unix + user-level isolation (see :ref:`deployment-hardening-for-improved-isolation`). + +**No cross-workload isolation in the Execution API** + All worker workloads authenticate to the same Execution API with tokens signed by the same key and + sharing the same audience. While the ``ti:self`` scope enforcement prevents a worker from accessing + another task's specific endpoints (heartbeat, state transitions), shared resources such as connections, + variables, and XComs are accessible to all tasks. There is no isolation between tasks belonging to + different teams or Dag authors at the Execution API level. + +**Token signing key might be a shared secret** + In symmetric key mode (``[api_auth] jwt_secret``), the same secret key is used to both generate and + validate tokens. Any component that has access to this secret can forge tokens with arbitrary claims, + including tokens for other task instances or with elevated scopes. This does not impact the security + of the system though if the secret is only available to api-server and scheduler via deployment + configuration. + +**Sensitive configuration values can be leaked through logs** + Dag authors can write code that prints environment variables or configuration values to task logs + (e.g., ``print(os.environ)``). Airflow masks known sensitive values in logs, but masking depends on + recognizing the value patterns. Dag authors who intentionally or accidentally log raw environment + variables may expose database credentials, JWT signing keys, Fernet keys, or other secrets in task + logs. Deployment Managers should restrict access to task logs and ensure that sensitive configuration + is only provided to components where it is needed (see the sensitive variables tables below). + +.. _deployment-hardening-for-improved-isolation: + +Deployment hardening for improved isolation +........................................... + +Deployment Managers who require stronger isolation between Dag authors and teams can take the following +measures. Note that these are deployment-specific actions that go beyond Airflow's built-in security +model — Airflow does not enforce these natively. + +**Mandatory code review of Dag files** + Implement a review process for all Dag submissions to Dag bundles. This can include: + + * Requiring pull request reviews before Dag files are deployed. + * Static analysis of Dag code to detect suspicious patterns (e.g., direct database access attempts, + reading environment variables, importing configuration modules). + * Automated linting rules that flag potentially dangerous code. + +**Restrict sensitive configuration to components that need them** + Do not share all configuration parameters across all components. In particular: + + * The JWT signing key (``[api_auth] jwt_secret`` or ``[api_auth] jwt_private_key_path``) should only + be available to components that need to generate tokens (Scheduler/Executor, API Server) and + components that need to validate tokens (API Server). Workers should not have access to the signing + key — they only need the tokens provided to them. + * Connection credentials for external systems (via Secrets Managers) should only be available to the API Server + (which serves them to workers via the Execution API), not to the Scheduler, Dag File Processor, + or Triggerer processes directly. This however limits some of the features of Airflow - such as Deadline + Alerts or triggers that need to authenticate with the external systems. + * Database connection strings should only be available to components that need direct database access + (API Server, Scheduler, Dag File Processor, Triggerer), not to workers. + +**Pass configuration via environment variables** + For higher security, pass sensitive configuration values via environment variables rather than + configuration files. Environment variables are inherently safer than configuration files in + Airflow's worker processes because of a built-in protection: on Linux, the supervisor process + calls ``prctl(PR_SET_DUMPABLE, 0)`` before forking the task process, and this flag is inherited + by the forked child. This marks both processes as non-dumpable, which prevents same-UID sibling + processes from reading ``/proc//environ``, ``/proc//mem``, or attaching via + ``ptrace``. In contrast, configuration files on disk are readable by any process running as + the same Unix user. Environment variables can also be scoped to individual processes or + containers, making it easier to restrict which components have access to which secrets. + + The following tables list all security-sensitive configuration variables (marked ``sensitive: true`` + in Airflow's configuration). Deployment Managers should review each variable and ensure it is only + provided to the components that need it. The "Needed by" column indicates which components + typically require the variable — but actual needs depend on the specific deployment topology and + features in use. + + .. START AUTOGENERATED CORE SENSITIVE VARS + + **Core Airflow sensitive configuration variables:** + + .. list-table:: + :header-rows: 1 + :widths: 40 30 30 + + * - Environment variable + - Description + - Needed by + * - ``AIRFLOW__API_AUTH__JWT_SECRET`` + - JWT signing key (symmetric mode) + - API Server, Scheduler + * - ``AIRFLOW__API__SECRET_KEY`` + - API secret key for log token signing + - API Server, Scheduler, Workers, Triggerer + * - ``AIRFLOW__CORE__ASSET_MANAGER_KWARGS`` + - Asset manager credentials + - Dag File Processor + * - ``AIRFLOW__CORE__FERNET_KEY`` + - Fernet encryption key for connections/variables at rest + - API Server, Scheduler, Workers, Dag File Processor, Triggerer + * - ``AIRFLOW__DATABASE__SQL_ALCHEMY_CONN`` + - Metadata database connection string + - API Server, Scheduler, Dag File Processor, Triggerer + * - ``AIRFLOW__DATABASE__SQL_ALCHEMY_CONN_ASYNC`` + - Async metadata database connection string + - API Server, Scheduler, Dag File Processor, Triggerer + * - ``AIRFLOW__DATABASE__SQL_ALCHEMY_ENGINE_ARGS`` + - SQLAlchemy engine parameters (may contain credentials) + - API Server, Scheduler, Dag File Processor, Triggerer + * - ``AIRFLOW__LOGGING__REMOTE_TASK_HANDLER_KWARGS`` + - Remote logging handler credentials + - Scheduler, Workers, Triggerer + * - ``AIRFLOW__SECRETS__BACKEND_KWARGS`` + - Secrets backend credentials (non-worker mode) + - Scheduler, Dag File Processor, Triggerer + * - ``AIRFLOW__SENTRY__SENTRY_DSN`` + - Sentry error reporting endpoint + - Scheduler, Triggerer + * - ``AIRFLOW__WORKERS__SECRETS_BACKEND_KWARGS`` + - Worker-specific secrets backend credentials + - Workers + + .. END AUTOGENERATED CORE SENSITIVE VARS + + Note that ``AIRFLOW__API_AUTH__JWT_PRIVATE_KEY_PATH`` (path to the JWT private key for asymmetric + signing) is not marked as ``sensitive`` in config.yml because it is a file path, not a secret + value itself. However, access to the file it points to should be restricted to the Scheduler + (which generates tokens) and the API Server (which validates them). + + .. START AUTOGENERATED PROVIDER SENSITIVE VARS + + **Provider-specific sensitive configuration variables:** + + The following variables are defined by Airflow providers and should only be set on components where + the corresponding provider functionality is needed. The decision of which components require these + variables depends on the Deployment Manager's choices about which providers and features are + enabled in each component. + + .. list-table:: + :header-rows: 1 + :widths: 40 30 30 + + * - Environment variable + - Provider + - Description + * - ``AIRFLOW__CELERY_BROKER_TRANSPORT_OPTIONS__SENTINEL_KWARGS`` + - celery + - Sentinel kwargs + * - ``AIRFLOW__CELERY_RESULT_BACKEND_TRANSPORT_OPTIONS__SENTINEL_KWARGS`` + - celery + - Sentinel kwargs + * - ``AIRFLOW__CELERY__BROKER_URL`` + - celery + - Broker url + * - ``AIRFLOW__CELERY__FLOWER_BASIC_AUTH`` + - celery + - Flower basic auth + * - ``AIRFLOW__CELERY__RESULT_BACKEND`` + - celery + - Result backend + * - ``AIRFLOW__KEYCLOAK_AUTH_MANAGER__CLIENT_SECRET`` + - keycloak + - Client secret + * - ``AIRFLOW__OPENSEARCH__PASSWORD`` + - opensearch + - Password + * - ``AIRFLOW__OPENSEARCH__USERNAME`` + - opensearch + - Username + + .. END AUTOGENERATED PROVIDER SENSITIVE VARS + + Deployment Managers should review the full configuration reference and identify any additional + parameters that contain credentials or secrets relevant to their specific deployment. + +**Use asymmetric keys for JWT signing** + Using asymmetric keys (``[api_auth] jwt_private_key_path`` with a JWKS endpoint) provides better + security than symmetric keys because: + + * The private key (used for signing) can be restricted to the Scheduler/Executor. + * The API Server only needs the public key (via JWKS) for validation. + * Workers cannot forge tokens even if they could access the JWKS endpoint, since they would + not have the private key. + +**Network-level isolation** + Use network policies, VPCs, or similar mechanisms to restrict which components can communicate + with each other. For example, workers should only be able to reach the Execution API endpoint, + not the metadata database or internal services directly. The Dag File Processor and Triggerer + child processes should ideally not have network access to the metadata database either, if + Unix user-level isolation is implemented. + +**Other measures and future improvements** + Deployment Managers may need to implement additional measures depending on their security + requirements. These may include monitoring and auditing of Execution API access patterns, + runtime sandboxing of Dag code, or dedicated infrastructure per team. + + Future versions of Airflow plan to address these limitations through two approaches: + + * **Strategic (longer-term)**: Move the Dag File Processor and Triggerer to communicate with + the metadata database exclusively through the API server (similar to how workers use the + Execution API today). This would eliminate the need for these components to have database + credentials at all, providing security by design rather than relying on deployment-level + measures. + * **Tactical (shorter-term)**: Native support for Unix user impersonation in the Dag File + Processor and Triggerer child processes, so that Dag author code runs as a different, low- + privilege user that cannot access the parent's credentials or the database. + + The Airflow community is actively working on these improvements. + + Custom RBAC limitations ----------------------- @@ -309,6 +625,8 @@ you trust them not to abuse the capabilities they have. You should also make sur properly configured the Airflow installation to prevent Dag authors from executing arbitrary code in the Scheduler and API Server processes. +.. _deploying-and-protecting-airflow-installation: + Deploying and protecting Airflow installation ............................................. @@ -354,13 +672,150 @@ Examples of fine-grained access control include (but are not limited to): * Access restrictions to views or Dags: Controlling user access to certain views or specific Dags, ensuring that users can only view or interact with authorized components. -Future: multi-tenancy isolation -............................... +Future: multi-team isolation +............................ These examples showcase ways in which Deployment Managers can refine and limit user privileges within Airflow, providing tighter control and ensuring that users have access only to the necessary components and functionalities based on their roles and responsibilities. However, fine-grained access control does not -provide full isolation and separation of access to allow isolation of different user groups in a -multi-tenant fashion yet. In future versions of Airflow, some fine-grained access control features could -become part of the Airflow security model, as the Airflow community is working on a multi-tenant model -currently. +yet provide full isolation and separation of access between different groups of users. + +The experimental multi-team feature (``[core] multi_team``) is a step towards cross-team isolation, but it +currently only enforces team-based isolation at the UI and REST API level. **Task-level isolation is not yet +guaranteed** — workloads from different teams share the same Execution API, JWT signing keys, and access to +connections, variables, and XComs. In deployments where additional hardening measures (described in +:ref:`deployment-hardening-for-improved-isolation`) are not implemented, a task belonging to one team can +potentially access shared resources available to tasks from other teams. Deployment Managers who enable the +multi-team feature should not rely on it alone for security-critical isolation between teams at the task +execution layer — a deep understanding of configuration and deployment security is required by Deployment +Managers to configure it in a way that can guarantee separation between teams. + +Future versions of Airflow will improve task-level isolation, including team-scoped Execution API enforcement, +finer-grained JWT token scopes, and better sandboxing of user-submitted code. The Airflow community is +actively working on these improvements. + + +What is NOT considered a security vulnerability +----------------------------------------------- + +The following scenarios are **not** considered security vulnerabilities in Airflow. They are either +intentional design choices, consequences of the trust model described above, or issues that fall +outside Airflow's threat model. Security researchers (and AI agents performing security analysis) +should review this section before reporting issues to the Airflow security team. + +For full details on reporting policies, see +`Airflow's Security Policy `_. + +Dag authors executing arbitrary code +..................................... + +Dag authors can execute arbitrary code on workers, the Dag File Processor, and the Triggerer. This +includes accessing credentials, environment variables, and (in the case of the Dag File Processor +and Triggerer) potentially the metadata database directly. This is the intended behavior as described in +:ref:`capabilities-of-dag-authors` — Dag authors are trusted users. Reports that a Dag author can +"achieve RCE" or "access the database" by writing Dag code are restating a documented capability, +not discovering a vulnerability. + +Dag author code passing unsanitized input to operators and hooks +................................................................ + +When a Dag author writes code that passes unsanitized UI user input (such as Dag run parameters, +variables, or connection configuration values) to operators, hooks, or third-party libraries, the +responsibility lies with the Dag author. Airflow's hooks and operators are low-level interfaces — +Dag authors are Python programmers who must sanitize inputs before passing them to these interfaces. + +SQL injection or command injection is only considered a vulnerability if it can be triggered by a +**non-Dag-author** user role (e.g., an authenticated UI user) **without** the Dag author deliberately +writing code that passes that input unsafely. If the only way to exploit the injection requires writing +or modifying a Dag file, it is not a vulnerability — the Dag author already has the ability to execute +arbitrary code. See also :doc:`/security/sql`. + +An exception exists when official Airflow documentation explicitly recommends a pattern that leads to +injection — in that case, the documentation guidance itself is the issue and may warrant an advisory. + +Dag File Processor and Triggerer potentially having database access +................................................................... + +The Dag File Processor potentially has direct database access to store serialized Dags. The Triggerer +potentially has direct database access to manage trigger state. Both components execute user-submitted +code (Dag files and trigger code respectively) and potentially bypass JWT authentication via an +in-process Execution API transport. These are intentional architectural choices, not vulnerabilities. +They are documented in :ref:`jwt-authentication-and-workload-isolation`. + +Workers accessing shared Execution API resources +................................................. + +Worker tasks can access connections, variables, and XComs via the Execution API using their JWT token. +While the ``ti:self`` scope prevents cross-task state manipulation, shared resources are accessible to +all tasks. This is the current design — not a vulnerability. Reports that "a task can read another +team's connection" are describing a known limitation of the current isolation model, documented in +:ref:`jwt-authentication-and-workload-isolation`. + +Execution API tokens not being revocable +........................................ + +Execution API tokens issued to workers are short-lived (default 10 minutes) with automatic refresh +and are intentionally not subject to revocation. This is a design choice documented in +:doc:`/security/jwt_token_authentication`, not a missing security control. + +Connection configuration capabilities +...................................... + +Users with the **Connection configuration** role can configure connections with arbitrary credentials +and connection parameters. When the ``test connection`` feature is enabled, these users can potentially +trigger RCE, arbitrary file reads, or Denial of Service through connection parameters. This is by +design — connection configuration users are highly privileged and must be trusted not to abuse these +capabilities. The ``test connection`` feature is disabled by default since Airflow 2.7.0, and enabling +it is an explicit Deployment Manager decision that acknowledges these risks. See +:ref:`connection-configuration-users` for details. + +Denial of Service by authenticated users +........................................ + +Airflow is not designed to be exposed to untrusted users on the public internet. All users who can +access the Airflow UI and API are authenticated and known. Denial of Service scenarios triggered by +authenticated users (such as creating very large Dag runs, submitting expensive queries, or flooding +the API) are not considered security vulnerabilities. They are operational concerns that Deployment +Managers should address through rate limiting, resource quotas, and monitoring — standard measures +for any internal application. See :ref:`deploying-and-protecting-airflow-installation`. + +Self-XSS by authenticated users +................................ + +Cross-site scripting (XSS) scenarios where the only victim is the user who injected the payload +(self-XSS) are not considered security vulnerabilities. Airflow's users are authenticated and +known, and self-XSS does not allow an attacker to compromise other users. If you discover an XSS +scenario where a lower-privileged user can inject a payload that executes in a higher-privileged +user's session without that user's action, that is a valid vulnerability and should be reported. + +Simple Auth Manager +................... + +The Simple Auth Manager is intended for development and testing only. This is clearly documented and +a prominent warning banner is displayed on the login page. Security issues specific to the Simple +Auth Manager (such as weak password handling, lack of rate limiting, or missing CSRF protections) are +not considered production security vulnerabilities. Production deployments must use a production-grade +auth manager. + +Third-party dependency vulnerabilities in Docker images +....................................................... + +Airflow's reference Docker images are built with the latest available dependencies at release time. +Vulnerabilities found by scanning these images against CVE databases are expected to appear over time +as new CVEs are published. These should **not** be reported to the Airflow security team. Instead, +users should build their own images with updated dependencies as described in the +`Docker image documentation `_. + +If you discover that a third-party dependency vulnerability is **actually exploitable** in Airflow +(with a proof-of-concept demonstrating the exploitation in Airflow's context), that is a valid +report and should be submitted following the security policy. + +Automated scanning results without human verification +..................................................... + +Automated security scanner reports that list findings without human verification against Airflow's +security model are not considered valid vulnerability reports. Airflow's trust model differs +significantly from typical web applications — many scanner findings (such as "admin user can execute +code" or "database credentials accessible in configuration") are expected behavior. Reports must +include a proof-of-concept that demonstrates how the finding violates the security model described +in this document, including identifying the specific user role involved and the attack scenario. diff --git a/airflow-core/docs/security/workload.rst b/airflow-core/docs/security/workload.rst index 31714aa21fbb2..0496cddc7f54a 100644 --- a/airflow-core/docs/security/workload.rst +++ b/airflow-core/docs/security/workload.rst @@ -50,3 +50,86 @@ not set. [core] default_impersonation = airflow + +.. _workload-isolation: + +Workload Isolation and Current Limitations +------------------------------------------ + +This section describes the current state of workload isolation in Apache Airflow, +including the protections that are in place, the known limitations, and planned improvements. + +For the full security model and deployment hardening guidance, see :doc:`/security/security_model`. +For details on the JWT authentication flows used by workers and internal components, see +:doc:`/security/jwt_token_authentication`. + +Worker process memory protection (Linux) +'''''''''''''''''''''''''''''''''''''''' + +On Linux, the supervisor process calls ``prctl(PR_SET_DUMPABLE, 0)`` at the start of +``supervise()`` before forking the task process. This flag is inherited by the forked +child. Marking processes as non-dumpable prevents same-UID sibling processes from reading +``/proc//mem``, ``/proc//environ``, or ``/proc//maps``, and blocks +``ptrace(PTRACE_ATTACH)``. This is critical because each supervisor holds a distinct JWT +token in memory — without this protection, a malicious task process running as the same +Unix user could steal tokens from sibling supervisor processes. + +This protection is one of the reasons that passing sensitive configuration via environment +variables is safer than via configuration files: environment variables are only readable +by the process itself (and root), whereas configuration files on disk are readable by any +process with filesystem access running as the same user. + +.. note:: + + This protection is Linux-specific. On non-Linux platforms, the + ``_make_process_nondumpable()`` call is a no-op. Deployment Managers running Airflow + on non-Linux platforms should implement alternative isolation measures. + +No cross-workload isolation +''''''''''''''''''''''''''' + +All worker workloads authenticate to the same Execution API with tokens that share the +same signing key, audience, and issuer. While the ``ti:self`` scope enforcement prevents +a worker from accessing *another task instance's* specific endpoints (e.g., heartbeat, +state transitions), the token grants access to shared resources such as connections, +variables, and XComs that are not scoped to individual tasks. + +No team-level isolation in Execution API (experimental multi-team feature) +'''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +The experimental multi-team feature (``[core] multi_team``) provides UI-level and REST +API-level RBAC isolation between teams, but **does not yet guarantee task-level isolation**. +At the Execution API level, there is no enforcement of team-based access boundaries. +A task from one team can access the same connections, variables, and XComs as a task from +another team. All workloads share the same JWT signing keys and audience regardless of team +assignment. + +In deployments where additional hardening measures are not implemented at the deployment +level, a task from one team can potentially access resources belonging to another team +(see :doc:`/security/security_model`). A deep understanding of configuration and deployment +security is required by Deployment Managers to configure it in a way that can guarantee +separation between teams. Task-level team isolation will be improved in future versions +of Airflow. + +Dag File Processor and Triggerer potentially bypass JWT and access the database +''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''''' + +As described in :doc:`/security/jwt_token_authentication`, the default deployment runs a +single Dag File Processor and a single Triggerer for all teams. Both potentially bypass +JWT authentication via in-process transport. For multi-team isolation, Deployment Managers +must run separate instances per team, but even then, each instance potentially retains +direct database access. A Dag author whose code runs in these components can potentially +access the database directly — including data belonging to other teams or the JWT signing +key configuration — unless the Deployment Manager restricts the database credentials and +configuration available to each instance. + +Planned improvements +'''''''''''''''''''' + +Future versions of Airflow will address these limitations with: + +- Finer-grained token scopes tied to specific resources (connections, variables) and teams. +- Enforcement of team-based isolation in the Execution API. +- Built-in support for per-team Dag File Processor and Triggerer instances. +- Improved sandboxing of user-submitted code in the Dag File Processor and Triggerer. +- Full task-level isolation for the multi-team feature. diff --git a/airflow-core/newsfragments/64552.improvement.rst b/airflow-core/newsfragments/64552.improvement.rst new file mode 100644 index 0000000000000..ae70554cd22ee --- /dev/null +++ b/airflow-core/newsfragments/64552.improvement.rst @@ -0,0 +1 @@ +Allow UI theme config with only CSS overrides, icon only, or empty ``{}`` to restore OSS defaults. The ``tokens`` field is now optional in the theme configuration. diff --git a/airflow-core/src/airflow/api_fastapi/common/types.py b/airflow-core/src/airflow/api_fastapi/common/types.py index bd4176a9fd927..7d2a944c82228 100644 --- a/airflow-core/src/airflow/api_fastapi/common/types.py +++ b/airflow-core/src/airflow/api_fastapi/common/types.py @@ -20,7 +20,7 @@ from dataclasses import dataclass from datetime import timedelta from enum import Enum -from typing import Annotated, Any, Literal +from typing import Annotated, Literal from pydantic import ( AfterValidator, @@ -208,15 +208,11 @@ def check_at_least_one_color(self) -> ThemeColors: raise ValueError("At least one color token must be provided: brand, gray, black, or white") return self - @model_serializer(mode="wrap") - def serialize_model(self, handler: Any) -> dict: - return {k: v for k, v in handler(self).items() if v is not None} - class Theme(BaseModel): """JSON to modify Chakra's theme.""" - tokens: dict[Literal["colors"], ThemeColors] + tokens: dict[Literal["colors"], ThemeColors] | None = None globalCss: dict[str, dict] | None = None icon: ThemeIconType = None icon_dark_mode: ThemeIconType = None diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/config.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/config.py index 96cd4aaad266a..a511b31142b22 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/config.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/config.py @@ -16,6 +16,8 @@ # under the License. from __future__ import annotations +from pydantic import ConfigDict, field_serializer + from airflow.api_fastapi.common.types import Theme, UIAlert from airflow.api_fastapi.core_api.base import BaseModel @@ -23,6 +25,8 @@ class ConfigResponse(BaseModel): """configuration serializer.""" + model_config = ConfigDict(json_schema_mode_override="validation") + fallback_page_limit: int auto_refresh_interval: int hide_paused_dags_by_default: bool @@ -36,3 +40,9 @@ class ConfigResponse(BaseModel): external_log_name: str | None = None theme: Theme | None multi_team: bool + + @field_serializer("theme") + def serialize_theme(self, theme: Theme | None) -> dict | None: + if theme is None: + return None + return theme.model_dump(exclude_none=True) diff --git a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/gantt.py b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/gantt.py index 57a96c8a0ad70..3b74e84b47f3d 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/gantt.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/datamodels/ui/gantt.py @@ -30,6 +30,8 @@ class GanttTaskInstance(BaseModel): task_display_name: str try_number: int state: TaskInstanceState | None + scheduled_dttm: datetime | None + queued_dttm: datetime | None start_date: datetime | None end_date: datetime | None is_group: bool = False diff --git a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml index 0ae06e4404641..e3bce4f8f15e2 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml +++ b/airflow-core/src/airflow/api_fastapi/core_api/openapi/_private_ui.yaml @@ -2469,6 +2469,18 @@ components: anyOf: - $ref: '#/components/schemas/TaskInstanceState' - type: 'null' + scheduled_dttm: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Scheduled Dttm + queued_dttm: + anyOf: + - type: string + format: date-time + - type: 'null' + title: Queued Dttm start_date: anyOf: - type: string @@ -2495,6 +2507,8 @@ components: - task_display_name - try_number - state + - scheduled_dttm + - queued_dttm - start_date - end_date title: GanttTaskInstance @@ -3492,11 +3506,13 @@ components: Theme: properties: tokens: - additionalProperties: - $ref: '#/components/schemas/ThemeColors' - propertyNames: - const: colors - type: object + anyOf: + - additionalProperties: + $ref: '#/components/schemas/ThemeColors' + propertyNames: + const: colors + type: object + - type: 'null' title: Tokens globalCss: anyOf: @@ -3517,13 +3533,80 @@ components: - type: 'null' title: Icon Dark Mode type: object - required: - - tokens title: Theme description: JSON to modify Chakra's theme. ThemeColors: - additionalProperties: true + properties: + brand: + anyOf: + - additionalProperties: + additionalProperties: + $ref: '#/components/schemas/OklchColor' + propertyNames: + const: value + type: object + propertyNames: + enum: + - '50' + - '100' + - '200' + - '300' + - '400' + - '500' + - '600' + - '700' + - '800' + - '900' + - '950' + type: object + - type: 'null' + title: Brand + gray: + anyOf: + - additionalProperties: + additionalProperties: + $ref: '#/components/schemas/OklchColor' + propertyNames: + const: value + type: object + propertyNames: + enum: + - '50' + - '100' + - '200' + - '300' + - '400' + - '500' + - '600' + - '700' + - '800' + - '900' + - '950' + type: object + - type: 'null' + title: Gray + black: + anyOf: + - additionalProperties: + $ref: '#/components/schemas/OklchColor' + propertyNames: + const: value + type: object + - type: 'null' + title: Black + white: + anyOf: + - additionalProperties: + $ref: '#/components/schemas/OklchColor' + propertyNames: + const: value + type: object + - type: 'null' + title: White type: object + title: ThemeColors + description: Color tokens for the UI theme. All fields are optional; at least + one must be provided. TokenType: type: string enum: diff --git a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/gantt.py b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/gantt.py index f33b12e6f7e8a..7807e3fd6bc0f 100644 --- a/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/gantt.py +++ b/airflow-core/src/airflow/api_fastapi/core_api/routes/ui/gantt.py @@ -67,6 +67,8 @@ def get_gantt_data( TaskInstance.task_display_name.label("task_display_name"), # type: ignore[attr-defined] TaskInstance.try_number.label("try_number"), TaskInstance.state.label("state"), + TaskInstance.scheduled_dttm.label("scheduled_dttm"), + TaskInstance.queued_dttm.label("queued_dttm"), TaskInstance.start_date.label("start_date"), TaskInstance.end_date.label("end_date"), ).where( @@ -81,6 +83,8 @@ def get_gantt_data( TaskInstanceHistory.task_display_name.label("task_display_name"), TaskInstanceHistory.try_number.label("try_number"), TaskInstanceHistory.state.label("state"), + TaskInstanceHistory.scheduled_dttm.label("scheduled_dttm"), + TaskInstanceHistory.queued_dttm.label("queued_dttm"), TaskInstanceHistory.start_date.label("start_date"), TaskInstanceHistory.end_date.label("end_date"), ).where( @@ -106,6 +110,8 @@ def get_gantt_data( task_display_name=row.task_display_name, try_number=row.try_number, state=row.state, + scheduled_dttm=row.scheduled_dttm, + queued_dttm=row.queued_dttm, start_date=row.start_date, end_date=row.end_date, ) diff --git a/airflow-core/src/airflow/config_templates/airflow_local_settings.py b/airflow-core/src/airflow/config_templates/airflow_local_settings.py index 48f14b0f9a9ee..0c54c55573575 100644 --- a/airflow-core/src/airflow/config_templates/airflow_local_settings.py +++ b/airflow-core/src/airflow/config_templates/airflow_local_settings.py @@ -334,34 +334,36 @@ def _default_conn_name_from(mod_path, hook_name): ) elif OPENSEARCH_HOST: - OPENSEARCH_END_OF_LOG_MARK: str = conf.get_mandatory_value("opensearch", "END_OF_LOG_MARK") - OPENSEARCH_PORT: str = conf.get_mandatory_value("opensearch", "PORT") + from airflow.providers.opensearch.log.os_task_handler import OpensearchRemoteLogIO + + OPENSEARCH_PORT = conf.getint("opensearch", "PORT", fallback=9200) OPENSEARCH_USERNAME: str = conf.get_mandatory_value("opensearch", "USERNAME") OPENSEARCH_PASSWORD: str = conf.get_mandatory_value("opensearch", "PASSWORD") OPENSEARCH_WRITE_STDOUT: bool = conf.getboolean("opensearch", "WRITE_STDOUT") + OPENSEARCH_WRITE_TO_OS: bool = conf.getboolean("opensearch", "WRITE_TO_OS") OPENSEARCH_JSON_FORMAT: bool = conf.getboolean("opensearch", "JSON_FORMAT") - OPENSEARCH_JSON_FIELDS: str = conf.get_mandatory_value("opensearch", "JSON_FIELDS") + OPENSEARCH_TARGET_INDEX: str = conf.get_mandatory_value("opensearch", "TARGET_INDEX") OPENSEARCH_HOST_FIELD: str = conf.get_mandatory_value("opensearch", "HOST_FIELD") OPENSEARCH_OFFSET_FIELD: str = conf.get_mandatory_value("opensearch", "OFFSET_FIELD") + OPENSEARCH_LOG_ID_TEMPLATE: str = conf.get("opensearch", "LOG_ID_TEMPLATE", fallback="") or ( + "{dag_id}-{task_id}-{run_id}-{map_index}-{try_number}" + ) - OPENSEARCH_REMOTE_HANDLERS: dict[str, dict[str, str | bool | None]] = { - "task": { - "class": "airflow.providers.opensearch.log.os_task_handler.OpensearchTaskHandler", - "formatter": "airflow", - "base_log_folder": BASE_LOG_FOLDER, - "end_of_log_mark": OPENSEARCH_END_OF_LOG_MARK, - "host": OPENSEARCH_HOST, - "port": OPENSEARCH_PORT, - "username": OPENSEARCH_USERNAME, - "password": OPENSEARCH_PASSWORD, - "write_stdout": OPENSEARCH_WRITE_STDOUT, - "json_format": OPENSEARCH_JSON_FORMAT, - "json_fields": OPENSEARCH_JSON_FIELDS, - "host_field": OPENSEARCH_HOST_FIELD, - "offset_field": OPENSEARCH_OFFSET_FIELD, - }, - } - DEFAULT_LOGGING_CONFIG["handlers"].update(OPENSEARCH_REMOTE_HANDLERS) + REMOTE_TASK_LOG = OpensearchRemoteLogIO( + host=OPENSEARCH_HOST, + port=OPENSEARCH_PORT, + username=OPENSEARCH_USERNAME, + password=OPENSEARCH_PASSWORD, + target_index=OPENSEARCH_TARGET_INDEX, + write_stdout=OPENSEARCH_WRITE_STDOUT, + write_to_opensearch=OPENSEARCH_WRITE_TO_OS, + offset_field=OPENSEARCH_OFFSET_FIELD, + host_field=OPENSEARCH_HOST_FIELD, + base_log_folder=BASE_LOG_FOLDER, + delete_local_copy=delete_local_copy, + json_format=OPENSEARCH_JSON_FORMAT, + log_id_template=OPENSEARCH_LOG_ID_TEMPLATE, + ) else: raise AirflowException( "Incorrect remote log configuration. Please check the configuration of option 'host' in " diff --git a/airflow-core/src/airflow/config_templates/config.yml b/airflow-core/src/airflow/config_templates/config.yml index 2f1c63a21c16c..4b44ce6c181e6 100644 --- a/airflow-core/src/airflow/config_templates/config.yml +++ b/airflow-core/src/airflow/config_templates/config.yml @@ -1987,8 +1987,14 @@ api_auth: description: | Secret key used to encode and decode JWTs to authenticate to public and private APIs. - It should be as random as possible. However, when running more than 1 instances of API services, - make sure all of them use the same ``jwt_secret`` otherwise calls will fail on authentication. + It should be as random as possible. This key must be consistent across all components that + generate or validate JWT tokens (Scheduler, API Server). For improved security, consider + using asymmetric keys (``jwt_private_key_path``) instead, which allow you to restrict the + signing key to only the components that need to generate tokens. + + For security-sensitive deployments, pass this value via environment variable + (``AIRFLOW__API_AUTH__JWT_SECRET``) rather than storing it in a configuration file, and + restrict it to only the components that need it. Mutually exclusive with ``jwt_private_key_path``. version_added: 3.0.0 diff --git a/airflow-core/src/airflow/migrations/utils.py b/airflow-core/src/airflow/migrations/utils.py index 985473cf80d28..4eeaf373c6a87 100644 --- a/airflow-core/src/airflow/migrations/utils.py +++ b/airflow-core/src/airflow/migrations/utils.py @@ -17,39 +17,8 @@ from __future__ import annotations import contextlib -from collections import defaultdict from contextlib import contextmanager -from sqlalchemy import text - - -def get_mssql_table_constraints(conn, table_name) -> dict[str, dict[str, list[str]]]: - """ - Return the primary and unique constraint along with column name. - - Some tables like `task_instance` are missing the primary key constraint - name and the name is auto-generated by the SQL server, so this function - helps to retrieve any primary or unique constraint name. - - :param conn: sql connection object - :param table_name: table name - :return: a dictionary of ((constraint name, constraint type), column name) of table - """ - query = text( - f"""SELECT tc.CONSTRAINT_NAME , tc.CONSTRAINT_TYPE, ccu.COLUMN_NAME - FROM INFORMATION_SCHEMA.TABLE_CONSTRAINTS AS tc - JOIN INFORMATION_SCHEMA.CONSTRAINT_COLUMN_USAGE AS ccu ON ccu.CONSTRAINT_NAME = tc.CONSTRAINT_NAME - WHERE tc.TABLE_NAME = '{table_name}' AND - (tc.CONSTRAINT_TYPE = 'PRIMARY KEY' or UPPER(tc.CONSTRAINT_TYPE) = 'UNIQUE' - or UPPER(tc.CONSTRAINT_TYPE) = 'FOREIGN KEY') - """ - ) - result = conn.execute(query).fetchall() - constraint_dict = defaultdict(lambda: defaultdict(list)) - for constraint, constraint_type, col_name in result: - constraint_dict[constraint_type][constraint].append(col_name) - return constraint_dict - @contextmanager def disable_sqlite_fkeys(op): @@ -86,224 +55,9 @@ def mysql_drop_foreignkey_if_exists(constraint_name, table_name, op): """) -def mysql_drop_index_if_exists(index_name, table_name, op): - """Older Mysql versions do not support DROP INDEX IF EXISTS.""" - op.execute(f""" - CREATE PROCEDURE DropIndexIfExists() - BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.STATISTICS - WHERE - TABLE_SCHEMA = DATABASE() AND - TABLE_NAME = '{table_name}' AND - INDEX_NAME = '{index_name}' - ) THEN - DROP INDEX `{index_name}` ON `{table_name}`; - END IF; - END; - CALL DropIndexIfExists(); - DROP PROCEDURE DropIndexIfExists; - """) - - def ignore_sqlite_value_error(): from alembic import op if op.get_bind().dialect.name == "sqlite": return contextlib.suppress(ValueError) return contextlib.nullcontext() - - -def get_dialect_name(op) -> str: - conn = op.get_bind() - return conn.dialect.name if conn is not None else op.get_context().dialect.name - - -def create_index_if_not_exists(op, index_name, table_name, columns, unique=False) -> None: - """ - Create an index if it does not already exist. - - MySQL does not support CREATE INDEX IF NOT EXISTS, so a stored procedure is used. - PostgreSQL and SQLite support it natively. - """ - dialect_name = get_dialect_name(op) - - if dialect_name == "mysql": - unique_kw = "UNIQUE " if unique else "" - col_list = ", ".join(f"`{c}`" for c in columns) - op.execute( - text(f""" - DROP PROCEDURE IF EXISTS CreateIndexIfNotExists; - CREATE PROCEDURE CreateIndexIfNotExists() - BEGIN - IF NOT EXISTS ( - SELECT 1 - FROM information_schema.STATISTICS - WHERE - TABLE_SCHEMA = DATABASE() AND - TABLE_NAME = '{table_name}' AND - INDEX_NAME = '{index_name}' - ) THEN - CREATE {unique_kw}INDEX `{index_name}` ON `{table_name}` ({col_list}); - END IF; - END; - CALL CreateIndexIfNotExists(); - DROP PROCEDURE IF EXISTS CreateIndexIfNotExists; - """) - ) - else: - op.create_index(index_name, table_name, columns, unique=unique, if_not_exists=True) - - -def drop_index_if_exists(op, index_name, table_name) -> None: - """ - Drop an index if it exists. - - Works in both online and offline mode by using raw SQL for PostgreSQL and MySQL. - SQLite and PostgreSQL support DROP INDEX IF EXISTS natively. - MySQL requires a stored procedure since it does not support IF EXISTS for DROP INDEX. - """ - dialect_name = get_dialect_name(op) - - if dialect_name == "mysql": - op.execute( - text(f""" - CREATE PROCEDURE DropIndexIfExists() - BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.STATISTICS - WHERE - TABLE_SCHEMA = DATABASE() AND - TABLE_NAME = '{table_name}' AND - INDEX_NAME = '{index_name}' - ) THEN - DROP INDEX `{index_name}` ON `{table_name}`; - END IF; - END; - CALL DropIndexIfExists(); - DROP PROCEDURE DropIndexIfExists; - """) - ) - else: - # PostgreSQL and SQLite both support DROP INDEX IF EXISTS - op.drop_index(index_name, table_name=table_name, if_exists=True) - - -def drop_unique_constraints_on_columns(op, table_name, columns) -> None: - """ - Drop all unique constraints covering any of the given columns, regardless of constraint name. - - Works in both online and offline mode by using raw SQL for PostgreSQL and MySQL. - SQLite falls back to batch mode and requires a live connection. - """ - import sqlalchemy as sa - - dialect_name = get_dialect_name(op) - - if dialect_name == "postgresql": - cols_array = ", ".join(f"'{c}'" for c in columns) - op.execute( - text(f""" - DO $$ - DECLARE r record; - BEGIN - FOR r IN - SELECT DISTINCT tc.constraint_name - FROM information_schema.table_constraints tc - JOIN information_schema.key_column_usage kcu - ON tc.constraint_name = kcu.constraint_name - AND tc.table_schema = kcu.table_schema - WHERE tc.table_name = '{table_name}' - AND tc.constraint_type = 'UNIQUE' - AND kcu.column_name = ANY(ARRAY[{cols_array}]::text[]) - LOOP - EXECUTE 'ALTER TABLE ' || quote_ident('{table_name}') || ' DROP CONSTRAINT IF EXISTS ' - || quote_ident(r.constraint_name); - END LOOP; - END $$ - """) - ) - elif dialect_name == "mysql": - cols_in = ", ".join(f"'{c}'" for c in columns) - op.execute( - text(f""" - CREATE PROCEDURE DropUniqueOnColumns() - BEGIN - DECLARE done INT DEFAULT FALSE; - DECLARE v_name VARCHAR(255); - DECLARE cur CURSOR FOR - SELECT DISTINCT kcu.CONSTRAINT_NAME - FROM information_schema.KEY_COLUMN_USAGE kcu - JOIN information_schema.TABLE_CONSTRAINTS tc - ON kcu.CONSTRAINT_NAME = tc.CONSTRAINT_NAME - AND kcu.TABLE_SCHEMA = tc.TABLE_SCHEMA - AND kcu.TABLE_NAME = tc.TABLE_NAME - WHERE kcu.TABLE_NAME = '{table_name}' - AND kcu.TABLE_SCHEMA = DATABASE() - AND tc.CONSTRAINT_TYPE = 'UNIQUE' - AND kcu.COLUMN_NAME IN ({cols_in}); - DECLARE CONTINUE HANDLER FOR NOT FOUND SET done = TRUE; - OPEN cur; - drop_loop: LOOP - FETCH cur INTO v_name; - IF done THEN LEAVE drop_loop; END IF; - SET @stmt = CONCAT('ALTER TABLE `{table_name}` DROP INDEX `', v_name, '`'); - PREPARE s FROM @stmt; - EXECUTE s; - DEALLOCATE PREPARE s; - END LOOP; - CLOSE cur; - END; - CALL DropUniqueOnColumns(); - DROP PROCEDURE DropUniqueOnColumns; - """) - ) - else: - # SQLite — batch mode rewrites the table; requires a live connection - with op.batch_alter_table(table_name, schema=None) as batch_op: - for uq in sa.inspect(op.get_bind()).get_unique_constraints(table_name): - if any(col in uq["column_names"] for col in columns): - batch_op.drop_constraint(uq["name"], type_="unique") - - -def drop_unique_constraint_if_exists(op, table_name, constraint_name) -> None: - """ - Drop a unique constraint by name if it exists. - - Works in both online and offline mode by using raw SQL for PostgreSQL and MySQL. - SQLite falls back to batch mode and requires a live connection. - """ - dialect_name = get_dialect_name(op) - - if dialect_name == "postgresql": - op.execute(text(f'ALTER TABLE "{table_name}" DROP CONSTRAINT IF EXISTS "{constraint_name}"')) - elif dialect_name == "mysql": - op.execute( - text(f""" - CREATE PROCEDURE DropUniqueIfExists() - BEGIN - IF EXISTS ( - SELECT 1 - FROM information_schema.TABLE_CONSTRAINTS - WHERE - CONSTRAINT_SCHEMA = DATABASE() AND - TABLE_NAME = '{table_name}' AND - CONSTRAINT_NAME = '{constraint_name}' AND - CONSTRAINT_TYPE = 'UNIQUE' - ) THEN - ALTER TABLE `{table_name}` DROP INDEX `{constraint_name}`; - ELSE - SELECT 1; - END IF; - END; - CALL DropUniqueIfExists(); - DROP PROCEDURE DropUniqueIfExists; - """) - ) - else: - # SQLite — batch mode rewrites the table; requires a live connection - with op.batch_alter_table(table_name, schema=None) as batch_op: - with contextlib.suppress(ValueError): - batch_op.drop_constraint(constraint_name, type_="unique") diff --git a/airflow-core/src/airflow/migrations/versions/0097_3_2_0_enforce_log_event_and_dag_is_stale_not_null.py b/airflow-core/src/airflow/migrations/versions/0097_3_2_0_enforce_log_event_and_dag_is_stale_not_null.py index 49fb4bcc3f7ac..5af4f7be74d62 100644 --- a/airflow-core/src/airflow/migrations/versions/0097_3_2_0_enforce_log_event_and_dag_is_stale_not_null.py +++ b/airflow-core/src/airflow/migrations/versions/0097_3_2_0_enforce_log_event_and_dag_is_stale_not_null.py @@ -40,13 +40,10 @@ def upgrade(): """Bring existing deployments in line with 0010 and 0067.""" - # Ensure `log.event` can safely transition to NOT NULL. - op.execute("UPDATE log SET event = '' WHERE event IS NULL") - - # Make sure DAG rows that survived the old 0067 path are not NULL. - op.execute("UPDATE dag SET is_stale = false WHERE is_stale IS NULL") - with disable_sqlite_fkeys(op): + op.execute("UPDATE log SET event = '' WHERE event IS NULL") + op.execute("UPDATE dag SET is_stale = false WHERE is_stale IS NULL") + with op.batch_alter_table("log") as batch_op: batch_op.alter_column("event", existing_type=sa.String(60), nullable=False) diff --git a/airflow-core/src/airflow/models/taskinstance.py b/airflow-core/src/airflow/models/taskinstance.py index 964c345a3bf6c..6fb2b38696cf1 100644 --- a/airflow-core/src/airflow/models/taskinstance.py +++ b/airflow-core/src/airflow/models/taskinstance.py @@ -794,6 +794,7 @@ def get_task_instance( select(TaskInstance) .options(lazyload(TaskInstance.dag_run)) # lazy load dag run to avoid locking it .filter_by( + dag_id=dag_id, run_id=run_id, task_id=task_id, map_index=map_index, diff --git a/airflow-core/src/airflow/triggers/base.py b/airflow-core/src/airflow/triggers/base.py index 7ca7ed20a7463..205e830c365aa 100644 --- a/airflow-core/src/airflow/triggers/base.py +++ b/airflow-core/src/airflow/triggers/base.py @@ -109,8 +109,26 @@ def task_instance(self, value: TaskInstance | None) -> None: if self.task_instance: self.task_id = self.task_instance.task_id if self.task: - self.template_fields = self.task.template_fields self.template_ext = self.task.template_ext + # Only keep operator template_fields that are also keys in + # start_trigger_args.trigger_kwargs *and* exist on the trigger. + # Using the full operator template_fields would cause + # AttributeError when the trigger does not have attributes with + # the same names as the operator (e.g. "bash_command"). + # + # When start_trigger_args is None (normal defer path), the triggerer + # does not build a template context, so render_template_fields is + # never called and empty template_fields is safe. + start_trigger_args = getattr(self.task, "start_trigger_args", None) + trigger_kwarg_keys = ( + set((start_trigger_args.trigger_kwargs or {}).keys()) if start_trigger_args else set() + ) + if trigger_kwarg_keys: + self.template_fields = tuple( + f for f in self.task.template_fields if f in trigger_kwarg_keys and hasattr(self, f) + ) + else: + self.template_fields = () def render_template_fields( self, @@ -127,7 +145,8 @@ def render_template_fields( """ if not jinja_env: jinja_env = self.get_template_env() - # We only need to render templated fields if templated fields are part of the start_trigger_args + # self.template_fields is already filtered (in the task_instance setter) to only + # include fields present in start_trigger_args.trigger_kwargs and on this trigger. self._do_render_template_fields(self, self.template_fields, context, jinja_env, set()) @abc.abstractmethod diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts index c37bb8400177f..2af71aa482e73 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/schemas.gen.ts @@ -8220,6 +8220,30 @@ export const $GanttTaskInstance = { } ] }, + scheduled_dttm: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Scheduled Dttm' + }, + queued_dttm: { + anyOf: [ + { + type: 'string', + format: 'date-time' + }, + { + type: 'null' + } + ], + title: 'Queued Dttm' + }, start_date: { anyOf: [ { @@ -8256,7 +8280,7 @@ export const $GanttTaskInstance = { } }, type: 'object', - required: ['task_id', 'task_display_name', 'try_number', 'state', 'start_date', 'end_date'], + required: ['task_id', 'task_display_name', 'try_number', 'state', 'scheduled_dttm', 'queued_dttm', 'start_date', 'end_date'], title: 'GanttTaskInstance', description: 'Task instance data for Gantt chart.' } as const; @@ -9087,13 +9111,20 @@ export const $TeamResponse = { export const $Theme = { properties: { tokens: { - additionalProperties: { - '$ref': '#/components/schemas/ThemeColors' - }, - propertyNames: { - const: 'colors' - }, - type: 'object', + anyOf: [ + { + additionalProperties: { + '$ref': '#/components/schemas/ThemeColors' + }, + propertyNames: { + const: 'colors' + }, + type: 'object' + }, + { + type: 'null' + } + ], title: 'Tokens' }, globalCss: { @@ -9135,14 +9166,96 @@ export const $Theme = { } }, type: 'object', - required: ['tokens'], title: 'Theme', description: "JSON to modify Chakra's theme." } as const; export const $ThemeColors = { - additionalProperties: true, - type: 'object' + properties: { + brand: { + anyOf: [ + { + additionalProperties: { + additionalProperties: { + '$ref': '#/components/schemas/OklchColor' + }, + propertyNames: { + const: 'value' + }, + type: 'object' + }, + propertyNames: { + enum: ['50', '100', '200', '300', '400', '500', '600', '700', '800', '900', '950'] + }, + type: 'object' + }, + { + type: 'null' + } + ], + title: 'Brand' + }, + gray: { + anyOf: [ + { + additionalProperties: { + additionalProperties: { + '$ref': '#/components/schemas/OklchColor' + }, + propertyNames: { + const: 'value' + }, + type: 'object' + }, + propertyNames: { + enum: ['50', '100', '200', '300', '400', '500', '600', '700', '800', '900', '950'] + }, + type: 'object' + }, + { + type: 'null' + } + ], + title: 'Gray' + }, + black: { + anyOf: [ + { + additionalProperties: { + '$ref': '#/components/schemas/OklchColor' + }, + propertyNames: { + const: 'value' + }, + type: 'object' + }, + { + type: 'null' + } + ], + title: 'Black' + }, + white: { + anyOf: [ + { + additionalProperties: { + '$ref': '#/components/schemas/OklchColor' + }, + propertyNames: { + const: 'value' + }, + type: 'object' + }, + { + type: 'null' + } + ], + title: 'White' + } + }, + type: 'object', + title: 'ThemeColors', + description: 'Color tokens for the UI theme. All fields are optional; at least one must be provided.' } as const; export const $TokenType = { diff --git a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts index 9a6c9fb79350a..5420e3e7505cd 100644 --- a/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts +++ b/airflow-core/src/airflow/ui/openapi-gen/requests/types.gen.ts @@ -2015,6 +2015,8 @@ export type GanttTaskInstance = { task_display_name: string; try_number: number; state: TaskInstanceState | null; + scheduled_dttm: string | null; + queued_dttm: string | null; start_date: string | null; end_date: string | null; is_group?: boolean; @@ -2241,9 +2243,9 @@ export type TeamResponse = { * JSON to modify Chakra's theme. */ export type Theme = { - tokens: { - [key: string]: ThemeColors; - }; + tokens?: { + [key: string]: ThemeColors; +} | null; globalCss?: { [key: string]: { [key: string]: unknown; @@ -2253,8 +2255,26 @@ export type Theme = { icon_dark_mode?: string | null; }; +/** + * Color tokens for the UI theme. All fields are optional; at least one must be provided. + */ export type ThemeColors = { - [key: string]: unknown; + brand?: { + [key: string]: { + [key: string]: OklchColor; + }; +} | null; + gray?: { + [key: string]: { + [key: string]: OklchColor; + }; +} | null; + black?: { + [key: string]: OklchColor; +} | null; + white?: { + [key: string]: OklchColor; +} | null; }; /** diff --git a/airflow-core/src/airflow/ui/package.json b/airflow-core/src/airflow/ui/package.json index 750bf47128dd5..f5ee19ce78505 100644 --- a/airflow-core/src/airflow/ui/package.json +++ b/airflow-core/src/airflow/ui/package.json @@ -38,7 +38,7 @@ "@visx/shape": "^3.12.0", "@xyflow/react": "^12.10.1", "anser": "^2.3.5", - "axios": "^1.13.6", + "axios": "^1.15.0", "chakra-react-select": "^6.1.1", "chart.js": "^4.5.1", "chartjs-adapter-dayjs-4": "^1.0.4", diff --git a/airflow-core/src/airflow/ui/pnpm-lock.yaml b/airflow-core/src/airflow/ui/pnpm-lock.yaml index c45e51feb1d6b..261aea68aaa4e 100644 --- a/airflow-core/src/airflow/ui/pnpm-lock.yaml +++ b/airflow-core/src/airflow/ui/pnpm-lock.yaml @@ -69,8 +69,8 @@ importers: specifier: ^2.3.5 version: 2.3.5 axios: - specifier: ^1.13.6 - version: 1.13.6 + specifier: ^1.15.0 + version: 1.15.0 chakra-react-select: specifier: ^6.1.1 version: 6.1.1(@chakra-ui/react@3.34.0(@emotion/react@11.14.0(@types/react@19.2.14)(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(@types/react@19.2.14)(next-themes@0.4.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4))(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -1916,8 +1916,8 @@ packages: resolution: {integrity: sha512-Xm7bpRXnDSX2YE2YFfBk2FnF0ep6tmG7xPh8iHee8MIcrgq762Nkce856dYtJYLkuIoYZvGfTs/PbZhideTcEg==} engines: {node: '>=4'} - axios@1.13.6: - resolution: {integrity: sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==} + axios@1.15.0: + resolution: {integrity: sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==} axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} @@ -3180,8 +3180,8 @@ packages: lru-cache@10.4.3: resolution: {integrity: sha512-JNAzZcXrCt42VGLuYz0zfAzDfAvJWW6AfYlDBQyDV5DClI2m5sAmK+OIO7s59XfsRsWHp02jAJrRadPRGTt6SQ==} - lru-cache@11.3.1: - resolution: {integrity: sha512-Y71HWT4hydF1IAG/2OPync4dgQ/J2iWye7eg6CuzJHI+E97tvqFPlADzxiNnjH6WSljg8ecfXMr9k6bfFuqA5w==} + lru-cache@11.3.3: + resolution: {integrity: sha512-JvNw9Y81y33E+BEYPr0U7omo+U9AySnsMsEiXgwT6yqd31VQWTLNQqmT4ou5eqPFUrTfIDFta2wKhB1hyohtAQ==} engines: {node: 20 || >=22} lru-cache@5.1.1: @@ -3652,8 +3652,9 @@ packages: proxy-compare@3.0.1: resolution: {integrity: sha512-V9plBAt3qjMlS1+nC8771KNf6oJ12gExvaxnNzN/9yVRLdTv/lc+oJlnSzrdYDAvBfTStPCoiaCOTmTs0adv7Q==} - proxy-from-env@1.1.0: - resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} proxy-memoize@3.0.1: resolution: {integrity: sha512-VDdG/VYtOgdGkWJx7y0o7p+zArSf2383Isci8C+BP3YXgMYDoPd3cCBjw0JdWb6YBb9sFiOPbAADDVTPJnh+9g==} @@ -6628,11 +6629,11 @@ snapshots: axe-core@4.10.3: {} - axios@1.13.6: + axios@1.15.0: dependencies: follow-redirects: 1.15.11 form-data: 4.0.5 - proxy-from-env: 1.1.0 + proxy-from-env: 2.1.0 transitivePeerDependencies: - debug @@ -8032,7 +8033,7 @@ snapshots: lru-cache@10.4.3: {} - lru-cache@11.3.1: {} + lru-cache@11.3.3: {} lru-cache@5.1.1: dependencies: @@ -8656,7 +8657,7 @@ snapshots: path-scurry@2.0.2: dependencies: - lru-cache: 11.3.1 + lru-cache: 11.3.3 minipass: 7.1.3 path-to-regexp@6.3.0: {} @@ -8731,7 +8732,7 @@ snapshots: proxy-compare@3.0.1: {} - proxy-from-env@1.1.0: {} + proxy-from-env@2.1.0: {} proxy-memoize@3.0.1: dependencies: diff --git a/airflow-core/src/airflow/ui/src/components/SearchBar.test.tsx b/airflow-core/src/airflow/ui/src/components/SearchBar.test.tsx index 0b149d37db63e..2d5be3990bd35 100644 --- a/airflow-core/src/airflow/ui/src/components/SearchBar.test.tsx +++ b/airflow-core/src/airflow/ui/src/components/SearchBar.test.tsx @@ -16,16 +16,22 @@ * specific language governing permissions and limitations * under the License. */ -import { fireEvent, render, screen, waitFor } from "@testing-library/react"; -import { describe, it, expect, vi } from "vitest"; +import { act, fireEvent, render, screen, waitFor } from "@testing-library/react"; +import { afterEach, describe, expect, it, vi } from "vitest"; import { Wrapper } from "src/utils/Wrapper"; import { SearchBar } from "./SearchBar"; +afterEach(() => { + vi.useRealTimers(); +}); + describe("Test SearchBar", () => { it("Renders and clear button works", async () => { - render(, { + const onChange = vi.fn(); + + render(, { wrapper: Wrapper, }); @@ -44,5 +50,35 @@ describe("Test SearchBar", () => { fireEvent.click(clearButton); expect((input as HTMLInputElement).value).toBe(""); + expect(onChange).toHaveBeenCalledWith(""); + }); + + it("cancels pending debounced changes when clearing", () => { + vi.useFakeTimers(); + + const onChange = vi.fn(); + + render(, { + wrapper: Wrapper, + }); + + const input = screen.getByTestId("search-dags"); + + fireEvent.change(input, { target: { value: "air" } }); + + expect((input as HTMLInputElement).value).toBe("air"); + expect(onChange).not.toHaveBeenCalled(); + + fireEvent.click(screen.getByTestId("clear-search")); + + expect((input as HTMLInputElement).value).toBe(""); + expect(onChange).toHaveBeenCalledTimes(1); + expect(onChange).toHaveBeenNthCalledWith(1, ""); + + act(() => { + vi.advanceTimersByTime(200); + }); + + expect(onChange).toHaveBeenCalledTimes(1); }); }); diff --git a/airflow-core/src/airflow/ui/src/components/SearchBar.tsx b/airflow-core/src/airflow/ui/src/components/SearchBar.tsx index 21bfca4bf50ae..91825004d45ed 100644 --- a/airflow-core/src/airflow/ui/src/components/SearchBar.tsx +++ b/airflow-core/src/airflow/ui/src/components/SearchBar.tsx @@ -50,6 +50,11 @@ export const SearchBar = ({ setValue(event.target.value); handleSearchChange(event.target.value); }; + const clearSearch = () => { + handleSearchChange.cancel(); + setValue(""); + onChange(""); + }; useHotkeys( "mod+k", @@ -70,10 +75,7 @@ export const SearchBar = ({ aria-label={translate("search.clear")} colorPalette="brand" data-testid="clear-search" - onClick={() => { - setValue(""); - onChange(""); - }} + onClick={clearSearch} size="xs" /> ) : undefined} diff --git a/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.test.ts b/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.test.ts new file mode 100644 index 0000000000000..a0e61cc857922 --- /dev/null +++ b/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.test.ts @@ -0,0 +1,350 @@ +/* eslint-disable max-lines */ + +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import type { ChartEvent, ActiveElement } from "chart.js"; +import dayjs from "dayjs"; +import type { TFunction } from "i18next"; +import { describe, it, expect } from "vitest"; + +import type { GanttDataItem } from "./utils"; +import { createChartOptions, transformGanttData } from "./utils"; + +// eslint-disable-next-line no-empty-function, @typescript-eslint/no-empty-function +const noop = () => {}; + +const defaultChartParams = { + gridColor: "#ccc", + handleBarClick: noop as (event: ChartEvent, elements: Array) => void, + handleBarHover: noop as (event: ChartEvent, elements: Array) => void, + hoveredId: undefined, + hoveredItemColor: "#eee", + labels: ["task_1", "task_2"], + selectedId: undefined, + selectedItemColor: "#ddd", + selectedTimezone: "UTC", + translate: ((key: string) => key) as unknown as TFunction, +}; + +describe("createChartOptions", () => { + describe("x-axis scale min/max with ISO date strings", () => { + it("should compute valid min/max for completed tasks with ISO dates", () => { + const data: Array = [ + { + state: "success", + taskId: "task_1", + x: ["2024-03-14T10:00:00.000Z", "2024-03-14T10:05:00.000Z"], + y: "task_1", + }, + { + state: "success", + taskId: "task_2", + x: ["2024-03-14T10:03:00.000Z", "2024-03-14T10:10:00.000Z"], + y: "task_2", + }, + ]; + + const options = createChartOptions({ + ...defaultChartParams, + data, + selectedRun: { + dag_id: "test_dag", + duration: 600, + end_date: "2024-03-14T10:10:00+00:00", + has_missed_deadline: false, + queued_at: "2024-03-14T09:59:00+00:00", + run_after: "2024-03-14T10:00:00+00:00", + run_id: "run_1", + run_type: "manual", + start_date: "2024-03-14T10:00:00+00:00", + state: "success", + }, + }); + + const xScale = options.scales.x; + + expect(xScale.min).toBeTypeOf("number"); + expect(xScale.max).toBeTypeOf("number"); + expect(Number.isNaN(xScale.min)).toBe(false); + expect(Number.isNaN(xScale.max)).toBe(false); + // max should be slightly beyond the latest end date (5% padding) + expect(xScale.max).toBeGreaterThan(new Date("2024-03-14T10:10:00.000Z").getTime()); + }); + + it("should compute valid min/max for running tasks", () => { + const now = dayjs().toISOString(); + const data: Array = [ + { + state: "success", + taskId: "task_1", + x: ["2024-03-14T10:00:00.000Z", "2024-03-14T10:05:00.000Z"], + y: "task_1", + }, + { + state: "running", + taskId: "task_2", + x: ["2024-03-14T10:05:00.000Z", now], + y: "task_2", + }, + ]; + + const options = createChartOptions({ + ...defaultChartParams, + data, + selectedRun: { + dag_id: "test_dag", + duration: 0, + // eslint-disable-next-line unicorn/no-null + end_date: null, + has_missed_deadline: false, + queued_at: "2024-03-14T09:59:00+00:00", + run_after: "2024-03-14T10:00:00+00:00", + run_id: "run_1", + run_type: "manual", + start_date: "2024-03-14T10:00:00+00:00", + state: "running", + }, + }); + + const xScale = options.scales.x; + + expect(xScale.min).toBeTypeOf("number"); + expect(xScale.max).toBeTypeOf("number"); + expect(Number.isNaN(xScale.min)).toBe(false); + expect(Number.isNaN(xScale.max)).toBe(false); + }); + + it("should handle empty data with running DagRun (fallback to formatted dates)", () => { + const options = createChartOptions({ + ...defaultChartParams, + data: [], + labels: [], + selectedRun: { + dag_id: "test_dag", + duration: 0, + // eslint-disable-next-line unicorn/no-null + end_date: null, + has_missed_deadline: false, + queued_at: "2024-03-14T09:59:00+00:00", + run_after: "2024-03-14T10:00:00+00:00", + run_id: "run_1", + run_type: "manual", + start_date: "2024-03-14T10:00:00+00:00", + state: "running", + }, + }); + + const xScale = options.scales.x; + + // With empty data, min/max are formatted date strings (fallback branch) + expect(xScale.min).toBeTypeOf("string"); + expect(xScale.max).toBeTypeOf("string"); + }); + }); +}); + +describe("transformGanttData", () => { + it("should skip tasks with null start_date", () => { + const result = transformGanttData({ + allTries: [ + { + // eslint-disable-next-line unicorn/no-null + end_date: null, + is_mapped: false, + // eslint-disable-next-line unicorn/no-null + queued_dttm: null, + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, + // eslint-disable-next-line unicorn/no-null + start_date: null, + // eslint-disable-next-line unicorn/no-null + state: null, + task_display_name: "task_1", + task_id: "task_1", + try_number: 1, + }, + ], + flatNodes: [{ depth: 0, id: "task_1", is_mapped: false, label: "task_1" }], + gridSummaries: [], + }); + + expect(result).toHaveLength(0); + }); + + it("should include running tasks with valid start_date and use current time as end", () => { + const before = dayjs(); + const result = transformGanttData({ + allTries: [ + { + // eslint-disable-next-line unicorn/no-null + end_date: null, + is_mapped: false, + // eslint-disable-next-line unicorn/no-null + queued_dttm: null, + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, + start_date: "2024-03-14T10:00:00+00:00", + state: "running", + task_display_name: "task_1", + task_id: "task_1", + try_number: 1, + }, + ], + flatNodes: [{ depth: 0, id: "task_1", is_mapped: false, label: "task_1" }], + gridSummaries: [], + }); + + expect(result).toHaveLength(1); + expect(result[0]?.state).toBe("running"); + // End time should be approximately now (ISO string) + const endTime = dayjs(result[0]?.x[1]); + + expect(endTime.valueOf()).toBeGreaterThanOrEqual(before.valueOf()); + }); + + it("should skip groups with null min_start_date or max_end_date", () => { + const result = transformGanttData({ + allTries: [], + flatNodes: [{ depth: 0, id: "group_1", is_mapped: false, isGroup: true, label: "group_1" }], + gridSummaries: [ + { + // eslint-disable-next-line unicorn/no-null + child_states: null, + // eslint-disable-next-line unicorn/no-null + max_end_date: null, + // eslint-disable-next-line unicorn/no-null + min_start_date: null, + // eslint-disable-next-line unicorn/no-null + state: null, + task_display_name: "group_1", + task_id: "group_1", + }, + ], + }); + + expect(result).toHaveLength(0); + }); + + it("should produce ISO date strings parseable by dayjs", () => { + const result = transformGanttData({ + allTries: [ + { + end_date: "2024-03-14T10:05:00+00:00", + is_mapped: false, + // eslint-disable-next-line unicorn/no-null + queued_dttm: null, + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, + start_date: "2024-03-14T10:00:00+00:00", + state: "success", + task_display_name: "task_1", + task_id: "task_1", + try_number: 1, + }, + ], + flatNodes: [{ depth: 0, id: "task_1", is_mapped: false, label: "task_1" }], + gridSummaries: [], + }); + + expect(result).toHaveLength(1); + // x values should be valid ISO strings that dayjs can parse without NaN + const start = dayjs(result[0]?.x[0]); + const end = dayjs(result[0]?.x[1]); + + expect(start.isValid()).toBe(true); + expect(end.isValid()).toBe(true); + expect(Number.isNaN(start.valueOf())).toBe(false); + expect(Number.isNaN(end.valueOf())).toBe(false); + }); + + it("should produce 3 segments when scheduled_dttm and queued_dttm are present", () => { + const result = transformGanttData({ + allTries: [ + { + end_date: "2024-03-14T10:05:00+00:00", + is_mapped: false, + queued_dttm: "2024-03-14T09:59:00+00:00", + scheduled_dttm: "2024-03-14T09:58:00+00:00", + start_date: "2024-03-14T10:00:00+00:00", + state: "success", + task_display_name: "task_1", + task_id: "task_1", + try_number: 1, + }, + ], + flatNodes: [{ depth: 0, id: "task_1", is_mapped: false, label: "task_1" }], + gridSummaries: [], + }); + + expect(result).toHaveLength(3); + expect(result[0]?.state).toBe("scheduled"); + expect(result[1]?.state).toBe("queued"); + expect(result[2]?.state).toBe("success"); + }); + + it("should produce 2 segments when only queued_dttm is present", () => { + const result = transformGanttData({ + allTries: [ + { + end_date: "2024-03-14T10:05:00+00:00", + is_mapped: false, + queued_dttm: "2024-03-14T09:59:00+00:00", + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, + start_date: "2024-03-14T10:00:00+00:00", + state: "success", + task_display_name: "task_1", + task_id: "task_1", + try_number: 1, + }, + ], + flatNodes: [{ depth: 0, id: "task_1", is_mapped: false, label: "task_1" }], + gridSummaries: [], + }); + + expect(result).toHaveLength(2); + expect(result[0]?.state).toBe("queued"); + expect(result[1]?.state).toBe("success"); + }); + + it("should produce 1 segment when scheduled_dttm and queued_dttm are null", () => { + const result = transformGanttData({ + allTries: [ + { + end_date: "2024-03-14T10:05:00+00:00", + is_mapped: false, + // eslint-disable-next-line unicorn/no-null + queued_dttm: null, + // eslint-disable-next-line unicorn/no-null + scheduled_dttm: null, + start_date: "2024-03-14T10:00:00+00:00", + state: "success", + task_display_name: "task_1", + task_id: "task_1", + try_number: 1, + }, + ], + flatNodes: [{ depth: 0, id: "task_1", is_mapped: false, label: "task_1" }], + gridSummaries: [], + }); + + expect(result).toHaveLength(1); + expect(result[0]?.state).toBe("success"); + }); +}); diff --git a/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.ts b/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.ts index 22df4eb28cffc..621f22e8ab923 100644 --- a/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.ts +++ b/airflow-core/src/airflow/ui/src/layouts/Details/Gantt/utils.ts @@ -122,14 +122,47 @@ export const transformGanttData = ({ if (tries && tries.length > 0) { return tries .filter((tryInstance) => tryInstance.start_date !== null) - .map((tryInstance) => { + .flatMap((tryInstance) => { const hasTaskRunning = isStatePending(tryInstance.state); const endTime = hasTaskRunning || tryInstance.end_date === null ? dayjs().toISOString() : tryInstance.end_date; - - return { + const items: Array = []; + + // Scheduled segment: from scheduled_dttm to queued_dttm (or start_date if no queued_dttm) + if (tryInstance.scheduled_dttm !== null) { + const scheduledEnd = tryInstance.queued_dttm ?? tryInstance.start_date ?? undefined; + + items.push({ + isGroup: false, + isMapped: tryInstance.is_mapped, + state: "scheduled" as TaskInstanceState, + taskId: tryInstance.task_id, + tryNumber: tryInstance.try_number, + x: [dayjs(tryInstance.scheduled_dttm).toISOString(), dayjs(scheduledEnd).toISOString()], + y: tryInstance.task_display_name, + }); + } + + // Queue segment: from queued_dttm to start_date + if (tryInstance.queued_dttm !== null) { + items.push({ + isGroup: false, + isMapped: tryInstance.is_mapped, + state: "queued" as TaskInstanceState, + taskId: tryInstance.task_id, + tryNumber: tryInstance.try_number, + x: [ + dayjs(tryInstance.queued_dttm).toISOString(), + dayjs(tryInstance.start_date ?? undefined).toISOString(), + ], + y: tryInstance.task_display_name, + }); + } + + // Execution segment: from start_date to end_date + items.push({ isGroup: false, isMapped: tryInstance.is_mapped, state: tryInstance.state, @@ -137,7 +170,9 @@ export const transformGanttData = ({ tryNumber: tryInstance.try_number, x: [dayjs(tryInstance.start_date).toISOString(), dayjs(endTime).toISOString()], y: tryInstance.task_display_name, - }; + }); + + return items; }); } } @@ -259,6 +294,11 @@ export const createChartOptions = ({ duration: 150, easing: "linear" as const, }, + datasets: { + bar: { + minBarLength: 4, + }, + }, indexAxis: "y" as const, maintainAspectRatio: false, onClick: handleBarClick, @@ -331,7 +371,7 @@ export const createChartOptions = ({ label(tooltipItem: TooltipItem<"bar">) { const taskInstance = data[tooltipItem.dataIndex]; - return `${translate("state")}: ${translate(`states.${taskInstance?.state}`)}`; + return `${translate("state")}: ${translate(`common:states.${taskInstance?.state}`)}`; }, }, }, @@ -347,8 +387,8 @@ export const createChartOptions = ({ max: data.length > 0 ? (() => { - const maxTime = Math.max(...data.map((item) => new Date(item.x[1] ?? "").getTime())); - const minTime = Math.min(...data.map((item) => new Date(item.x[0] ?? "").getTime())); + const maxTime = Math.max(...data.map((item) => dayjs(item.x[1]).valueOf())); + const minTime = Math.min(...data.map((item) => dayjs(item.x[0]).valueOf())); const totalDuration = maxTime - minTime; // add 5% to the max time to avoid the last tick being cut off @@ -358,8 +398,8 @@ export const createChartOptions = ({ min: data.length > 0 ? (() => { - const maxTime = Math.max(...data.map((item) => new Date(item.x[1] ?? "").getTime())); - const minTime = Math.min(...data.map((item) => new Date(item.x[0] ?? "").getTime())); + const maxTime = Math.max(...data.map((item) => dayjs(item.x[1]).valueOf())); + const minTime = Math.min(...data.map((item) => dayjs(item.x[0]).valueOf())); const totalDuration = maxTime - minTime; // subtract 2% from min time so background color shows before data diff --git a/airflow-core/src/airflow/ui/src/layouts/Details/Grid/TaskNames.tsx b/airflow-core/src/airflow/ui/src/layouts/Details/Grid/TaskNames.tsx index bd76a6663bebf..510d8be714321 100644 --- a/airflow-core/src/airflow/ui/src/layouts/Details/Grid/TaskNames.tsx +++ b/airflow-core/src/airflow/ui/src/layouts/Details/Grid/TaskNames.tsx @@ -67,6 +67,22 @@ export const TaskNames = ({ nodes, onRowClick, virtualItems }: Props) => { } }; + const onClick = (event: MouseEvent) => { + const groupNodeId = event.currentTarget.dataset.groupId; + + if (groupNodeId === undefined || groupNodeId === "") { + return; + } + + const id = groupNodeId; + const isViewingSameGroup = typeof groupId === "string" && groupId === id; + + if (isViewingSameGroup) { + toggleGroupId(id); + } + onRowClick?.(); + }; + const search = searchParams.toString(); // If virtualItems is provided, use virtualization; otherwise render all items @@ -109,7 +125,8 @@ export const TaskNames = ({ nodes, onRowClick, virtualItems }: Props) => { {node.isGroup ? ( { const link = authLinks?.extra_menu_items.find((mi) => mi.text.toLowerCase().replace(" ", "-") === page); const navigate = useNavigate(); + // Track when we are already redirecting so that setting iframe.src = "about:blank" + // (which fires another onLoad event) does not trigger a second navigate call. + const isRedirecting = useRef(false); const onLoad = () => { + if (isRedirecting.current) { + return; + } + const iframe: HTMLIFrameElement | null = document.querySelector("#security-iframe"); if (iframe?.contentWindow) { const base = new URL(document.baseURI).pathname.replace(/\/$/u, ""); // Remove trailing slash if exists if (!iframe.contentWindow.location.pathname.startsWith(`${base}/auth/`)) { + // Clear the iframe immediately so that the React app does not render its own + // navigation sidebar inside the iframe, which would produce a duplicate nav bar. + isRedirecting.current = true; + iframe.src = "about:blank"; void navigate("/"); } } diff --git a/airflow-core/src/airflow/ui/src/pages/TaskInstance/ExtraLinks.test.tsx b/airflow-core/src/airflow/ui/src/pages/TaskInstance/ExtraLinks.test.tsx new file mode 100644 index 0000000000000..75c65e8abcc6d --- /dev/null +++ b/airflow-core/src/airflow/ui/src/pages/TaskInstance/ExtraLinks.test.tsx @@ -0,0 +1,124 @@ +/*! + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +import "@testing-library/jest-dom"; +import { render, screen } from "@testing-library/react"; +import { useParams } from "react-router-dom"; +import { describe, it, expect, vi, beforeEach, afterEach } from "vitest"; + +import * as queries from "openapi/queries"; +import { Wrapper } from "src/utils/Wrapper"; + +import { ExtraLinks } from "./ExtraLinks"; + +vi.mock("openapi/queries"); +vi.mock("react-router-dom", async () => { + const actual = await vi.importActual("react-router-dom"); + + return { + ...actual, + useParams: vi.fn(), + }; +}); + +describe("ExtraLinks Component", () => { + beforeEach(() => { + vi.clearAllMocks(); + vi.mocked(useParams).mockReturnValue({ + dagId: "test-dag", + mapIndex: "-1", + runId: "test-run", + taskId: "test-task", + }); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + }); + + it("renders internal links with target='_self'", () => { + vi.mocked(queries.useTaskInstanceServiceGetExtraLinks).mockReturnValue({ + data: { + extra_links: { + "Internal Link": "/dags/test/runs/run1", + "Same Origin": "http://localhost:3000/some-path", + }, + }, + } as unknown as ReturnType); + + // Mock window.location.origin + vi.stubGlobal("location", { origin: "http://localhost:3000" }); + + render( + + + , + ); + + const internalLink = screen.getByText("Internal Link"); + + expect(internalLink.closest("a")).toHaveAttribute("target", "_self"); + + const sameOriginLink = screen.getByText("Same Origin"); + + expect(sameOriginLink.closest("a")).toHaveAttribute("target", "_self"); + }); + + it("renders external links with target='_blank'", () => { + vi.mocked(queries.useTaskInstanceServiceGetExtraLinks).mockReturnValue({ + data: { + extra_links: { + "External Link": "https://www.google.com", + }, + }, + } as unknown as ReturnType); + + // Mock window.location.origin + vi.stubGlobal("location", { origin: "http://localhost:3000" }); + + render( + + + , + ); + + const externalLink = screen.getByText("External Link"); + + expect(externalLink.closest("a")).toHaveAttribute("target", "_blank"); + }); + + it("filters out null urls", () => { + vi.mocked(queries.useTaskInstanceServiceGetExtraLinks).mockReturnValue({ + data: { + extra_links: { + Invalid: null, + Valid: "http://localhost:3000/valid", + }, + }, + } as unknown as ReturnType); + + render( + + + , + ); + + expect(screen.getByText("Valid")).toBeInTheDocument(); + expect(screen.queryByText("Invalid")).not.toBeInTheDocument(); + }); +}); diff --git a/airflow-core/src/airflow/ui/src/pages/TaskInstance/ExtraLinks.tsx b/airflow-core/src/airflow/ui/src/pages/TaskInstance/ExtraLinks.tsx index 2ff7876dfc0fa..da4ab0342d727 100644 --- a/airflow-core/src/airflow/ui/src/pages/TaskInstance/ExtraLinks.tsx +++ b/airflow-core/src/airflow/ui/src/pages/TaskInstance/ExtraLinks.tsx @@ -26,6 +26,16 @@ type ExtraLinksProps = { readonly refetchInterval: number | false; }; +const getTarget = (url: string) => { + try { + return new URL(url, globalThis.location.origin).origin === globalThis.location.origin + ? "_self" + : "_blank"; + } catch { + return "_blank"; + } +}; + export const ExtraLinks = ({ refetchInterval }: ExtraLinksProps) => { const { t: translate } = useTranslation("dag"); const { dagId = "", mapIndex = "-1", runId = "", taskId = "" } = useParams(); @@ -47,15 +57,21 @@ export const ExtraLinks = ({ refetchInterval }: ExtraLinksProps) => { {translate("extraLinks")} - {Object.entries(data.extra_links).map(([key, value], _) => - value === null ? undefined : ( + {Object.entries(data.extra_links).map(([key, url]) => { + if (url === null) { + return undefined; + } + + const target = getTarget(url); + + return ( - ), - )} + ); + })} ) : undefined; diff --git a/airflow-core/src/airflow/ui/src/queries/useBulkDeleteConnections.tsx b/airflow-core/src/airflow/ui/src/queries/useBulkDeleteConnections.tsx index d56f8e0fa0be5..bfd4359a08745 100644 --- a/airflow-core/src/airflow/ui/src/queries/useBulkDeleteConnections.tsx +++ b/airflow-core/src/airflow/ui/src/queries/useBulkDeleteConnections.tsx @@ -54,7 +54,9 @@ export const useBulkDeleteConnections = ({ clearSelections, onSuccessConfirm }: keys: success.join(", "), resourceName: translate("admin:connections.connection_other"), }), - title: translate("toaster.bulkDelete.success.title"), + title: translate("toaster.bulkDelete.success.title", { + resourceName: translate("admin:connections.connection_other"), + }), type: "success", }); clearSelections(); diff --git a/airflow-core/src/airflow/ui/src/theme.ts b/airflow-core/src/airflow/ui/src/theme.ts index fc9c07a99b892..15b34dda2c9dc 100644 --- a/airflow-core/src/airflow/ui/src/theme.ts +++ b/airflow-core/src/airflow/ui/src/theme.ts @@ -406,16 +406,20 @@ const defaultAirflowTheme = { export const createTheme = (userTheme?: Theme) => { const defaultAirflowConfig = defineConfig({ theme: defaultAirflowTheme }); - const userConfig = defineConfig( - userTheme - ? { - theme: { tokens: userTheme.tokens }, + const userConfig = userTheme + ? defineConfig({ + ...(userTheme.tokens !== undefined && { + theme: { tokens: userTheme.tokens as Record }, + }), + ...(userTheme.globalCss !== undefined && { globalCss: userTheme.globalCss as Record, - } - : {}, - ); + }), + }) + : undefined; - const mergedConfig = mergeConfigs(defaultConfig, defaultAirflowConfig, userConfig); + const mergedConfig = userConfig + ? mergeConfigs(defaultConfig, defaultAirflowConfig, userConfig) + : mergeConfigs(defaultConfig, defaultAirflowConfig); return createSystem(mergedConfig); }; diff --git a/airflow-core/src/airflow/ui/tests/e2e/pages/DagCalendarTab.ts b/airflow-core/src/airflow/ui/tests/e2e/pages/DagCalendarTab.ts index 7aee20db7c083..79e00c2226637 100644 --- a/airflow-core/src/airflow/ui/tests/e2e/pages/DagCalendarTab.ts +++ b/airflow-core/src/airflow/ui/tests/e2e/pages/DagCalendarTab.ts @@ -50,9 +50,35 @@ export class DagCalendarTab extends BasePage { for (let i = 0; i < count; i++) { const cell = this.activeCells.nth(i); - const bg = await cell.evaluate((el) => window.getComputedStyle(el).backgroundColor); + const computedColor = await cell.evaluate((el: Element) => { + const getRenderableColor = (element: Element): string => { + const color = window.getComputedStyle(element).backgroundColor; - colors.push(bg); + return color && color !== "rgba(0, 0, 0, 0)" && color !== "transparent" ? color : ""; + }; + + const cellColor = getRenderableColor(el); + + if (cellColor) { + return cellColor; + } + + const children = [...el.querySelectorAll("*")]; + + for (const child of children) { + const childColor = getRenderableColor(child); + + if (childColor) { + return childColor; + } + } + + return ""; + }); + + if (computedColor) { + colors.push(computedColor); + } } return colors; @@ -88,7 +114,7 @@ export class DagCalendarTab extends BasePage { public async navigateToCalendar(dagId: string) { await expect(async () => { await this.safeGoto(`/dags/${dagId}/calendar`); - await this.page.getByTestId("dag-calendar-root").waitFor({ state: "visible", timeout: 5000 }); + await expect(this.page.getByTestId("dag-calendar-root")).toBeVisible({ timeout: 5000 }); }).toPass({ intervals: [2000], timeout: 60_000 }); await this.waitForCalendarReady(); } @@ -100,7 +126,7 @@ export class DagCalendarTab extends BasePage { public async switchToHourly() { await this.hourlyToggle.click(); - await this.page.getByTestId("calendar-hourly-view").waitFor({ state: "visible", timeout: 30_000 }); + await expect(this.page.getByTestId("calendar-hourly-view")).toBeVisible({ timeout: 30_000 }); } public async switchToTotalView() { @@ -112,22 +138,16 @@ export class DagCalendarTab extends BasePage { } private async waitForCalendarReady(): Promise { - await this.page.getByTestId("dag-calendar-root").waitFor({ state: "visible", timeout: 120_000 }); + await expect(this.page.getByTestId("dag-calendar-root")).toBeVisible({ timeout: 120_000 }); - await this.page.getByTestId("calendar-current-period").waitFor({ state: "visible", timeout: 120_000 }); + await expect(this.page.getByTestId("calendar-current-period")).toBeVisible({ timeout: 120_000 }); + await expect(this.page.getByTestId("calendar-grid")).toBeVisible({ timeout: 120_000 }); const overlay = this.page.getByTestId("calendar-loading-overlay"); - if (await overlay.isVisible().catch(() => false)) { - await overlay.waitFor({ state: "hidden", timeout: 120_000 }); - } - - await this.page.getByTestId("calendar-grid").waitFor({ state: "visible", timeout: 120_000 }); - - await this.page.waitForFunction(() => { - const cells = document.querySelectorAll('[data-testid="calendar-cell"]'); + await expect(overlay).toBeHidden({ timeout: 120_000 }); + const cells = this.page.getByTestId("calendar-cell"); - return cells.length > 0; - }); + await expect(cells.first()).toBeVisible({ timeout: 120_000 }); } } diff --git a/airflow-core/tests/system/example_empty.py b/airflow-core/tests/system/example_empty.py index a452cd6382376..1e226db6b0b5b 100644 --- a/airflow-core/tests/system/example_empty.py +++ b/airflow-core/tests/system/example_empty.py @@ -44,5 +44,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/airflow-core/tests/unit/api_fastapi/common/test_types.py b/airflow-core/tests/unit/api_fastapi/common/test_types.py index 3476f6a529394..da17ca505cc76 100644 --- a/airflow-core/tests/unit/api_fastapi/common/test_types.py +++ b/airflow-core/tests/unit/api_fastapi/common/test_types.py @@ -147,7 +147,7 @@ def test_invalid_shade_key_rejected(self): def test_serialization_excludes_none_fields(self): colors = ThemeColors.model_validate({"brand": _BRAND_SCALE}) - dumped = colors.model_dump() + dumped = colors.model_dump(exclude_none=True) assert "brand" in dumped assert "gray" not in dumped assert "black" not in dumped @@ -200,10 +200,37 @@ def test_empty_colors_rejected(self): def test_serialization_round_trip(self): """Verify None color fields are excluded and OklchColor values are serialized as strings.""" theme = Theme.model_validate({"tokens": {"colors": {"brand": _BRAND_SCALE}}}) - dumped = theme.model_dump() + dumped = theme.model_dump(exclude_none=True) colors = dumped["tokens"]["colors"] assert "brand" in colors assert "gray" not in colors assert "black" not in colors assert "white" not in colors assert colors["brand"]["50"]["value"] == "oklch(0.975 0.007 298.0)" + + def test_globalcss_only_theme(self): + """tokens is optional; globalCss alone is sufficient.""" + theme = Theme.model_validate({"globalCss": {"button": {"text-transform": "uppercase"}}}) + assert theme.tokens is None + assert theme.globalCss == {"button": {"text-transform": "uppercase"}} + + def test_icon_only_theme(self): + """tokens is optional; an icon URL alone is sufficient.""" + theme = Theme.model_validate({"icon": "https://example.com/logo.svg"}) + assert theme.tokens is None + assert theme.icon == "https://example.com/logo.svg" + + def test_empty_theme(self): + """An empty theme object is valid — it means 'use OSS defaults'.""" + theme = Theme.model_validate({}) + assert theme.tokens is None + assert theme.globalCss is None + assert theme.icon is None + assert theme.icon_dark_mode is None + + def test_theme_serialization_excludes_none_tokens(self): + """When tokens is None it must not appear in the serialized output.""" + theme = Theme.model_validate({"globalCss": {"a": {"color": "red"}}}) + dumped = theme.model_dump(exclude_none=True) + assert "tokens" not in dumped + assert dumped == {"globalCss": {"a": {"color": "red"}}} diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_config.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_config.py index dbc3c0eb64937..8b9982fc47b91 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_config.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_config.py @@ -136,6 +136,19 @@ def mock_config_data_all_colors(): yield +THEME_CSS_ONLY = { + "globalCss": { + "button": {"text-transform": "uppercase"}, + } +} + + +@pytest.fixture +def mock_config_data_css_only(): + with conf_vars(_theme_conf_vars(THEME_CSS_ONLY)): + yield + + class TestGetConfig: def test_should_response_200(self, mock_config_data, test_client): """ @@ -170,3 +183,14 @@ def test_should_response_200_with_all_color_tokens(self, mock_config_data_all_co assert "white" in colors assert colors["black"] == {"value": "oklch(0.22 0.025 288.6)"} assert colors["white"] == {"value": "oklch(0.985 0.002 264.0)"} + + def test_should_response_200_with_css_only_theme(self, mock_config_data_css_only, test_client): + """Theme with only globalCss (no tokens) is valid and round-trips correctly.""" + response = test_client.get("/config") + + assert response.status_code == 200 + theme = response.json()["theme"] + assert "tokens" not in theme + assert theme["globalCss"] == {"button": {"text-transform": "uppercase"}} + assert "icon" not in theme + assert "icon_dark_mode" not in theme diff --git a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_gantt.py b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_gantt.py index 162c82682afcf..0e2be9e277cae 100644 --- a/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_gantt.py +++ b/airflow-core/tests/unit/api_fastapi/core_api/routes/ui/test_gantt.py @@ -51,6 +51,8 @@ "task_display_name": TASK_DISPLAY_NAME, "try_number": 1, "state": "success", + "scheduled_dttm": "2024-11-30T09:50:00Z", + "queued_dttm": "2024-11-30T09:55:00Z", "start_date": "2024-11-30T10:00:00Z", "end_date": "2024-11-30T10:05:00Z", "is_group": False, @@ -62,6 +64,8 @@ "task_display_name": TASK_DISPLAY_NAME_2, "try_number": 1, "state": "failed", + "scheduled_dttm": "2024-11-30T10:02:00Z", + "queued_dttm": "2024-11-30T10:03:00Z", "start_date": "2024-11-30T10:05:00Z", "end_date": "2024-11-30T10:10:00Z", "is_group": False, @@ -73,6 +77,8 @@ "task_display_name": TASK_DISPLAY_NAME_3, "try_number": 1, "state": "running", + "scheduled_dttm": None, + "queued_dttm": None, "start_date": "2024-11-30T10:10:00Z", "end_date": None, "is_group": False, @@ -116,16 +122,22 @@ def setup(dag_maker, session=None): if ti.task_id == TASK_ID: ti.state = TaskInstanceState.SUCCESS ti.try_number = 1 + ti.scheduled_dttm = pendulum.DateTime(2024, 11, 30, 9, 50, 0, tzinfo=pendulum.UTC) + ti.queued_dttm = pendulum.DateTime(2024, 11, 30, 9, 55, 0, tzinfo=pendulum.UTC) ti.start_date = pendulum.DateTime(2024, 11, 30, 10, 0, 0, tzinfo=pendulum.UTC) ti.end_date = pendulum.DateTime(2024, 11, 30, 10, 5, 0, tzinfo=pendulum.UTC) elif ti.task_id == TASK_ID_2: ti.state = TaskInstanceState.FAILED ti.try_number = 1 + ti.scheduled_dttm = pendulum.DateTime(2024, 11, 30, 10, 2, 0, tzinfo=pendulum.UTC) + ti.queued_dttm = pendulum.DateTime(2024, 11, 30, 10, 3, 0, tzinfo=pendulum.UTC) ti.start_date = pendulum.DateTime(2024, 11, 30, 10, 5, 0, tzinfo=pendulum.UTC) ti.end_date = pendulum.DateTime(2024, 11, 30, 10, 10, 0, tzinfo=pendulum.UTC) elif ti.task_id == TASK_ID_3: ti.state = TaskInstanceState.RUNNING ti.try_number = 1 + ti.scheduled_dttm = None + ti.queued_dttm = None ti.start_date = pendulum.DateTime(2024, 11, 30, 10, 10, 0, tzinfo=pendulum.UTC) ti.end_date = None @@ -306,6 +318,18 @@ def test_sorted_by_task_id_and_try_number(self, test_client): sorted_tis = sorted(task_instances, key=lambda x: (x["task_id"], x["try_number"])) assert task_instances == sorted_tis + def test_timing_fields_are_returned(self, test_client): + response = test_client.get(f"/gantt/{DAG_ID}/run_1") + assert response.status_code == 200 + data = response.json() + tis = {ti["task_id"]: ti for ti in data["task_instances"]} + assert tis[TASK_ID]["scheduled_dttm"] == "2024-11-30T09:50:00Z" + assert tis[TASK_ID]["queued_dttm"] == "2024-11-30T09:55:00Z" + assert tis[TASK_ID_2]["scheduled_dttm"] == "2024-11-30T10:02:00Z" + assert tis[TASK_ID_2]["queued_dttm"] == "2024-11-30T10:03:00Z" + assert tis[TASK_ID_3]["scheduled_dttm"] is None + assert tis[TASK_ID_3]["queued_dttm"] is None + def test_should_response_401(self, unauthenticated_test_client): response = unauthenticated_test_client.get(f"/gantt/{DAG_ID}/run_1") assert response.status_code == 401 diff --git a/airflow-core/tests/unit/core/test_configuration.py b/airflow-core/tests/unit/core/test_configuration.py index 2d53d69c1d554..abc42c26433dd 100644 --- a/airflow-core/tests/unit/core/test_configuration.py +++ b/airflow-core/tests/unit/core/test_configuration.py @@ -1855,6 +1855,7 @@ def test_sensitive_values(): ("database", "sql_alchemy_conn"), ("database", "sql_alchemy_conn_async"), ("core", "fernet_key"), + ("core", "sql_alchemy_conn"), # NOTE: Added for 3.2.1 ("api_auth", "jwt_secret"), ("api", "secret_key"), ("secrets", "backend_kwargs"), diff --git a/airflow-core/tests/unit/models/test_taskinstance.py b/airflow-core/tests/unit/models/test_taskinstance.py index 9eba07daaa130..d874b6163a039 100644 --- a/airflow-core/tests/unit/models/test_taskinstance.py +++ b/airflow-core/tests/unit/models/test_taskinstance.py @@ -2605,6 +2605,81 @@ def test_task_instance_history_is_created_when_ti_goes_for_retry(self, dag_maker # the new try_id should be different from what's recorded in tih assert tih[0].task_instance_id == try_id + @pytest.mark.parametrize( + ("first_ti", "second_ti"), + [ + pytest.param( + ("dag_1", "run_1", "task_1", -1), + ("dag_2", "run_1", "task_1", -1), + id="tasks_with_different_dags", + ), + pytest.param( + ("dag_1", "run_1", "task_1", -1), + ("dag_1", "run_2", "task_1", -1), + id="tasks_with_different_runs", + ), + # There are no cases with equal dag_id/run_id because create_task_instance() + # creates a DagRun each time, and DagRun has a unique (dag_id, run_id) constraint. + ], + ) + def test_get_task_instance_disambiguates_by_dag_id_and_run_id( + self, create_task_instance, session, first_ti, second_ti + ): + dag_id_1, run_id_1, task_id_1, map_index_1 = first_ti + dag_id_2, run_id_2, task_id_2, map_index_2 = second_ti + + ti1 = create_task_instance( + dag_id=dag_id_1, + run_id=run_id_1, + task_id=task_id_1, + map_index=map_index_1, + session=session, + ) + ti2 = create_task_instance( + dag_id=dag_id_2, + run_id=run_id_2, + task_id=task_id_2, + map_index=map_index_2, + session=session, + ) + + # Regression setup for #64957: if dag_id is ignored, this lookup key becomes ambiguous. + if dag_id_1 != dag_id_2: + ambiguous_count = session.scalar( + select(func.count()) + .select_from(TI) + .filter_by(run_id=run_id_1, task_id=task_id_1, map_index=map_index_1) + ) + assert ambiguous_count == 2, "Setup failure: expected two TIs matching without dag_id filter" + + # This case does not target the original regression directly (run_id was already filtered), + # but we keep it as defense-in-depth against future changes. + found_1 = TI.get_task_instance( + dag_id=dag_id_1, + run_id=run_id_1, + task_id=task_id_1, + map_index=map_index_1, + session=session, + ) + found_2 = TI.get_task_instance( + dag_id=dag_id_2, + run_id=run_id_2, + task_id=task_id_2, + map_index=map_index_2, + session=session, + ) + + assert found_1 is not None + assert found_2 is not None + + assert found_1.id == ti1.id + assert found_2.id == ti2.id + + # Keep dag_id assertions explicit to document the regression intent (#64957): + # get_task_instance() must disambiguate identical run/task/map_index by dag_id. + assert found_1.dag_id == dag_id_1 + assert found_2.dag_id == dag_id_2 + @pytest.mark.parametrize("pool_override", [None, "test_pool2"]) @pytest.mark.parametrize("queue_by_policy", [None, "forced_queue"]) diff --git a/airflow-core/tests/unit/triggers/test_base_trigger.py b/airflow-core/tests/unit/triggers/test_base_trigger.py index 53066c46f6a14..d9e38385a2052 100644 --- a/airflow-core/tests/unit/triggers/test_base_trigger.py +++ b/airflow-core/tests/unit/triggers/test_base_trigger.py @@ -27,6 +27,18 @@ class DummyOperator(BaseOperator): template_fields = ("name",) +class OperatorWithExtraTemplateFields(BaseOperator): + """Operator whose template_fields do NOT all exist on the trigger.""" + + template_fields = ("bash_command", "env", "name") + + def __init__(self, bash_command="", env=None, name="", **kwargs): + super().__init__(**kwargs) + self.bash_command = bash_command + self.env = env + self.name = name + + class DummyTrigger(BaseTrigger): def __init__(self, name: str, **kwargs): super().__init__(**kwargs) @@ -67,3 +79,62 @@ def test_render_template_fields(create_task_instance): trigger.render_template_fields(context={"name": "world"}) assert trigger.name == "Hello world" + + +@pytest.mark.db_test +def test_render_template_fields_filters_to_trigger_kwargs(create_task_instance): + """Only fields present in both trigger_kwargs and on the trigger should be rendered. + + Operator template_fields like 'bash_command' and 'env' that don't exist on the + trigger must be excluded to avoid AttributeError. + """ + op = OperatorWithExtraTemplateFields( + task_id="extra_fields_task", + bash_command="echo hello", + env={"KEY": "val"}, + name="static", + ) + ti = create_task_instance( + task=op, + start_from_trigger=True, + start_trigger_args=StartTriggerArgs( + trigger_cls=f"{DummyTrigger.__module__}.{DummyTrigger.__qualname__}", + next_method="resume_method", + trigger_kwargs={"name": "Hello {{ name }}"}, + ), + ) + + trigger = DummyTrigger(name="Hello {{ name }}") + trigger.task_instance = ti + + # Only 'name' should be in template_fields; 'bash_command' and 'env' are excluded + # because they aren't keys in trigger_kwargs or don't exist on the trigger. + assert trigger.template_fields == ("name",) + + # Rendering must not raise AttributeError for missing operator fields + trigger.render_template_fields(context={"name": "world"}) + assert trigger.name == "Hello world" + + +@pytest.mark.db_test +def test_render_template_fields_empty_when_no_trigger_kwargs(create_task_instance): + """When start_trigger_args has no trigger_kwargs, template_fields should be empty.""" + op = DummyOperator(task_id="no_kwargs_task") + ti = create_task_instance( + task=op, + start_from_trigger=True, + start_trigger_args=StartTriggerArgs( + trigger_cls=f"{DummyTrigger.__module__}.{DummyTrigger.__qualname__}", + next_method="resume_method", + trigger_kwargs=None, + ), + ) + + trigger = DummyTrigger(name="Hello {{ name }}") + trigger.task_instance = ti + + assert trigger.template_fields == () + + # Rendering with empty template_fields is a no-op + trigger.render_template_fields(context={"name": "world"}) + assert trigger.name == "Hello {{ name }}" diff --git a/airflow-ctl-tests/tests/airflowctl_tests/test_airflowctl_commands.py b/airflow-ctl-tests/tests/airflowctl_tests/test_airflowctl_commands.py index 4aea60dca681e..de07d0d3751a9 100644 --- a/airflow-ctl-tests/tests/airflowctl_tests/test_airflowctl_commands.py +++ b/airflow-ctl-tests/tests/airflowctl_tests/test_airflowctl_commands.py @@ -51,6 +51,7 @@ def date_param(): TEST_COMMANDS = [ # Auth commands f"auth token {CREDENTIAL_SUFFIX}", + "auth list-envs", # Assets commands "assets list", "assets get --asset-id=1", @@ -124,6 +125,9 @@ def date_param(): "variables delete --variable-key=test_import_var_with_desc", # Version command "version --remote", + # Plugins command + "plugins list", + "plugins list-import-errors", ] DATE_PARAM_1 = date_param() @@ -168,9 +172,7 @@ def test_hardcoded_xcom_key_would_collide(): ) def test_airflowctl_commands(command: str, run_command): """Test airflowctl commands using docker-compose environment.""" - env_vars = {"AIRFLOW_CLI_DEBUG_MODE": "true"} - - run_command(command, env_vars, skip_login=True) + run_command(command=command, env_vars={"AIRFLOW_CLI_DEBUG_MODE": "true"}, skip_login=True) @pytest.mark.parametrize( @@ -180,9 +182,12 @@ def test_airflowctl_commands(command: str, run_command): ) def test_airflowctl_commands_skip_keyring(command: str, api_token: str, run_command): """Test airflowctl commands using docker-compose environment without using keyring.""" - env_vars = {} - env_vars["AIRFLOW_CLI_TOKEN"] = api_token - env_vars["AIRFLOW_CLI_DEBUG_MODE"] = "false" - env_vars["AIRFLOW_CLI_ENVIRONMENT"] = "nokeyring" - - run_command(command, env_vars, skip_login=True) + run_command( + command=command, + env_vars={ + "AIRFLOW_CLI_TOKEN": api_token, + "AIRFLOW_CLI_DEBUG_MODE": "false", + "AIRFLOW_CLI_ENVIRONMENT": "nokeyring", + }, + skip_login=True, + ) diff --git a/airflow-ctl/.pre-commit-config.yaml b/airflow-ctl/.pre-commit-config.yaml index c45a1985ec129..a5773e94aabca 100644 --- a/airflow-ctl/.pre-commit-config.yaml +++ b/airflow-ctl/.pre-commit-config.yaml @@ -50,3 +50,12 @@ repos: (?x) ^src/airflowctl/api/operations\.py$| ^docs/images/command_hashes.txt$ + - id: check-airflowctl-help-texts + name: Check airflowctl CLI commands have help texts + entry: ../scripts/ci/prek/check_airflowctl_help_texts.py + language: python + pass_filenames: false + files: + (?x) + ^src/airflowctl/api/operations\.py$| + ^src/airflowctl/ctl/help_texts\.yaml$ diff --git a/airflow-ctl/docs/images/command_hashes.txt b/airflow-ctl/docs/images/command_hashes.txt index b0089d41d1f91..c824ba2f9abe0 100644 --- a/airflow-ctl/docs/images/command_hashes.txt +++ b/airflow-ctl/docs/images/command_hashes.txt @@ -1,14 +1,15 @@ -main:65249416abad6ad24c276fb44326ae15 -assets:b3ae2b933e54528bf486ff28e887804d +main:27a22c00dcf32e7a1a4f06672dc8e3c8 +assets:6e2d3f0f73df1bd794a6b7d8fefffdc3 auth:d79e9c7d00c432bdbcbc2a86e2e32053 -backfill:bbce9859a2d1ce054ad22db92dea8c05 -config:cb175bedf29e8a2c2c6a2ebd13d770a7 -connections:e34b6b93f64714986139958c1f370428 -dags:287a128a71c97d2b537e09a5c7c73c09 -dagrun:f47ed2a89ed0f8c71f79dba53a3a3882 -jobs:7f8680afff230eb9940bc7fca727bd52 -pools:03fc7d948cbecf16ff8d640eb8f0ce43 -providers:1c0afb2dff31d93ab2934b032a2250ab -variables:0354f8f4b0dde1c3771ed1568692c6ae +backfill:41e008e4bc78d44e69bd9769098ba3b0 +config:a3d936cb15fe3b547bf6c82cf93d923f +connections:942f9f88cb908c28bf5c19159fc5065b +dags:d9d0b3460097db0b9fbf8ae42bf500c3 +dagrun:0e46473ad2f3dfa1ee9ee27678dde57e +jobs:a5b644c5da8889443bb40ee10b599270 +pools:19efe105b9515ab1926ebcaf0e028d71 +providers:34502fe09dc0b8b0a13e7e46efdffda6 +variables:f8fc76d3d398b2780f4e97f7cd816646 version:31f4efdf8de0dbaaa4fac71ff7efecc3 +plugins:4864fd8f356704bd2b3cd1aec3567e35 auth login:9fe2bb1dd5c602beea2eefb33a2b20a8 diff --git a/airflow-ctl/docs/images/output_assets.svg b/airflow-ctl/docs/images/output_assets.svg index 0e2783bab8da1..07e78ea7c1e0a 100644 --- a/airflow-ctl/docs/images/output_assets.svg +++ b/airflow-ctl/docs/images/output_assets.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + + + + - + - + - - Usage:airflowctl assets [-h] COMMAND... - -Perform Assets operations - -Positional Arguments: -COMMAND -create-eventPerform create_event operation -delete-dag-queued-events -Perform delete_dag_queued_events  -operation -delete-queued-event -Perform delete_queued_event operation -delete-queued-events -Perform delete_queued_events operation -getPerform get operation -get-by-aliasPerform get_by_alias operation -get-dag-queued-event -Perform get_dag_queued_event operation -get-dag-queued-events -Perform get_dag_queued_events operation -get-queued-eventsPerform get_queued_events operation -listPerform list operation -list-by-aliasPerform list_by_alias operation -materializePerform materialize operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl assets [-h] COMMAND... + +Perform Assets operations + +Positional Arguments: +COMMAND +create-eventCreate an event for a given asset +delete-dag-queued-events +Delete all queued asset events for a given DAG +delete-queued-event +Delete a specific queued asset event for a given  +DAG and asset +delete-queued-events +Delete all queued events for a given asset +getRetrieve an asset by its ID +get-by-aliasRetrieve an asset by its alias +get-dag-queued-event +Retrieve a specific queued asset event for a given  +DAG and asset +get-dag-queued-events +List queued asset events for a given DAG +get-queued-eventsList queued events for a given asset +listList all assets +list-by-aliasList all asset aliases +materializeTrigger materialization of an asset by its ID + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_backfill.svg b/airflow-ctl/docs/images/output_backfill.svg index 239a37ab8ea0d..4119e5058af9c 100644 --- a/airflow-ctl/docs/images/output_backfill.svg +++ b/airflow-ctl/docs/images/output_backfill.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - Usage:airflowctl backfill [-h] COMMAND... - -Perform Backfill operations - -Positional Arguments: -COMMAND -cancelPerform cancel operation -createPerform create operation -create-dry-run -Perform create_dry_run operation -getPerform get operation -listPerform list operation -pausePerform pause operation -unpausePerform unpause operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl backfill [-h] COMMAND... + +Perform Backfill operations + +Positional Arguments: +COMMAND +cancelCancel a backfill job +createCreate a backfill job for a given DAG ID and date range +create-dry-run +Preview a backfill job without executing it +getRetrieve details of a backfill job by its ID +listList all backfill jobs for a given DAG +pausePause an active backfill job +unpauseResume a paused backfill job + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_config.svg b/airflow-ctl/docs/images/output_config.svg index b2c917e8d3d95..efed77719ff6f 100644 --- a/airflow-ctl/docs/images/output_config.svg +++ b/airflow-ctl/docs/images/output_config.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - Usage:airflowctl config [-h] COMMAND... - -Perform Config operations - -Positional Arguments: -COMMAND -getPerform get operation -lintLint options for the configuration changes while  -migrating from Airflow 2 to Airflow 3 -listPerform list operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl config [-h] COMMAND... + +Perform Config operations + +Positional Arguments: +COMMAND +getRetrieve the value of a specific configuration option +lintLint options for the configuration changes while migrating  +from Airflow 2 to Airflow 3 +listList all configuration sections and their options + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_connections.svg b/airflow-ctl/docs/images/output_connections.svg index 7dc48cce16a6c..ea110f8190dab 100644 --- a/airflow-ctl/docs/images/output_connections.svg +++ b/airflow-ctl/docs/images/output_connections.svg @@ -19,78 +19,78 @@ font-weight: 700; } - .terminal-1162206820-matrix { + .terminal-1848112235-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-1162206820-title { + .terminal-1848112235-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-1162206820-r1 { fill: #ff8700 } -.terminal-1162206820-r2 { fill: #c5c8c6 } -.terminal-1162206820-r3 { fill: #808080 } -.terminal-1162206820-r4 { fill: #68a0b3 } + .terminal-1848112235-r1 { fill: #ff8700 } +.terminal-1848112235-r2 { fill: #c5c8c6 } +.terminal-1848112235-r3 { fill: #808080 } +.terminal-1848112235-r4 { fill: #68a0b3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -102,27 +102,27 @@ - + - - Usage:airflowctl connections [-h] COMMAND... - -Perform Connections operations - -Positional Arguments: -COMMAND -createPerform create operation -create-defaults -Perform create_defaults operation -deletePerform delete operation -getPerform get operation -importImport connections from a file exported with local CLI. -listPerform list operation -testPerform test operation -updatePerform update operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl connections [-h] COMMAND... + +Perform Connections operations + +Positional Arguments: +COMMAND +createCreate a new connection +create-defaults +Populate default connections for installed providers +deleteDelete a connection by its ID +getRetrieve a connection by its ID +importImport connections from a file exported with local CLI. +listList all configured connections +testTest connectivity for a given connection +updateUpdate an existing connection + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_dagrun.svg b/airflow-ctl/docs/images/output_dagrun.svg index a56fba6d5e5f2..a03fb094b3e25 100644 --- a/airflow-ctl/docs/images/output_dagrun.svg +++ b/airflow-ctl/docs/images/output_dagrun.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - Usage:airflowctl dagrun [-h] COMMAND... - -Perform DagRun operations - -Positional Arguments: -COMMAND -getPerform get operation -listPerform list operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl dagrun [-h] COMMAND... + +Perform DagRun operations + +Positional Arguments: +COMMAND +getRetrieve a DAG run by DAG ID and run ID +listList DAG runs, optionally filtered by state and date range + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_dags.svg b/airflow-ctl/docs/images/output_dags.svg index 6863827b594ed..0bbe6f479d743 100644 --- a/airflow-ctl/docs/images/output_dags.svg +++ b/airflow-ctl/docs/images/output_dags.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - Usage:airflowctl dags [-h] COMMAND... - -Perform Dags operations - -Positional Arguments: -COMMAND -deletePerform delete operation -getPerform get operation -get-detailsPerform get_details operation -get-import-errorPerform get_import_error operation -get-statsPerform get_stats operation -get-tagsPerform get_tags operation -get-versionPerform get_version operation -listPerform list operation -list-import-errors -Perform list_import_errors operation -list-versionPerform list_version operation -list-warningPerform list_warning operation -pausePause a Dag -triggerPerform trigger operation -unpauseUnpause a Dag -updatePerform update operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl dags [-h] COMMAND... + +Perform Dags operations + +Positional Arguments: +COMMAND +deleteDelete a DAG by its ID +getRetrieve a DAG by its ID +get-detailsRetrieve detailed information for a DAG +get-import-errorRetrieve a DAG import error by its ID +get-statsRetrieve run statistics for one or more DAGs +get-tagsList all tags used across DAGs +get-versionRetrieve a specific version of a DAG +listList all DAGs +list-import-errors +List all DAG import errors +list-versionList all versions of a DAG +list-warningList all DAG warnings +pausePause a Dag +triggerTrigger a new DAG run +unpauseUnpause a Dag +updateUpdate properties of a DAG + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_jobs.svg b/airflow-ctl/docs/images/output_jobs.svg index a844eea541592..13b31d2caedc4 100644 --- a/airflow-ctl/docs/images/output_jobs.svg +++ b/airflow-ctl/docs/images/output_jobs.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - Usage:airflowctl jobs [-h] COMMAND... - -Perform Jobs operations - -Positional Arguments: -COMMAND -listPerform list operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl jobs [-h] COMMAND... + +Perform Jobs operations + +Positional Arguments: +COMMAND +listList scheduler, triggerer, and other Airflow jobs + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_main.svg b/airflow-ctl/docs/images/output_main.svg index 8e4ef71bdb016..f586877bce8eb 100644 --- a/airflow-ctl/docs/images/output_main.svg +++ b/airflow-ctl/docs/images/output_main.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + - + - + - - Usage:airflowctl [-h] GROUP_OR_COMMAND... - -Positional Arguments: -GROUP_OR_COMMAND - -    Groups -assetsPerform Assets operations -authManage authentication for CLI. Either pass token from -environment variable/parameter or pass username and -password. -backfillPerform Backfill operations -configPerform Config operations -connectionsPerform Connections operations -dagrunPerform DagRun operations -dagsPerform Dags operations -jobsPerform Jobs operations -poolsPerform Pools operations -providersPerform Providers operations -variablesPerform Variables operations -xcomPerform XCom operations - -    Commands: -versionShow version information - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl [-h] GROUP_OR_COMMAND... + +Positional Arguments: +GROUP_OR_COMMAND + +    Groups +assetsPerform Assets operations +authManage authentication for CLI. Either pass token from +environment variable/parameter or pass username and +password. +backfillPerform Backfill operations +configPerform Config operations +connectionsPerform Connections operations +dagrunPerform DagRun operations +dagsPerform Dags operations +jobsPerform Jobs operations +pluginsPerform Plugins operations +poolsPerform Pools operations +providersPerform Providers operations +variablesPerform Variables operations +xcomPerform XCom operations + +    Commands: +versionShow version information + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_plugins.svg b/airflow-ctl/docs/images/output_plugins.svg new file mode 100644 index 0000000000000..7d91abc3e55dc --- /dev/null +++ b/airflow-ctl/docs/images/output_plugins.svg @@ -0,0 +1,105 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Usage:airflowctl plugins [-h] COMMAND... + +Perform Plugins operations + +Positional Arguments: +COMMAND +listList all installed Airflow plugins +list-import-errors +List all plugin import errors + +Options: +-h, --helpshow this help message and exit + + + + diff --git a/airflow-ctl/docs/images/output_pools.svg b/airflow-ctl/docs/images/output_pools.svg index 9423ec2d7c5dd..f30061e3deced 100644 --- a/airflow-ctl/docs/images/output_pools.svg +++ b/airflow-ctl/docs/images/output_pools.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - Usage:airflowctl pools [-h] COMMAND... - -Perform Pools operations - -Positional Arguments: -COMMAND -createPerform create operation -deletePerform delete operation -exportExport all pools -getPerform get operation -importImport pools -listPerform list operation -updatePerform update operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl pools [-h] COMMAND... + +Perform Pools operations + +Positional Arguments: +COMMAND +createCreate a new pool +deleteDelete a pool by its name +exportExport all pools +getRetrieve a pool by its name +importImport pools +listList all pools +updateUpdate an existing pool + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_providers.svg b/airflow-ctl/docs/images/output_providers.svg index ff4a2c8e4d243..f4ce50e83cd57 100644 --- a/airflow-ctl/docs/images/output_providers.svg +++ b/airflow-ctl/docs/images/output_providers.svg @@ -1,4 +1,4 @@ - + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - - + + - + - + - - Usage:airflowctl providers [-h] COMMAND... - -Perform Providers operations - -Positional Arguments: -COMMAND -listPerform list operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl providers [-h] COMMAND... + +Perform Providers operations + +Positional Arguments: +COMMAND +listList all installed Airflow providers + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/docs/images/output_variables.svg b/airflow-ctl/docs/images/output_variables.svg index 78c24d31074b6..a8833a923899d 100644 --- a/airflow-ctl/docs/images/output_variables.svg +++ b/airflow-ctl/docs/images/output_variables.svg @@ -19,69 +19,69 @@ font-weight: 700; } - .terminal-938427142-matrix { + .terminal-1391084489-matrix { font-family: Fira Code, monospace; font-size: 20px; line-height: 24.4px; font-variant-east-asian: full-width; } - .terminal-938427142-title { + .terminal-1391084489-title { font-size: 18px; font-weight: bold; font-family: arial; } - .terminal-938427142-r1 { fill: #ff8700 } -.terminal-938427142-r2 { fill: #c5c8c6 } -.terminal-938427142-r3 { fill: #808080 } -.terminal-938427142-r4 { fill: #68a0b3 } + .terminal-1391084489-r1 { fill: #ff8700 } +.terminal-1391084489-r2 { fill: #c5c8c6 } +.terminal-1391084489-r3 { fill: #808080 } +.terminal-1391084489-r4 { fill: #68a0b3 } - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + @@ -93,24 +93,24 @@ - + - - Usage:airflowctl variables [-h] COMMAND... - -Perform Variables operations - -Positional Arguments: -COMMAND -createPerform create operation -deletePerform delete operation -getPerform get operation -importImport variables from a file exported with local CLI. -listPerform list operation -updatePerform update operation - -Options: --h, --helpshow this help message and exit + + Usage:airflowctl variables [-h] COMMAND... + +Perform Variables operations + +Positional Arguments: +COMMAND +createCreate a new variable +deleteDelete a variable by its key +getRetrieve a variable by its key +importImport variables from a file exported with local CLI. +listList all variables +updateUpdate an existing variable + +Options: +-h, --helpshow this help message and exit diff --git a/airflow-ctl/src/airflowctl/api/client.py b/airflow-ctl/src/airflowctl/api/client.py index 0ef5d7cb16441..f3ca3f673f119 100644 --- a/airflow-ctl/src/airflowctl/api/client.py +++ b/airflow-ctl/src/airflowctl/api/client.py @@ -26,6 +26,7 @@ import sys from collections.abc import Callable from functools import wraps +from pathlib import Path from typing import TYPE_CHECKING, Any, Literal, ParamSpec, TypeVar, cast import httpx @@ -52,6 +53,7 @@ DagsOperations, JobsOperations, LoginOperations, + PluginsOperations, PoolsOperations, ProvidersOperations, ServerResponseError, @@ -61,6 +63,7 @@ ) from airflowctl.exceptions import ( AirflowCtlCredentialNotFoundException, + AirflowCtlException, AirflowCtlKeyringException, AirflowCtlNotFoundException, ) @@ -143,6 +146,17 @@ def _bounded_get_new_password() -> str: ) +def _safe_path_under_airflow_home(airflow_home: str, filename: str) -> str: + base = Path(airflow_home).resolve() + target = (base / filename).resolve() + if not target.is_relative_to(base): + raise AirflowCtlException( + f"Security Error: Path traversal detected in '{filename}'. " + f"The resolved path must stay within AIRFLOW_HOME." + ) + return str(target) + + # Credentials for the API class Credentials: """Credentials for the API.""" @@ -160,14 +174,26 @@ def __init__( ): self.api_url = api_url self.api_token = api_token - self.api_environment = os.getenv("AIRFLOW_CLI_ENVIRONMENT") or api_environment self.client_kind = client_kind + raw_env = os.getenv("AIRFLOW_CLI_ENVIRONMENT") or api_environment + if "/" in raw_env or "\\" in raw_env or ".." in raw_env: + raise AirflowCtlException( + f"Invalid environment name: '{raw_env}'. " + f"Environment names cannot contain path separators ('/', '\\') or '..'." + ) + + self.api_environment = raw_env @property def input_cli_config_file(self) -> str: """Generate path for the CLI config file.""" return f"{self.api_environment}.json" + @staticmethod + def token_key_for_environment(api_environment: str) -> str: + """Build the keyring/debug token key for a given environment name.""" + return f"api_token_{api_environment}" + def save(self, skip_keyring: bool = False): """ Save the credentials to keyring and URL to disk as a file. @@ -177,15 +203,17 @@ def save(self, skip_keyring: bool = False): """ default_config_dir = os.environ.get("AIRFLOW_HOME", os.path.expanduser("~/airflow")) os.makedirs(default_config_dir, exist_ok=True) - with open(os.path.join(default_config_dir, self.input_cli_config_file), "w") as f: + config_path = _safe_path_under_airflow_home(default_config_dir, self.input_cli_config_file) + with open(config_path, "w") as f: json.dump({"api_url": self.api_url}, f) try: if os.getenv("AIRFLOW_CLI_DEBUG_MODE") == "true": - with open( - os.path.join(default_config_dir, f"debug_creds_{self.input_cli_config_file}"), "w" - ) as f: - json.dump({f"api_token_{self.api_environment}": self.api_token}, f) + debug_path = _safe_path_under_airflow_home( + default_config_dir, f"debug_creds_{self.input_cli_config_file}" + ) + with open(debug_path, "w") as f: + json.dump({self.token_key_for_environment(self.api_environment): self.api_token}, f) else: if skip_keyring: return @@ -198,7 +226,11 @@ def save(self, skip_keyring: bool = False): for candidate in candidates: if hasattr(candidate, "_get_new_password"): candidate._get_new_password = _bounded_get_new_password - keyring.set_password("airflowctl", f"api_token_{self.api_environment}", self.api_token) # type: ignore[arg-type] + keyring.set_password( + "airflowctl", + self.token_key_for_environment(self.api_environment), + self.api_token, # type: ignore[arg-type] + ) except (NoKeyringError, NotImplementedError) as e: log.error(e) raise AirflowCtlKeyringException( @@ -216,7 +248,7 @@ def save(self, skip_keyring: bool = False): def load(self) -> Credentials: """Load the credentials from keyring and URL from disk file.""" default_config_dir = os.environ.get("AIRFLOW_HOME", os.path.expanduser("~/airflow")) - config_path = os.path.join(default_config_dir, self.input_cli_config_file) + config_path = _safe_path_under_airflow_home(default_config_dir, self.input_cli_config_file) try: with open(config_path) as f: credentials = json.load(f) @@ -224,16 +256,26 @@ def load(self) -> Credentials: if self.api_token is not None: return self if os.getenv("AIRFLOW_CLI_DEBUG_MODE") == "true": - debug_creds_path = os.path.join( + debug_creds_path = _safe_path_under_airflow_home( default_config_dir, f"debug_creds_{self.input_cli_config_file}" ) - with open(debug_creds_path) as df: - debug_credentials = json.load(df) - self.api_token = debug_credentials.get(f"api_token_{self.api_environment}") + try: + with open(debug_creds_path) as df: + debug_credentials = json.load(df) + self.api_token = debug_credentials.get( + self.token_key_for_environment(self.api_environment) + ) + except FileNotFoundError as e: + if self.client_kind == ClientKind.CLI: + raise AirflowCtlCredentialNotFoundException( + f"Debug credentials file not found: {debug_creds_path}. " + "Set AIRFLOW_CLI_DEBUG_MODE=false or log in with debug mode enabled first." + ) from e + self.api_token = None else: try: self.api_token = keyring.get_password( - "airflowctl", f"api_token_{self.api_environment}" + "airflowctl", self.token_key_for_environment(self.api_environment) ) except ValueError as e: # Incorrect keyring password @@ -415,6 +457,12 @@ def xcom(self): """Operations related to XComs.""" return XComOperations(self) + @lru_cache() # type: ignore[prop-decorator] + @property + def plugins(self): + """Operations related to plugins.""" + return PluginsOperations(self) + # API Client Decorator for CLI Actions @contextlib.contextmanager diff --git a/airflow-ctl/src/airflowctl/api/operations.py b/airflow-ctl/src/airflowctl/api/operations.py index 6b38225250559..3ce196c10cb32 100644 --- a/airflow-ctl/src/airflowctl/api/operations.py +++ b/airflow-ctl/src/airflowctl/api/operations.py @@ -59,6 +59,8 @@ ImportErrorCollectionResponse, ImportErrorResponse, JobCollectionResponse, + PluginCollectionResponse, + PluginImportErrorCollectionResponse, PoolBody, PoolCollectionResponse, PoolPatchBody, @@ -644,10 +646,20 @@ class JobsOperations(BaseOperations): """Job operations.""" def list( - self, job_type: str, hostname: str, is_alive: bool + self, + job_type: str | None = None, + hostname: str | None = None, + is_alive: bool | None = None, ) -> JobCollectionResponse | ServerResponseError: """List all jobs.""" - params = {"job_type": job_type, "hostname": hostname, "is_alive": is_alive} + params: dict[str, Any] = {} + if job_type: + params["job_type"] = job_type + if hostname: + params["hostname"] = hostname + if is_alive is not None: + params["is_alive"] = is_alive + return super().execute_list(path="jobs", data_model=JobCollectionResponse, params=params) @@ -893,3 +905,19 @@ def delete( return key except ServerResponseError as e: raise e + + +class PluginsOperations(BaseOperations): + """Plugins operations.""" + + def list(self) -> PluginCollectionResponse | ServerResponseError: + """List all plugins from the API server.""" + return super().execute_list(path="plugins", data_model=PluginCollectionResponse) + + def list_import_errors(self) -> PluginImportErrorCollectionResponse | ServerResponseError: + """List plugin import errors from the API server.""" + try: + self.response = self.client.get("plugins/importErrors") + return PluginImportErrorCollectionResponse.model_validate_json(self.response.content) + except ServerResponseError as e: + raise e diff --git a/airflow-ctl/src/airflowctl/ctl/cli_config.py b/airflow-ctl/src/airflowctl/ctl/cli_config.py index 5f17c60335734..466ee671b61b2 100755 --- a/airflow-ctl/src/airflowctl/ctl/cli_config.py +++ b/airflow-ctl/src/airflowctl/ctl/cli_config.py @@ -25,6 +25,7 @@ import datetime import inspect import os +import sys from argparse import Namespace from collections.abc import Callable, Iterable from enum import Enum @@ -34,6 +35,7 @@ import httpx import rich +import yaml import airflowctl.api.datamodels.generated as generated_datamodels from airflowctl.api.client import NEW_API_CLIENT, Client, ClientKind, provide_api_client @@ -64,8 +66,6 @@ def command(*args, **kwargs): def safe_call_command(function: Callable, args: Iterable[Arg]) -> None: - import sys - if os.getenv("AIRFLOW_CLI_DEBUG_MODE") == "true": rich.print( "[yellow]Debug mode is enabled. Please be aware that your credentials are not secure.\n" @@ -90,10 +90,12 @@ def safe_call_command(function: Callable, args: Iterable[Arg]) -> None: f"[red]Server response error: {e}. " "Please check if the server is running and the API URL is correct.[/red]" ) + sys.exit(1) except httpx.ReadTimeout as e: rich.print(f"[red]Read timeout error: {e}[/red]") if "timed out" in str(e): rich.print("[red]Please check if the server is running and the API ready to accept calls.[/red]") + sys.exit(1) except ServerResponseError as e: rich.print(f"Server response error: {e}") if "Client error message:" in str(e): @@ -102,6 +104,7 @@ def safe_call_command(function: Callable, args: Iterable[Arg]) -> None: "Please check the command and its parameters. " "If you need help, run the command with --help." ) + sys.exit(1) class DefaultHelpParser(argparse.ArgumentParser): @@ -192,6 +195,14 @@ def string_lower_type(val): return val.strip().lower() +def _load_help_texts_yaml() -> dict[str, dict[str, str]]: + """Load the help texts yaml for the auto-generated commands.""" + help_texts_path = Path(__file__).parent / "help_texts.yaml" + with open(help_texts_path) as yaml_file: + help_texts = yaml.safe_load(yaml_file) + return help_texts + + # Common Positional Arguments ARG_FILE = Arg( flags=("file",), @@ -368,6 +379,7 @@ class CommandFactory: output_command_list: list[str] exclude_operation_names: list[str] exclude_method_names: list[str] + help_texts: dict[str, dict[str, str]] def __init__(self, file_path: str | Path | None = None): self.datamodels_extended_map = {} @@ -376,6 +388,7 @@ def __init__(self, file_path: str | Path | None = None): self.args_map = {} self.commands_map = {} self.group_commands_list = [] + self.help_texts = _load_help_texts_yaml() self.file_path = inspect.getfile(BaseOperations) if file_path is None else file_path # Excluded Lists are in Class Level for further usage and avoid searching them # Exclude parameters that are not needed for CLI from datamodels @@ -567,14 +580,15 @@ def _create_args_map_from_operation(self): for parameter in operation.get("parameters"): for parameter_key, parameter_type in parameter.items(): if self._is_primitive_type(type_name=parameter_type): - is_bool = parameter_type == "bool" + base_parameter_type = parameter_type.replace(" | None", "").strip() + is_bool = base_parameter_type == "bool" args.append( self._create_arg( arg_flags=("--" + self._sanitize_arg_parameter_key(parameter_key),), arg_type=self._python_type_from_string(parameter_type), arg_action=argparse.BooleanOptionalAction if is_bool else None, arg_help=f"{parameter_key} for {operation.get('name')} operation in {operation.get('parent').name}", - arg_default=False if is_bool else None, + arg_default=None, ) ) else: @@ -718,12 +732,15 @@ def _create_group_commands_from_operation(self): for operation in self.operations: operation_name = operation["name"] operation_group_name = operation["parent"].name + help_text = self.help_texts.get(operation_group_name.replace("Operations", "").lower(), {}).get( + operation_name.replace("_", "-") + ) if operation_group_name not in self.commands_map: self.commands_map[operation_group_name] = [] self.commands_map[operation_group_name].append( ActionCommand( name=operation["name"].replace("_", "-"), - help=f"Perform {operation_name} operation", + help=help_text, func=self.func_map[(operation_name, operation_group_name)], args=self.args_map[(operation_name, operation_group_name)], ) diff --git a/airflow-ctl/src/airflowctl/ctl/commands/auth_command.py b/airflow-ctl/src/airflowctl/ctl/commands/auth_command.py index 236b8d5c6b8de..cf521cbe7eea0 100644 --- a/airflow-ctl/src/airflowctl/ctl/commands/auth_command.py +++ b/airflow-ctl/src/airflowctl/ctl/commands/auth_command.py @@ -144,7 +144,7 @@ def list_envs(args) -> None: if filename.startswith("debug_creds_") or filename.endswith("_generated.json"): continue - env_name = filename.replace(".json", "") + env_name, _ = os.path.splitext(filename) # Try to read config file api_url = None @@ -168,11 +168,11 @@ def list_envs(args) -> None: if os.path.exists(debug_path): with open(debug_path) as f: debug_creds = json.load(f) - if f"api_token_{env_name}" in debug_creds: + if Credentials.token_key_for_environment(env_name) in debug_creds: token_status = "authenticated" else: # Check keyring - token = keyring.get_password("airflowctl", f"api_token_{env_name}") + token = keyring.get_password("airflowctl", Credentials.token_key_for_environment(env_name)) if token: token_status = "authenticated" except NoKeyringError: diff --git a/airflow-ctl/src/airflowctl/ctl/help_texts.yaml b/airflow-ctl/src/airflowctl/ctl/help_texts.yaml new file mode 100644 index 0000000000000..3dac52be0bc99 --- /dev/null +++ b/airflow-ctl/src/airflowctl/ctl/help_texts.yaml @@ -0,0 +1,102 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +--- +assets: + get: "Retrieve an asset by its ID" + get-by-alias: "Retrieve an asset by its alias" + list: "List all assets" + list-by-alias: "List all asset aliases" + create-event: "Create an event for a given asset" + materialize: "Trigger materialization of an asset by its ID" + get-queued-events: "List queued events for a given asset" + get-dag-queued-events: "List queued asset events for a given DAG" + get-dag-queued-event: "Retrieve a specific queued asset event for a given DAG and asset" + delete-queued-events: "Delete all queued events for a given asset" + delete-dag-queued-events: "Delete all queued asset events for a given DAG" + delete-queued-event: "Delete a specific queued asset event for a given DAG and asset" + +backfill: + create: "Create a backfill job for a given DAG ID and date range" + create-dry-run: "Preview a backfill job without executing it" + get: "Retrieve details of a backfill job by its ID" + list: "List all backfill jobs for a given DAG" + pause: "Pause an active backfill job" + unpause: "Resume a paused backfill job" + cancel: "Cancel a backfill job" + +config: + get: "Retrieve the value of a specific configuration option" + list: "List all configuration sections and their options" + +connections: + get: "Retrieve a connection by its ID" + list: "List all configured connections" + create: "Create a new connection" + create-defaults: "Populate default connections for installed providers" + delete: "Delete a connection by its ID" + update: "Update an existing connection" + test: "Test connectivity for a given connection" + +dags: + get: "Retrieve a DAG by its ID" + get-details: "Retrieve detailed information for a DAG" + get-tags: "List all tags used across DAGs" + list: "List all DAGs" + update: "Update properties of a DAG" + delete: "Delete a DAG by its ID" + get-import-error: "Retrieve a DAG import error by its ID" + list-import-errors: "List all DAG import errors" + get-stats: "Retrieve run statistics for one or more DAGs" + get-version: "Retrieve a specific version of a DAG" + list-version: "List all versions of a DAG" + list-warning: "List all DAG warnings" + trigger: "Trigger a new DAG run" + +dagrun: + get: "Retrieve a DAG run by DAG ID and run ID" + list: "List DAG runs, optionally filtered by state and date range" + +jobs: + list: "List scheduler, triggerer, and other Airflow jobs" + +pools: + get: "Retrieve a pool by its name" + list: "List all pools" + create: "Create a new pool" + delete: "Delete a pool by its name" + update: "Update an existing pool" + +providers: + list: "List all installed Airflow providers" + +variables: + get: "Retrieve a variable by its key" + list: "List all variables" + create: "Create a new variable" + delete: "Delete a variable by its key" + update: "Update an existing variable" + +xcom: + get: "Retrieve an XCom entry for a specific task instance" + list: "List XCom entries for a specific task instance" + add: "Add a new XCom entry for a specific task instance" + edit: "Update an existing XCom entry for a specific task instance" + delete: "Delete an XCom entry for a specific task instance" + +plugins: + list: "List all installed Airflow plugins" + list-import-errors: "List all plugin import errors" diff --git a/airflow-ctl/tests/airflow_ctl/api/test_client.py b/airflow-ctl/tests/airflow_ctl/api/test_client.py index 0617d62276a1c..f2d216fcc82fb 100644 --- a/airflow-ctl/tests/airflow_ctl/api/test_client.py +++ b/airflow-ctl/tests/airflow_ctl/api/test_client.py @@ -30,7 +30,11 @@ from airflowctl.api.client import Client, ClientKind, Credentials, _bounded_get_new_password from airflowctl.api.operations import ServerResponseError -from airflowctl.exceptions import AirflowCtlCredentialNotFoundException, AirflowCtlKeyringException +from airflowctl.exceptions import ( + AirflowCtlCredentialNotFoundException, + AirflowCtlException, + AirflowCtlKeyringException, +) def make_client_w_responses(responses: list[httpx.Response]) -> Client: @@ -376,3 +380,34 @@ def test_retry_handling_ok(self): response = client.get("http://error") assert response.status_code == 200 assert len(responses) == 1 + + def test_debug_mode_missing_debug_creds_reports_correct_error(self, monkeypatch, tmp_path): + monkeypatch.setenv("AIRFLOW_HOME", str(tmp_path)) + monkeypatch.setenv("AIRFLOW_CLI_DEBUG_MODE", "true") + monkeypatch.setenv("AIRFLOW_CLI_ENVIRONMENT", "TEST_DEBUG") + + config_path = tmp_path / "TEST_DEBUG.json" + config_path.write_text(json.dumps({"api_url": "http://localhost:8080"}), encoding="utf-8") + # Intentionally do not create debug_creds_TEST_DEBUG.json to simulate a missing file + + creds = Credentials(client_kind=ClientKind.CLI, api_environment="TEST_DEBUG") + with pytest.raises(AirflowCtlCredentialNotFoundException, match="Debug credentials file not found"): + creds.load() + + +def test_credentials_accepts_safe_env(): + creds = Credentials(client_kind=ClientKind.CLI, api_environment="prod-us_1") + assert creds.api_environment == "prod-us_1" + + +@pytest.mark.parametrize("api_environment", ["../evil", "..\\evil", "a/b", "a\\b"]) +def test_credentials_rejects_unsafe_env_argument(api_environment): + with pytest.raises(AirflowCtlException, match="environment"): + Credentials(client_kind=ClientKind.CLI, api_environment=api_environment) + + +@pytest.mark.parametrize("api_environment", ["../evil", "..\\evil", "a/b", "a\\b"]) +def test_credentials_rejects_unsafe_env_from_environment_variable(monkeypatch, api_environment): + monkeypatch.setenv("AIRFLOW_CLI_ENVIRONMENT", api_environment) + with pytest.raises(AirflowCtlException, match="environment"): + Credentials(client_kind=ClientKind.CLI) diff --git a/airflow-ctl/tests/airflow_ctl/api/test_operations.py b/airflow-ctl/tests/airflow_ctl/api/test_operations.py index 44c5ab3231b69..aa559f174214b 100644 --- a/airflow-ctl/tests/airflow_ctl/api/test_operations.py +++ b/airflow-ctl/tests/airflow_ctl/api/test_operations.py @@ -79,6 +79,10 @@ ImportErrorResponse, JobCollectionResponse, JobResponse, + PluginCollectionResponse, + PluginImportErrorCollectionResponse, + PluginImportErrorResponse, + PluginResponse, PoolBody, PoolCollectionResponse, PoolResponse, @@ -1154,6 +1158,11 @@ class TestJobsOperations: def test_list(self): def handle_request(request: httpx.Request) -> httpx.Response: assert request.url.path == "/api/v2/jobs" + params = dict(request.url.params) + assert params["job_type"] == "job_type" + assert params["hostname"] == "hostname" + assert params["is_alive"] == "true" + assert params["limit"] == "50" return httpx.Response(200, json=json.loads(self.job_collection_response.model_dump_json())) client = make_api_client(transport=httpx.MockTransport(handle_request)) @@ -1164,6 +1173,32 @@ def handle_request(request: httpx.Request) -> httpx.Response: ) assert response == self.job_collection_response + @pytest.mark.parametrize( + ("job_type", "hostname", "is_alive", "expected_subset"), + [ + (None, None, None, {}), + ("scheduler", None, None, {"job_type": "scheduler"}), + (None, "host-a", None, {"hostname": "host-a"}), + (None, None, False, {"is_alive": "false"}), + ], + ) + def test_list_omits_empty_filters(self, job_type, hostname, is_alive, expected_subset): + def handle_request(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v2/jobs" + params = dict(request.url.params) + assert params["limit"] == "50" + for key, value in expected_subset.items(): + assert params[key] == value + + assert ("job_type" in params) is ("job_type" in expected_subset) + assert ("hostname" in params) is ("hostname" in expected_subset) + assert ("is_alive" in params) is ("is_alive" in expected_subset) + return httpx.Response(200, json=json.loads(self.job_collection_response.model_dump_json())) + + client = make_api_client(transport=httpx.MockTransport(handle_request)) + response = client.jobs.list(job_type=job_type, hostname=hostname, is_alive=is_alive) + assert response == self.job_collection_response + class TestPoolsOperations: pool_name = "pool_name" @@ -1732,3 +1767,53 @@ def handle_request(request: httpx.Request) -> httpx.Response: map_index=self.map_index, ) assert response == self.key + + +class TestPluginsOperations: + plugin_response = PluginResponse( + name="test-plugin", + macros=[], + flask_blueprints=[], + fastapi_apps=[], + fastapi_root_middlewares=[], + external_views=[], + react_apps=[], + appbuilder_views=[], + appbuilder_menu_items=[], + global_operator_extra_links=[], + operator_extra_links=[], + source="test-source", + listeners=[], + timetables=[], + ) + plugin_collection_response = PluginCollectionResponse(plugins=[plugin_response], total_entries=1) + plugin_import_error_response = PluginImportErrorResponse( + source="plugins/test_plugin.py", error="something went wrong" + ) + plugin_import_error_collection_response = PluginImportErrorCollectionResponse( + import_errors=[plugin_import_error_response], total_entries=1 + ) + + def test_list(self): + """Test listing plugins""" + + def handle_request(request: httpx.Request) -> httpx.Response: + assert request.url.path == ("/api/v2/plugins") + return httpx.Response(200, json=json.loads(self.plugin_collection_response.model_dump_json())) + + client = make_api_client(transport=httpx.MockTransport(handle_request)) + response = client.plugins.list() + assert response == self.plugin_collection_response + + def test_list_import_errors(self): + """Test listing plugin import errors""" + + def handle_request(request: httpx.Request) -> httpx.Response: + assert request.url.path == "/api/v2/plugins/importErrors" + return httpx.Response( + 200, json=json.loads(self.plugin_import_error_collection_response.model_dump_json()) + ) + + client = make_api_client(transport=httpx.MockTransport(handle_request)) + response = client.plugins.list_import_errors() + assert response == self.plugin_import_error_collection_response diff --git a/airflow-ctl/tests/airflow_ctl/ctl/commands/test_auth_command.py b/airflow-ctl/tests/airflow_ctl/ctl/commands/test_auth_command.py index e76fafc28adf9..2bda56b0fdc18 100644 --- a/airflow-ctl/tests/airflow_ctl/ctl/commands/test_auth_command.py +++ b/airflow-ctl/tests/airflow_ctl/ctl/commands/test_auth_command.py @@ -477,3 +477,21 @@ def test_list_envs_filters_special_files(self, monkeypatch): # Only production environment should be checked, not the special files mock_get_password.assert_called_once_with("airflowctl", "api_token_production") + + def test_list_envs_environment_name_with_json_substring(self, monkeypatch): + """Test list-envs keeps '.json' substrings in environment name for key lookup.""" + with ( + tempfile.TemporaryDirectory() as temp_airflow_home, + patch("keyring.get_password") as mock_get_password, + ): + monkeypatch.setenv("AIRFLOW_HOME", temp_airflow_home) + + with open(os.path.join(temp_airflow_home, "prod.json.region.json"), "w") as f: + json.dump({"api_url": "http://localhost:8080"}, f) + + mock_get_password.return_value = "test_token" + + args = self.parser.parse_args(["auth", "list-envs"]) + auth_command.list_envs(args) + + mock_get_password.assert_called_once_with("airflowctl", "api_token_prod.json.region") diff --git a/airflow-ctl/tests/airflow_ctl/ctl/test_cli_config.py b/airflow-ctl/tests/airflow_ctl/ctl/test_cli_config.py index 117f874c34651..e0278cd7c5348 100644 --- a/airflow-ctl/tests/airflow_ctl/ctl/test_cli_config.py +++ b/airflow-ctl/tests/airflow_ctl/ctl/test_cli_config.py @@ -21,8 +21,10 @@ from argparse import BooleanOptionalAction from textwrap import dedent +import httpx import pytest +from airflowctl.api.operations import ServerResponseError from airflowctl.ctl.cli_config import ( ARG_AUTH_TOKEN, ActionCommand, @@ -31,6 +33,13 @@ GroupCommand, add_auth_token_to_all_commands, merge_commands, + safe_call_command, +) +from airflowctl.exceptions import ( + AirflowCtlConnectionException, + AirflowCtlCredentialNotFoundException, + AirflowCtlKeyringException, + AirflowCtlNotFoundException, ) @@ -287,8 +296,95 @@ def delete(self, backfill_id: str) -> ServerResponseError | None: assert arg.kwargs["default"] == test_arg[1]["default"] assert arg.kwargs["type"] == test_arg[1]["type"] + def test_command_factory_optional_bool_uses_boolean_optional_action(self): + """Optional bool parameters should support --flag and --no-flag forms.""" + temp_file = "test_command.py" + self._save_temp_operations_py( + temp_file=temp_file, + file_content=""" + class JobsOperations(BaseOperations): + def list(self, is_alive: bool | None = None) -> JobCollectionResponse | ServerResponseError: + self.response = self.client.get("jobs") + return JobCollectionResponse.model_validate_json(self.response.content) + """, + ) + + command_factory = CommandFactory(file_path=temp_file) + generated_group_commands = command_factory.group_commands + + jobs_list_args = [] + for generated_group_command in generated_group_commands: + if generated_group_command.name != "jobs": + continue + for sub_command in generated_group_command.subcommands: + if sub_command.name == "list": + jobs_list_args = list(sub_command.args) + break + + is_alive_arg = next(arg for arg in jobs_list_args if arg.flags == ("--is-alive",)) + assert is_alive_arg.kwargs["action"] == BooleanOptionalAction + assert is_alive_arg.kwargs["default"] is None + assert is_alive_arg.kwargs["type"] is bool + class TestCliConfigMethods: + @pytest.mark.parametrize( + "raised_exception", + [ + AirflowCtlCredentialNotFoundException("missing credentials"), + AirflowCtlConnectionException("connection failed"), + AirflowCtlKeyringException("keyring failure"), + AirflowCtlNotFoundException("resource not found"), + ], + ids=["credential-not-found", "connection-error", "keyring-error", "not-found"], + ) + def test_safe_call_command_exits_non_zero_for_airflowctl_exceptions(self, raised_exception): + def raise_error(_args): + raise raised_exception + + with pytest.raises(SystemExit) as ctx: + safe_call_command(raise_error, args=argparse.Namespace()) + + assert ctx.value.code == 1 + + @pytest.mark.parametrize( + "raised_exception", + [ + httpx.RemoteProtocolError("remote protocol error"), + httpx.ReadError("read error"), + ], + ids=["remote-protocol-error", "read-error"], + ) + def test_safe_call_command_exits_non_zero_for_httpx_protocol_errors(self, raised_exception): + def raise_error(_args): + raise raised_exception + + with pytest.raises(SystemExit) as ctx: + safe_call_command(raise_error, args=argparse.Namespace()) + + assert ctx.value.code == 1 + + def test_safe_call_command_exits_non_zero_for_httpx_read_timeout(self): + def raise_error(_args): + raise httpx.ReadTimeout("timed out") + + with pytest.raises(SystemExit) as ctx: + safe_call_command(raise_error, args=argparse.Namespace()) + + assert ctx.value.code == 1 + + def test_safe_call_command_exits_non_zero_for_server_response_error(self): + request = httpx.Request("GET", "http://localhost:8080/api/v2/dags") + response = httpx.Response(500, request=request, json={"detail": "boom"}) + + def raise_error(_args): + raise ServerResponseError("server error", request=request, response=response) + + with pytest.raises(SystemExit) as ctx: + safe_call_command(raise_error, args=argparse.Namespace()) + + assert ctx.value.code == 1 + def test_add_to_parser_drops_type_for_boolean_optional_action(self): """Test add_to_parser removes type for BooleanOptionalAction.""" parser = argparse.ArgumentParser() @@ -554,3 +650,22 @@ def test_apply_datamodel_defaults_other_datamodel(self): # Should return params unchanged for other datamodels assert result == params, "Params should be unchanged for non-TriggerDAGRunPostBody datamodels" + + @pytest.mark.parametrize( + ("group_name", "subcommand_name", "expected_help"), + [ + ("assets", "get", "Retrieve an asset by its ID"), + ("connections", "get", "Retrieve a connection by its ID"), + ], + ) + def test_help_texts_used_for_auto_generated_commands(self, group_name, subcommand_name, expected_help): + """Test that help texts from YAML are used for auto-generated commands.""" + command_factory = CommandFactory() + for group_command in command_factory.group_commands: + if group_command.name == group_name: + for subcommand in group_command.subcommands: + if subcommand.name == subcommand_name: + assert subcommand.help == expected_help, ( + "Help message should match the help_text.yaml" + ) + return diff --git a/chart/Chart.yaml b/chart/Chart.yaml index 4756fd1896ebf..1f6104014b895 100644 --- a/chart/Chart.yaml +++ b/chart/Chart.yaml @@ -20,7 +20,7 @@ apiVersion: v2 name: airflow version: 1.21.0 -appVersion: 3.1.8 +appVersion: 3.2.0 description: The official Helm chart to deploy Apache Airflow, a platform to programmatically author, schedule, and monitor workflows home: https://airflow.apache.org/ @@ -47,21 +47,21 @@ annotations: url: https://airflow.apache.org/docs/helm-chart/1.21.0/ artifacthub.io/screenshots: | - title: Home Page - url: https://airflow.apache.org/docs/apache-airflow/3.1.8/_images/home_dark.png + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/home_dark.png - title: DAG Overview Dashboard - url: https://airflow.apache.org/docs/apache-airflow/3.1.8/_images/dag_overview_dashboard.png + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dag_overview_dashboard.png - title: DAGs View - url: https://airflow.apache.org/docs/apache-airflow/3.1.8/_images/dags.png + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dags.png - title: Assets View - url: https://airflow.apache.org/docs/apache-airflow/3.1.8/_images/asset_view.png + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/asset_view.png - title: Grid View - url: https://airflow.apache.org/docs/apache-airflow/3.1.8/_images/dag_overview_grid.png + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dag_overview_grid.png - title: Graph View - url: https://airflow.apache.org/docs/apache-airflow/3.1.8/_images/dag_overview_graph.png + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dag_overview_graph.png - title: Variable View - url: https://airflow.apache.org/docs/apache-airflow/3.1.8/_images/variable_hidden.png + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/variable_hidden.png - title: Code View - url: https://airflow.apache.org/docs/apache-airflow/3.1.8/_images/dag_overview_code.png + url: https://airflow.apache.org/docs/apache-airflow/3.2.0/_images/dag_overview_code.png artifacthub.io/changes: | - description: Add missing deprecation warnings for workers section kind: changed @@ -115,7 +115,7 @@ annotations: links: - name: '#62334' url: https://github.com/apache/airflow/pull/62334 - - description: Default airflow version to 3.1.8 + - description: Default airflow version to 3.2.0 kind: changed links: - name: '#63392' diff --git a/chart/docs/customizing-labels.rst b/chart/docs/customizing-labels.rst index 978dbec2152ca..80b56cee80924 100644 --- a/chart/docs/customizing-labels.rst +++ b/chart/docs/customizing-labels.rst @@ -42,7 +42,7 @@ You can also set specific labels for individual Airflow components, which will b If the same label key exists in both global and component-specific labels, the component-specific value takes precedence (overrides the global value). This allows you to customize labels for specific components while still maintaining common global labels across all resources. -For example, to add specific labels to different components: +For example, to add specific labels to different components like scheduler or api-server: .. code-block:: yaml :caption: values.yaml @@ -56,11 +56,6 @@ For example, to add specific labels to different components: labels: role: scheduler - # Worker specific labels - workers: - labels: - role: worker - # API Server specific labels apiServer: labels: diff --git a/chart/docs/production-guide.rst b/chart/docs/production-guide.rst index 94c72aa5cb1fd..ee8f7efd16eef 100644 --- a/chart/docs/production-guide.rst +++ b/chart/docs/production-guide.rst @@ -265,6 +265,17 @@ generated using the secret key has a short expiry time though. Make sure that ti that you run Airflow components on is synchronized (for example using ntpd). You might get "forbidden" errors when the logs are accessed otherwise. +JWT Secret +---------- + +You should set a static JWT Secret key when deploying with Airflow chart as it will increase environment +stability. It can be achieved by using ``jwtSecretName`` field in the ``values.yaml`` file. + +.. note:: + + For increase security of production setup, consider creating custom JWT Secret rollover procedure which will + not cause failures in dag runs due to mismatch in tokens. + Eviction configuration ---------------------- When running Airflow along with the `Kubernetes Cluster Autoscaler `_, it is important to configure whether pods can be safely evicted. diff --git a/chart/docs/setting-resources-for-containers.rst b/chart/docs/setting-resources-for-containers.rst index 600b690778ddd..1402d465d79cf 100644 --- a/chart/docs/setting-resources-for-containers.rst +++ b/chart/docs/setting-resources-for-containers.rst @@ -31,7 +31,7 @@ Possible containers where resources can be configured include: * Main Airflow containers and their sidecars. You can add the resources for these containers through the following parameters: * ``workers.resources`` - * ``workers.logGroomerSidecar.resources`` + * ``workers.celery.logGroomerSidecar.resources`` * ``workers.kerberosSidecar.resources`` * ``workers.kerberosInitContainer.resources`` * ``scheduler.resources`` diff --git a/chart/docs/using-additional-containers.rst b/chart/docs/using-additional-containers.rst index 60ce810492655..af339d683ea2d 100644 --- a/chart/docs/using-additional-containers.rst +++ b/chart/docs/using-additional-containers.rst @@ -22,7 +22,7 @@ Sidecar Containers ------------------ If you want to deploy your own sidecar container, you can add it through the ``extraContainers`` parameter. -You can define different containers for the scheduler, webserver, api server, worker, triggerer, dag processor, flower, create user job and migrate database job pods. +You can define different containers for the scheduler, webserver/api-server, Kubernetes/Celery workers, triggerer, dag processor, flower, create user job and migrate database job pods. For example, sidecars that sync Dags from object storage: @@ -34,15 +34,17 @@ For example, sidecars that sync Dags from object storage: - name: s3-sync image: my-company/s3-sync:latest imagePullPolicy: Always + workers: - extraContainers: - - name: s3-sync - image: my-company/s3-sync:latest - imagePullPolicy: Always + kubernetes: + extraContainers: + - name: s3-sync + image: my-company/s3-sync:latest + imagePullPolicy: Always .. note:: - If you use ``workers.extraContainers`` with ``KubernetesExecutor``, you are responsible for signaling + If you use ``workers.kubernetes.extraContainers`` (dedicated for ``KubernetesExecutor``), you are responsible for signaling sidecars to exit when the main container finishes so Airflow can continue the worker shutdown process. @@ -50,7 +52,7 @@ Init Containers --------------- You can also deploy extra init containers through the ``extraInitContainers`` parameter. -You can define different containers for the scheduler, webserver, api server, worker, triggerer, dag processor, create user job and migrate database job pods. +You can define different containers for the scheduler, webserver/api-server, Celery/Kubernetes workers, triggerer, dag processor, create user job and migrate database job pods. For example, an init container that just says hello: diff --git a/chart/files/pod-template-file.kubernetes-helm-yaml b/chart/files/pod-template-file.kubernetes-helm-yaml index 902391cd1a9e5..26b1b8505d0c1 100644 --- a/chart/files/pod-template-file.kubernetes-helm-yaml +++ b/chart/files/pod-template-file.kubernetes-helm-yaml @@ -18,9 +18,9 @@ */}} --- {{- $nodeSelector := or .Values.workers.kubernetes.nodeSelector .Values.workers.nodeSelector .Values.nodeSelector }} -{{- $affinity := or .Values.workers.affinity .Values.affinity }} -{{- $tolerations := or .Values.workers.tolerations .Values.tolerations }} -{{- $topologySpreadConstraints := or .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }} +{{- $affinity := or .Values.workers.kubernetes.affinity .Values.workers.affinity .Values.affinity }} +{{- $tolerations := or .Values.workers.kubernetes.tolerations .Values.workers.tolerations .Values.tolerations }} +{{- $topologySpreadConstraints := or .Values.workers.kubernetes.topologySpreadConstraints .Values.workers.topologySpreadConstraints .Values.topologySpreadConstraints }} {{- $securityContext := include "airflowPodSecurityContext" (list .Values.workers.kubernetes .Values.workers .Values) }} {{- $containerSecurityContextKerberosSidecar := include "containerSecurityContext" (list .Values.workers.kubernetes.kerberosSidecar .Values.workers.kerberosSidecar .Values) }} {{- $containerLifecycleHooksKerberosSidecar := or .Values.workers.kubernetes.kerberosSidecar.containerLifecycleHooks .Values.workers.kerberosSidecar.containerLifecycleHooks .Values.containerLifecycleHooks }} @@ -29,7 +29,7 @@ {{- $containerSecurityContext := include "containerSecurityContext" (list .Values.workers.kubernetes .Values.workers .Values) }} {{- $containerLifecycleHooks := or .Values.workers.kubernetes.containerLifecycleHooks .Values.workers.containerLifecycleHooks .Values.containerLifecycleHooks }} {{- $safeToEvict := dict "cluster-autoscaler.kubernetes.io/safe-to-evict" (or .Values.workers.kubernetes.safeToEvict (and (not (has .Values.workers.kubernetes.safeToEvict (list true false))) .Values.workers.safeToEvict) | toString) }} -{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations) $safeToEvict .Values.workers.podAnnotations }} +{{- $podAnnotations := mergeOverwrite (deepCopy .Values.airflowPodAnnotations) $safeToEvict (.Values.workers.kubernetes.podAnnotations | default .Values.workers.podAnnotations) }} {{- $schedulerName := or .Values.workers.kubernetes.schedulerName .Values.workers.schedulerName .Values.schedulerName }} apiVersion: v1 kind: Pod @@ -39,8 +39,8 @@ metadata: tier: airflow component: worker release: {{ .Release.Name }} - {{- if or (.Values.labels) (.Values.workers.labels) }} - {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} + {{- if or .Values.labels .Values.workers.labels .Values.workers.kubernetes.labels }} + {{- mustMerge (.Values.workers.kubernetes.labels | default .Values.workers.labels) .Values.labels | toYaml | nindent 4 }} {{- end }} annotations: {{- tpl (toYaml $podAnnotations) . | nindent 4 }} @@ -52,8 +52,8 @@ spec: {{- if and .Values.dags.gitSync.enabled (not .Values.dags.persistence.enabled) }} {{- include "git_sync_container" (dict "Values" .Values "is_init" "true" "Template" .Template) | nindent 4 }} {{- end }} - {{- if .Values.workers.extraInitContainers }} - {{- tpl (toYaml .Values.workers.extraInitContainers) . | nindent 4 }} + {{- if or .Values.workers.kubernetes.extraInitContainers .Values.workers.extraInitContainers }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraInitContainers | default .Values.workers.extraInitContainers)) . | nindent 4 }} {{- end }} {{- if or .Values.workers.kubernetes.kerberosInitContainer.enabled .Values.workers.kerberosInitContainer.enabled }} - name: kerberos-init @@ -86,8 +86,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 8 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }} + {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} {{- end }} {{- if semverCompare ">=3.0.0" .Values.airflowVersion }} {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} @@ -120,7 +120,7 @@ spec: {{- end }} {{- include "standard_airflow_environment" . | indent 6}} {{- include "custom_airflow_environment" . | indent 6 }} - {{- include "container_extra_envs" (list . .Values.workers.env) | indent 6 }} + {{- include "container_extra_envs" (list . (.Values.workers.kubernetes.env | default .Values.workers.env)) | indent 6 }} image: {{ template "pod_template_image" . }} imagePullPolicy: {{ .Values.images.pod_template.pullPolicy }} securityContext: {{ $containerSecurityContext | nindent 8 }} @@ -145,8 +145,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 8 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }} + {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} {{- end }} {{- if .Values.kerberos.enabled }} - name: kerberos-keytab @@ -192,8 +192,8 @@ spec: {{- if .Values.volumeMounts }} {{- toYaml .Values.volumeMounts | nindent 8 }} {{- end }} - {{- if .Values.workers.extraVolumeMounts }} - {{- tpl (toYaml .Values.workers.extraVolumeMounts) . | nindent 8 }} + {{- if or .Values.workers.extraVolumeMounts .Values.workers.kubernetes.extraVolumeMounts }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumeMounts | default .Values.workers.extraVolumeMounts)) . | nindent 8 }} {{- end }} {{- if semverCompare ">=3.0.0" .Values.airflowVersion }} {{- if or .Values.apiServer.apiServerConfig .Values.apiServer.apiServerConfigConfigMapName }} @@ -213,8 +213,8 @@ spec: {{- include "custom_airflow_environment" . | indent 6 }} {{- include "standard_airflow_environment" . | indent 6 }} {{- end }} - {{- if .Values.workers.extraContainers }} - {{- tpl (toYaml .Values.workers.extraContainers) . | nindent 4 }} + {{- if or .Values.workers.kubernetes.extraContainers .Values.workers.extraContainers }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraContainers | default .Values.workers.extraContainers)) . | nindent 4 }} {{- end }} {{- if or .Values.workers.kubernetes.priorityClassName .Values.workers.priorityClassName }} priorityClassName: {{ .Values.workers.kubernetes.priorityClassName | default .Values.workers.priorityClassName }} @@ -236,7 +236,11 @@ spec: terminationGracePeriodSeconds: {{ .Values.workers.kubernetes.terminationGracePeriodSeconds | default .Values.workers.terminationGracePeriodSeconds }} tolerations: {{- toYaml $tolerations | nindent 4 }} topologySpreadConstraints: {{- toYaml $topologySpreadConstraints | nindent 4 }} + {{- if .Values.workers.kubernetes.serviceAccount.create }} + serviceAccountName: {{ include "worker.kubernetes.serviceAccountName" . }} + {{- else }} serviceAccountName: {{ include "worker.serviceAccountName" . }} + {{- end }} volumes: {{- if .Values.dags.persistence.enabled }} - name: dags @@ -283,6 +287,6 @@ spec: - name: kerberos-ccache emptyDir: {} {{- end }} - {{- if .Values.workers.extraVolumes }} - {{- tpl (toYaml .Values.workers.extraVolumes) . | nindent 2 }} + {{- if or .Values.workers.kubernetes.extraVolumes .Values.workers.extraVolumes }} + {{- tpl (toYaml (.Values.workers.kubernetes.extraVolumes | default .Values.workers.extraVolumes)) . | nindent 2 }} {{- end }} diff --git a/chart/newsfragments/62054.significant.rst b/chart/newsfragments/62054.significant.rst new file mode 100644 index 0000000000000..7b564a363f48d --- /dev/null +++ b/chart/newsfragments/62054.significant.rst @@ -0,0 +1 @@ +``workers.waitForMigrations`` section is now deprecated in favor of ``workers.celery.waitForMigrations``. Please update your configuration accordingly. diff --git a/chart/newsfragments/64730.significant.rst b/chart/newsfragments/64730.significant.rst new file mode 100644 index 0000000000000..6213c4e8f92fb --- /dev/null +++ b/chart/newsfragments/64730.significant.rst @@ -0,0 +1 @@ +``workers.serviceAccount`` section is now deprecated in favor of ``workers.celery.serviceAccount`` and ``workers.kubernetes.serviceAccount``. Please update your configuration accordingly. diff --git a/chart/newsfragments/64734.significant.rst b/chart/newsfragments/64734.significant.rst new file mode 100644 index 0000000000000..52c2ebaee4382 --- /dev/null +++ b/chart/newsfragments/64734.significant.rst @@ -0,0 +1 @@ +``workers.hpa`` section is now deprecated in favor of ``workers.celery.hpa``. Please update your configuration accordingly. diff --git a/chart/newsfragments/64739.significant.rst b/chart/newsfragments/64739.significant.rst new file mode 100644 index 0000000000000..3ae49c41234e6 --- /dev/null +++ b/chart/newsfragments/64739.significant.rst @@ -0,0 +1 @@ +``workers.extraContainers`` field is now deprecated in favor of ``workers.celery.extraContainers`` and ``workers.kubernetes.extraContainers``. Please update your configuration accordingly. diff --git a/chart/newsfragments/64741.significant.rst b/chart/newsfragments/64741.significant.rst new file mode 100644 index 0000000000000..18a90f8f1da9c --- /dev/null +++ b/chart/newsfragments/64741.significant.rst @@ -0,0 +1 @@ +``workers.extraInitContainers`` field is now deprecated in favor of ``workers.celery.extraInitContainers`` and ``workers.kubernetes.extraInitContainers``. Please update your configuration accordingly. diff --git a/chart/newsfragments/64746.significant.rst b/chart/newsfragments/64746.significant.rst new file mode 100644 index 0000000000000..9aafcef769685 --- /dev/null +++ b/chart/newsfragments/64746.significant.rst @@ -0,0 +1 @@ +``workers.extraVolumes`` field is now deprecated in favor of ``workers.celery.extraVolumes`` and ``workers.kubernetes.extraVolumes``. Please update your configuration accordingly. diff --git a/chart/newsfragments/64841.significant.rst b/chart/newsfragments/64841.significant.rst new file mode 100644 index 0000000000000..de33b33a0c013 --- /dev/null +++ b/chart/newsfragments/64841.significant.rst @@ -0,0 +1,3 @@ +Default Airflow image is updated to ``3.2.0`` + +The default Airflow image that is used with the Chart is now ``3.2.0``, previously it was ``3.1.8``. diff --git a/chart/newsfragments/64860.significant.rst b/chart/newsfragments/64860.significant.rst new file mode 100644 index 0000000000000..672bd787a514c --- /dev/null +++ b/chart/newsfragments/64860.significant.rst @@ -0,0 +1 @@ +``workers.affinity`` field is now deprecated in favor of ``workers.celery.affinity`` and ``workers.kubernetes.affinity``. Please update your configuration accordingly. diff --git a/chart/newsfragments/64976.significant.rst b/chart/newsfragments/64976.significant.rst new file mode 100644 index 0000000000000..2a43592ae27c3 --- /dev/null +++ b/chart/newsfragments/64976.significant.rst @@ -0,0 +1 @@ +``workers.tolerations`` field is now deprecated in favor of ``workers.celery.tolerations`` and ``workers.kubernetes.tolerations``. Please update your configuration accordingly. diff --git a/chart/newsfragments/64980.significant.rst b/chart/newsfragments/64980.significant.rst new file mode 100644 index 0000000000000..1fd365a31d0dd --- /dev/null +++ b/chart/newsfragments/64980.significant.rst @@ -0,0 +1 @@ +``workers.topologySpreadConstraints`` field is now deprecated in favor of ``workers.celery.topologySpreadConstraints`` and ``workers.kubernetes.topologySpreadConstraints``. Please update your configuration accordingly. diff --git a/chart/newsfragments/64982.significant.rst b/chart/newsfragments/64982.significant.rst new file mode 100644 index 0000000000000..d850632175130 --- /dev/null +++ b/chart/newsfragments/64982.significant.rst @@ -0,0 +1 @@ +``workers.annotations`` field is now deprecated in favor of ``workers.celery.annotations``. Please update your configuration accordingly. diff --git a/chart/newsfragments/65027.significant.rst b/chart/newsfragments/65027.significant.rst new file mode 100644 index 0000000000000..861f12339888b --- /dev/null +++ b/chart/newsfragments/65027.significant.rst @@ -0,0 +1 @@ +``workers.podAnnotations`` field is now deprecated in favor of ``workers.celery.podAnnotations`` and ``workers.kubernetes.podAnnotations``. Please update your configuration accordingly. diff --git a/chart/newsfragments/65030.significant.rst b/chart/newsfragments/65030.significant.rst new file mode 100644 index 0000000000000..055bfa7651919 --- /dev/null +++ b/chart/newsfragments/65030.significant.rst @@ -0,0 +1 @@ +``workers.labels`` field is now deprecated in favor of ``workers.celery.labels`` and ``workers.kubernetes.labels``. Please update your configuration accordingly. diff --git a/chart/newsfragments/65033.significant.rst b/chart/newsfragments/65033.significant.rst new file mode 100644 index 0000000000000..ee4ec2bfd2092 --- /dev/null +++ b/chart/newsfragments/65033.significant.rst @@ -0,0 +1 @@ +``workers.logGroomerSidecar`` section is now deprecated in favor of ``workers.celery.logGroomerSidecar``. Please update your configuration accordingly. diff --git a/chart/newsfragments/65056.significant.rst b/chart/newsfragments/65056.significant.rst new file mode 100644 index 0000000000000..6e919a246200e --- /dev/null +++ b/chart/newsfragments/65056.significant.rst @@ -0,0 +1 @@ +``workers.env`` field is now deprecated in favor of ``workers.celery.env`` and ``workers.kubernetes.env``. Please update your configuration accordingly. diff --git a/chart/newsfragments/65059.significant.rst b/chart/newsfragments/65059.significant.rst new file mode 100644 index 0000000000000..b45e0d13108de --- /dev/null +++ b/chart/newsfragments/65059.significant.rst @@ -0,0 +1 @@ +``workers.extraVolumeMounts`` field is now deprecated in favor of ``workers.celery.extraVolumeMounts`` and ``workers.kubernetes.extraVolumeMounts``. Please update your configuration accordingly. diff --git a/chart/templates/NOTES.txt b/chart/templates/NOTES.txt index f752f3f5ef0cb..57d94edf22cb7 100644 --- a/chart/templates/NOTES.txt +++ b/chart/templates/NOTES.txt @@ -413,6 +413,38 @@ DEPRECATION WARNING: {{- end }} +{{- if not .Values.workers.serviceAccount.automountServiceAccountToken }} + + DEPRECATION WARNING: + `workers.serviceAccount.automountServiceAccountToken` has been renamed to `workers.celery.serviceAccount.automountServiceAccountToken`/`workers.kubernetes.serviceAccount.automountServiceAccountToken`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.workers.serviceAccount.create }} + + DEPRECATION WARNING: + `workers.serviceAccount.create` has been renamed to `workers.celery.serviceAccount.create`/`workers.kubernetes.serviceAccount.create`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.serviceAccount.name) }} + + DEPRECATION WARNING: + `workers.serviceAccount.name` has been renamed to `workers.celery.serviceAccount.name`/`workers.kubernetes.serviceAccount.name`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.serviceAccount.annotations) }} + + DEPRECATION WARNING: + `workers.serviceAccount.annotations` has been renamed to `workers.celery.serviceAccount.annotations`/`workers.kubernetes.serviceAccount.annotations`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if .Values.workers.keda.enabled }} DEPRECATION WARNING: @@ -485,6 +517,46 @@ DEPRECATION WARNING: {{- end }} +{{- if .Values.workers.hpa.enabled }} + + DEPRECATION WARNING: + `workers.hpa.enabled` has been renamed to `workers.celery.hpa.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.hpa.minReplicaCount) 0 }} + + DEPRECATION WARNING: + `workers.hpa.minReplicaCount` has been renamed to `workers.celery.hpa.minReplicaCount`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.hpa.maxReplicaCount) 5 }} + + DEPRECATION WARNING: + `workers.hpa.maxReplicaCount` has been renamed to `workers.celery.hpa.maxReplicaCount`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (toJson .Values.workers.hpa.metrics | quote) (toJson "[{\"resource\":{\"name\":\"cpu\",\"target\":{\"averageUtilization\":80,\"type\":\"Utilization\"}},\"type\":\"Resource\"}]") }} + + DEPRECATION WARNING: + `workers.hpa.metrics` has been renamed to `workers.celery.hpa.metrics`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.hpa.behavior) }} + + DEPRECATION WARNING: + `workers.hpa.behavior` has been renamed to `workers.celery.hpa.behavior`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not .Values.workers.persistence.enabled }} DEPRECATION WARNING: @@ -629,6 +701,38 @@ DEPRECATION WARNING: {{- end }} +{{- if not (empty .Values.workers.extraContainers) }} + + DEPRECATION WARNING: + `workers.extraContainers` has been renamed to `workers.celery.extraContainers`/`workers.kubernetes.extraContainers`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.extraInitContainers) }} + + DEPRECATION WARNING: + `workers.extraInitContainers` has been renamed to `workers.celery.extraInitContainers`/`workers.kubernetes.extraInitContainers`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.extraVolumes) }} + + DEPRECATION WARNING: + `workers.extraVolumes` has been renamed to `workers.celery.extraVolumes`/`workers.kubernetes.extraVolumes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.extraVolumeMounts) }} + + DEPRECATION WARNING: + `workers.extraVolumeMounts` has been renamed to `workers.celery.extraVolumeMounts`/`workers.kubernetes.extraVolumeMounts`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not (empty .Values.workers.runtimeClassName) }} DEPRECATION WARNING: @@ -645,6 +749,30 @@ DEPRECATION WARNING: {{- end }} +{{- if not (empty .Values.workers.affinity) }} + + DEPRECATION WARNING: + `workers.affinity` has been renamed to `workers.celery.affinity`/`workers.kubernetes.affinity`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.tolerations) }} + + DEPRECATION WARNING: + `workers.tolerations` has been renamed to `workers.celery.tolerations`/`workers.kubernetes.tolerations`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.topologySpreadConstraints) }} + + DEPRECATION WARNING: + `workers.topologySpreadConstraints` has been renamed to `workers.celery.topologySpreadConstraints`/`workers.kubernetes.topologySpreadConstraints`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not (empty .Values.workers.nodeSelector) }} DEPRECATION WARNING: @@ -669,6 +797,30 @@ DEPRECATION WARNING: {{- end }} +{{- if not (empty .Values.workers.annotations) }} + + DEPRECATION WARNING: + `workers.annotations` has been renamed to `workers.celery.annotations`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.podAnnotations) }} + + DEPRECATION WARNING: + `workers.podAnnotations` has been renamed to `workers.celery.podAnnotations`/`workers.kubernetes.podAnnotations`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.labels) }} + + DEPRECATION WARNING: + `workers.labels` has been renamed to `workers.celery.labels`/`workers.kubernetes.labels`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not (empty .Values.workers.volumeClaimTemplates) }} DEPRECATION WARNING: @@ -685,6 +837,134 @@ DEPRECATION WARNING: {{- end }} +{{- if not .Values.workers.logGroomerSidecar.enabled }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.enabled` has been renamed to `workers.celery.logGroomerSidecar.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.command) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.command` has been renamed to `workers.celery.logGroomerSidecar.command`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (.Values.workers.logGroomerSidecar.args | toJson) (list "bash" "/clean-logs" | toJson) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.args` has been renamed to `workers.celery.logGroomerSidecar.args`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.retentionDays) 15 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.retentionDays` has been renamed to `workers.celery.logGroomerSidecar.retentionDays`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.retentionMinutes) 0 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.retentionMinutes` has been renamed to `workers.celery.logGroomerSidecar.retentionMinutes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.frequencyMinutes) 15 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.frequencyMinutes` has been renamed to `workers.celery.logGroomerSidecar.frequencyMinutes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.maxSizeBytes) 0 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.maxSizeBytes` has been renamed to `workers.celery.logGroomerSidecar.maxSizeBytes`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if ne (int .Values.workers.logGroomerSidecar.maxSizePercent) 0 }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.maxSizePercent` has been renamed to `workers.celery.logGroomerSidecar.maxSizePercent`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.resources) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.resources` has been renamed to `workers.celery.logGroomerSidecar.resources`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.securityContexts.container) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.securityContexts.container` has been renamed to `workers.celery.logGroomerSidecar.securityContexts.container`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.env) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.env` has been renamed to `workers.celery.logGroomerSidecar.env`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.logGroomerSidecar.containerLifecycleHooks) }} + + DEPRECATION WARNING: + `workers.logGroomerSidecar.containerLifecycleHooks` has been renamed to `workers.celery.logGroomerSidecar.containerLifecycleHooks`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not .Values.workers.waitForMigrations.enabled }} + + DEPRECATION WARNING: + `workers.waitForMigrations.enabled` has been renamed to `workers.celery.waitForMigrations.enabled`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.waitForMigrations.env) }} + + DEPRECATION WARNING: + `workers.waitForMigrations.env` has been renamed to `workers.celery.waitForMigrations.env`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.waitForMigrations.securityContexts.container) }} + + DEPRECATION WARNING: + `workers.waitForMigrations.securityContexts.container` has been renamed to `workers.celery.waitForMigrations.securityContexts.container`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + +{{- if not (empty .Values.workers.env) }} + + DEPRECATION WARNING: + `workers.env` has been renamed to `workers.celery.env`/`workers.kubernetes.env`. + Please change your values as support for the old name will be dropped in a future release. + +{{- end }} + {{- if not (empty .Values.webserver.defaultUser) }} DEPRECATION WARNING: diff --git a/chart/templates/_helpers.yaml b/chart/templates/_helpers.yaml index 5ef1759f604fc..b9a7587b6b28f 100644 --- a/chart/templates/_helpers.yaml +++ b/chart/templates/_helpers.yaml @@ -50,6 +50,14 @@ If release name contains chart name it will be used as a full name. {{- end }} {{- end }} +{{- define "airflow.tplDict" -}} + {{- $rendered := dict -}} + {{- range $key, $value := .values }} + {{- $_ := set $rendered $key (tpl (toString $value) $.context) -}} + {{- end }} + {{- toYaml $rendered -}} +{{- end }} + {{/* Standard Airflow environment variables */}} {{- define "standard_airflow_environment" }} # Hard Coded Airflow Envs @@ -497,8 +505,8 @@ If release name contains chart name it will be used as a full name. {{ $pgMetadataHost := .Values.data.metadataConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} {{ $pgResultBackendHost := $resultBackendConnection.host | default (printf "%s-%s.%s" .Release.Name "postgresql" .Release.Namespace) }} [databases] -{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ .Values.data.metadataConnection.db }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }} {{ .Values.pgbouncer.extraIniMetadata | default "" }} -{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ $resultBackendConnection.db }} port={{ $resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }} {{ .Values.pgbouncer.extraIniResultBackend | default "" }} +{{ .Release.Name }}-metadata = host={{ $pgMetadataHost }} dbname={{ tpl .Values.data.metadataConnection.db . }} port={{ .Values.data.metadataConnection.port }} pool_size={{ .Values.pgbouncer.metadataPoolSize }} {{ .Values.pgbouncer.extraIniMetadata | default "" }} +{{ .Release.Name }}-result-backend = host={{ $pgResultBackendHost }} dbname={{ tpl $resultBackendConnection.db . }} port={{ $resultBackendConnection.port }} pool_size={{ .Values.pgbouncer.resultBackendPoolSize }} {{ .Values.pgbouncer.extraIniResultBackend | default "" }} [pgbouncer] pool_mode = transaction @@ -506,7 +514,7 @@ listen_port = {{ .Values.ports.pgbouncer }} listen_addr = * auth_type = {{ .Values.pgbouncer.auth_type }} auth_file = {{ .Values.pgbouncer.auth_file }} -stats_users = {{ .Values.data.metadataConnection.user }} +stats_users = {{ tpl .Values.data.metadataConnection.user . }} ignore_startup_parameters = extra_float_digits max_client_conn = {{ .Values.pgbouncer.maxClientConn }} verbose = {{ .Values.pgbouncer.verbose }} @@ -533,8 +541,8 @@ server_tls_key_file = /etc/pgbouncer/server.key {{ define "pgbouncer_users" }} {{- $resultBackendConnection := .Values.data.resultBackendConnection | default .Values.data.metadataConnection }} -{{ .Values.data.metadataConnection.user | quote }} {{ .Values.data.metadataConnection.pass | quote }} -{{ $resultBackendConnection.user | quote }} {{ $resultBackendConnection.pass | quote }} +{{ tpl .Values.data.metadataConnection.user . | quote }} {{ .Values.data.metadataConnection.pass | quote }} +{{ tpl $resultBackendConnection.user . | quote }} {{ $resultBackendConnection.pass | quote }} {{- end }} {{- define "airflow_logs" -}} @@ -599,7 +607,11 @@ server_tls_key_file = /etc/pgbouncer/server.key {{- end }} {{- define "airflow_webserver_config_configmap_name" -}} - {{- default (printf "%s-webserver-config" (include "airflow.fullname" .)) .Values.webserver.webserverConfigConfigMapName }} + {{- if .Values.webserver.webserverConfigConfigMapName }} + {{- tpl .Values.webserver.webserverConfigConfigMapName . }} + {{- else }} + {{- printf "%s-webserver-config" (include "airflow.fullname" .) }} + {{- end }} {{- end }} {{- define "airflow_webserver_config_mount" -}} @@ -610,7 +622,11 @@ server_tls_key_file = /etc/pgbouncer/server.key {{- end }} {{- define "airflow_api_server_config_configmap_name" -}} - {{- default (printf "%s-api-server-config" (include "airflow.fullname" .)) .Values.apiServer.apiServerConfigConfigMapName }} + {{- if .Values.apiServer.apiServerConfigConfigMapName }} + {{- tpl .Values.apiServer.apiServerConfigConfigMapName . }} + {{- else }} + {{- printf "%s-api-server-config" (include "airflow.fullname" .) }} + {{- end }} {{- end }} {{- define "airflow_api_server_config_mount" -}} @@ -641,13 +657,23 @@ server_tls_key_file = /etc/pgbouncer/server.key {{- end }} {{- end }} -{{/* Helper to generate service account name respecting .Values.$section.serviceAccount flags */}} +{{/* Helper for service account name generation */}} +{{- define "_serviceAccountNameGen" -}} + {{- if .sa.create }} + {{- default (printf "%s-%s" (include "airflow.serviceAccountName" .) (default .key .nameSuffix)) .sa.name | quote }} + {{- else }} + {{- default "default" .sa.name | quote }} + {{- end }} +{{- end }} + +{{/* Helper to generate service account name respecting .Values.$section.serviceAccount or .Values.$section.$subSection.serviceAccount flags */}} {{- define "_serviceAccountName" -}} - {{- $sa := get (get .Values .key) "serviceAccount" }} - {{- if $sa.create }} - {{- default (printf "%s-%s" (include "airflow.serviceAccountName" .) (default .key .nameSuffix )) $sa.name | quote }} + {{- if .subKey }} + {{- $sa := get (get (get .Values .key) .subKey) "serviceAccount" -}} + {{- include "_serviceAccountNameGen" (merge (dict "sa" $sa "key" .key "nameSuffix" .nameSuffix) .) }} {{- else }} - {{- default "default" $sa.name | quote }} + {{- $sa := get (get .Values .key) "serviceAccount" }} + {{- include "_serviceAccountNameGen" (merge (dict "sa" $sa "key" .key "nameSuffix" .nameSuffix) .) }} {{- end }} {{- end }} @@ -700,6 +726,11 @@ server_tls_key_file = /etc/pgbouncer/server.key {{- end }} {{- end }} +{{/* Create the name of the worker kubernetes service account to use */}} +{{- define "worker.kubernetes.serviceAccountName" -}} + {{- include "_serviceAccountName" (merge (dict "key" "workers" "subKey" "kubernetes" "nameSuffix" "worker-kubernetes") .) -}} +{{- end }} + {{/* Create the name of the triggerer service account to use */}} {{- define "triggerer.serviceAccountName" -}} {{- include "_serviceAccountName" (merge (dict "key" "triggerer") .) -}} diff --git a/chart/templates/api-server/api-server-serviceaccount.yaml b/chart/templates/api-server/api-server-serviceaccount.yaml index 0cd9984df96ec..f9bbaf8df5775 100644 --- a/chart/templates/api-server/api-server-serviceaccount.yaml +++ b/chart/templates/api-server/api-server-serviceaccount.yaml @@ -36,6 +36,7 @@ metadata: {{- mustMerge .Values.apiServer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.apiServer.serviceAccount.annotations }} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/dag-processor/dag-processor-serviceaccount.yaml b/chart/templates/dag-processor/dag-processor-serviceaccount.yaml index e47511617f12f..89b71eb40f45e 100644 --- a/chart/templates/dag-processor/dag-processor-serviceaccount.yaml +++ b/chart/templates/dag-processor/dag-processor-serviceaccount.yaml @@ -40,6 +40,7 @@ metadata: {{- mustMerge .Values.dagProcessor.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.dagProcessor.serviceAccount.annotations}} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/database-cleanup/database-cleanup-cronjob.yaml b/chart/templates/database-cleanup/database-cleanup-cronjob.yaml index f22db1fc47dc4..ae506f7080004 100644 --- a/chart/templates/database-cleanup/database-cleanup-cronjob.yaml +++ b/chart/templates/database-cleanup/database-cleanup-cronjob.yaml @@ -56,6 +56,9 @@ spec: jobTemplate: spec: backoffLimit: 1 + {{- if not (kindIs "invalid" .Values.databaseCleanup.ttlSecondsAfterFinished) }} + ttlSecondsAfterFinished: {{ .Values.databaseCleanup.ttlSecondsAfterFinished }} + {{- end }} template: metadata: labels: diff --git a/chart/templates/pgbouncer/pgbouncer-serviceaccount.yaml b/chart/templates/pgbouncer/pgbouncer-serviceaccount.yaml index c9f757eb10390..53712ff3884b4 100644 --- a/chart/templates/pgbouncer/pgbouncer-serviceaccount.yaml +++ b/chart/templates/pgbouncer/pgbouncer-serviceaccount.yaml @@ -36,6 +36,7 @@ metadata: {{- mustMerge .Values.pgbouncer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.pgbouncer.serviceAccount.annotations }} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/scheduler/scheduler-serviceaccount.yaml b/chart/templates/scheduler/scheduler-serviceaccount.yaml index b87a5149d4f47..0142c367c9a03 100644 --- a/chart/templates/scheduler/scheduler-serviceaccount.yaml +++ b/chart/templates/scheduler/scheduler-serviceaccount.yaml @@ -39,8 +39,6 @@ metadata: {{- end }} {{- with .Values.scheduler.serviceAccount.annotations }} annotations: - {{- range $key, $value := . }} - {{- printf "%s: %s" $key (tpl $value $ | quote) | nindent 4 }} - {{- end }} + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/secrets/jwt-secret.yaml b/chart/templates/secrets/jwt-secret.yaml index 04f04a3a70aa2..288a5490038f9 100644 --- a/chart/templates/secrets/jwt-secret.yaml +++ b/chart/templates/secrets/jwt-secret.yaml @@ -40,5 +40,5 @@ metadata: {{- end }} type: Opaque data: - jwt-secret: {{ .Values.jwtSecret | default (randAlphaNum 32) | b64enc | quote }} + jwt-secret: {{ .Values.jwtSecret | default (randAlphaNum 128) | b64enc | quote }} {{- end }} diff --git a/chart/templates/secrets/metadata-connection-secret.yaml b/chart/templates/secrets/metadata-connection-secret.yaml index d64637d805eab..71441755d4bae 100644 --- a/chart/templates/secrets/metadata-connection-secret.yaml +++ b/chart/templates/secrets/metadata-connection-secret.yaml @@ -27,7 +27,8 @@ {{- $host := ternary $pgbouncerHost $metadataHost .Values.pgbouncer.enabled }} {{- $metadataPort := .Values.data.metadataConnection.port | toString }} {{- $port := (ternary .Values.ports.pgbouncer $metadataPort .Values.pgbouncer.enabled) | toString }} -{{- $metadataDatabase := .Values.data.metadataConnection.db }} +{{- $metadataUser := tpl .Values.data.metadataConnection.user . }} +{{- $metadataDatabase := tpl .Values.data.metadataConnection.db . }} {{- $database := ternary (printf "%s-%s" .Release.Name "metadata") $metadataDatabase .Values.pgbouncer.enabled }} {{- $query := ternary (printf "sslmode=%s" .Values.data.metadataConnection.sslmode) "" (eq .Values.data.metadataConnection.protocol "postgresql") }} {{- $kedaEnabled := .Values.workers.keda.enabled }} @@ -55,15 +56,15 @@ metadata: type: Opaque data: {{- with .Values.data.metadataConnection }} - connection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }} + connection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $host $port) "path" (printf "/%s" $database) "query" $query) | b64enc | quote }} {{- end }} {{- if and $kedaEnabled .Values.pgbouncer.enabled (not $kedaUsePgBouncer) }} {{- with .Values.data.metadataConnection }} - kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc | quote }} + kedaConnection: {{ urlJoin (dict "scheme" .protocol "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "%s:%s" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | b64enc | quote }} {{- end }} {{- else if and (or $kedaEnabled .Values.triggerer.keda.enabled) (eq .Values.data.metadataConnection.protocol "mysql") }} {{- with .Values.data.metadataConnection }} - kedaConnection: {{ urlJoin (dict "userinfo" (printf "%s:%s" (.user | urlquery) (.pass | urlquery) ) "host" (printf "tcp(%s:%s)" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | trimPrefix "//" | b64enc | quote }} + kedaConnection: {{ urlJoin (dict "userinfo" (printf "%s:%s" ($metadataUser | urlquery) (.pass | urlquery) ) "host" (printf "tcp(%s:%s)" $metadataHost $metadataPort) "path" (printf "/%s" $metadataDatabase) "query" $query) | trimPrefix "//" | b64enc | quote }} {{- end }} {{- end }} {{- end }} diff --git a/chart/templates/triggerer/triggerer-serviceaccount.yaml b/chart/templates/triggerer/triggerer-serviceaccount.yaml index 27fd76d08021e..e2fed78e3329a 100644 --- a/chart/templates/triggerer/triggerer-serviceaccount.yaml +++ b/chart/templates/triggerer/triggerer-serviceaccount.yaml @@ -36,6 +36,7 @@ metadata: {{- mustMerge .Values.triggerer.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.triggerer.serviceAccount.annotations}} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/webserver/webserver-serviceaccount.yaml b/chart/templates/webserver/webserver-serviceaccount.yaml index e105dbde0a039..c3ae6aff97838 100644 --- a/chart/templates/webserver/webserver-serviceaccount.yaml +++ b/chart/templates/webserver/webserver-serviceaccount.yaml @@ -36,6 +36,7 @@ metadata: {{- mustMerge .Values.webserver.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.webserver.serviceAccount.annotations }} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} diff --git a/chart/templates/workers/worker-deployment.yaml b/chart/templates/workers/worker-deployment.yaml index 0d221fd003a22..340c9aaea3e13 100644 --- a/chart/templates/workers/worker-deployment.yaml +++ b/chart/templates/workers/worker-deployment.yaml @@ -32,11 +32,11 @@ {{- $workers := (include "workersMergeValues" (list $mergedWorkers $workerSet "" list) | fromYaml) -}} {{- $_ := set $globals.Values "workers" $workers -}} {{- with $globals -}} +{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }} +--- {{- $persistence := or .Values.workers.persistence.enabled }} {{- $keda := .Values.workers.keda.enabled }} {{- $hpa := and .Values.workers.hpa.enabled (not .Values.workers.keda.enabled) }} -{{- if or (contains "CeleryExecutor" .Values.executor) (contains "CeleryKubernetesExecutor" .Values.executor) }} ---- {{- $nodeSelector := or .Values.workers.nodeSelector .Values.nodeSelector }} {{- $affinity := or .Values.workers.affinity .Values.affinity }} {{- $tolerations := or .Values.workers.tolerations .Values.tolerations }} diff --git a/chart/templates/workers/worker-kubernetes-serviceaccount.yaml b/chart/templates/workers/worker-kubernetes-serviceaccount.yaml new file mode 100644 index 0000000000000..b74474b42f288 --- /dev/null +++ b/chart/templates/workers/worker-kubernetes-serviceaccount.yaml @@ -0,0 +1,41 @@ +{{/* + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. +*/}} + +########################################### +## Airflow Worker Kubernetes ServiceAccount +########################################### +{{- if and .Values.workers.kubernetes.serviceAccount.create (contains "KubernetesExecutor" .Values.executor) }} +apiVersion: v1 +kind: ServiceAccount +automountServiceAccountToken: {{ or .Values.workers.kubernetes.serviceAccount.automountServiceAccountToken (and (not (has .Values.workers.kubernetes.serviceAccount.automountServiceAccountToken (list true false))) .Values.workers.serviceAccount.automountServiceAccountToken) }} +metadata: + name: {{ include "worker.kubernetes.serviceAccountName" . }} + labels: + tier: airflow + component: worker + release: {{ .Release.Name }} + chart: "{{ .Chart.Name }}-{{ .Chart.Version }}" + heritage: {{ .Release.Service }} + {{- if or .Values.labels .Values.workers.labels .Values.workers.kubernetes.labels }} + {{- mustMerge (.Values.workers.kubernetes.labels | default .Values.workers.labels) .Values.labels | toYaml | nindent 4 }} + {{- end }} + {{- with (.Values.workers.kubernetes.serviceAccount.annotations | default .Values.workers.serviceAccount.annotations) }} + annotations: {{- toYaml . | nindent 4 }} + {{- end }} +{{- end }} diff --git a/chart/templates/workers/worker-serviceaccount.yaml b/chart/templates/workers/worker-serviceaccount.yaml index d8e377f56f401..cbcf95381e834 100644 --- a/chart/templates/workers/worker-serviceaccount.yaml +++ b/chart/templates/workers/worker-serviceaccount.yaml @@ -49,7 +49,8 @@ metadata: {{- mustMerge .Values.workers.labels .Values.labels | toYaml | nindent 4 }} {{- end }} {{- with .Values.workers.serviceAccount.annotations}} - annotations: {{- toYaml . | nindent 4 }} + annotations: + {{- include "airflow.tplDict" (dict "values" . "context" $) | nindent 4 }} {{- end }} {{- end }} {{- end }} diff --git a/chart/values.schema.json b/chart/values.schema.json index 17eae47f3e377..da245664137fd 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -78,7 +78,7 @@ "defaultAirflowTag": { "description": "Default airflow tag to deploy.", "type": "string", - "default": "3.1.8", + "default": "3.2.0", "x-docsSection": "Common" }, "defaultAirflowDigest": { @@ -93,7 +93,7 @@ "airflowVersion": { "description": "Airflow version (Used to make some decisions based on Airflow Version being deployed). Version 2.11.0 and above is supported.", "type": "string", - "default": "3.1.8", + "default": "3.2.0", "x-docsSection": "Common" }, "securityContext": { @@ -1617,7 +1617,7 @@ "default": null }, "jwtSecret": { - "description": "Secret key used to encode and decode JWTs to authenticate to public and private APIs (can only be set during install, not upgrade).", + "description": "Secret key used to encode and decode JWTs to authenticate to public and private APIs. Note: It is not recommended to use in production as during helm upgrade it will be changed which can cause dag failures during component rollover.", "type": [ "string", "null" @@ -1848,21 +1848,21 @@ } }, "serviceAccount": { - "description": "Create ServiceAccount for Airflow Celery workers and pods created with pod-template-file.", + "description": "Create ServiceAccount for Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.serviceAccount`` and/or ``workers.kubernetes.serviceAccount`` instead).", "type": "object", "properties": { "automountServiceAccountToken": { - "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods", + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods (deprecated, use ``workers.celery.serviceAccount.automountServiceAccountToken`` and/or ``workers.kubernetes.serviceAccount.automountServiceAccountToken`` instead)", "type": "boolean", "default": true }, "create": { - "description": "Specifies whether a ServiceAccount should be created.", + "description": "Specifies whether a ServiceAccount should be created (deprecated, use ``workers.celery.serviceAccount.create`` and/or ``workers.kubernetes.serviceAccount.create`` instead).", "type": "boolean", "default": true }, "name": { - "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "description": "The name of the ServiceAccount to use (deprecated, use ``workers.celery.serviceAccount.name`` and/or ``workers.kubernetes.serviceAccount.name`` instead). If not set and create is true, a name is generated using the release name.", "type": [ "string", "null" @@ -1870,7 +1870,7 @@ "default": null }, "annotations": { - "description": "Annotations to add to the worker Kubernetes ServiceAccount.", + "description": "Annotations to add to the worker Kubernetes ServiceAccount (deprecated, use ``workers.celery.serviceAccount.annotations`` and/or ``workers.kubernetes.serviceAccount.annotations`` instead).", "type": "object", "default": {}, "additionalProperties": { @@ -1951,27 +1951,27 @@ } }, "hpa": { - "description": "HPA configuration for Airflow Celery workers.", + "description": "HPA configuration for Airflow Celery workers (deprecated, use ``workers.celery.hpa`` instead).", "type": "object", "additionalProperties": false, "properties": { "enabled": { - "description": "Allow HPA autoscaling (KEDA must be disabled).", + "description": "Allow HPA autoscaling (KEDA must be disabled) (deprecated, use ``workers.celery.hpa.enabled`` instead).", "type": "boolean", "default": false }, "minReplicaCount": { - "description": "Minimum number of Airflow Celery workers created by HPA.", + "description": "Minimum number of Airflow Celery workers created by HPA (deprecated, use ``workers.celery.hpa.minReplicaCount`` instead).", "type": "integer", "default": 0 }, "maxReplicaCount": { - "description": "Maximum number of Airflow Celery workers created by HPA.", + "description": "Maximum number of Airflow Celery workers created by HPA (deprecated, use ``workers.celery.hpa.maxReplicaCount`` instead).", "type": "integer", "default": 5 }, "metrics": { - "description": "Specifications for which to use to calculate the desired replica count.", + "description": "Specifications for which to use to calculate the desired replica count (deprecated, use ``workers.celery.hpa.metrics`` instead).", "type": "array", "default": [ { @@ -1990,7 +1990,7 @@ } }, "behavior": { - "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.", + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target (deprecated, use ``workers.celery.hpa.behavior`` instead).", "type": "object", "default": {}, "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" @@ -2293,7 +2293,7 @@ "default": false }, "extraContainers": { - "description": "Launch additional containers into Airflow Celery workers and pods created with pod-template-file (templated). Note, if used with KubernetesExecutor, you are responsible for signaling sidecars to exit when the main container finishes so Airflow can continue the worker shutdown process!", + "description": "Launch additional containers into Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.extraContainers`` and/or ``workers.kubernetes.extraContainers`` instead). Note, if used with KubernetesExecutor, you are responsible for signaling sidecars to exit when the main container finishes so Airflow can continue the worker shutdown process!", "type": "array", "default": [], "items": { @@ -2301,7 +2301,7 @@ } }, "extraInitContainers": { - "description": "Add additional init containers into Airflow Celery workers and pods created with pod-template-file (templated).", + "description": "Add additional init containers into Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.extraInitContainers`` and/or ``workers.kubernetes.extraInitContainers`` instead).", "type": "array", "default": [], "items": { @@ -2317,7 +2317,7 @@ } }, "extraVolumes": { - "description": "Additional volumes attached to the Airflow Celery workers and pods created with pod-template-file.", + "description": "Additional volumes attached to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.extraVolumes`` and/or ``workers.kubernetes.extraVolumes`` instead).", "type": "array", "default": [], "items": { @@ -2325,7 +2325,7 @@ } }, "extraVolumeMounts": { - "description": "Additional volume mounts attached to the Airflow Celery workers and pods created with pod-template-file.", + "description": "Additional volume mounts attached to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.extraVolumeMounts`` and/or ``workers.kubernetes.extraVolumeMounts`` instead).", "type": "array", "default": [], "items": { @@ -2357,13 +2357,13 @@ "default": null }, "affinity": { - "description": "Specify scheduling constraints for Airflow Celery worker pods and pods created with pod-template-file.", + "description": "Specify scheduling constraints for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.affinity`` and/or ``workers.kubernetes.affinity`` instead).", "type": "object", "default": "See values.yaml", "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" }, "tolerations": { - "description": "Specify Tolerations for Airflow Celery worker pods and pods created with pod-template-file.", + "description": "Specify Tolerations for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.tolerations`` and/or ``workers.kubernetes.tolerations`` instead).", "type": "array", "default": [], "items": { @@ -2372,7 +2372,7 @@ } }, "topologySpreadConstraints": { - "description": "Specify topology spread constraints for Airflow Celery worker pods and pods created with pod-template-file.", + "description": "Specify topology spread constraints for Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.topologySpreadConstraints`` and/or ``workers.kubernetes.topologySpreadConstraints`` instead).", "type": "array", "default": [], "items": { @@ -2403,7 +2403,7 @@ ] }, "annotations": { - "description": "Annotations to add to the Airflow Celery worker deployment.", + "description": "Annotations to add to the Airflow Celery worker deployment (deprecated, use ``workers.celery.annotations`` instead).", "type": "object", "default": {}, "additionalProperties": { @@ -2411,7 +2411,7 @@ } }, "podAnnotations": { - "description": "Annotations to add to the Airflow Celery workers and pods created with pod-template-file (templated).", + "description": "Annotations to add to the Airflow Celery workers and pods created with pod-template-file (templated) (deprecated, use ``workers.celery.podAnnotations`` and/or ``workers.kubernetes.podAnnotations`` instead).", "type": "object", "default": {}, "additionalProperties": { @@ -2428,7 +2428,7 @@ "x-docsSection": "Common" }, "labels": { - "description": "Labels to add to the Airflow Celery workers objects and pods created with pod-template-file.", + "description": "Labels to add to the Airflow Celery workers objects and pods created with pod-template-file (deprecated, use ``workers.celery.labels`` and/or ``workers.kubernetes.labels`` instead).", "type": "object", "default": {}, "additionalProperties": { @@ -2436,8 +2436,150 @@ } }, "logGroomerSidecar": { - "$ref": "#/definitions/logGroomerConfigType", - "description": "Configuration for Airflow Celery worker log groomer sidecar" + "description": "Configuration for Airflow Celery worker log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar`` instead).", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether to deploy the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.enabled`` instead).", + "type": "boolean", + "default": true + }, + "command": { + "description": "Command to use when running the Airflow log groomer sidecar (templated) (deprecated, use ``workers.celery.logGroomerSidecar.command`` instead).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow log groomer sidecar (templated) (deprecated, use ``workers.celery.logGroomerSidecar.args`` instead).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [ + "bash", + "/clean-logs" + ] + }, + "retentionDays": { + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionDays`` instead). Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": "integer", + "default": 15 + }, + "retentionMinutes": { + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.retentionMinutes`` instead). Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": "integer", + "default": 0 + }, + "frequencyMinutes": { + "description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.frequencyMinutes`` instead).", + "type": "integer", + "default": 15 + }, + "maxSizeBytes": { + "description": "Max size of logs directory in bytes (deprecated, use ``workers.celery.logGroomerSidecar.maxSizeBytes`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", + "type": "integer", + "default": 0, + "minimum": 0 + }, + "maxSizePercent": { + "description": "Max size of logs as a percentage of total disk space (deprecated, use ``workers.celery.logGroomerSidecar.maxSizePercent`` instead). When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", + "type": "integer", + "default": 0, + "minimum": 0, + "maximum": 100 + }, + "env": { + "description": "Add additional env vars to log groomer sidecar container (templated) (deprecated, use ``workers.celery.logGroomerSidecar.env`` instead).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "type": "array", + "default": [], + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + }, + "resources": { + "description": "Resources for Airflow log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.resources`` instead).", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.containerLifecycleHooks`` instead). If not set, the values from global `containerLifecycleHooks` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the log groomer sidecar (deprecated, use ``workers.celery.logGroomerSidecar.securityContexts.container`` instead).", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } }, "securityContext": { "description": "Security context for the Airflow Celery worker pods and pods created with pod-template-file (deprecated, use ``workers.celery.securityContexts`` and/or ``workers.kubernetes.securityContexts`` instead). If not set, the values from `securityContext` will be used.", @@ -2520,17 +2662,17 @@ } }, "waitForMigrations": { - "description": "Configuration of wait-for-airflow-migration init container for Airflow Celery workers.", + "description": "Configuration of wait-for-airflow-migration init container for Airflow Celery workers (deprecated, use ``workers.celery.waitForMigrations`` instead).", "type": "object", "additionalProperties": false, "properties": { "enabled": { - "description": "Enable wait-for-airflow-migrations init container.", + "description": "Enable wait-for-airflow-migrations init container (deprecated, use ``workers.celery.waitForMigrations.enabled`` instead).", "type": "boolean", "default": true }, "env": { - "description": "Add additional env vars to wait-for-airflow-migrations init container.", + "description": "Add additional env vars to wait-for-airflow-migrations init container (deprecated, use ``workers.celery.waitForMigrations.env`` instead).", "type": "array", "default": [], "items": { @@ -2551,12 +2693,12 @@ } }, "securityContexts": { - "description": "Security context definition for the wait-for-airflow-migrations container. If not set, the values from global `securityContexts` will be used.", + "description": "Security context definition for the wait-for-airflow-migrations container (deprecated, use ``workers.celery.waitForMigrations.securityContexts`` instead). If not set, the values from global `securityContexts` will be used.", "type": "object", "x-docsSection": "Kubernetes", "properties": { "container": { - "description": "Container security context definition for the wait-for-airflow-migrations container.", + "description": "Container security context definition for the wait-for-airflow-migrations container (deprecated, use ``workers.celery.waitForMigrations.securityContexts.container`` instead).", "type": "object", "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", "default": {}, @@ -2577,7 +2719,7 @@ } }, "env": { - "description": "Add additional env vars to the Airflow Celery workers and pods created with pod-template-file.", + "description": "Add additional env vars to the Airflow Celery workers and pods created with pod-template-file (deprecated, use ``workers.celery.env`` and/or ``workers.kubernetes.env`` instead).", "type": "array", "default": [], "items": { @@ -2921,6 +3063,44 @@ } } }, + "serviceAccount": { + "description": "Create ServiceAccount for Airflow Celery workers.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and create is true, a name is generated using the release name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the worker Kubernetes ServiceAccount.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, "keda": { "description": "KEDA configuration of Airflow Celery workers.", "type": "object", @@ -3013,6 +3193,54 @@ } } }, + "hpa": { + "description": "HPA configuration for Airflow Celery workers.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Allow HPA autoscaling (KEDA must be disabled).", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "minReplicaCount": { + "description": "Minimum number of Airflow Celery workers created by HPA.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "maxReplicaCount": { + "description": "Maximum number of Airflow Celery workers created by HPA.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "metrics": { + "description": "Specifications for which to use to calculate the desired replica count.", + "type": [ + "array", + "null" + ], + "default": null, + "items": { + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.MetricSpec" + } + }, + "behavior": { + "description": "HorizontalPodAutoscalerBehavior configures the scaling behavior of the target.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.autoscaling.v2.HorizontalPodAutoscalerBehavior" + } + } + }, "persistence": { "description": "Persistence configuration for Airflow Celery workers.", "type": "object", @@ -3231,16 +3459,405 @@ ] } } - } - ] + } + ] + }, + "securityContexts": { + "description": "Security context definition for the kerberos init container. If not set, the values from `workers.kerberosInitContainer.securityContexts` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the kerberos init container.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "resources": { + "description": "Resource configuration for Airflow Celery workers.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "terminationGracePeriodSeconds": { + "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "safeToEvict": { + "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "extraContainers": { + "description": "Launch additional containers into Airflow Celery worker (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into Airflow Celery workers (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Additional volumes attached to the Airflow Celery workers.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Additional volume mounts attached to the Airflow Celery workers.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, + "extraPorts": { + "description": "Expose additional ports of Airflow Celery worker container.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.ContainerPort" + } + }, + "nodeSelector": { + "description": "Select certain nodes for Airflow Celery worker pods.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "runtimeClassName": { + "description": "Specify runtime for Airflow Celery worker pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "priorityClassName": { + "description": "Specify priority for Airflow Celery worker pods.", + "type": [ + "string", + "null" + ], + "default": null + }, + "affinity": { + "description": "Specify scheduling constraints for Airflow Celery worker pods.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for Airflow Celery worker pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for Airflow Celery worker pods.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, + "hostAliases": { + "description": "Specify HostAliases for Airflow Celery worker pods.", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" + }, + "type": "array", + "default": [], + "examples": [ + { + "ip": "127.0.0.2", + "hostnames": [ + "test.hostname.one" + ] + }, + { + "ip": "127.0.0.3", + "hostnames": [ + "test.hostname.two" + ] + } + ] + }, + "annotations": { + "description": "Annotations to add to the Airflow Celery worker deployment.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "podAnnotations": { + "description": "Annotations to add to the Airflow Celery workers (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the Airflow Celery workers objects.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "logGroomerSidecar": { + "description": "Configuration for Airflow Celery worker log groomer sidecar.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether to deploy the Airflow log groomer sidecar.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "command": { + "description": "Command to use when running the Airflow log groomer sidecar (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": null + }, + "args": { + "description": "Args to use when running the Airflow log groomer sidecar (templated).", + "type": [ + "array", + "null" + ], + "items": { + "type": "string" + }, + "default": [] + }, + "retentionDays": { + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "retentionMinutes": { + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "frequencyMinutes": { + "description": "Number of minutes between attempts to groom the Airflow logs in log groomer sidecar.", + "type": [ + "integer", + "null" + ], + "default": null + }, + "maxSizeBytes": { + "description": "Max size of logs directory in bytes. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled.", + "type": [ + "integer", + "null" + ], + "default": null, + "minimum": 0 + }, + "maxSizePercent": { + "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", + "type": [ + "integer", + "null" + ], + "default": null, + "minimum": 0, + "maximum": 100 + }, + "env": { + "description": "Add additional env vars to log groomer sidecar container (templated).", + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.EnvVar" + }, + "type": "array", + "default": [], + "x-kubernetes-patch-merge-key": "name", + "x-kubernetes-patch-strategy": "merge" + }, + "resources": { + "description": "Resources for Airflow log groomer sidecar.", + "type": "object", + "default": {}, + "examples": [ + { + "limits": { + "cpu": "100m", + "memory": "128Mi" + }, + "requests": { + "cpu": "100m", + "memory": "128Mi" + } + } + ], + "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" + }, + "containerLifecycleHooks": { + "description": "Container Lifecycle Hooks definition for the log groomer sidecar. If not set, the values from ``workers.containerLifecycleHooks`` will be used.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Lifecycle", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "postStart": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo postStart handler > /usr/share/message" + ] + } + }, + "preStop": { + "exec": { + "command": [ + "/bin/sh", + "-c", + "echo preStop handler > /usr/share/message" + ] + } + } + } + ] + }, + "securityContexts": { + "description": "Security context definition for the log groomer sidecar. If not set, the values from ``workers.securityContexts`` will be used.", + "type": "object", + "x-docsSection": "Kubernetes", + "properties": { + "container": { + "description": "Container security context definition for the log groomer sidecar.", + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", + "default": {}, + "x-docsSection": "Kubernetes", + "examples": [ + { + "allowPrivilegeEscalation": false, + "capabilities": { + "drop": [ + "ALL" + ] + } + } + ] + } + } + } + } + }, + "waitForMigrations": { + "description": "Configuration of wait-for-airflow-migration init container for Airflow Celery workers.", + "type": "object", + "additionalProperties": false, + "properties": { + "enabled": { + "description": "Whether to create init container to wait for db migrations.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "env": { + "description": "Add additional env vars to wait-for-airflow-migrations init container.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + } + }, + "required": [ + "name", + "value" + ], + "additionalProperties": false + } }, "securityContexts": { - "description": "Security context definition for the kerberos init container. If not set, the values from `workers.kerberosInitContainer.securityContexts` will be used.", + "description": "Security context definition for the wait-for-airflow-migrations container. If not set, the values from ``workers.waitForMigrations.securityContexts`` will be used.", "type": "object", "x-docsSection": "Kubernetes", "properties": { "container": { - "description": "Container security context definition for the kerberos init container.", + "description": "Container security context definition for the wait-for-airflow-migrations container.", "type": "object", "$ref": "#/definitions/io.k8s.api.core.v1.SecurityContext", "default": {}, @@ -3260,93 +3877,62 @@ } } }, - "resources": { - "description": "Resource configuration for Airflow Celery workers.", - "type": "object", - "default": {}, - "examples": [ - { - "limits": { - "cpu": "100m", - "memory": "128Mi" - }, - "requests": { - "cpu": "100m", - "memory": "128Mi" - } - } - ], - "$ref": "#/definitions/io.k8s.api.core.v1.ResourceRequirements" - }, - "terminationGracePeriodSeconds": { - "description": "Grace period for tasks to finish after SIGTERM is sent from Kubernetes.", - "type": [ - "integer", - "null" - ], - "default": null - }, - "safeToEvict": { - "description": "This setting tells Kubernetes that it's ok to evict when it wants to scale a node down.", - "type": [ - "boolean", - "null" - ], - "default": null - }, - "extraPorts": { - "description": "Expose additional ports of Airflow Celery worker container.", + "env": { + "description": "Add additional env vars to the Airflow Celery workers.", "type": "array", "default": [], "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.ContainerPort" - } - }, - "nodeSelector": { - "description": "Select certain nodes for Airflow Celery worker pods.", - "type": "object", - "default": {}, - "additionalProperties": { - "type": "string" - } - }, - "runtimeClassName": { - "description": "Specify runtime for Airflow Celery worker pods.", - "type": [ - "string", - "null" - ], - "default": null - }, - "priorityClassName": { - "description": "Specify priority for Airflow Celery worker pods.", - "type": [ - "string", - "null" - ], - "default": null - }, - "hostAliases": { - "description": "Specify HostAliases for Airflow Celery worker pods.", - "items": { - "$ref": "#/definitions/io.k8s.api.core.v1.HostAlias" - }, - "type": "array", - "default": [], - "examples": [ - { - "ip": "127.0.0.2", - "hostnames": [ - "test.hostname.one" - ] + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } }, - { - "ip": "127.0.0.3", - "hostnames": [ - "test.hostname.two" - ] - } - ] + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } }, "volumeClaimTemplates": { "description": "Specify additional volume claim template for Airflow Celery workers.", @@ -3474,6 +4060,44 @@ } ] }, + "serviceAccount": { + "description": "Create ServiceAccount for pods created with pod-template-file. When this section is specified, the Service Account is created from ``templates/workers/worker-kubernetes-serviceaccount.yaml`` file.", + "type": "object", + "properties": { + "automountServiceAccountToken": { + "description": "Specifies if ServiceAccount's API credentials should be mounted onto Pods. If not specified, the ``workers.serviceAccount.automountServiceAccountToken`` value will be taken.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "create": { + "description": "Specifies whether a ServiceAccount should be created. If not specified, the ServiceAccount will be generated and used from ``templates/workers/worker-serviceaccount.yaml`` file if ``workers.serviceAccount.create`` will be 'true'.", + "type": [ + "boolean", + "null" + ], + "default": null + }, + "name": { + "description": "The name of the ServiceAccount to use. If not set and ``create`` is 'true', a name is generated using the release name with kubernetes dedicated name.", + "type": [ + "string", + "null" + ], + "default": null + }, + "annotations": { + "description": "Annotations to add to the worker Kubernetes ServiceAccount. If not specified, the ``workers.serviceAccount.annotations`` value will be taken.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + } + } + }, "kerberosSidecar": { "description": "Kerberos sidecar for pods created with pod-template-file.", "type": "object", @@ -3680,6 +4304,38 @@ ], "default": null }, + "extraContainers": { + "description": "Launch additional containers into pods created with pod-template-file (templated). Note, you are responsible for signaling sidecars to exit when the main container finishes so Airflow can continue the worker shutdown process!", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraInitContainers": { + "description": "Add additional init containers into pods created with pod-template-file (templated).", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Container" + } + }, + "extraVolumes": { + "description": "Additional volumes attached to the pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.Volume" + } + }, + "extraVolumeMounts": { + "description": "Additional volume mounts attached to the pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "$ref": "#/definitions/io.k8s.api.core.v1.VolumeMount" + } + }, "nodeSelector": { "description": "Select certain nodes for pods created with pod-template-file.", "type": "object", @@ -3704,6 +4360,30 @@ ], "default": null }, + "affinity": { + "description": "Specify scheduling constraints for pods created with pod-template-file.", + "type": "object", + "default": {}, + "$ref": "#/definitions/io.k8s.api.core.v1.Affinity" + }, + "tolerations": { + "description": "Specify Tolerations for pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.Toleration" + } + }, + "topologySpreadConstraints": { + "description": "Specify topology spread constraints for pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "type": "object", + "$ref": "#/definitions/io.k8s.api.core.v1.TopologySpreadConstraint" + } + }, "hostAliases": { "description": "Specify HostAliases for pods created with pod-template-file.", "items": { @@ -3726,6 +4406,79 @@ } ] }, + "podAnnotations": { + "description": "Annotations to add to the pods created with pod-template-file (templated).", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "labels": { + "description": "Labels to add to the pods created with pod-template-file.", + "type": "object", + "default": {}, + "additionalProperties": { + "type": "string" + } + }, + "env": { + "description": "Add additional env vars to the pods created with pod-template-file.", + "type": "array", + "default": [], + "items": { + "type": "object", + "properties": { + "name": { + "type": "string" + }, + "value": { + "type": "string" + }, + "valueFrom": { + "type": "object", + "properties": { + "configMapKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.ConfigMapKeySelector", + "description": "Selects a key of a ConfigMap." + }, + "secretKeyRef": { + "$ref": "#/definitions/io.k8s.api.core.v1.SecretKeySelector", + "description": "Selects a key of a secret in the pod's namespace" + } + }, + "anyOf": [ + { + "required": [ + "configMapKeyRef" + ] + }, + { + "required": [ + "secretKeyRef" + ] + } + ] + } + }, + "required": [ + "name" + ], + "anyOf": [ + { + "required": [ + "value" + ] + }, + { + "required": [ + "valueFrom" + ] + } + ], + "additionalProperties": false + } + }, "schedulerName": { "description": "Specify kube scheduler name for pods created with pod-template-file.", "type": [ @@ -10628,6 +11381,15 @@ ], "default": 1, "x-docsSection": "Kubernetes" + }, + "ttlSecondsAfterFinished": { + "description": "The number of seconds after a finished Job is eligible to be automatically deleted.", + "type": [ + "integer", + "null" + ], + "default": null, + "x-docsSection": "Kubernetes" } } }, @@ -14369,12 +15131,12 @@ ] }, "retentionDays": { - "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is retentionDays + retentionMinutes.", + "description": "Number of days to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", "type": "integer", "default": 15 }, "retentionMinutes": { - "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is retentionDays + retentionMinutes.", + "description": "Number of minutes to retain the logs when running the Airflow log groomer sidecar. Total retention time is ``retentionDays`` + ``retentionMinutes``.", "type": "integer", "default": 0 }, @@ -14390,7 +15152,7 @@ "minimum": 0 }, "maxSizePercent": { - "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if maxSizeBytes is set.", + "description": "Max size of logs as a percentage of total disk space. When exceeded, the log groomer reduces retention until size is under limit. 0 = disabled. Ignored if ``maxSizeBytes`` is set.", "type": "integer", "default": 0, "minimum": 0, diff --git a/chart/values.yaml b/chart/values.yaml index e07901ba03a29..3bdf20b516154 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -68,14 +68,14 @@ airflowHome: /opt/airflow defaultAirflowRepository: apache/airflow # Default Airflow tag to deploy -defaultAirflowTag: "3.1.8" +defaultAirflowTag: "3.2.0" # Default Airflow digest. If specified, it takes precedence over tag defaultAirflowDigest: ~ # Airflow version (Used to make some decisions based on Airflow Version being deployed) # Version 2.11.0 and above is supported. -airflowVersion: "3.1.8" +airflowVersion: "3.2.0" images: airflow: @@ -580,6 +580,8 @@ apiSecretKeySecretName: ~ # api-secret-key: # Secret key used to encode and decode JWTs: '[api_auth] jwt_secret' in airflow.cfg +# Note: It is not advised to use in production as during helm upgrade it will be changed +# which can cause dag failures during component rollouts jwtSecret: ~ # Add custom annotations to the JWT secret @@ -770,19 +772,36 @@ workers: # (deprecated, use `workers.celery.podDisruptionBudget.config.minAvailable` instead) # minAvailable: 1 - # Create ServiceAccount for Airflow Celery workers and pods created with pod-template-file + # Create Service Account for Airflow Celery workers and pods created with pod-template-file + # (deprecated, use `workers.celery.serviceAccount` and/or `workers.kubernetes.serviceAccount` instead) serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + # (deprecated, use + # `workers.celery.serviceAccount.automountServiceAccountToken` and/or + # `workers.kubernetes.serviceAccount.automountServiceAccountToken` + # instead) automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created + # (deprecated, use + # `workers.celery.serviceAccount.create` and/or + # `workers.kubernetes.serviceAccount.create` + # instead) create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name + # (deprecated, use + # `workers.celery.serviceAccount.name` and/or + # `workers.kubernetes.serviceAccount.name` + # instead) name: ~ # Annotations to add to worker Kubernetes Service Account. + # (deprecated, use + # `workers.celery.serviceAccount.annotations` and/or + # `workers.kubernetes.serviceAccount.annotations` + # instead) annotations: {} # Allow KEDA autoscaling for Airflow Celery workers @@ -848,16 +867,21 @@ workers: usePgbouncer: true # Allow HPA for Airflow Celery workers (KEDA must be disabled) + # (deprecated, use `workers.celery.hpa` instead) hpa: + # (deprecated, use `workers.celery.hpa.enabled` instead) enabled: false # Minimum number of Airflow Celery workers created by HPA + # (deprecated, use `workers.celery.hpa.minReplicaCount` instead) minReplicaCount: 0 # Maximum number of Airflow Celery workers created by HPA + # (deprecated, use `workers.celery.hpa.maxReplicaCount` instead) maxReplicaCount: 5 # Specifications for which to use to calculate the desired replica count + # (deprecated, use `workers.celery.hpa.metrics` instead) metrics: - type: Resource resource: @@ -867,6 +891,7 @@ workers: averageUtilization: 80 # Scaling behavior of the target in both Up and Down directions + # (deprecated, use `workers.celery.hpa.behavior` instead) behavior: {} # Persistence volume configuration for Airflow Celery workers @@ -1019,18 +1044,26 @@ workers: # Launch additional containers into Airflow Celery worker # and pods created with pod-template-file (templated). + # (deprecated, use + # `workers.celery.extraContainers` and/or + # `workers.kubernetes.extraContainers` + # instead) # Note: If used with KubernetesExecutor, you are responsible for signaling sidecars to exit when the main # container finishes so Airflow can continue the worker shutdown process! extraContainers: [] # Add additional init containers into Airflow Celery workers # and pods created with pod-template-file (templated). + # (deprecated, use + # `workers.celery.extraInitContainers` and/or + # `workers.kubernetes.extraInitContainers` + # instead) extraInitContainers: [] - # Additional volumes and volume mounts attached to the - # Airflow Celery workers and pods created with pod-template-file + # Additional volumes attached to the Airflow Celery workers + # and pods created with pod-template-file + # (deprecated, use `workers.celery.extraVolumes` and/or `workers.kubernetes.extraVolumes` instead) extraVolumes: [] - extraVolumeMounts: [] # Mount additional volumes into workers pods. It can be templated like in the following example: # extraVolumes: # - name: my-templated-extra-volume @@ -1038,7 +1071,15 @@ workers: # secretName: '{{ include "my_secret_template" . }}' # defaultMode: 0640 # optional: true - # + + # Additional volume mounts attached to the Airflow Celery workers + # and pods created with pod-template-file + # (deprecated, use + # `workers.celery.extraVolumeMounts` and/or + # `workers.kubernetes.extraVolumeMounts` + # instead) + extraVolumeMounts: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: # extraVolumeMounts: # - name: my-templated-extra-volume # mountPath: "{{ .Values.my_custom_path }}" @@ -1058,6 +1099,7 @@ workers: # (deprecated, use `workers.celery.priorityClassName` and/or `workers.kubernetes.priorityClassName` instead) priorityClassName: ~ + # (deprecated, use `workers.celery.affinity` and/or `workers.kubernetes.affinity` instead) affinity: {} # Default Airflow Celery worker affinity is: # podAntiAffinity: @@ -1069,7 +1111,13 @@ workers: # topologyKey: kubernetes.io/hostname # weight: 100 + # (deprecated, use `workers.celery.tolerations` and/or `workers.kubernetes.tolerations` instead) tolerations: [] + + # (deprecated, use + # `workers.celery.topologySpreadConstraints` and/or + # `workers.kubernetes.topologySpreadConstraints` + # instead) topologySpreadConstraints: [] # hostAliases to use in Airflow Celery worker pods and pods created with pod-template-file @@ -1085,42 +1133,55 @@ workers: # - "test.hostname.two" # Annotations for the Airflow Celery worker resource + # (deprecated, use `workers.celery.annotations` instead) annotations: {} # Pod annotations for the Airflow Celery workers and pods created with pod-template-file (templated) + # (deprecated, use `workers.celery.podAnnotations` and/or `workers.kubernetes.podAnnotations` instead) podAnnotations: {} # Labels specific to Airflow Celery workers objects and pods created with pod-template-file + # (deprecated, use `workers.celery.labels` and/or `workers.kubernetes.labels` instead) labels: {} # Log groomer configuration for Airflow Celery workers + # (deprecated, use `workers.celery.logGroomerSidecar` instead) logGroomerSidecar: # Whether to deploy the Airflow Celery worker log groomer sidecar + # (deprecated, use `workers.celery.logGroomerSidecar.enabled` instead) enabled: true # Command to use when running the Airflow Celery worker log groomer sidecar (templated) + # (deprecated, use `workers.celery.logGroomerSidecar.command` instead) command: ~ # Args to use when running the Airflow Celery worker log groomer sidecar (templated) + # (deprecated, use `workers.celery.logGroomerSidecar.args` instead) args: ["bash", "/clean-logs"] # Number of days to retain logs + # (deprecated, use `workers.celery.logGroomerSidecar.retentionDays` instead) retentionDays: 15 # Number of minutes to retain logs. # This can be used for finer granularity than days. - # Total retention is retentionDays + retentionMinutes. + # Total retention is `retentionDays` + `retentionMinutes`. + # (deprecated, use `workers.celery.logGroomerSidecar.retentionMinutes` instead) retentionMinutes: 0 # Frequency to attempt to groom logs (in minutes) + # (deprecated, use `workers.celery.logGroomerSidecar.frequencyMinutes` instead) frequencyMinutes: 15 # Max size of logs in bytes. 0 = disabled + # (deprecated, use `workers.celery.logGroomerSidecar.maxSizeBytes` instead) maxSizeBytes: 0 - # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. + # (deprecated, use `workers.celery.logGroomerSidecar.maxSizePercent` instead) maxSizePercent: 0 + # (deprecated, use `workers.celery.logGroomerSidecar.resources` instead) resources: {} # limits: # cpu: 100m @@ -1129,24 +1190,37 @@ workers: # cpu: 100m # memory: 128Mi - # Detailed default security context for logGroomerSidecar for container level + # Detailed default security context for `logGroomerSidecar` for container level + # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts` instead) securityContexts: + # (deprecated, use `workers.celery.logGroomerSidecar.securityContexts.container` instead) container: {} + # (deprecated, use `workers.celery.logGroomerSidecar.env` instead) env: [] + # Container level lifecycle hooks + # (deprecated, use `workers.celery.logGroomerSidecar.containerLifecycleHooks` instead) + containerLifecycleHooks: {} + # Configuration of wait-for-airflow-migration init container for Airflow Celery workers + # (deprecated, use `workers.celery.waitForMigrations` instead) waitForMigrations: # Whether to create init container to wait for db migrations + # (deprecated, use `workers.celery.waitForMigrations.enabled` instead) enabled: true + # (deprecated, use `workers.celery.waitForMigrations.env` instead) env: [] # Detailed default security context for wait-for-airflow-migrations container + # (deprecated, use `workers.celery.waitForMigrations.securityContexts` instead) securityContexts: + # (deprecated, use `workers.celery.waitForMigrations.securityContexts.container` instead) container: {} # Additional env variable configuration for Airflow Celery workers and pods created with pod-template-file + # (deprecated, use `workers.celery.env` and/or `workers.kubernetes.env` instead) env: [] # Additional volume claim templates for Airflow Celery workers. @@ -1260,6 +1334,21 @@ workers: maxUnavailable: ~ # minAvailable: ~ + # Create Service Account for Airflow Celery workers + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + automountServiceAccountToken: ~ + + # Specifies whether a Service Account should be created + create: ~ + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + name: ~ + + # Annotations to add to worker Kubernetes Service Account. + annotations: {} + # Allow KEDA autoscaling for Airflow Celery workers keda: enabled: ~ @@ -1297,6 +1386,22 @@ workers: # This configuration will be ignored if PGBouncer is not enabled usePgbouncer: ~ + # Allow HPA for Airflow Celery workers (KEDA must be disabled) + hpa: + enabled: ~ + + # Minimum number of Airflow Celery workers created by HPA + minReplicaCount: ~ + + # Maximum number of Airflow Celery workers created by HPA + maxReplicaCount: ~ + + # Specifications for which to use to calculate the desired replica count + metrics: ~ + + # Scaling behavior of the target in both Up and Down directions + behavior: {} + # Persistence volume configuration for Airflow Celery workers persistence: # Enable persistent volumes @@ -1383,6 +1488,30 @@ workers: # This setting tells Kubernetes that its ok to evict when it wants to scale a node down safeToEvict: ~ + # Launch additional containers into Airflow Celery worker (templated) + extraContainers: [] + + # Add additional init containers into Airflow Celery workers (templated) + extraInitContainers: [] + + # Additional volumes attached to the Airflow Celery workers + extraVolumes: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + + # Additional volume mounts attached to the Airflow Celery workers + extraVolumeMounts: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + # Expose additional ports of Airflow Celery workers. These can be used for additional metric collection. extraPorts: [] @@ -1393,6 +1522,21 @@ workers: priorityClassName: ~ + affinity: {} + # Default Airflow Celery worker affinity is: + # podAntiAffinity: + # preferredDuringSchedulingIgnoredDuringExecution: + # - podAffinityTerm: + # labelSelector: + # matchLabels: + # component: worker + # topologyKey: kubernetes.io/hostname + # weight: 100 + + tolerations: [] + + topologySpreadConstraints: [] + # hostAliases to use in Airflow Celery worker pods # See: # https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ @@ -1404,6 +1548,74 @@ workers: # hostnames: # - "test.hostname.two" + # Annotations for the Airflow Celery worker resource + annotations: {} + + # Pod annotations for the Airflow Celery workers (templated) + podAnnotations: {} + + # Labels specific to Airflow Celery workers objects + labels: {} + + # Log groomer configuration for Airflow Celery workers + logGroomerSidecar: + # Whether to deploy the Airflow Celery worker log groomer sidecar + enabled: ~ + + # Command to use when running the Airflow Celery worker log groomer sidecar (templated) + command: ~ + + # Args to use when running the Airflow Celery worker log groomer sidecar (templated) + args: [] + + # Number of days to retain logs + retentionDays: ~ + + # Number of minutes to retain logs. + # This can be used for finer granularity than days. + # Total retention is `retentionDays` + `retentionMinutes`. + retentionMinutes: ~ + + # Frequency to attempt to groom logs (in minutes) + frequencyMinutes: ~ + + # Max size of logs in bytes. 0 = disabled + maxSizeBytes: ~ + + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. + maxSizePercent: ~ + + resources: {} + # limits: + # cpu: 100m + # memory: 128Mi + # requests: + # cpu: 100m + # memory: 128Mi + + # Detailed default security context for `logGroomerSidecar` for container level + securityContexts: + container: {} + + env: [] + + # Container level lifecycle hooks + containerLifecycleHooks: {} + + # Configuration of wait-for-airflow-migration init container for Airflow Celery workers + waitForMigrations: + # Whether to create init container to wait for db migrations + enabled: ~ + + env: [] + + # Detailed default security context for wait-for-airflow-migrations container + securityContexts: + container: {} + + # Additional env variable configuration for Airflow Celery workers + env: [] + # Additional volume claim templates for Airflow Celery workers. # Requires mounting of specified volumes under extraVolumeMounts. volumeClaimTemplates: [] @@ -1443,6 +1655,29 @@ workers: # Container level Lifecycle Hooks definition for pods created with pod-template-file containerLifecycleHooks: {} + # Create Service Account for pods created with pod-template-file + # When this section is specified, the Service Account is created from + # 'templates/workers/worker-kubernetes-serviceaccount.yaml' file + serviceAccount: + # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ + # If not specified, the `workers.serviceAccount.automountServiceAccountToken` value will be taken + automountServiceAccountToken: ~ + + # Specifies whether a Service Account should be created. + # If not specified, the Service Account will be generated and used from + # 'templates/workers/worker-serviceaccount.yaml' file if `workers.serviceAccount.create` + # will be 'true' + create: ~ + + # The name of the Service Account to use. + # If not set and `create` is 'true', a name is generated using the release name + # with Kubernetes dedicated name + name: ~ + + # Annotations to add to worker Kubernetes Service Account. + # If not specified, the `workers.serviceAccount.annotations` value will be taken + annotations: {} + # Kerberos sidecar configuration for pods created with pod-template-file kerberosSidecar: # Enable kerberos sidecar @@ -1500,6 +1735,32 @@ workers: # This setting tells Kubernetes that its ok to evict when it wants to scale a node down safeToEvict: ~ + # Launch additional containers into pods created with pod-template-file (templated). + # Note: You are responsible for signaling sidecars to exit when the main + # container finishes so Airflow can continue the worker shutdown process! + extraContainers: [] + + # Add additional init containers into pods created with pod-template-file (templated) + extraInitContainers: [] + + # Additional volumes attached to the pods created with pod-template-file + extraVolumes: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumes: + # - name: my-templated-extra-volume + # secret: + # secretName: '{{ include "my_secret_template" . }}' + # defaultMode: 0640 + # optional: true + + # Additional volume mounts attached to the pods created with pod-template-file + extraVolumeMounts: [] + # Mount additional volumes into workers pods. It can be templated like in the following example: + # extraVolumeMounts: + # - name: my-templated-extra-volume + # mountPath: "{{ .Values.my_custom_path }}" + # readOnly: true + # Select certain nodes for pods created with pod-template-file nodeSelector: {} @@ -1507,6 +1768,12 @@ workers: priorityClassName: ~ + affinity: {} + + tolerations: [] + + topologySpreadConstraints: [] + # hostAliases to use in pods created with pod-template-file # See: # https://kubernetes.io/docs/concepts/services-networking/add-entries-to-pod-etc-hosts-with-host-aliases/ @@ -1518,6 +1785,15 @@ workers: # hostnames: # - "test.hostname.two" + # Pod annotations for the pods created with pod-template-file (templated) + podAnnotations: {} + + # Labels specific to pods created with pod-template-file + labels: {} + + # Additional env variable configuration for pods created with pod-template-file + env: [] + schedulerName: ~ # Airflow scheduler settings @@ -1585,16 +1861,16 @@ scheduler: # Grace period for tasks to finish after SIGTERM is sent from Kubernetes terminationGracePeriodSeconds: 10 - # Create ServiceAccount + # Create Service Account serviceAccount: # Affects all executors that launch pods # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -1713,7 +1989,7 @@ scheduler: # Max size of logs in bytes. 0 = disabled maxSizeBytes: 0 - # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if maxSizeBytes is set. + # Max size of logs as a percent of disk usage. 0 = disabled. Ignored if `maxSizeBytes` is set. maxSizePercent: 0 resources: {} @@ -1724,7 +2000,7 @@ scheduler: # cpu: 100m # memory: 128Mi - # Detailed default security context for logGroomerSidecar for container level + # Detailed default security context for `logGroomerSidecar` for container level securityContexts: container: {} @@ -1814,15 +2090,15 @@ createUserJob: # Container level lifecycle hooks containerLifecycleHooks: {} - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -1918,15 +2194,15 @@ migrateDatabaseJob: # Container level lifecycle hooks containerLifecycleHooks: {} - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -2034,10 +2310,10 @@ apiServer: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -2264,15 +2540,15 @@ webserver: # Scaling behavior of the target in both Up and Down directions behavior: {} - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -2478,15 +2754,15 @@ triggerer: periodSeconds: 60 command: ~ - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -2647,7 +2923,7 @@ triggerer: # cpu: 100m # memory: 128Mi - # Detailed default security context for logGroomerSidecar for container level + # Detailed default security context for `logGroomerSidecar` for container level securityContexts: container: {} @@ -2764,15 +3040,15 @@ dagProcessor: periodSeconds: 60 command: ~ - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -2996,15 +3272,15 @@ flower: # Container level lifecycle hooks containerLifecycleHooks: {} - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -3136,15 +3412,15 @@ statsd: # Grace period for StatsD to finish after SIGTERM is sent from Kubernetes terminationGracePeriodSeconds: 30 - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -3241,15 +3517,15 @@ pgbouncer: # Add custom annotations to the PgBouncer certificates secret certificatesSecretAnnotations: {} - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -3435,15 +3711,15 @@ redis: # Annotations for Redis Statefulset annotations: {} - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -3655,15 +3931,15 @@ cleanup: # cpu: 100m # memory: 128Mi - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -3755,15 +4031,15 @@ databaseCleanup: # cpu: 100m # memory: 128Mi - # Create ServiceAccount + # Create Service Account serviceAccount: # ref: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ automountServiceAccountToken: true - # Specifies whether a ServiceAccount should be created + # Specifies whether a Service Account should be created create: true - # The name of the ServiceAccount to use. + # The name of the Service Account to use. # If not set and `create` is 'true', a name is generated using the release name name: ~ @@ -3785,6 +4061,9 @@ databaseCleanup: failedJobsHistoryLimit: 1 successfulJobsHistoryLimit: 1 + # Time to live (in seconds) for Jobs created by this CronJob after they finish. + ttlSecondsAfterFinished: ~ + # Configuration for postgresql subchart # Uses bitnamilegacy images to avoid Bitnami licensing restrictions # Not recommended for production - use external database instead diff --git a/clients/python/CHANGELOG.md b/clients/python/CHANGELOG.md index 8ea0d3d5ce215..ed62a402ffaea 100644 --- a/clients/python/CHANGELOG.md +++ b/clients/python/CHANGELOG.md @@ -17,6 +17,53 @@ under the License. --> +# v3.2.0 + +## New Features: + +- Add Asset Partitioning support: backfill for partitioned DAGs ([#61464](https://github.com/apache/airflow/pull/61464)) +- Add `partition_key` to `DagRunAssetReference` ([#61725](https://github.com/apache/airflow/pull/61725)) +- Add partition key column and filter to DAG Runs list ([#61939](https://github.com/apache/airflow/pull/61939)) +- Add `DagRunType` for asset materializations ([#62276](https://github.com/apache/airflow/pull/62276)) +- Add `allowed_run_types` to allowlist specific DAG run types ([#61833](https://github.com/apache/airflow/pull/61833)) +- Add DAG runs filters by `bundleVersion` ([#62810](https://github.com/apache/airflow/pull/62810)) +- Expose `timetable_partitioned` in UI API ([#62777](https://github.com/apache/airflow/pull/62777)) +- Add base React plugin destination ([#62530](https://github.com/apache/airflow/pull/62530)) +- Add `team_name` to Pool APIs ([#60952](https://github.com/apache/airflow/pull/60952)) +- Add `team_name` to connection public APIs ([#59336](https://github.com/apache/airflow/pull/59336)) +- Add `team_id` to variable APIs ([#57102](https://github.com/apache/airflow/pull/57102)) +- Add team selector to list variables and list connections pages ([#60995](https://github.com/apache/airflow/pull/60995)) +- Support OR operator in search parameters ([#60008](https://github.com/apache/airflow/pull/60008)) +- Add wildcard support for `dag_id` and `dag_run_id` in bulk task instance endpoint ([#57441](https://github.com/apache/airflow/pull/57441)) +- Add `operator_name_pattern`, `pool_pattern`, `queue_pattern` as search filters for task instances ([#57571](https://github.com/apache/airflow/pull/57571)) +- Add filters to Task Instances tab ([#56920](https://github.com/apache/airflow/pull/56920)) +- Add API support for filtering DAGs by timetable type ([#58852](https://github.com/apache/airflow/pull/58852)) +- Enable triggerer queues ([#59239](https://github.com/apache/airflow/pull/59239)) +- Add ability to add, edit, and delete XComs directly from UI ([#58921](https://github.com/apache/airflow/pull/58921)) +- Add HITL detail history ([#55952](https://github.com/apache/airflow/pull/55952), [#56760](https://github.com/apache/airflow/pull/56760)) +- Add update_mask support for bulk PATCH APIs ([#54597](https://github.com/apache/airflow/pull/54597)) +- Introduce named asset watchers ([#55643](https://github.com/apache/airflow/pull/55643)) +- Add DAG ID pattern search functionality to DAG Runs and Task Instances ([#55691](https://github.com/apache/airflow/pull/55691)) +- Add UI to allow creation of DAG Runs with partition key ([#58004](https://github.com/apache/airflow/pull/58004)) +- Support retry multiplier parameter ([#56866](https://github.com/apache/airflow/pull/56866)) +- Display active DAG runs count in header with auto-refresh ([#58332](https://github.com/apache/airflow/pull/58332)) +- Add checkbox before clear task confirmation to prevent rerun of tasks in Running state ([#56351](https://github.com/apache/airflow/pull/56351)) + +## Improvements: + +- Upgrade FastAPI and conform OpenAPI schema changes ([#61476](https://github.com/apache/airflow/pull/61476)) +- Use SQLAlchemy native `Uuid`/`JSON` types instead of `sqlalchemy-utils` ([#61532](https://github.com/apache/airflow/pull/61532)) +- Remove team ID and use team name as primary key ([#59109](https://github.com/apache/airflow/pull/59109)) +- Update `BulkDeleteAction` to use generic type ([#59207](https://github.com/apache/airflow/pull/59207)) +- Add link to API docs ([#53346](https://github.com/apache/airflow/pull/53346)) + +## Bug Fixes: + +- Fix null `dag_run_conf` in `BackfillResponse` serialization ([#63259](https://github.com/apache/airflow/pull/63259)) +- Fix missing `dag_id` filter on DAG Run query ([#62750](https://github.com/apache/airflow/pull/62750)) +- Fix `HITLResponse` data model name ([#57795](https://github.com/apache/airflow/pull/57795)) +- Remove unused parameter in logout ([#58045](https://github.com/apache/airflow/pull/58045)) + # v3.1.8 ## Bug Fixes: diff --git a/clients/python/version.txt b/clients/python/version.txt index c848fb9cb4360..944880fa15e85 100644 --- a/clients/python/version.txt +++ b/clients/python/version.txt @@ -1 +1 @@ -3.1.8 +3.2.0 diff --git a/contributing-docs/10_working_with_git.rst b/contributing-docs/10_working_with_git.rst index 5551b1ea513dd..32536bfde3fef 100644 --- a/contributing-docs/10_working_with_git.rst +++ b/contributing-docs/10_working_with_git.rst @@ -181,7 +181,10 @@ we will be adding the remote as "apache" so you can refer to it easily push your changes to your repository. That should trigger the build in our CI if you have a Pull Request (PR) opened already -8. While rebasing you might have conflicts. Read carefully what git tells you when it prints information +8. When you have conflicts with ``uv.lock`` when rebasing, simply delete the ``uv.lock`` file and run + ``uv lock`` to regenerate it. This is the recommended way to solve conflicts in ``uv.lock`` file. + +9. While rebasing you might have conflicts. Read carefully what git tells you when it prints information about the conflicts. You need to solve the conflicts manually. This is sometimes the most difficult part and requires deliberately correcting your code and looking at what has changed since you developed your changes @@ -195,7 +198,7 @@ we will be adding the remote as "apache" so you can refer to it easily you have a very intuitive and helpful merge tool. For more information, see `Resolve conflicts `_. -9. After you've solved your conflict run +10. After you've solved your conflict run ``git rebase --continue`` diff --git a/contributing-docs/12_provider_distributions.rst b/contributing-docs/12_provider_distributions.rst index 523ad2a8fca40..832d89128a8a3 100644 --- a/contributing-docs/12_provider_distributions.rst +++ b/contributing-docs/12_provider_distributions.rst @@ -297,7 +297,7 @@ You can see for example ``google`` provider which has very comprehensive documen * `Documentation <../../providers/google/docs>`_ * `System tests/Example Dags <../providers/google/tests/system/google/>`_ -Part of the documentation are example dags (placed in the ``tests/system`` folder). The reason why +Part of the documentation are example dags (placed in the ``providers//tests/system`` folder). The reason why they are in ``tests/system`` is because we are using the example dags for various purposes: * showing real examples of how your provider classes (Operators/Sensors/Transfers) can be used diff --git a/contributing-docs/testing/system_tests.rst b/contributing-docs/testing/system_tests.rst index 19341c7ba5e67..16709663e45a0 100644 --- a/contributing-docs/testing/system_tests.rst +++ b/contributing-docs/testing/system_tests.rst @@ -114,7 +114,7 @@ There are multiple ways of running system tests. Each system test is a self-cont other Dag. Some tests may require access to external services, enabled APIs or specific permissions. Make sure to prepare your environment correctly, depending on the system tests you want to run - some may require additional configuration which should be documented by the relevant providers in their subdirectory -``tests/system//README.md``. +``providers//tests/system//README.md``. Running as Airflow Dags ....................... @@ -125,8 +125,8 @@ your Airflow instance as Dags and they will be automatically triggered. If the s how to set up the environment is documented in each provider's system tests directory. Make sure that all resource required by the tests are also imported. -Running via Pytest + Breezee -............................ +Running via Pytest + Breeze +........................... Running system tests with pytest is the easiest with `Breeze `_. Breeze makes sure that the environment is pre-configured, and all additional required services are started, diff --git a/dev/README_RELEASE_PROVIDERS.md b/dev/README_RELEASE_PROVIDERS.md index 6b7b9018a0a27..e871a7ef5b3c1 100644 --- a/dev/README_RELEASE_PROVIDERS.md +++ b/dev/README_RELEASE_PROVIDERS.md @@ -30,6 +30,7 @@ - [Perform review of security issues that are marked for the release](#perform-review-of-security-issues-that-are-marked-for-the-release) - [Convert commits to changelog entries and bump provider versions](#convert-commits-to-changelog-entries-and-bump-provider-versions) - [Update versions of dependent providers to the next version](#update-versions-of-dependent-providers-to-the-next-version) + - [Create a PR with the changes](#create-a-pr-with-the-changes) - [Apply incremental changes and merge the PR](#apply-incremental-changes-and-merge-the-pr) - [(Optional) Apply template updates](#optional-apply-template-updates) - [Build Provider distributions for SVN apache upload](#build-provider-distributions-for-svn-apache-upload) @@ -234,6 +235,22 @@ removed. breeze release-management update-providers-next-version ``` +## Create a PR with the changes + +Make sure to set labels: `allow provider dependency bump` and `skip common compat check` to the PR, +so that the PR is not blocked by selective checks. + +You can do it for example this way: + +```shell script +gh pr create \ + --title "Prepare providers release ${RELEASE_DATE}" \ + --label "allow provider dependency bump" \ + --label "skip common compat check" \ + --body "Prepare providers release ${RELEASE_DATE}" \ + --web +``` + ## Apply incremental changes and merge the PR When those changes are generated, you should commit the changes, create a PR and get it reviewed. diff --git a/dev/breeze/doc/09_release_management_tasks.rst b/dev/breeze/doc/09_release_management_tasks.rst index 657b523f0bba7..3d5faef5e216a 100644 --- a/dev/breeze/doc/09_release_management_tasks.rst +++ b/dev/breeze/doc/09_release_management_tasks.rst @@ -209,7 +209,7 @@ Generating Airflow core Issue You can use Breeze to generate a Airflow core issue when you release new airflow. -.. image:: ./images/output_release-management_generate-issue-content-providers.svg +.. image:: ./images/output_release-management_generate-issue-content-core.svg :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_generate-issue-content-core.svg :width: 100% :alt: Breeze generate-issue-content-core @@ -416,7 +416,7 @@ Generating helm chart Issue You can use Breeze to generate a helm chart issue when you release new helm chart. -.. image:: ./images/output_release-management_generate-issue-content-providers.svg +.. image:: ./images/output_release-management_generate-issue-content-helm-chart.svg :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_generate-issue-content-helm-chart.svg :width: 100% :alt: Breeze generate-issue-content-helm-chart @@ -894,6 +894,16 @@ If you pass ``--tag`` fag, the distribution will create a source tarball release :width: 100% :alt: Breeze release-management prepare-airflow-ctl-distributions +Generating airflow-ctl issue +"""""""""""""""""""""""""""" + +You can use Breeze to generate an airflow-ctl issue when you release new airflow-ctl. + +.. image:: ./images/output_release-management_generate-issue-content-airflow-ctl.svg + :target: https://raw.githubusercontent.com/apache/airflow/main/dev/breeze/doc/images/output_release-management_generate-issue-content-airflow-ctl.svg + :width: 100% + :alt: Breeze generate-issue-content-airflow-ctl + Publishing the documentation to S3 """""""""""""""""""""""""""""""""" diff --git a/dev/breeze/doc/images/output_build-docs.svg b/dev/breeze/doc/images/output_build-docs.svg index ebdd2875195d0..4dae99c17ce0b 100644 --- a/dev/breeze/doc/images/output_build-docs.svg +++ b/dev/breeze/doc/images/output_build-docs.svg @@ -241,7 +241,7 @@ microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis |    salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard |    -tableau | task-sdk | telegram | teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                     +tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk]...             Build documents. diff --git a/dev/breeze/doc/images/output_build-docs.txt b/dev/breeze/doc/images/output_build-docs.txt index ea24111dadf71..31a7c75e0ac57 100644 --- a/dev/breeze/doc/images/output_build-docs.txt +++ b/dev/breeze/doc/images/output_build-docs.txt @@ -1 +1 @@ -5023b820002e3f33104ae46d617645c4 +5a672545cd009db3d79db5c526c3bbd6 diff --git a/dev/breeze/doc/images/output_ci_upgrade.svg b/dev/breeze/doc/images/output_ci_upgrade.svg index fdc6374d40260..6bca01160d013 100644 --- a/dev/breeze/doc/images/output_ci_upgrade.svg +++ b/dev/breeze/doc/images/output_ci_upgrade.svg @@ -1,4 +1,4 @@ - + │main](TEXT)│ │--create-pr/--no-create-prAutomatically create a PR with the upgrade changes (if not specified, will    │ │ask)                                                                          │ -│--switch-to-base/--no-switch-to-baseAutomatically switch to the base branch if not already on it (if not          │ -│specified, will ask)                                                          │ -│--airflow-site                      Path to airflow-site checkout for publishing K8s schemas [default: │ -│../airflow-site](DIRECTORY)│ -│--force-k8s-schema-sync             Force syncing K8s schemas to airflow-site even if all versions appear         │ -│published                                                                     │ -│--github-token                      The token used to authenticate to GitHub. (TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Upgrade steps ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--autoupdate/--no-autoupdateRun prek autoupdate to update hook revisions [default: │ -│autoupdate]│ -│--update-chart-dependencies/--no-update-chart-dependencieRun update-chart-dependencies to update Helm chart       │ -│sdependencies [default: update-chart-dependencies]│ -│--upgrade-important-versions/--no-upgrade-important-versiRun upgrade-important-versions to bump key dependency    │ -│onsversions [default: upgrade-important-versions]│ -│--update-uv-lock/--no-update-uv-lockRun update-uv-lock to regenerate uv.lock with latest     │ -│resolutions inside Breeze CI image [default: │ -│update-uv-lock]│ -│--k8s-schema-sync/--no-k8s-schema-syncSync K8s JSON schemas to airflow-site [default: │ -│k8s-schema-sync]│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--answer -aForce answer to questions. (y | n | q | yes | no | quit)│ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help   -hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--draft/--no-draftCreate the PR as a draft (useful for scheduled CI runs where a human undrafts │ +│to trigger CI) [default: no-draft]│ +│--switch-to-base/--no-switch-to-baseAutomatically switch to the base branch if not already on it (if not          │ +│specified, will ask)                                                          │ +│--airflow-site                      Path to airflow-site checkout for publishing K8s schemas [default: │ +│../airflow-site](DIRECTORY)│ +│--force-k8s-schema-sync             Force syncing K8s schemas to airflow-site even if all versions appear         │ +│published                                                                     │ +│--github-token                      The token used to authenticate to GitHub. (TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Upgrade steps ──────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--autoupdate/--no-autoupdateRun prek autoupdate to update hook revisions [default: │ +│autoupdate]│ +│--update-chart-dependencies/--no-update-chart-dependencieRun update-chart-dependencies to update Helm chart       │ +│sdependencies [default: update-chart-dependencies]│ +│--upgrade-important-versions/--no-upgrade-important-versiRun upgrade-important-versions to bump key dependency    │ +│onsversions [default: upgrade-important-versions]│ +│--update-uv-lock/--no-update-uv-lockRun update-uv-lock to regenerate uv.lock with latest     │ +│resolutions inside Breeze CI image [default: │ +│update-uv-lock]│ +│--k8s-schema-sync/--no-k8s-schema-syncSync K8s JSON schemas to airflow-site [default: │ +│k8s-schema-sync]│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--answer -aForce answer to questions. (y | n | q | yes | no | quit)│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help   -hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_ci_upgrade.txt b/dev/breeze/doc/images/output_ci_upgrade.txt index e6eb6d802f0f8..d26a86502dcd4 100644 --- a/dev/breeze/doc/images/output_ci_upgrade.txt +++ b/dev/breeze/doc/images/output_ci_upgrade.txt @@ -1 +1 @@ -74dd8bf3d8af445737f1788dac83740d +2982b71d3fe990ce8f59410e96621f4d diff --git a/dev/breeze/doc/images/output_pr_auto-triage.svg b/dev/breeze/doc/images/output_pr_auto-triage.svg index 23937ddcfc8cb..acb22d594e0ef 100644 --- a/dev/breeze/doc/images/output_pr_auto-triage.svg +++ b/dev/breeze/doc/images/output_pr_auto-triage.svg @@ -400,12 +400,12 @@ │| provider:sendgrid | provider:sftp | provider:singularity | provider:slack | provider:smtp│ │| provider:snowflake | provider:sqlite | provider:ssh | provider:standard | │ │provider:tableau | provider:telegram | provider:teradata | provider:trino | │ -│provider:vertica | provider:weaviate | provider:yandex | provider:ydb | provider:zendesk | │ -│translation:ar | translation:ca | translation:de | translation:default | translation:el | │ -│translation:es | translation:fr | translation:he | translation:hi | translation:hu | │ -│translation:it | translation:ja | translation:ko | translation:nl | translation:pl | │ -│translation:pt | translation:ru | translation:th | translation:tr | translation:zh-CN | │ -│translation:zh-TW)│ +│provider:vertica | provider:vespa | provider:weaviate | provider:yandex | provider:ydb | │ +│provider:zendesk | translation:ar | translation:ca | translation:de | translation:default |│ +│translation:el | translation:es | translation:fr | translation:he | translation:hi | │ +│translation:hu | translation:it | translation:ja | translation:ko | translation:nl | │ +│translation:pl | translation:pt | translation:ru | translation:th | translation:tr | │ +│translation:zh-CN | translation:zh-TW)│ │--exclude-label        Exclude PRs with this label. Supports wildcards. Can be repeated. (TEXT)│ │--created-after        Only PRs created on or after this date (YYYY-MM-DD). (TEXT)│ │--created-before       Only PRs created on or before this date (YYYY-MM-DD). (TEXT)│ diff --git a/dev/breeze/doc/images/output_pr_auto-triage.txt b/dev/breeze/doc/images/output_pr_auto-triage.txt index 2fb0e6f62779d..d9b1de5b3a0db 100644 --- a/dev/breeze/doc/images/output_pr_auto-triage.txt +++ b/dev/breeze/doc/images/output_pr_auto-triage.txt @@ -1 +1 @@ -15636cdbbdf6fde36f2ae6e5c43da51e +aea67302554bf746d0490edd816b5568 diff --git a/dev/breeze/doc/images/output_release-management.svg b/dev/breeze/doc/images/output_release-management.svg index f125c70fe38c5..b6def918b8562 100644 --- a/dev/breeze/doc/images/output_release-management.svg +++ b/dev/breeze/doc/images/output_release-management.svg @@ -1,4 +1,4 @@ - + │prepare-task-sdk-distributions            Prepare sdist/whl distributions of Airflow Task SDK.                    │ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ ╭─ airflowctl release commands ────────────────────────────────────────────────────────────────────────────────────────╮ -│prepare-airflow-ctl-distributions               Prepare sdist/whl distributions of airflowctl.                    │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ -│add-back-references     Command to add back references for documentation to make it backward compatible.          │ -│prepare-python-client   Prepares python client packages.                                                          │ -│publish-docs            Command to publish generated documentation to airflow-site                                │ -│generate-constraints    Generates pinned constraint files with all extras from pyproject.toml in parallel.        │ -│update-constraints      Update released constraints with manual changes.                                          │ -│publish-docs-to-s3      Publishes docs to S3.                                                                     │ -│verify-rc-by-pmc        [EXPERIMENTAL] Verify a release candidate for PMC voting.                                 │ -│check-release-files     Verify that all expected packages are present in Apache Airflow svn.                      │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│constraints-version-checkCheck constraints against released versions of packages.                                 │ -│merge-prod-images        Merge production images in DockerHub based on digest files (needs DockerHub permissions).│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│prepare-airflow-ctl-distributions           Prepare sdist/whl distributions of airflowctl.                        │ +│generate-issue-content-airflow-ctl          Generates content for issue to test airflow-ctl release.              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Other release commands ─────────────────────────────────────────────────────────────────────────────────────────────╮ +│add-back-references     Command to add back references for documentation to make it backward compatible.          │ +│prepare-python-client   Prepares python client packages.                                                          │ +│publish-docs            Command to publish generated documentation to airflow-site                                │ +│generate-constraints    Generates pinned constraint files with all extras from pyproject.toml in parallel.        │ +│update-constraints      Update released constraints with manual changes.                                          │ +│publish-docs-to-s3      Publishes docs to S3.                                                                     │ +│verify-rc-by-pmc        [EXPERIMENTAL] Verify a release candidate for PMC voting.                                 │ +│check-release-files     Verify that all expected packages are present in Apache Airflow svn.                      │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Commands ───────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│constraints-version-checkCheck constraints against released versions of packages.                                 │ +│merge-prod-images        Merge production images in DockerHub based on digest files (needs DockerHub permissions).│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management.txt b/dev/breeze/doc/images/output_release-management.txt index 37bc4826d0a9b..ebe977dad7570 100644 --- a/dev/breeze/doc/images/output_release-management.txt +++ b/dev/breeze/doc/images/output_release-management.txt @@ -1 +1 @@ -79e5925d47d1fdbf49a06ba80f113d05 +8f4b47cae96d73578872d9454722e7c0 diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.svg b/dev/breeze/doc/images/output_release-management_add-back-references.svg index 297996f63b099..89fe5aebeb475 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.svg +++ b/dev/breeze/doc/images/output_release-management_add-back-references.svg @@ -156,7 +156,7 @@ microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis |    salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard |    -tableau | task-sdk | telegram | teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                     +tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk]...             Command to add back references for documentation to make it backward compatible. diff --git a/dev/breeze/doc/images/output_release-management_add-back-references.txt b/dev/breeze/doc/images/output_release-management_add-back-references.txt index d8b21fbf0ba31..c42c85324f095 100644 --- a/dev/breeze/doc/images/output_release-management_add-back-references.txt +++ b/dev/breeze/doc/images/output_release-management_add-back-references.txt @@ -1 +1 @@ -c4f3137fd042c7fe7f6cf21479523c5d +4212e26e30b312eabb2bec58b55e9019 diff --git a/dev/breeze/doc/images/output_release-management_constraints-version-check.svg b/dev/breeze/doc/images/output_release-management_constraints-version-check.svg index df0f126560056..5bc64a7545922 100644 --- a/dev/breeze/doc/images/output_release-management_constraints-version-check.svg +++ b/dev/breeze/doc/images/output_release-management_constraints-version-check.svg @@ -1,4 +1,4 @@ - + │(full|diff-all|diff-constraints)│ │--package                     Only check specific package(s). Can be used multiple times. (TEXT)│ │--explain-why/--no-explain-whyShow explanations for outdated packages.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Build options. ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--builderBuildx builder used to perform `docker buildx build` commands. [default: autodetect](TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help   -hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│--cooldown-days               Ignore package versions released within this many days (cooldown period). [default: │ +│4](INTEGER)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Build options. ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--builderBuildx builder used to perform `docker buildx build` commands. [default: autodetect](TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help   -hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_constraints-version-check.txt b/dev/breeze/doc/images/output_release-management_constraints-version-check.txt index 5f4e91344580c..5f03ceac2f59e 100644 --- a/dev/breeze/doc/images/output_release-management_constraints-version-check.txt +++ b/dev/breeze/doc/images/output_release-management_constraints-version-check.txt @@ -1 +1 @@ -69595e717626710cce603db050e76a21 +9997441e5cb12ba360db1756028718d3 diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-airflow-ctl.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-airflow-ctl.svg new file mode 100644 index 0000000000000..efb6cda10a69f --- /dev/null +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-airflow-ctl.svg @@ -0,0 +1,134 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Command: release-management generate-issue-content-airflow-ctl + + + + + + + + + + +Usage:breeze release-management generate-issue-content-airflow-ctl[OPTIONS] + +Generates content for issue to test airflow-ctl release. + +╭─ Generate issue flags ───────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-token    GitHub token used to authenticate. You can set omit it if you have GITHUB_TOKEN env variable │ +│set. Can be generated with:                                                                  │ +│https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status (TEXT)│ +│*--previous-releasecommit reference (for example hash or tag) of the previous release. [required](TEXT)│ +│*--current-release commit reference (for example hash or tag) of the current release. [required](TEXT)│ +│--excluded-pr-listComa-separated list of PRs to exclude from the issue. (TEXT)│ +│--limit-pr-count  Limit PR count processes (useful for testing small subset of PRs). (INTEGER)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--help   -hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + + + diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-airflow-ctl.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-airflow-ctl.txt new file mode 100644 index 0000000000000..0e7d8691f2b2d --- /dev/null +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-airflow-ctl.txt @@ -0,0 +1 @@ +9ccebae692bfa80fcd28341dc5aeb359 diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg index 66c37e87ed9f5..377de72e45414 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.svg @@ -147,7 +147,7 @@ microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |       opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce |    samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau |       -telegram | teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                                          +telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                  Generates content for issue to test the release. diff --git a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt index bb8e773660edc..3d046af2db40c 100644 --- a/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt +++ b/dev/breeze/doc/images/output_release-management_generate-issue-content-providers.txt @@ -1 +1 @@ -c8b768e8bdd550b61a4ed50f08a5166d +8842218524208746ad1cffa81b926604 diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg index f3a910fd3027e..3700192f51557 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.svg @@ -177,7 +177,7 @@ │| neo4j | odbc | openai | openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | │ │papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce | samba | │ │segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | │ -│tableau | telegram | teradata | trino | vertica | weaviate | yandex | ydb | zendesk)│ +│tableau | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ │--provider-versionProvider version to generate metadata for. Only used when --provider-id is specified. Limits    │ │running metadata generation to only this version of the provider. (TEXT)│ ╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt index 111ec617ead5d..e1d6f576303dc 100644 --- a/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt +++ b/dev/breeze/doc/images/output_release-management_generate-providers-metadata.txt @@ -1 +1 @@ -86362717620d15b504b32ed3a904881e +5e7b2d2037b6a8588b7c0b0bf0a70b3a diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg index f4ce6bc52a0e6..09753afc56a70 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.svg @@ -189,7 +189,7 @@ microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |       opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce |    samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau |       -telegram | teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                                          +telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                  Prepare sdist/whl distributions of Airflow Providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt index 72ce17860c85a..32193df2e7937 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-distributions.txt @@ -1 +1 @@ -9d80a1e75041f4ea3a67cb3e05c11720 +e521f889a0553cda7737d292aceae195 diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg index 4c669df9e6d37..848ffe1194eeb 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.svg @@ -216,7 +216,7 @@ microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage | opensearch |       opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis | salesforce |    samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau |       -telegram | teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                                          +telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk]...                                  Prepare CHANGELOG, README and COMMITS information for providers. diff --git a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt index eb24b754104af..ecfbecc0b92bf 100644 --- a/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt +++ b/dev/breeze/doc/images/output_release-management_prepare-provider-documentation.txt @@ -1 +1 @@ -2f1aac6a4d4f8825e7b93d4e3add2af2 +2e38d3826e90845ac9776d70023cc638 diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.svg b/dev/breeze/doc/images/output_release-management_publish-docs.svg index f3f41e8890694..7bb8567554d3b 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.svg +++ b/dev/breeze/doc/images/output_release-management_publish-docs.svg @@ -195,7 +195,7 @@ microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis |    salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard |    -tableau | task-sdk | telegram | teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                     +tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk]...             Command to publish generated documentation to airflow-site diff --git a/dev/breeze/doc/images/output_release-management_publish-docs.txt b/dev/breeze/doc/images/output_release-management_publish-docs.txt index 12e06cfa472c4..c5e04a1b1ecc4 100644 --- a/dev/breeze/doc/images/output_release-management_publish-docs.txt +++ b/dev/breeze/doc/images/output_release-management_publish-docs.txt @@ -1 +1 @@ -7967a912265c05eb878af2ba3ddf2057 +2a2b639f232bc385cade07f37da8645b diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg index aef687a4db08c..dd0ea273eca17 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.svg @@ -191,7 +191,7 @@ │openfaas | openlineage | opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | │ │pinecone | postgres | presto | qdrant | redis | salesforce | samba | segment | sendgrid | sftp |│ │singularity | slack | smtp | snowflake | sqlite | ssh | standard | tableau | telegram | teradata│ -│| trino | vertica | weaviate | yandex | ydb | zendesk)│ +│| trino | vertica | vespa | weaviate | yandex | ydb | zendesk)│ │--provider-versionProvider version to generate the requirements for i.e `2.1.0`. `latest` is also a supported     │ │value to account for the most recent version of the provider (TEXT)│ │--force           Force update providers requirements even if they already exist.│ diff --git a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt index 0c39f82ba3755..158f228b6629c 100644 --- a/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt +++ b/dev/breeze/doc/images/output_sbom_generate-providers-requirements.txt @@ -1 +1 @@ -cb4611abc6764a8d7c1aacad63da03e3 +c5e62520c600c1704df855523f6b02ad diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg index b43dd779612d1..bd3b4cdb980ce 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.svg @@ -210,8 +210,8 @@ │registry:publish-versions | release-management | release-management:add-back-references | │ │release-management:check-release-files | release-management:clean-old-provider-artifacts | │ │release-management:constraints-version-check | release-management:create-minor-branch | │ -│release-management:generate-constraints | release-management:generate-issue-content-core | │ -│release-management:generate-issue-content-helm-chart | │ +│release-management:generate-constraints | release-management:generate-issue-content-airflow-ctl | │ +│release-management:generate-issue-content-core | release-management:generate-issue-content-helm-chart | │ │release-management:generate-issue-content-providers | release-management:generate-providers-metadata | │ │release-management:install-provider-distributions | release-management:merge-prod-images | │ │release-management:prepare-airflow-ctl-distributions | release-management:prepare-airflow-distributions |│ diff --git a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt index bd0ef796d4b0a..f0e39386d5f24 100644 --- a/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt +++ b/dev/breeze/doc/images/output_setup_check-all-params-in-groups.txt @@ -1 +1 @@ -eba55480dd7f88affd08c6f609ad8c40 +bbfdc83b48ce5463c194360826da069a diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg index 2d6b2bae76c9f..03edcf75bdf66 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.svg +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.svg @@ -1,4 +1,4 @@ - + │release-management:add-back-references | release-management:check-release-files | │ │release-management:clean-old-provider-artifacts | release-management:constraints-version-check | │ │release-management:create-minor-branch | release-management:generate-constraints | │ -│release-management:generate-issue-content-core | release-management:generate-issue-content-helm-chart │ -│| release-management:generate-issue-content-providers | release-management:generate-providers-metadata│ -│| release-management:install-provider-distributions | release-management:merge-prod-images | │ -│release-management:prepare-airflow-ctl-distributions | │ -│release-management:prepare-airflow-distributions | release-management:prepare-helm-chart-package | │ -│release-management:prepare-helm-chart-tarball | release-management:prepare-provider-distributions | │ -│release-management:prepare-provider-documentation | release-management:prepare-python-client | │ -│release-management:prepare-tarball | release-management:prepare-task-sdk-distributions | │ -│release-management:publish-docs | release-management:publish-docs-to-s3 | │ -│release-management:release-prod-images | release-management:start-rc-process | │ -│release-management:start-release | release-management:tag-providers | │ -│release-management:update-constraints | release-management:update-providers-next-version | │ -│release-management:verify-provider-distributions | release-management:verify-rc-by-pmc | run | sbom | │ -│sbom:build-all-airflow-images | sbom:export-dependency-information | │ -│sbom:generate-providers-requirements | sbom:update-sbom-information | setup | setup:autocomplete | │ -│setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images | setup:self-upgrade│ -│| setup:synchronize-local-mounts | setup:version | shell | start-airflow | testing | │ -│testing:airflow-ctl-integration-tests | testing:airflow-ctl-tests | testing:airflow-e2e-tests | │ -│testing:core-integration-tests | testing:core-tests | testing:docker-compose-tests | │ -│testing:helm-tests | testing:providers-integration-tests | testing:providers-tests | │ -│testing:python-api-client-tests | testing:system-tests | testing:task-sdk-integration-tests | │ -│testing:task-sdk-tests | testing:ui-e2e-tests | ui | ui:check-translation-completeness | │ -│ui:compile-assets | workflow-run | workflow-run:publish-docs)│ -│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used  │ -│together with --command flag or --force.                                                              │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--help   -hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +│release-management:generate-issue-content-airflow-ctl | release-management:generate-issue-content-core│ +│| release-management:generate-issue-content-helm-chart | │ +│release-management:generate-issue-content-providers | release-management:generate-providers-metadata |│ +│release-management:install-provider-distributions | release-management:merge-prod-images | │ +│release-management:prepare-airflow-ctl-distributions | │ +│release-management:prepare-airflow-distributions | release-management:prepare-helm-chart-package | │ +│release-management:prepare-helm-chart-tarball | release-management:prepare-provider-distributions | │ +│release-management:prepare-provider-documentation | release-management:prepare-python-client | │ +│release-management:prepare-tarball | release-management:prepare-task-sdk-distributions | │ +│release-management:publish-docs | release-management:publish-docs-to-s3 | │ +│release-management:release-prod-images | release-management:start-rc-process | │ +│release-management:start-release | release-management:tag-providers | │ +│release-management:update-constraints | release-management:update-providers-next-version | │ +│release-management:verify-provider-distributions | release-management:verify-rc-by-pmc | run | sbom | │ +│sbom:build-all-airflow-images | sbom:export-dependency-information | │ +│sbom:generate-providers-requirements | sbom:update-sbom-information | setup | setup:autocomplete | │ +│setup:check-all-params-in-groups | setup:config | setup:regenerate-command-images | setup:self-upgrade│ +│| setup:synchronize-local-mounts | setup:version | shell | start-airflow | testing | │ +│testing:airflow-ctl-integration-tests | testing:airflow-ctl-tests | testing:airflow-e2e-tests | │ +│testing:core-integration-tests | testing:core-tests | testing:docker-compose-tests | │ +│testing:helm-tests | testing:providers-integration-tests | testing:providers-tests | │ +│testing:python-api-client-tests | testing:system-tests | testing:task-sdk-integration-tests | │ +│testing:task-sdk-tests | testing:ui-e2e-tests | ui | ui:check-translation-completeness | │ +│ui:compile-assets | workflow-run | workflow-run:publish-docs)│ +│--check-onlyOnly check if some images need to be regenerated. Return 0 if no need or 1 if needed. Cannot be used  │ +│together with --command flag or --force.                                                              │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Common options ─────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--help   -hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt index 45cb2a115ab16..5f1cbe61c45f9 100644 --- a/dev/breeze/doc/images/output_setup_regenerate-command-images.txt +++ b/dev/breeze/doc/images/output_setup_regenerate-command-images.txt @@ -1 +1 @@ -3b18680cf71a6fe2ac14af44b029a537 +c3aa8a568d6aaa55aae183f1da0697a1 diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index d57016971c488..c3ca46d7e26d0 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -201,7 +201,7 @@ microsoft.mssql | microsoft.psrp | microsoft.winrm | mongo | mysql | neo4j | odbc | openai | openfaas | openlineage |  opensearch | opsgenie | oracle | pagerduty | papermill | pgvector | pinecone | postgres | presto | qdrant | redis |    salesforce | samba | segment | sendgrid | sftp | singularity | slack | smtp | snowflake | sqlite | ssh | standard |    -tableau | task-sdk | telegram | teradata | trino | vertica | weaviate | yandex | ydb | zendesk]...                     +tableau | task-sdk | telegram | teradata | trino | vertica | vespa | weaviate | yandex | ydb | zendesk]...             Trigger publish docs to S3 workflow diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt index 3a57f0bb6d668..f032e9220ced5 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.txt +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.txt @@ -1 +1 @@ -7915334135094723635d68ce396033bf +58e5b5dd710b9517c1309e7b4a24a9dc diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_commands.py index 68c184d2423f0..b8d4138371345 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands.py @@ -536,6 +536,13 @@ def _sync_k8s_schemas_to_airflow_site(airflow_site: Path, force: bool, command_e help="Automatically create a PR with the upgrade changes (if not specified, will ask)", is_flag=True, ) +@click.option( + "--draft/--no-draft", + default=False, + show_default=True, + help="Create the PR as a draft (useful for scheduled CI runs where a human undrafts to trigger CI)", + is_flag=True, +) @click.option( "--switch-to-base/--no-switch-to-base", default=None, @@ -592,6 +599,7 @@ def _sync_k8s_schemas_to_airflow_site(airflow_site: Path, force: bool, command_e def upgrade( target_branch: str, create_pr: bool | None, + draft: bool, switch_to_base: bool | None, airflow_site: Path, force_k8s_schema_sync: bool, @@ -828,12 +836,8 @@ def upgrade( should_create_pr = user_confirm("Do you want to create a PR with the upgrade changes?") == Answer.YES if should_create_pr: - # Get current HEAD commit hash for unique branch name - head_result = run_command( - ["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, check=False - ) - commit_hash = head_result.stdout.strip() if head_result.returncode == 0 else "unknown" - branch_name = f"ci-upgrade-{commit_hash}" + # Use a stable branch name based on target branch so scheduled runs can reuse/update the same PR + branch_name = f"ci-upgrade-{target_branch}" # Check if branch already exists and delete it branch_check = run_command( @@ -845,7 +849,23 @@ def upgrade( run_command(["git", "checkout", "-b", branch_name]) run_command(["git", "add", "."]) - run_command(["git", "commit", "-m", "CI: Upgrade important CI environment"]) + try: + run_command( + ["git", "commit", "--message", f"[{target_branch}] CI: Upgrade important CI environment"] + ) + except subprocess.CalledProcessError: + console_print("[info]Commit failed, assume some auto-fixes might have been made...[/]") + run_command(["git", "add", "."]) + run_command( + [ + "git", + "commit", + # postpone pre-commit checks to CI, not to fail in automation if e.g. mypy changes force code checks + "--no-verify", + "--message", + f"[{target_branch}] CI: Upgrade important CI environment", + ] + ) # Push the branch to origin (use detected origin or fallback to 'origin') push_remote = origin_remote_name if origin_remote_name else "origin" @@ -875,33 +895,88 @@ def upgrade( head_ref = branch_name console_print("[warning]Could not determine fork repository. Using branch name only.[/]") - pr_result = run_command( + pr_title = f"[{target_branch}] Upgrade important CI environment" + pr_body = "This PR upgrades important dependencies of the CI environment." + + # Check if there's already an open PR for this branch + existing_pr_result = run_command( [ "gh", "pr", - "create", - "-w", + "list", "--repo", "apache/airflow", "--head", head_ref, "--base", target_branch, - "--title", - f"[{target_branch}] Upgrade important CI environment", - "--body", - "This PR upgrades important dependencies of the CI environment.", + "--state", + "open", + "--json", + "number,url", + "--jq", + ".[0]", ], capture_output=True, text=True, check=False, env=command_env, ) - if pr_result.returncode != 0: - console_print(f"[error]Failed to create PR:\n{pr_result.stdout}\n{pr_result.stderr}[/]") - sys.exit(1) - pr_url = pr_result.stdout.strip() if pr_result.returncode == 0 else "" - console_print(f"[success]PR created successfully: {pr_url}.[/]") + + existing_pr = existing_pr_result.stdout.strip() if existing_pr_result.returncode == 0 else "" + + if existing_pr and existing_pr != "null" and existing_pr != "": + console_print(f"[success]Existing PR found and updated with force push: {existing_pr}[/]") + if draft: + # Convert back to draft so a human must undraft to trigger CI + run_command( + [ + "gh", + "pr", + "ready", + "--repo", + "apache/airflow", + "--undo", + head_ref, + ], + capture_output=True, + text=True, + check=False, + env=command_env, + ) + console_print("[info]Existing PR converted back to draft.[/]") + else: + # Create a new PR + gh_create_cmd = [ + "gh", + "pr", + "create", + "--repo", + "apache/airflow", + "--head", + head_ref, + "--base", + target_branch, + "--title", + pr_title, + "--body", + pr_body, + ] + if draft: + gh_create_cmd.append("--draft") + + pr_result = run_command( + gh_create_cmd, + capture_output=True, + text=True, + check=False, + env=command_env, + ) + if pr_result.returncode != 0: + console_print(f"[error]Failed to create PR:\n{pr_result.stdout}\n{pr_result.stderr}[/]") + sys.exit(1) + pr_url = pr_result.stdout.strip() if pr_result.returncode == 0 else "" + console_print(f"[success]PR created successfully: {pr_url}.[/]") # Switch back to appropriate branch and delete the temporary branch console_print(f"[info]Cleaning up temporary branch {branch_name}...[/]") diff --git a/dev/breeze/src/airflow_breeze/commands/ci_commands_config.py b/dev/breeze/src/airflow_breeze/commands/ci_commands_config.py index 8c02d6c9c06cc..5e3593f18cf9a 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_commands_config.py @@ -74,6 +74,7 @@ "options": [ "--target-branch", "--create-pr", + "--draft", "--switch-to-base", "--airflow-site", "--force-k8s-schema-sync", diff --git a/dev/breeze/src/airflow_breeze/commands/pr_commands.py b/dev/breeze/src/airflow_breeze/commands/pr_commands.py index c2f92873642a7..5eddc266a7d7c 100644 --- a/dev/breeze/src/airflow_breeze/commands/pr_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/pr_commands.py @@ -58,11 +58,13 @@ from airflow_breeze.utils.pr_cache import ( classification_cache as _classification_cache, get_cached_assessment as _get_cached_assessment, + get_cached_author_profile as _get_cached_author_profile, get_cached_classification as _get_cached_classification, get_cached_review as _get_cached_review, get_cached_status as _get_cached_status, review_cache as _review_cache, save_assessment_cache as _save_assessment_cache, + save_author_profile as _save_author_profile, save_classification_cache as _save_classification_cache, save_review_cache as _save_review_cache, save_status_cache as _save_status_cache, @@ -210,10 +212,12 @@ def _cached_assess_pr( pr_body: str, check_status_summary: str, llm_model: str, + diff_text: str | None = None, ) -> PRAssessment: """Run assess_pr with caching keyed by PR number + commit hash. Returns cached PRAssessment when the commit hash matches, avoiding redundant LLM calls. + When *diff_text* is provided, generates directed review questions from it. """ from airflow_breeze.utils.github import PRAssessment, Violation from airflow_breeze.utils.llm_utils import assess_pr @@ -243,6 +247,16 @@ def _cached_assess_pr( result._from_cache = True # type: ignore[attr-defined] return result + # Generate directed review questions from the diff if available. + # In the TUI, diff_text is passed when the diff has been fetched by the + # background executor before the LLM submission. In the non-TUI flow, + # it is passed explicitly during sequential review. + review_questions: list[str] | None = None + if diff_text: + from airflow_breeze.utils.pr_vault import generate_review_questions + + review_questions = generate_review_questions(diff_text, pr_body) or None + t_start = time.monotonic() last_err: Exception | None = None attempts_made = 0 @@ -255,6 +269,7 @@ def _cached_assess_pr( pr_body=pr_body, check_status_summary=check_status_summary, llm_model=llm_model, + review_questions=review_questions, ) if not result.error: break @@ -1016,7 +1031,14 @@ def _fetch_check_status_counts(token: str, github_repository: str, head_sha: str """Fetch counts of checks by status for a commit. Returns a dict like {"SUCCESS": 5, "FAILURE": 2, ...}. Also includes an "IN_PROGRESS" key for checks still running. + Tries the local vault first; falls back to the GitHub API. """ + from airflow_breeze.utils.pr_vault import load_check_status, save_check_status + + cached = load_check_status(github_repository, head_sha) + if cached is not None: + return cached + owner, repo = github_repository.split("/", 1) counts: dict[str, int] = {} cursor: str | None = None @@ -1053,6 +1075,10 @@ def _fetch_check_status_counts(token: str, github_repository: str, head_sha: str break cursor = page_info.get("endCursor") + # Persist to vault for reuse (same SHA = same results) + if counts: + save_check_status(github_repository, head_sha, counts) + return counts @@ -1788,6 +1814,11 @@ def _fetch_prs_graphql( ) ) + # Persist fetched PRs to vault for reuse across sessions + from airflow_breeze.utils.pr_vault import save_prs_batch + + save_prs_batch(github_repository, prs) + return prs, has_next_page, end_cursor, search_data["issueCount"] @@ -1828,7 +1859,44 @@ def _fetch_single_pr_graphql(token: str, github_repository: str, pr_number: int) ) +def _load_pr_from_vault(github_repository: str, pr_number: int) -> PRData | None: + """Try to load a PR from the vault. Returns None on miss or expired TTL. + + The returned PRData has ``unresolved_threads=[]``, ``review_decisions=[]``, + and ``has_collaborator_review=False``. These are backfilled by + ``_enrich_candidate_details`` which runs during triage/review regardless + of whether the PR came from vault or the API. + """ + from airflow_breeze.utils.pr_vault import load_pr + + data = load_pr(github_repository, pr_number) + if data is None: + return None + return PRData( + number=data["number"], + title=data["title"], + body=data.get("body", ""), + url=data["url"], + created_at=data["created_at"], + updated_at=data["updated_at"], + node_id=data.get("node_id", ""), + author_login=data["author_login"], + author_association=data.get("author_association", "NONE"), + head_sha=data["head_sha"], + base_ref=data.get("base_ref", "main"), + check_summary=data.get("check_summary", ""), + checks_state=data.get("checks_state", "UNKNOWN"), + failed_checks=data.get("failed_checks", []), + commits_behind=data.get("commits_behind", 0), + is_draft=data.get("is_draft", False), + mergeable=data.get("mergeable", "UNKNOWN"), + labels=data.get("labels", []), + unresolved_threads=[], + ) + + _author_profile_cache: dict[str, dict] = {} +_author_profile_lock = threading.Lock() def _compute_author_scoring( @@ -1904,10 +1972,18 @@ def _compute_author_scoring( def _fetch_author_profile(token: str, login: str, github_repository: str) -> dict: """Fetch author profile info via GraphQL: account age, PR counts, contributed repos. - Results are cached per login so the same author is only queried once. + Results are cached in memory (per session) and on disk (across sessions, 7-day TTL). + Thread-safe: uses a lock to avoid redundant API calls from concurrent workers. """ - if login in _author_profile_cache: - return _author_profile_cache[login] + with _author_profile_lock: + if login in _author_profile_cache: + return _author_profile_cache[login] + + # Try disk cache before hitting the API + disk_profile = _get_cached_author_profile(github_repository, login) + if disk_profile: + _author_profile_cache[login] = disk_profile + return disk_profile repo_prefix = f"repo:{github_repository} type:pr author:{login}" global_prefix = f"type:pr author:{login}" @@ -1939,7 +2015,8 @@ def _fetch_author_profile(token: str, login: str, github_repository: str) -> dic "contributed_repos": [], "contributed_repos_total": 0, } - _author_profile_cache[login] = profile + with _author_profile_lock: + _author_profile_cache[login] = profile return profile user_data = data.get("user") or {} created_at = user_data.get("createdAt", "unknown") @@ -1989,7 +2066,12 @@ def _fetch_author_profile(token: str, login: str, github_repository: str) -> dic contrib_total, ), } - _author_profile_cache[login] = profile + with _author_profile_lock: + _author_profile_cache[login] = profile + + # Persist to disk for reuse across sessions + _save_author_profile(github_repository, login, profile) + return profile @@ -5276,6 +5358,7 @@ def _ensure_diff_for_pr(tui_ref: TriageTUI, pr_number: int, pr_url: str) -> None pr_body=cur_pr.body, check_status_summary=cur_pr.check_summary, llm_model=llm_model, + diff_text=diff_cache.get(cur_pr.number), ) ctx.llm_future_to_pr[fut] = cur_pr # Keep as PASSING with LLM in progress @@ -7885,7 +7968,14 @@ def _find_workflow_runs_by_status( """Find workflow runs with a given status for a commit SHA. Common statuses: ``action_required``, ``in_progress``, ``queued``. + Tries the local vault first (10-minute TTL); falls back to the GitHub REST API. """ + from airflow_breeze.utils.pr_vault import load_workflow_runs, save_workflow_runs + + cached = load_workflow_runs(github_repository, head_sha, status) + if cached is not None: + return cached + import requests url = f"https://api.github.com/repos/{github_repository}/actions/runs" @@ -7900,7 +7990,10 @@ def _find_workflow_runs_by_status( return [] if response.status_code != 200: return [] - return response.json().get("workflow_runs", []) + runs = response.json().get("workflow_runs", []) + + save_workflow_runs(github_repository, head_sha, status, runs) + return runs def _find_pending_workflow_runs(token: str, github_repository: str, head_sha: str) -> list[dict]: @@ -9985,9 +10078,15 @@ def _fetch_initial_prs( _initial_review_requested_user: str | None = None if review_mode else review_requested_user if pr_number: - if not quiet: - console_print(f"[info]Fetching PR #{pr_number} via GraphQL...[/]") - all_prs = [_fetch_single_pr_graphql(token, github_repository, pr_number)] + cached = _load_pr_from_vault(github_repository, pr_number) + if cached is not None: + if not quiet: + console_print(f"[info]Loaded PR #{pr_number} from vault cache.[/]") + all_prs = [cached] + else: + if not quiet: + console_print(f"[info]Fetching PR #{pr_number} via GraphQL...[/]") + all_prs = [_fetch_single_pr_graphql(token, github_repository, pr_number)] total_matching_prs = 1 elif len(review_requested_users) > 1 and not review_mode: if not quiet: diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py index 893f4167ebb0a..a0db16d5a86a2 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands.py @@ -2741,6 +2741,7 @@ def get_git_log_command( from_commit: str | None = None, to_commit: str | None = None, is_helm_chart: bool = True, + is_airflow_ctl: bool = False, ) -> list[str]: git_cmd = [ "git", @@ -2754,6 +2755,8 @@ def get_git_log_command( git_cmd.append(from_commit) if is_helm_chart: git_cmd.extend(["--", "chart/"]) + elif is_airflow_ctl: + git_cmd.extend(["--", "airflow-ctl/"]) else: git_cmd.extend(["--", "."]) if verbose: @@ -2794,6 +2797,7 @@ def get_changes( previous_release: str, current_release: str, is_helm_chart: bool = False, + is_airflow_ctl: bool = False, ) -> list[Change]: print(MY_DIR_PATH, SOURCE_DIR_PATH) change_strings = subprocess.check_output( @@ -2802,6 +2806,7 @@ def get_changes( from_commit=previous_release, to_commit=current_release, is_helm_chart=is_helm_chart, + is_airflow_ctl=is_airflow_ctl, ), cwd=SOURCE_DIR_PATH, text=True, @@ -2835,12 +2840,17 @@ def print_issue_content( linked_issues, users: dict[int, set[str]], is_helm_chart: bool = False, + is_airflow_ctl: bool = False, ): link = f"https://pypi.org/project/apache-airflow/{current_release}/" link_text = f"Apache Airflow RC {current_release}" if is_helm_chart: link = f"https://dist.apache.org/repos/dist/dev/airflow/{current_release}" link_text = f"Apache Airflow Helm Chart {current_release.split('/')[-1]}" + elif is_airflow_ctl: + link = f"https://pypi.org/project/apache-airflow-ctl/{current_release.split('/')[-1]}/" + link_text = f"Apache Airflow CTL RC {current_release.split('/')[-1]}" + # Only include PRs that have corresponding user data to avoid KeyError in template pr_list = sorted([pr for pr in pull_requests.keys() if pr in users]) user_logins: dict[int, str] = {pr: " ".join(f"@{u}" for u in uu) for pr, uu in users.items()} @@ -2970,6 +2980,58 @@ def generate_issue_content_core( ) +@release_management_group.command( + name="generate-issue-content-airflow-ctl", help="Generates content for issue to test airflow-ctl release." +) +@click.option( + "--github-token", + envvar="GITHUB_TOKEN", + help=textwrap.dedent( + """ + GitHub token used to authenticate. + You can set omit it if you have GITHUB_TOKEN env variable set. + Can be generated with: + https://github.com/settings/tokens/new?description=Read%20sssues&scopes=repo:status""" + ), +) +@click.option( + "--previous-release", + type=str, + help="commit reference (for example hash or tag) of the previous release.", + required=True, +) +@click.option( + "--current-release", + type=str, + help="commit reference (for example hash or tag) of the current release.", + required=True, +) +@click.option("--excluded-pr-list", type=str, help="Coma-separated list of PRs to exclude from the issue.") +@click.option( + "--limit-pr-count", + type=int, + default=None, + help="Limit PR count processes (useful for testing small subset of PRs).", +) +@option_verbose +def generate_issue_content_airflow_ctl( + github_token: str, + previous_release: str, + current_release: str, + excluded_pr_list: str, + limit_pr_count: int | None, +): + generate_issue_content( + github_token, + previous_release, + current_release, + excluded_pr_list, + limit_pr_count, + is_helm_chart=False, + is_airflow_ctl=True, + ) + + @release_management_group.command( name="generate-providers-metadata", help="Generates metadata for providers." ) @@ -4044,6 +4106,7 @@ def generate_issue_content( excluded_pr_list: str, limit_pr_count: int | None, is_helm_chart: bool, + is_airflow_ctl: bool = False, ): from github import Github, Issue, PullRequest, UnknownObjectException @@ -4053,7 +4116,7 @@ def generate_issue_content( previous = previous_release current = current_release - changes = get_changes(verbose, previous, current, is_helm_chart) + changes = get_changes(verbose, previous, current, is_helm_chart, is_airflow_ctl) change_prs = [change.pr for change in changes] if excluded_pr_list: excluded_prs = [int(pr) for pr in excluded_pr_list.split(",")] @@ -4138,7 +4201,7 @@ def generate_issue_content( users[pr_number].add(linked_issue.user.login) progress.advance(task) - print_issue_content(current, pull_requests, linked_issues, users, is_helm_chart) + print_issue_content(current, pull_requests, linked_issues, users, is_helm_chart, is_airflow_ctl) @release_management_group.command(name="publish-docs-to-s3", help="Publishes docs to S3.") @@ -4253,6 +4316,13 @@ def publish_docs_to_s3( default=False, help="Show explanations for outdated packages.", ) +@click.option( + "--cooldown-days", + type=int, + default=4, + show_default=True, + help="Ignore package versions released within this many days (cooldown period).", +) @option_github_token @option_github_repository @option_verbose @@ -4263,6 +4333,7 @@ def version_check( diff_mode, package: tuple[str], explain_why: bool, + cooldown_days: int, github_token: str, github_repository: str, builder: str, @@ -4288,6 +4359,7 @@ def version_check( explain_why=explain_why, github_token=github_token, github_repository=github_repository, + cooldown_days=cooldown_days, ) diff --git a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py index e6a1fbfa9bf01..aed28fb03bcb3 100644 --- a/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py +++ b/dev/breeze/src/airflow_breeze/commands/release_management_commands_config.py @@ -62,9 +62,7 @@ RELEASE_AIRFLOW_CTL_COMMANDS: dict[str, str | list[str]] = { "name": "airflowctl release commands", - "commands": [ - "prepare-airflow-ctl-distributions", - ], + "commands": ["prepare-airflow-ctl-distributions", "generate-issue-content-airflow-ctl"], } RELEASE_OTHER_COMMANDS: dict[str, str | list[str]] = { @@ -118,6 +116,18 @@ ], } ], + "breeze release-management generate-issue-content-airflow-ctl": [ + { + "name": "Generate issue flags", + "options": [ + "--github-token", + "--previous-release", + "--current-release", + "--excluded-pr-list", + "--limit-pr-count", + ], + } + ], "breeze release-management prepare-helm-chart-tarball": [ { "name": "Package flags", @@ -515,6 +525,7 @@ "--diff-mode", "--package", "--explain-why", + "--cooldown-days", ], }, { diff --git a/dev/breeze/src/airflow_breeze/global_constants.py b/dev/breeze/src/airflow_breeze/global_constants.py index efba238c4dee4..2e3a45505f351 100644 --- a/dev/breeze/src/airflow_breeze/global_constants.py +++ b/dev/breeze/src/airflow_breeze/global_constants.py @@ -767,7 +767,7 @@ def get_airflow_extras(): { "python-version": "3.10", "airflow-version": "2.11.1", - "remove-providers": "common.messaging edge3 fab git keycloak informatica common.ai", + "remove-providers": "common.messaging edge3 fab git keycloak informatica common.ai opensearch", "run-unit-tests": "true", }, { @@ -782,13 +782,19 @@ def get_airflow_extras(): "remove-providers": "", "run-unit-tests": "true", }, + { + "python-version": "3.10", + "airflow-version": "3.2.0", + "remove-providers": "", + "run-unit-tests": "true", + }, ] ALL_PYTHON_VERSION_TO_PATCHLEVEL_VERSION: dict[str, str] = { "3.10": "3.10.20", "3.11": "3.11.15", "3.12": "3.12.13", - "3.13": "3.13.12", + "3.13": "3.13.13", "3.14": "3.14.3", } diff --git a/dev/breeze/src/airflow_breeze/utils/check_release_files.py b/dev/breeze/src/airflow_breeze/utils/check_release_files.py index 81ea3c0563b34..e4fb0f5b0226c 100644 --- a/dev/breeze/src/airflow_breeze/utils/check_release_files.py +++ b/dev/breeze/src/airflow_breeze/utils/check_release_files.py @@ -26,7 +26,8 @@ FROM ghcr.io/apache/airflow/main/ci/python3.10 RUN cd airflow-core; uv sync --no-sources -# Install providers +# Install providers with providers pre-releases allowed +COPY pyproject.toml . {} """ diff --git a/dev/breeze/src/airflow_breeze/utils/click_utils.py b/dev/breeze/src/airflow_breeze/utils/click_utils.py index 92d8d7b7cea15..25a9de3886832 100644 --- a/dev/breeze/src/airflow_breeze/utils/click_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/click_utils.py @@ -16,7 +16,31 @@ # under the License. from __future__ import annotations +from typing import TYPE_CHECKING + try: - from rich_click import RichGroup as BreezeGroup + from rich_click import RichCommand as _BaseCommand, RichGroup as _BaseGroup except ImportError: - from click import Group as BreezeGroup # type: ignore[assignment] # noqa: F401 + from click import ( # type: ignore[assignment] + Command as _BaseCommand, + Group as _BaseGroup, + ) + +if TYPE_CHECKING: + import click + + +class BreezeCommand(_BaseCommand): + """Breeze CLI command that automatically prints reproduction instructions in CI.""" + + def invoke(self, ctx: click.Context) -> None: + try: + return super().invoke(ctx) + finally: + from airflow_breeze.utils.reproduce_ci import maybe_print_reproduction + + maybe_print_reproduction(ctx) + + +class BreezeGroup(_BaseGroup): + command_class = BreezeCommand diff --git a/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py b/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py index aea56854f85ca..9114b2d108943 100755 --- a/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py +++ b/dev/breeze/src/airflow_breeze/utils/constraints_version_check.py @@ -24,7 +24,7 @@ import tempfile import urllib.request from contextlib import contextmanager -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from typing import TYPE_CHECKING, Any from urllib.error import HTTPError, URLError @@ -168,6 +168,38 @@ def should_show_package(releases, latest_version, constraints_date, mode, is_lat return True +def get_latest_version_with_cooldown(releases: dict[str, Any], cooldown_days: int) -> str | None: + """Find the latest non-prerelease version whose release date is outside the cooldown period. + + Returns the version string, or None if no version qualifies. + """ + from packaging import version + + cutoff = datetime.now() - timedelta(days=cooldown_days) + candidates: list[tuple[version.Version, str]] = [] + for v, release_files in releases.items(): + if not release_files: + continue + try: + parsed_v = version.parse(v) + except version.InvalidVersion: + continue + if parsed_v.is_prerelease or parsed_v.is_devrelease: + continue + try: + upload_time = datetime.fromisoformat( + release_files[0]["upload_time_iso_8601"].replace("Z", "+00:00") + ).replace(tzinfo=None) + except (KeyError, IndexError, ValueError): + continue + if upload_time <= cutoff: + candidates.append((parsed_v, v)) + if not candidates: + return None + candidates.sort(key=lambda x: x[0], reverse=True) + return candidates[0][1] + + def get_first_newer_release_date_str(releases, current_version): from packaging import version @@ -210,9 +242,11 @@ def constraints_version_check( explain_why: bool = False, github_token: str | None = None, github_repository: str | None = None, + cooldown_days: int = 4, ): console_print(f"[bold cyan]Python version:[/] [white]{python}[/]") - console_print(f"[bold cyan]Constraints mode:[/] [white]{airflow_constraints_mode}[/]\n") + console_print(f"[bold cyan]Constraints mode:[/] [white]{airflow_constraints_mode}[/]") + console_print(f"[bold cyan]Cooldown period:[/] [white]{cooldown_days} days[/]\n") with tempfile.TemporaryDirectory() as temp_dir: constraints_file = Path(temp_dir) / "constraints.txt" download_constraints_file( @@ -248,6 +282,7 @@ def constraints_version_check( python_version=python, airflow_constraints_mode=airflow_constraints_mode, github_repository=github_repository, + cooldown_days=cooldown_days, ) print_table_footer( @@ -386,6 +421,7 @@ def process_packages( python_version: str, airflow_constraints_mode: str, github_repository: str | None, + cooldown_days: int = 4, ) -> tuple[int, int, list[str], dict[str, int]]: @contextmanager def preserve_pyproject_file(pyproject_path: Path): @@ -417,8 +453,9 @@ def get_release_dates(releases: dict, version: str) -> str: for pkg, pinned_version in packages: try: data = fetch_pypi_data(pkg) - latest_version = data["info"]["version"] releases = data["releases"] + latest_version_with_cooldown = get_latest_version_with_cooldown(releases, cooldown_days) + latest_version = latest_version_with_cooldown or data["info"]["version"] latest_release_date = get_release_dates(releases, latest_version) constraint_release_date = get_release_dates(releases, pinned_version) is_latest_version = pinned_version == latest_version diff --git a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py index 0162168166982..068d73d5b6f93 100644 --- a/dev/breeze/src/airflow_breeze/utils/docs_publisher.py +++ b/dev/breeze/src/airflow_breeze/utils/docs_publisher.py @@ -108,6 +108,10 @@ def publish(self, override_versioned: bool, airflow_site_dir: str): return 1, f"Skipping {self.package_name}: Previously existing directory" # If output directory exists and is not versioned, delete it shutil.rmtree(output_dir) + if not os.path.exists(self._build_dir): + get_console(output=self.output).print(f"Build directory {self._build_dir} does not exist!") + get_console(output=self.output).print() + return 0, f"Skipping {self.package_name}: Build directory does not exist" shutil.copytree(self._build_dir, output_dir) if self.is_versioned: with open(os.path.join(output_dir, "..", "stable.txt"), "w") as stable_file: diff --git a/dev/breeze/src/airflow_breeze/utils/llm_utils.py b/dev/breeze/src/airflow_breeze/utils/llm_utils.py index ee38e251e07d3..d15e203730af7 100644 --- a/dev/breeze/src/airflow_breeze/utils/llm_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/llm_utils.py @@ -151,16 +151,22 @@ def _build_user_message( pr_title: str, pr_body: str, check_status_summary: str, + review_questions: list[str] | None = None, ) -> str: truncated_body = pr_body[:MAX_PR_BODY_CHARS] if pr_body else "(empty)" if pr_body and len(pr_body) > MAX_PR_BODY_CHARS: truncated_body += "\n... (truncated)" - return ( + msg = ( f"PR #{pr_number}\n" f"Title: {pr_title}\n\n" f"Description:\n{truncated_body}\n\n" f"Check status summary:\n{check_status_summary}\n" ) + if review_questions: + msg += "\nDirected verification questions (address each one):\n" + for i, q in enumerate(review_questions, 1): + msg += f" {i}. {q}\n" + return msg def _extract_json(text: str) -> str: @@ -645,10 +651,13 @@ def assess_pr( pr_body: str, check_status_summary: str, llm_model: str, + review_questions: list[str] | None = None, ) -> PRAssessment: """Assess a PR using an LLM CLI tool. Returns PRAssessment. llm_model must be in "provider/model" format (e.g. "claude/claude-3-opus" or "codex/gpt-5.3-codex"). + When *review_questions* is provided, they are appended to the user message so the LLM + addresses each one in its assessment. """ provider, model = _resolve_cli_provider(llm_model) caller = _CLI_CALLERS.get(provider) @@ -658,7 +667,9 @@ def assess_pr( _check_cli_available(provider) system_prompt = get_system_prompt() - user_message = _build_user_message(pr_number, pr_title, pr_body, check_status_summary) + user_message = _build_user_message( + pr_number, pr_title, pr_body, check_status_summary, review_questions=review_questions + ) try: raw = caller(model, system_prompt, user_message) diff --git a/dev/breeze/src/airflow_breeze/utils/path_utils.py b/dev/breeze/src/airflow_breeze/utils/path_utils.py index 03877d6476163..7f2625a0bcca9 100644 --- a/dev/breeze/src/airflow_breeze/utils/path_utils.py +++ b/dev/breeze/src/airflow_breeze/utils/path_utils.py @@ -418,22 +418,26 @@ def cleanup_python_generated_files(): if get_verbose(): console_print("[info]Cleaning .pyc and __pycache__") permission_errors = [] - for path in AIRFLOW_ROOT_PATH.rglob("*.pyc"): - try: - path.unlink() - except FileNotFoundError: - # File has been removed in the meantime. - pass - except PermissionError: - permission_errors.append(path) - for path in AIRFLOW_ROOT_PATH.rglob("__pycache__"): - try: - shutil.rmtree(path) - except FileNotFoundError: - # File has been removed in the meantime. - pass - except PermissionError: - permission_errors.append(path) + for dirpath, dirnames, filenames in os.walk(AIRFLOW_ROOT_PATH): + # Skip node_modules and hidden directories (.*) — modify in place to prune os.walk + dirnames[:] = [d for d in dirnames if d != "node_modules" and not d.startswith(".")] + for filename in filenames: + if filename.endswith(".pyc"): + path = Path(dirpath) / filename + try: + path.unlink() + except FileNotFoundError: + pass + except PermissionError: + permission_errors.append(path) + if Path(dirpath).name == "__pycache__": + try: + shutil.rmtree(dirpath) + except FileNotFoundError: + pass + except PermissionError: + permission_errors.append(Path(dirpath)) + dirnames.clear() if permission_errors: if platform.uname().system.lower() == "linux": console_print("[warning]There were files that you could not clean-up:\n") diff --git a/dev/breeze/src/airflow_breeze/utils/pr_cache.py b/dev/breeze/src/airflow_breeze/utils/pr_cache.py index c1ca137e283ee..fb93d7834607b 100644 --- a/dev/breeze/src/airflow_breeze/utils/pr_cache.py +++ b/dev/breeze/src/airflow_breeze/utils/pr_cache.py @@ -65,10 +65,31 @@ def get(self, github_repository: str, key: str, *, match: dict[str, str] | None return data def save(self, github_repository: str, key: str, data: dict) -> None: - """Save *data* as JSON. Automatically adds ``cached_at`` when TTL is configured.""" + """Save *data* as JSON. Automatically adds ``cached_at`` when TTL is configured. + + Uses atomic write (temp file + os.replace) to avoid corrupt reads when + multiple threads write the same key concurrently. + """ + import os + import tempfile + if self._ttl_seconds: + # time.time() is intentional here: monotonic clocks reset across process + # restarts, so wall-clock time is the only option for persistent TTLs. data = {**data, "cached_at": time.time()} - self._file(github_repository, key).write_text(json.dumps(data, indent=2)) + target = self._file(github_repository, key) + fd, tmp_path = tempfile.mkstemp(dir=target.parent, suffix=".tmp") + closed = False + try: + os.write(fd, json.dumps(data, indent=2).encode()) + os.close(fd) + closed = True + os.replace(tmp_path, target) + except BaseException: + if not closed: + os.close(fd) + Path(tmp_path).unlink(missing_ok=True) + raise # Concrete cache stores — one per domain @@ -77,6 +98,7 @@ def save(self, github_repository: str, key: str, data: dict) -> None: triage_cache = CacheStore("triage_cache") status_cache = CacheStore("status_cache", ttl_seconds=4 * 3600) stats_interaction_cache = CacheStore("stats_interaction_cache") +author_cache = CacheStore("author_cache", ttl_seconds=7 * 24 * 3600) # Convenience functions for common cache operations @@ -142,6 +164,23 @@ def save_status_cache(github_repository: str, cache_key: str, payload: dict | li status_cache.save(github_repository, cache_key, {"payload": payload}) +def get_cached_author_profile(github_repository: str, login: str) -> dict | None: + """Load a cached author profile. Returns None if missing or expired (7-day TTL). + + Strips the internal ``cached_at`` field so callers get the same shape + regardless of whether the profile came from disk or the API. + """ + data = author_cache.get(github_repository, f"author_{login}") + if data is not None: + data.pop("cached_at", None) + return data + + +def save_author_profile(github_repository: str, login: str, profile: dict) -> None: + """Persist an author profile to disk.""" + author_cache.save(github_repository, f"author_{login}", profile) + + # PR-keyed caches that store head_sha and should be validated on startup _PR_CACHES: list[CacheStore] = [review_cache, classification_cache, triage_cache] diff --git a/dev/breeze/src/airflow_breeze/utils/pr_vault.py b/dev/breeze/src/airflow_breeze/utils/pr_vault.py new file mode 100644 index 0000000000000..00208ec249e16 --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/pr_vault.py @@ -0,0 +1,210 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Vault storage for PR triage data — persist across sessions to reduce API calls.""" + +from __future__ import annotations + +import re + +from airflow_breeze.utils.pr_cache import CacheStore + +# ── PR metadata vault ──────────────────────────────────────────── +# 4-hour TTL: PR metadata can change (labels, checks, mergeable status) +# but re-fetching every run is wasteful for PRs that haven't been updated. +_pr_vault = CacheStore("pr_vault", ttl_seconds=4 * 3600) + +# Fields from PRData that are safe to serialize to JSON. +_VAULT_FIELDS = ( + "number", + "title", + "body", + "url", + "created_at", + "updated_at", + "node_id", + "author_login", + "author_association", + "head_sha", + "base_ref", + "check_summary", + "checks_state", + "failed_checks", + "commits_behind", + "is_draft", + "mergeable", + "labels", +) + + +def save_pr(github_repository: str, pr) -> None: + """Persist a PRData instance to the vault.""" + data = {field: getattr(pr, field) for field in _VAULT_FIELDS} + _pr_vault.save(github_repository, f"pr_{pr.number}", data) + + +def load_pr(github_repository: str, pr_number: int, *, head_sha: str | None = None) -> dict | None: + """Load a PR from the vault. Returns None when not cached, expired, or SHA mismatch.""" + match = {"head_sha": head_sha} if head_sha else None + data = _pr_vault.get(github_repository, f"pr_{pr_number}", match=match) + if data is not None: + data.pop("cached_at", None) + return data + + +def save_prs_batch(github_repository: str, prs) -> int: + """Persist a batch of PRData instances. Returns count saved.""" + for pr in prs: + save_pr(github_repository, pr) + return len(prs) + + +# ── Check status vault ─────────────────────────────────────────── +# Keyed by head_sha. Only caches fully-completed check results (no +# IN_PROGRESS or QUEUED). Uses a 4-hour TTL because checks can be +# re-run on the same commit without a force push. +_check_vault = CacheStore("check_vault", ttl_seconds=4 * 3600) + +# Statuses that indicate checks are still running +_INCOMPLETE_STATUSES = {"IN_PROGRESS", "QUEUED", "PENDING"} + + +def save_check_status(github_repository: str, head_sha: str, counts: dict[str, int]) -> None: + """Persist check status counts for a commit. + + Skips caching when any checks are still in progress — partial results + would cause future sessions to see an incomplete picture. + """ + if _INCOMPLETE_STATUSES & set(counts.keys()): + return + _check_vault.save(github_repository, f"checks_{head_sha}", {"head_sha": head_sha, "counts": counts}) + + +def load_check_status(github_repository: str, head_sha: str) -> dict[str, int] | None: + """Load cached check status counts for a commit. Returns None if not cached.""" + data = _check_vault.get(github_repository, f"checks_{head_sha}", match={"head_sha": head_sha}) + return data.get("counts") if data else None + + +# ── Workflow runs vault ────────────────────────────────────────── +# 10-minute TTL: workflow status changes frequently but not instantly. +_workflow_vault = CacheStore("workflow_vault", ttl_seconds=600) + + +def save_workflow_runs(github_repository: str, head_sha: str, status: str, runs: list[dict]) -> None: + """Persist workflow runs for a commit + status combination.""" + _workflow_vault.save( + github_repository, + f"wf_{head_sha}_{status}", + {"head_sha": head_sha, "status": status, "runs": runs}, + ) + + +def load_workflow_runs(github_repository: str, head_sha: str, status: str) -> list[dict] | None: + """Load cached workflow runs. Returns None if not cached or expired.""" + data = _workflow_vault.get( + github_repository, + f"wf_{head_sha}_{status}", + match={"head_sha": head_sha, "status": status}, + ) + return data.get("runs") if data else None + + +# ── Directed review questions ──────────────────────────────────── + + +def generate_review_questions(diff_text: str, pr_body: str) -> list[str]: + """Generate verification questions from a PR diff and body. + + These are deterministic checks that don't require an LLM. They can be + appended to the LLM prompt to focus the assessment on concrete issues. + """ + questions: list[str] = [] + + if not diff_text: + return questions + + # Extract only added lines for content analysis (avoids false positives + # from removed lines that contain keywords like "deprecated"). + added_lines = "\n".join( + line[1:] for line in diff_text.splitlines() if line.startswith("+") and not line.startswith("+++") + ) + + # Count changes + added = len(re.findall(r"^\+[^+]", diff_text, re.MULTILINE)) + removed = len(re.findall(r"^-[^-]", diff_text, re.MULTILINE)) + total = added + removed + + # Large PR warning + if total > 500: + questions.append( + f"LARGE PR: {total} changed lines (+{added}/-{removed}). " + f"Should this be split into smaller, focused PRs?" + ) + + # Source files without test changes + src_files: set[str] = set() + test_files: set[str] = set() + for match in re.finditer(r"^diff --git a/(.+?) b/", diff_text, re.MULTILINE): + path = match.group(1) + if "test" in path.lower(): + test_files.add(path) + elif path.endswith((".py", ".js", ".ts", ".java", ".go", ".rs")): + src_files.add(path) + if src_files and not test_files: + questions.append( + f"TEST COVERAGE: {len(src_files)} source file(s) modified but no test files changed. " + f"Is test coverage needed?" + ) + + # Version fields referencing already-released versions (only in added lines) + version_matches = re.findall(r"version_added:\s*[\"']?(\d+\.\d+\.\d+)", added_lines) + if version_matches: + questions.append( + f"VERSION CHECK: version_added references {', '.join(set(version_matches))}. " + f"Verify these are unreleased versions." + ) + + # Breaking change indicators (only in added lines to avoid false positives + # from removed deprecation notices) + breaking_signals = [ + "breaking", + "backward", + "deprecat", + "behaviour change", + "behavior change", + "BREAKING CHANGE", + "incompatible", + ] + added_lower = added_lines.lower() + found_signals = [s for s in breaking_signals if s in added_lower] + if found_signals: + questions.append( + "BREAKING CHANGE: This diff contains breaking change indicators " + f"({', '.join(found_signals)}). Has this been discussed in an issue or " + "on the mailing list?" + ) + + # Multiple exception types (only in added lines) + exceptions = re.findall(r"raise (\w+(?:Error|Exception))\(", added_lines) + unique_exceptions = set(exceptions) + if len(unique_exceptions) > 3: + questions.append( + f"CONSISTENCY: {len(unique_exceptions)} different exception types raised " + f"({', '.join(sorted(unique_exceptions)[:5])}). Should these be consolidated?" + ) + + return questions diff --git a/dev/breeze/src/airflow_breeze/utils/reproduce_ci.py b/dev/breeze/src/airflow_breeze/utils/reproduce_ci.py new file mode 100644 index 0000000000000..4a429f556ab8d --- /dev/null +++ b/dev/breeze/src/airflow_breeze/utils/reproduce_ci.py @@ -0,0 +1,239 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Helpers for printing local reproduction instructions in CI logs.""" + +from __future__ import annotations + +import os +import shlex +from dataclasses import dataclass + +import click +from click.core import ParameterSource +from rich.markup import escape + +from airflow_breeze.global_constants import APACHE_AIRFLOW_GITHUB_REPOSITORY +from airflow_breeze.utils.console import get_console +from airflow_breeze.utils.run_utils import commit_sha + +# Options that are side-effect-only or not meaningful for reproduction (safety net; +# expose_value=False options like --verbose/--dry-run/--answer are already excluded +# automatically because they don't appear in ctx.params). +_EXCLUDED_PARAMS: frozenset[str] = frozenset( + { + "verbose", + "dry_run", + "answer", + "include_success_outputs", + "debug_resources", + } +) + +# These sources represent values explicitly provided by the user or CI. +_EXPLICIT_SOURCES: frozenset[ParameterSource] = frozenset( + { + ParameterSource.COMMANDLINE, + ParameterSource.ENVIRONMENT, + ParameterSource.PROMPT, + } +) + + +@dataclass +class ReproductionCommand: + argv: list[str] + comment: str | None = None + + +def build_reproduction_command_from_context( + ctx: click.Context, + *, + comment: str = "Run the same Breeze command locally", +) -> ReproductionCommand: + """Reconstruct the CLI invocation from the current click Context. + + Iterates over every parameter defined on the command, uses + ``ctx.get_parameter_source()`` to identify explicitly-provided values + (COMMANDLINE / ENVIRONMENT / PROMPT), and emits only those. DEFAULT + and DEFAULT_MAP values are omitted to keep the output concise. + + This removes the need for per-command builder functions. + """ + argv: list[str] = ctx.command_path.split() + + for param in ctx.command.params: + if not getattr(param, "expose_value", True): + continue + if param.name is None or param.name in _EXCLUDED_PARAMS: + continue + + value = ctx.params.get(param.name) + source = ctx.get_parameter_source(param.name) + + if isinstance(param, click.Argument): + continue # collected after options + + if not isinstance(param, click.Option): + continue + + # Flag pair (e.g. --force-sa-warnings/--no-force-sa-warnings): + # emit the appropriate side only when explicitly provided. + if param.is_flag and param.secondary_opts: + if source in _EXPLICIT_SOURCES: + # Prefer long-form alias for both sides of the flag pair. + flag = param.opts[-1] if value else param.secondary_opts[-1] + argv.append(flag) + continue + + # Simple boolean flag (no secondary_opts) + if param.is_flag: + if value and source in _EXPLICIT_SOURCES: + argv.append(param.opts[-1]) + continue + + # Non-flag option: only emit explicitly-provided values + if source not in _EXPLICIT_SOURCES: + continue + if value is None: + continue + + flag = param.opts[-1] # prefer long form + + # Multiple option (e.g. --package-filter repeated) + if param.multiple: + for item in value: + argv.extend([flag, str(item)]) + continue + + argv.extend([flag, str(value)]) + + # Append positional arguments at the end + for param in ctx.command.params: + if isinstance(param, click.Argument) and param.name is not None: + value = ctx.params.get(param.name) + if value is not None: + if isinstance(value, (list, tuple)): + argv.extend(str(v) for v in value) + else: + argv.append(str(value)) + + return ReproductionCommand(argv=argv, comment=comment) + + +def build_checkout_reproduction_commands(github_repository: str) -> list[ReproductionCommand]: + """Build git commands needed to reproduce the current CI checkout locally.""" + current_commit_sha = os.environ.get("GITHUB_SHA") or os.environ.get("COMMIT_SHA") or commit_sha() + github_ref = os.environ.get("GITHUB_REF", "") + github_ref_parts = github_ref.split("/") + if len(github_ref_parts) == 4 and github_ref_parts[:2] == ["refs", "pull"]: + pull_request_number = github_ref_parts[2] + pull_request_ref_kind = github_ref_parts[3] + return [ + ReproductionCommand( + argv=[ + "git", + "fetch", + f"https://github.com/{github_repository}.git", + github_ref, + ], + comment=f"Fetch the same code as CI (pull request {pull_request_ref_kind} ref)" + f" — or: gh pr checkout {pull_request_number}", + ), + ReproductionCommand( + argv=["git", "checkout", current_commit_sha], + ), + ] + + if not current_commit_sha or current_commit_sha == "COMMIT_SHA_NOT_FOUND": + return [] + return [ + ReproductionCommand( + argv=["git", "checkout", current_commit_sha], + comment="Check out the same commit", + ) + ] + + +def build_ci_image_reproduction_command( + *, + github_repository: str = APACHE_AIRFLOW_GITHUB_REPOSITORY, + platform: str = "linux/amd64", + python: str = "", +) -> ReproductionCommand: + """Build the CI image preparation command for local reproduction.""" + if not python: + from airflow_breeze.global_constants import DEFAULT_PYTHON_MAJOR_MINOR_VERSION + + python = DEFAULT_PYTHON_MAJOR_MINOR_VERSION + command = ["breeze", "ci-image", "build"] + if github_repository != APACHE_AIRFLOW_GITHUB_REPOSITORY: + command.extend(["--github-repository", github_repository]) + command.extend(["--platform", platform, "--python", python]) + return ReproductionCommand( + argv=command, + comment="Build the CI image locally", + ) + + +def should_print_local_reproduction() -> bool: + """Return True when local reproduction instructions should be printed.""" + return ( + os.environ.get("CI", "").lower() == "true" and os.environ.get("GITHUB_ACTIONS", "").lower() == "true" + ) + + +def print_local_reproduction(commands: list[ReproductionCommand]) -> None: + """Print local reproduction commands in CI logs.""" + if not should_print_local_reproduction() or not commands: + return + lines: list[str] = [] + step_number = 0 + for command in commands: + if command.comment: + if lines: + lines.append("") + step_number += 1 + lines.append(f"# {step_number}. {command.comment}") + lines.append(shlex.join(command.argv)) + rendered = "\n".join(lines) + ruler = "─" * 80 + console = get_console() + console.print(f"\n[warning]{ruler}[/]") + console.print("[warning]HOW TO REPRODUCE LOCALLY[/]\n") + console.print(f"[info]{escape(rendered)}[/]\n", soft_wrap=True) + console.print(f"[warning]{ruler}[/]\n") + + +def maybe_print_reproduction(ctx: click.Context) -> None: + """Called by BreezeCommand.invoke() — prints reproduction instructions in CI.""" + if not should_print_local_reproduction(): + return + + github_repository = ctx.params.get("github_repository", APACHE_AIRFLOW_GITHUB_REPOSITORY) + commands = build_checkout_reproduction_commands(github_repository) + # Skip CI image prelude for image build commands themselves — it would be redundant. + command_path = ctx.command_path + if not command_path.startswith(("breeze ci-image", "breeze prod-image")): + commands.append( + build_ci_image_reproduction_command( + github_repository=github_repository, + python=ctx.params.get("python", ""), + platform=ctx.params.get("platform", "linux/amd64"), + ) + ) + commands.append(build_reproduction_command_from_context(ctx)) + print_local_reproduction(commands) diff --git a/dev/breeze/src/airflow_breeze/utils/selective_checks.py b/dev/breeze/src/airflow_breeze/utils/selective_checks.py index 6a756289db649..19506d490b6a5 100644 --- a/dev/breeze/src/airflow_breeze/utils/selective_checks.py +++ b/dev/breeze/src/airflow_breeze/utils/selective_checks.py @@ -302,6 +302,7 @@ def __hash__(self): r"^chart/templates/.*", r"^providers/.*/src/.*", r"^providers/.*/tests/.*", + r"^shared/.*\.py$", r"^task-sdk/src/.*", r"^task-sdk/tests/.*", r"^devel-common/src/.*", diff --git a/dev/breeze/src/airflow_breeze/utils/tui_display.py b/dev/breeze/src/airflow_breeze/utils/tui_display.py index eb408bfae4a80..159f1661e8b47 100644 --- a/dev/breeze/src/airflow_breeze/utils/tui_display.py +++ b/dev/breeze/src/airflow_breeze/utils/tui_display.py @@ -1753,7 +1753,7 @@ def search_jump(self) -> bool: matching entry. Pressing Escape cancels. Returns True if the cursor moved. """ width, height = _get_terminal_size() - prompt = "/ Jump to PR #: " + prompt = "/ Search (PR#, title, author): " query = "" while True: @@ -1786,18 +1786,28 @@ def search_jump(self) -> bool: if not query: return False - # Match by PR number only + # Try exact PR number match first + stripped = query.lstrip("#") try: - target_num = int(query.lstrip("#")) + target_num = int(stripped) except ValueError: - return False - + target_num = None + + if target_num is not None: + for idx, entry in enumerate(self.entries): + if entry.pr.number == target_num: + self.cursor = idx + self.scroll_offset = idx + self._focus = _FocusPanel.PR_LIST + return True + + # Fall back to text search on title/author (also for numeric queries + # that didn't match any PR number) + query_lower = query.lower() for idx, entry in enumerate(self.entries): - if entry.pr.number == target_num: + if query_lower in entry.pr.title.lower() or query_lower in entry.pr.author_login.lower(): self.cursor = idx - # Put the matched entry at the top of the visible list self.scroll_offset = idx - # Switch focus to PR list so the selection is highlighted self._focus = _FocusPanel.PR_LIST return True diff --git a/dev/breeze/tests/test_author_cache.py b/dev/breeze/tests/test_author_cache.py new file mode 100644 index 0000000000000..d2ad1c1bd35a3 --- /dev/null +++ b/dev/breeze/tests/test_author_cache.py @@ -0,0 +1,80 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +import time +from unittest import mock + +import pytest + +from airflow_breeze.utils.pr_cache import ( + author_cache, + get_cached_author_profile, + save_author_profile, +) + + +@pytest.fixture +def _fake_cache_dir(tmp_path): + """Redirect CacheStore to a temporary directory.""" + with mock.patch( + "airflow_breeze.utils.pr_cache.CacheStore.cache_dir", + return_value=tmp_path, + ): + yield tmp_path + + +class TestAuthorProfilePersistence: + def test_save_and_load(self, _fake_cache_dir): + profile = { + "login": "testuser", + "account_age": "2 years", + "repo_total_prs": 10, + "repo_merged_prs": 8, + } + save_author_profile("apache/airflow", "testuser", profile) + loaded = get_cached_author_profile("apache/airflow", "testuser") + assert loaded is not None + assert loaded["login"] == "testuser" + assert loaded["repo_total_prs"] == 10 + + def test_returns_none_when_missing(self, _fake_cache_dir): + assert get_cached_author_profile("apache/airflow", "nobody") is None + + def test_ttl_expiration(self, _fake_cache_dir): + profile = {"login": "olduser", "repo_total_prs": 5} + save_author_profile("apache/airflow", "olduser", profile) + + # Manually backdate the cached_at timestamp + cache_file = _fake_cache_dir / "author_olduser.json" + data = json.loads(cache_file.read_text()) + data["cached_at"] = time.time() - (8 * 24 * 3600) # 8 days ago + cache_file.write_text(json.dumps(data)) + + assert get_cached_author_profile("apache/airflow", "olduser") is None + + def test_fresh_cache_not_expired(self, _fake_cache_dir): + profile = {"login": "freshuser", "repo_total_prs": 3} + save_author_profile("apache/airflow", "freshuser", profile) + + loaded = get_cached_author_profile("apache/airflow", "freshuser") + assert loaded is not None + assert loaded["login"] == "freshuser" + + def test_ttl_is_7_days(self): + assert author_cache._ttl_seconds == 7 * 24 * 3600 diff --git a/dev/breeze/tests/test_cache_validation.py b/dev/breeze/tests/test_cache_validation.py new file mode 100644 index 0000000000000..3924d8f478e8e --- /dev/null +++ b/dev/breeze/tests/test_cache_validation.py @@ -0,0 +1,117 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import pytest + +from airflow_breeze.utils.pr_cache import ( + invalidate_stale_caches, + review_cache, + save_review_cache, + scan_cached_pr_numbers, + triage_cache, +) + + +@pytest.fixture +def _fake_cache_dirs(tmp_path): + """Each CacheStore gets its own subdir under tmp_path.""" + + def _make_dir(self, github_repository): + safe = github_repository.replace("/", "_") + d = tmp_path / self._cache_name / safe + d.mkdir(parents=True, exist_ok=True) + return d + + with mock.patch( + "airflow_breeze.utils.pr_cache.CacheStore.cache_dir", + _make_dir, + ): + yield tmp_path + + +class TestScanCachedPrNumbers: + def test_scans_across_caches(self, _fake_cache_dirs): + save_review_cache("apache/airflow", 100, "sha_aaa", {"summary": "ok"}) + save_review_cache("apache/airflow", 200, "sha_bbb", {"summary": "ok"}) + + result = scan_cached_pr_numbers("apache/airflow") + assert 100 in result + assert 200 in result + assert result[100]["review_cache"] == "sha_aaa" + + def test_empty_cache(self, _fake_cache_dirs): + result = scan_cached_pr_numbers("apache/airflow") + assert result == {} + + def test_ignores_non_pr_files(self, _fake_cache_dirs): + # Create a file that doesn't match pr_.json + cache_dir = review_cache.cache_dir("apache/airflow") + (cache_dir / "other_data.json").write_text('{"key": "value"}') + + result = scan_cached_pr_numbers("apache/airflow") + assert result == {} + + def test_handles_corrupt_json(self, _fake_cache_dirs): + cache_dir = review_cache.cache_dir("apache/airflow") + (cache_dir / "pr_999.json").write_text("not json{{{") + + result = scan_cached_pr_numbers("apache/airflow") + assert 999 not in result + + +class TestInvalidateStaleCaches: + def test_removes_stale_entries(self, _fake_cache_dirs): + save_review_cache("apache/airflow", 100, "old_sha", {"summary": "ok"}) + + removed = invalidate_stale_caches("apache/airflow", {100: "new_sha"}) + assert removed == 1 + + # Cache file should be gone + assert not (review_cache.cache_dir("apache/airflow") / "pr_100.json").exists() + + def test_keeps_fresh_entries(self, _fake_cache_dirs): + save_review_cache("apache/airflow", 100, "same_sha", {"summary": "ok"}) + + removed = invalidate_stale_caches("apache/airflow", {100: "same_sha"}) + assert removed == 0 + + # Cache file should still exist + assert (review_cache.cache_dir("apache/airflow") / "pr_100.json").exists() + + def test_handles_missing_pr(self, _fake_cache_dirs): + # PR 999 has no cache entry + removed = invalidate_stale_caches("apache/airflow", {999: "any_sha"}) + assert removed == 0 + + def test_removes_corrupt_files(self, _fake_cache_dirs): + cache_dir = review_cache.cache_dir("apache/airflow") + (cache_dir / "pr_100.json").write_text("corrupt{{{") + + removed = invalidate_stale_caches("apache/airflow", {100: "any_sha"}) + assert removed == 1 + assert not (cache_dir / "pr_100.json").exists() + + def test_removes_across_multiple_caches(self, _fake_cache_dirs): + save_review_cache("apache/airflow", 100, "old_sha", {"summary": "ok"}) + # Also save to triage cache + triage_cache.save("apache/airflow", "pr_100", {"head_sha": "old_sha", "assessment": {}}) + + removed = invalidate_stale_caches("apache/airflow", {100: "new_sha"}) + assert removed == 2 diff --git a/dev/breeze/tests/test_pr_vault.py b/dev/breeze/tests/test_pr_vault.py new file mode 100644 index 0000000000000..4565f0a15d9a2 --- /dev/null +++ b/dev/breeze/tests/test_pr_vault.py @@ -0,0 +1,253 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import json +import time +from dataclasses import dataclass +from unittest import mock + +import pytest + +from airflow_breeze.utils.pr_vault import ( + generate_review_questions, + load_check_status, + load_pr, + load_workflow_runs, + save_check_status, + save_pr, + save_prs_batch, + save_workflow_runs, +) + + +@dataclass +class _FakePR: + """Minimal PR-like object for testing vault serialization.""" + + number: int = 12345 + title: str = "Fix something" + body: str = "Description" + url: str = "https://github.com/apache/airflow/pull/12345" + created_at: str = "2026-03-01T00:00:00Z" + updated_at: str = "2026-03-02T00:00:00Z" + node_id: str = "PR_abc" + author_login: str = "testuser" + author_association: str = "CONTRIBUTOR" + head_sha: str = "sha123" + base_ref: str = "main" + check_summary: str = "3 checks: 2 success, 1 failure" + checks_state: str = "FAILURE" + failed_checks: list | None = None + commits_behind: int = 5 + is_draft: bool = False + mergeable: str = "MERGEABLE" + labels: list | None = None + + def __post_init__(self): + if self.failed_checks is None: + self.failed_checks = ["mypy"] + if self.labels is None: + self.labels = ["area:core"] + + +@pytest.fixture +def _fake_cache_dir(tmp_path): + with mock.patch( + "airflow_breeze.utils.pr_cache.CacheStore.cache_dir", + return_value=tmp_path, + ): + yield tmp_path + + +class TestPRVault: + def test_save_and_load(self, _fake_cache_dir): + pr = _FakePR() + save_pr("apache/airflow", pr) + loaded = load_pr("apache/airflow", 12345) + assert loaded is not None + assert loaded["number"] == 12345 + assert loaded["title"] == "Fix something" + assert loaded["head_sha"] == "sha123" + assert loaded["labels"] == ["area:core"] + + def test_load_missing(self, _fake_cache_dir): + assert load_pr("apache/airflow", 99999) is None + + def test_load_with_matching_sha(self, _fake_cache_dir): + pr = _FakePR() + save_pr("apache/airflow", pr) + loaded = load_pr("apache/airflow", 12345, head_sha="sha123") + assert loaded is not None + assert loaded["number"] == 12345 + + def test_load_with_stale_sha(self, _fake_cache_dir): + pr = _FakePR() + save_pr("apache/airflow", pr) + loaded = load_pr("apache/airflow", 12345, head_sha="different_sha") + assert loaded is None + + def test_ttl_expiration(self, _fake_cache_dir): + pr = _FakePR() + save_pr("apache/airflow", pr) + + cache_file = _fake_cache_dir / "pr_12345.json" + data = json.loads(cache_file.read_text()) + data["cached_at"] = time.time() - (5 * 3600) # 5 hours ago, past 4h TTL + cache_file.write_text(json.dumps(data)) + + assert load_pr("apache/airflow", 12345) is None + + def test_fresh_not_expired(self, _fake_cache_dir): + pr = _FakePR() + save_pr("apache/airflow", pr) + assert load_pr("apache/airflow", 12345) is not None + + def test_save_batch(self, _fake_cache_dir): + prs = [_FakePR(number=100), _FakePR(number=200), _FakePR(number=300)] + count = save_prs_batch("apache/airflow", prs) + assert count == 3 + assert load_pr("apache/airflow", 100) is not None + assert load_pr("apache/airflow", 200) is not None + assert load_pr("apache/airflow", 300) is not None + + def test_does_not_serialize_unresolved_threads(self, _fake_cache_dir): + pr = _FakePR() + save_pr("apache/airflow", pr) + loaded = load_pr("apache/airflow", 12345) + assert "unresolved_threads" not in loaded + + def test_strips_cached_at(self, _fake_cache_dir): + pr = _FakePR() + save_pr("apache/airflow", pr) + loaded = load_pr("apache/airflow", 12345) + assert "cached_at" not in loaded + + +class TestCheckStatusVault: + def test_save_and_load(self, _fake_cache_dir): + counts = {"SUCCESS": 5, "FAILURE": 2} + save_check_status("apache/airflow", "sha_abc", counts) + loaded = load_check_status("apache/airflow", "sha_abc") + assert loaded == {"SUCCESS": 5, "FAILURE": 2} + + def test_load_missing(self, _fake_cache_dir): + assert load_check_status("apache/airflow", "nonexistent") is None + + def test_different_sha_returns_none(self, _fake_cache_dir): + save_check_status("apache/airflow", "sha_abc", {"SUCCESS": 1}) + assert load_check_status("apache/airflow", "sha_different") is None + + def test_ttl_expires_stale_results(self, _fake_cache_dir): + """Check vault uses 4h TTL so re-run results are picked up.""" + save_check_status("apache/airflow", "sha_abc", {"SUCCESS": 1}) + loaded = load_check_status("apache/airflow", "sha_abc") + assert loaded is not None + + # Simulate expiry by backdating cached_at + import json + + cache_file = _fake_cache_dir / "checks_sha_abc.json" + data = json.loads(cache_file.read_text()) + data["cached_at"] = data["cached_at"] - 5 * 3600 # 5 hours ago + cache_file.write_text(json.dumps(data)) + + assert load_check_status("apache/airflow", "sha_abc") is None + + +class TestWorkflowRunsVault: + def test_save_and_load(self, _fake_cache_dir): + runs = [{"id": 1, "name": "Tests", "status": "action_required"}] + save_workflow_runs("apache/airflow", "sha_abc", "action_required", runs) + loaded = load_workflow_runs("apache/airflow", "sha_abc", "action_required") + assert loaded == runs + + def test_load_missing(self, _fake_cache_dir): + assert load_workflow_runs("apache/airflow", "sha_abc", "action_required") is None + + def test_different_status_returns_none(self, _fake_cache_dir): + runs = [{"id": 1}] + save_workflow_runs("apache/airflow", "sha_abc", "action_required", runs) + assert load_workflow_runs("apache/airflow", "sha_abc", "in_progress") is None + + def test_ttl_expiration(self, _fake_cache_dir): + runs = [{"id": 1}] + save_workflow_runs("apache/airflow", "sha_abc", "action_required", runs) + + cache_file = _fake_cache_dir / "wf_sha_abc_action_required.json" + data = json.loads(cache_file.read_text()) + data["cached_at"] = time.time() - 700 # past 600s TTL + cache_file.write_text(json.dumps(data)) + + assert load_workflow_runs("apache/airflow", "sha_abc", "action_required") is None + + +class TestGenerateReviewQuestions: + def test_large_pr(self): + diff = "\n".join([f"+line{i}" for i in range(600)]) + questions = generate_review_questions(diff, "") + assert any("LARGE PR" in q for q in questions) + + def test_no_tests(self): + diff = "diff --git a/src/foo.py b/src/foo.py\n+new code\n" + questions = generate_review_questions(diff, "") + assert any("TEST COVERAGE" in q for q in questions) + + def test_with_tests(self): + diff = ( + "diff --git a/src/foo.py b/src/foo.py\n+code\n" + "diff --git a/tests/test_foo.py b/tests/test_foo.py\n+test\n" + ) + questions = generate_review_questions(diff, "") + assert not any("TEST COVERAGE" in q for q in questions) + + def test_version_added(self): + diff = '+ version_added: "2.8.0"\n' + questions = generate_review_questions(diff, "") + assert any("VERSION CHECK" in q for q in questions) + + def test_breaking_change(self): + diff = "+# BREAKING CHANGE: removed old API\n" + questions = generate_review_questions(diff, "") + assert any("BREAKING CHANGE" in q for q in questions) + + def test_empty_diff(self): + assert generate_review_questions("", "") == [] + + def test_small_clean_pr(self): + diff = ( + "diff --git a/src/foo.py b/src/foo.py\n+one line\n" + "diff --git a/tests/test_foo.py b/tests/test_foo.py\n+test\n" + ) + questions = generate_review_questions(diff, "") + assert questions == [] + + def test_multiple_exceptions(self): + diff = "+raise ValueError(\n+raise TypeError(\n+raise KeyError(\n+raise RuntimeError(\n" + questions = generate_review_questions(diff, "") + assert any("CONSISTENCY" in q for q in questions) + + def test_removed_deprecated_no_false_positive(self): + """Removing a deprecation notice should not trigger BREAKING CHANGE.""" + diff = "-# This is deprecated and will be removed\n+# Updated comment\n" + questions = generate_review_questions(diff, "") + assert not any("BREAKING CHANGE" in q for q in questions) + + def test_added_deprecated_triggers(self): + diff = "+# deprecated: use new_function instead\n" + questions = generate_review_questions(diff, "") + assert any("BREAKING CHANGE" in q for q in questions) diff --git a/dev/breeze/tests/test_pytest_args_for_test_types.py b/dev/breeze/tests/test_pytest_args_for_test_types.py index 91a2a54b328a4..423c96eb10f20 100644 --- a/dev/breeze/tests/test_pytest_args_for_test_types.py +++ b/dev/breeze/tests/test_pytest_args_for_test_types.py @@ -70,6 +70,7 @@ def _find_all_integration_folders() -> list[str]: "providers/microsoft/mssql/tests/integration", "providers/mongo/tests/integration", "providers/openlineage/tests/integration", + "providers/opensearch/tests/integration", "providers/qdrant/tests/integration", "providers/redis/tests/integration", "providers/trino/tests/integration", diff --git a/dev/breeze/tests/test_reproduce_ci.py b/dev/breeze/tests/test_reproduce_ci.py new file mode 100644 index 0000000000000..13d9c1b58e626 --- /dev/null +++ b/dev/breeze/tests/test_reproduce_ci.py @@ -0,0 +1,432 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +from unittest import mock + +import click +import click.testing +import pytest + +from airflow_breeze.utils.click_utils import BreezeGroup +from airflow_breeze.utils.reproduce_ci import ( + ReproductionCommand, + build_checkout_reproduction_commands, + build_ci_image_reproduction_command, + build_reproduction_command_from_context, + print_local_reproduction, + should_print_local_reproduction, +) + + +@pytest.mark.parametrize( + ("env_vars", "expected"), + [ + ({"CI": "true", "GITHUB_ACTIONS": "true"}, True), + ({"CI": "true", "GITHUB_ACTIONS": "false"}, False), + ({"CI": "false", "GITHUB_ACTIONS": "true"}, False), + ({}, False), + ], +) +def test_should_print_local_reproduction_only_in_github_actions(env_vars, expected, monkeypatch): + monkeypatch.delenv("CI", raising=False) + monkeypatch.delenv("GITHUB_ACTIONS", raising=False) + for key, value in env_vars.items(): + monkeypatch.setenv(key, value) + assert should_print_local_reproduction() is expected + + +def test_build_ci_image_reproduction_command_with_custom_repo(): + result = build_ci_image_reproduction_command( + github_repository="someone/airflow", + platform="linux/arm64", + python="3.11", + ) + assert result.comment == "Build the CI image locally" + assert result.argv == [ + "breeze", + "ci-image", + "build", + "--github-repository", + "someone/airflow", + "--platform", + "linux/arm64", + "--python", + "3.11", + ] + + +def test_build_ci_image_reproduction_command_default_repo(): + result = build_ci_image_reproduction_command(platform="linux/amd64", python="3.10") + assert result.argv == [ + "breeze", + "ci-image", + "build", + "--platform", + "linux/amd64", + "--python", + "3.10", + ] + + +@pytest.mark.parametrize("pr_ref_kind", ["merge", "head"]) +def test_build_checkout_reproduction_commands_fetches_pull_request_ref(pr_ref_kind, monkeypatch): + github_ref = f"refs/pull/42/{pr_ref_kind}" + monkeypatch.setenv("GITHUB_REF", github_ref) + monkeypatch.setenv("GITHUB_SHA", "merge-sha") + + commands = build_checkout_reproduction_commands("someone/airflow") + + assert [command.comment for command in commands] == [ + f"Fetch the same code as CI (pull request {pr_ref_kind} ref) — or: gh pr checkout 42", + None, + ] + assert commands[0].argv == [ + "git", + "fetch", + "https://github.com/someone/airflow.git", + github_ref, + ] + assert commands[1].argv == ["git", "checkout", "merge-sha"] + + +def test_build_checkout_reproduction_commands_plain_sha(monkeypatch): + monkeypatch.delenv("GITHUB_REF", raising=False) + monkeypatch.setenv("GITHUB_SHA", "def456") + + commands = build_checkout_reproduction_commands("apache/airflow") + + assert len(commands) == 1 + assert commands[0].argv == ["git", "checkout", "def456"] + + +@mock.patch("airflow_breeze.utils.reproduce_ci.get_console", autospec=True) +def test_print_local_reproduction_renders_copyable_commands(mock_get_console, monkeypatch): + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("GITHUB_ACTIONS", "true") + + print_local_reproduction( + [ + ReproductionCommand(argv=["git", "checkout", "abc123"], comment="Check out the same commit"), + ReproductionCommand( + argv=["breeze", "build-docs", "--docs-only"], + comment="Run the same Breeze command locally", + ), + ] + ) + + assert mock_get_console.return_value.print.call_count == 4 + ruler_line = mock_get_console.return_value.print.call_args_list[0].args[0] + assert "─" * 80 in ruler_line + rendered_output = mock_get_console.return_value.print.call_args_list[2].args[0] + assert "# 1. Check out the same commit" in rendered_output + assert "git checkout abc123" in rendered_output + assert "breeze build-docs --docs-only" in rendered_output + bottom_ruler = mock_get_console.return_value.print.call_args_list[3].args[0] + assert "─" * 80 in bottom_ruler + + +def _invoke_and_capture_reproduction(cli, args, monkeypatch, *, env: dict[str, str] | None = None): + captured_commands: list[list[ReproductionCommand]] = [] + + monkeypatch.setenv("CI", "true") + monkeypatch.setenv("GITHUB_ACTIONS", "true") + monkeypatch.setenv("GITHUB_SHA", "abc123") + monkeypatch.delenv("GITHUB_REF", raising=False) + if env: + for key, value in env.items(): + monkeypatch.setenv(key, value) + monkeypatch.setattr( + "airflow_breeze.utils.reproduce_ci.print_local_reproduction", + lambda commands: captured_commands.append(commands), + ) + + result = click.testing.CliRunner().invoke(cli, args) + + assert result.exit_code == 0, result.output + assert len(captured_commands) == 1 + return captured_commands[0] + + +def test_breeze_command_smoke_skip_cleanup_is_included_in_rendered_command(monkeypatch): + @click.group(cls=BreezeGroup, name="breeze") + def cli(): + pass + + @cli.command(name="parallel-task") + @click.option("--skip-cleanup", is_flag=True) + def parallel_task(skip_cleanup: bool): + del skip_cleanup + + captured_commands = _invoke_and_capture_reproduction( + cli, ["parallel-task", "--skip-cleanup"], monkeypatch + ) + + assert captured_commands[-1].argv == ["breeze", "parallel-task", "--skip-cleanup"] + + +# --------------------------------------------------------------------------- +# Tests for build_reproduction_command_from_context +# --------------------------------------------------------------------------- + + +def _build_test_command(**options): + """Build a simple click command with the given options for testing.""" + + @click.command("test-cmd") + def cmd(**kwargs): + pass + + for _name, opt in options.items(): + cmd = opt(cmd) + return cmd + + +def _invoke_and_get_context(cmd, args, env=None): + """Invoke a click command and capture the context.""" + captured_ctx = {} + + original_invoke = cmd.invoke + + def patched_invoke(ctx): + captured_ctx["ctx"] = ctx + return original_invoke(ctx) + + cmd.invoke = patched_invoke + runner = click.testing.CliRunner(env=env or {}) + result = runner.invoke(cmd, args, catch_exceptions=False) + assert result.exit_code == 0, result.output + return captured_ctx["ctx"] + + +class TestBuildReproductionCommandFromContext: + """Tests for the generic Click context-based command renderer.""" + + def test_simple_bool_flag_emitted_when_true(self): + @click.command("my-cmd") + @click.option("--verbose-output", is_flag=True, default=False) + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["--verbose-output"]) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd", "--verbose-output"] + + def test_simple_bool_flag_omitted_when_default(self): + @click.command("my-cmd") + @click.option("--verbose-output", is_flag=True, default=False) + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, []) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd"] + + def test_flag_pair_emits_positive_side(self): + @click.command("my-cmd") + @click.option("--force/--no-force", default=False) + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["--force"]) + result = build_reproduction_command_from_context(ctx) + assert "--force" in result.argv + assert "--no-force" not in result.argv + + def test_flag_pair_emits_negative_side(self): + @click.command("my-cmd") + @click.option("--force/--no-force", default=True) + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["--no-force"]) + result = build_reproduction_command_from_context(ctx) + assert "--no-force" in result.argv + assert result.argv.count("--force") == 0 + + def test_flag_pair_omitted_when_default(self): + @click.command("my-cmd") + @click.option("--force/--no-force", default=True) + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, []) + result = build_reproduction_command_from_context(ctx) + assert "--force" not in result.argv + assert "--no-force" not in result.argv + + def test_flag_pair_prefers_long_form(self): + @click.command("my-cmd") + @click.option("-f", "--force/--no-force", default=False) + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["-f"]) + result = build_reproduction_command_from_context(ctx) + assert "--force" in result.argv + assert "-f" not in result.argv + + def test_string_option_emitted_when_explicit(self): + @click.command("my-cmd") + @click.option("--backend", default="sqlite") + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["--backend", "postgres"]) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd", "--backend", "postgres"] + + def test_string_option_omitted_when_default(self): + @click.command("my-cmd") + @click.option("--backend", default="sqlite") + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, []) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd"] + + def test_multiple_option_repeats_flag(self): + @click.command("my-cmd") + @click.option("--package-filter", multiple=True) + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["--package-filter", "foo", "--package-filter", "bar"]) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd", "--package-filter", "foo", "--package-filter", "bar"] + + def test_positional_arguments_appended_at_end(self): + @click.command("my-cmd") + @click.option("--flag", is_flag=True) + @click.argument("files", nargs=-1) + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["--flag", "file1.py", "file2.py"]) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd", "--flag", "file1.py", "file2.py"] + + def test_expose_value_false_option_excluded(self): + @click.command("my-cmd") + @click.option("--verbose", is_flag=True, expose_value=False) + @click.option("--backend", default="sqlite") + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["--verbose", "--backend", "postgres"]) + result = build_reproduction_command_from_context(ctx) + assert "--verbose" not in result.argv + assert "--backend" in result.argv + + def test_excluded_params_filtered_out(self): + @click.command("my-cmd") + @click.option("--debug-resources", is_flag=True) + @click.option("--backend", default="sqlite") + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["--debug-resources", "--backend", "postgres"]) + result = build_reproduction_command_from_context(ctx) + assert "--debug-resources" not in result.argv + assert "--backend" in result.argv + + def test_envvar_source_included(self): + @click.command("my-cmd") + @click.option("--backend", default="sqlite", envvar="BACKEND") + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, [], env={"BACKEND": "postgres"}) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd", "--backend", "postgres"] + + def test_envvar_same_as_default_still_included(self): + """When envvar explicitly sets the same value as default, it should still be emitted.""" + + @click.command("my-cmd") + @click.option("--backend", default="sqlite", envvar="BACKEND") + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, [], env={"BACKEND": "sqlite"}) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd", "--backend", "sqlite"] + + def test_custom_comment(self): + @click.command("my-cmd") + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, []) + result = build_reproduction_command_from_context(ctx, comment="Custom comment") + assert result.comment == "Custom comment" + + def test_default_comment(self): + @click.command("my-cmd") + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, []) + result = build_reproduction_command_from_context(ctx) + assert result.comment == "Run the same Breeze command locally" + + def test_subcommand_path(self): + @click.group() + def grp(): + pass + + @grp.command("sub-cmd") + @click.option("--flag", is_flag=True) + def sub_cmd(**kwargs): + pass + + captured_ctx = {} + + original_invoke = sub_cmd.invoke + + def patched_invoke(ctx): + captured_ctx["ctx"] = ctx + return original_invoke(ctx) + + sub_cmd.invoke = patched_invoke + runner = click.testing.CliRunner() + result = runner.invoke(grp, ["sub-cmd", "--flag"], catch_exceptions=False) + assert result.exit_code == 0 + ctx = captured_ctx["ctx"] + repro = build_reproduction_command_from_context(ctx) + assert repro.argv == ["grp", "sub-cmd", "--flag"] + + def test_integer_option_converted_to_string(self): + @click.command("my-cmd") + @click.option("--timeout", type=int, default=60) + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["--timeout", "120"]) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd", "--timeout", "120"] + + def test_prefers_long_option_form(self): + @click.command("my-cmd") + @click.option("-b", "--backend", default="sqlite") + def cmd(**kwargs): + pass + + ctx = _invoke_and_get_context(cmd, ["-b", "postgres"]) + result = build_reproduction_command_from_context(ctx) + assert result.argv == ["my-cmd", "--backend", "postgres"] diff --git a/dev/breeze/tests/test_selective_checks.py b/dev/breeze/tests/test_selective_checks.py index 90e5551ce3377..3838f779e4c2d 100644 --- a/dev/breeze/tests/test_selective_checks.py +++ b/dev/breeze/tests/test_selective_checks.py @@ -1297,6 +1297,16 @@ def assert_outputs_are_printed(expected_outputs: dict[str, str], stderr: str): id="Shared logging library changes enable both remote logging e2e jobs", ) ), + ( + pytest.param( + ("shared/timezones/src/airflow_shared/timezones/timezone.py",), + { + "ci-image-build": "true", + "run-unit-tests": "true", + }, + id="Shared library python changes trigger unit tests", + ) + ), ], ) def test_expected_output_pull_request_main( diff --git a/dev/breeze/uv.lock b/dev/breeze/uv.lock index fb1411563144f..00d098597a420 100644 --- a/dev/breeze/uv.lock +++ b/dev/breeze/uv.lock @@ -7,7 +7,7 @@ resolution-markers = [ ] [options] -exclude-newer = "2026-04-02T16:36:23.460955321Z" +exclude-newer = "2026-04-04T19:04:26.94801064Z" exclude-newer-span = "P4D" [[package]] @@ -264,30 +264,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.42.81" +version = "1.42.83" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/9d/4d/40029c26b535c41333a0b11573127cfc548fdcb1cbcd1798ea7046c56bab/boto3-1.42.81.tar.gz", hash = "sha256:e5c0d57229763007151be6d388319514a040ccdc922fbb27e37c3100a7fbc01a", size = 112785, upload-time = "2026-04-01T19:35:34.293Z" } +sdist = { url = "https://files.pythonhosted.org/packages/9f/87/1ed88eaa1e814841a37e71fee74c2b74341d14b791c0c6038b7ba914bea1/boto3-1.42.83.tar.gz", hash = "sha256:cc5621e603982cb3145b7f6c9970e02e297a1a0eb94637cc7f7b69d3017640ee", size = 112719, upload-time = "2026-04-03T19:34:21.254Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/e5/a1a8e8bbaaa258645fe04bb6a39d7d57b6a12650312f880b8e9add638a56/boto3-1.42.81-py3-none-any.whl", hash = "sha256:216f43e308f1f65e69f57784e5042ffcb2eb6a45e370d118ea384510c148fde7", size = 140554, upload-time = "2026-04-01T19:35:32.71Z" }, + { url = "https://files.pythonhosted.org/packages/c1/b1/8a066bc8f02937d49783c0b3948ab951d8284e6fde436cab9f359dbd4d93/boto3-1.42.83-py3-none-any.whl", hash = "sha256:544846fdb10585bb7837e409868e8e04c6b372fa04479ba1597ce82cf1242076", size = 140555, upload-time = "2026-04-03T19:34:17.935Z" }, ] [[package]] name = "botocore" -version = "1.42.81" +version = "1.42.83" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/fa/5f/b0bb9a8768398fb131e1fe722c9cc5b18f74d21ca1970efe8576912b2c6e/botocore-1.42.81.tar.gz", hash = "sha256:48e6f6f52de1cc107a34810309b8ca998ea9bb719a3fe4c06f903a604b3138cb", size = 15129980, upload-time = "2026-04-01T19:35:23.439Z" } +sdist = { url = "https://files.pythonhosted.org/packages/4e/01/b46a3f8b6e9362258f78f1890db1a96d4ed73214d6a36420dc158dcfd221/botocore-1.42.83.tar.gz", hash = "sha256:34bc8cb64b17ac17f8901f073fe4fc9572a5cac9393a37b2b3ea372a83b87f4a", size = 15140337, upload-time = "2026-04-03T19:34:08.779Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/d7/33/c7a01649a6cb7219b233d2ed071ab925e52cdb64e15ce935024c0007376f/botocore-1.42.81-py3-none-any.whl", hash = "sha256:bcef8c93c20ebeba95e4f8b9edfbffbc78a0e11235425a92ee32e48fd8e03c37", size = 14807198, upload-time = "2026-04-01T19:35:20.437Z" }, + { url = "https://files.pythonhosted.org/packages/a3/97/0d6f50822dc8c1df7f3eadb0bc6822fc0f98f02287c4efc7c7c88fde129a/botocore-1.42.83-py3-none-any.whl", hash = "sha256:ec0c3ecb3772936ed22a3bdda09883b34858933f71004686d460d829bab39d8e", size = 14818388, upload-time = "2026-04-03T19:34:03.333Z" }, ] [[package]] @@ -488,14 +488,14 @@ wheels = [ [[package]] name = "click" -version = "8.3.1" +version = "8.3.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "colorama", marker = "sys_platform == 'win32'" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/3d/fa/656b739db8587d7b5dfa22e22ed02566950fbfbcdc20311993483657a5c0/click-8.3.1.tar.gz", hash = "sha256:12ff4785d337a1bb490bb7e9c2b1ee5da3112e94a8622f26a6c77f5d2fc6842a", size = 295065, upload-time = "2025-11-15T20:45:42.706Z" } +sdist = { url = "https://files.pythonhosted.org/packages/57/75/31212c6bf2503fdf920d87fee5d7a86a2e3bcf444984126f13d8e4016804/click-8.3.2.tar.gz", hash = "sha256:14162b8b3b3550a7d479eafa77dfd3c38d9dc8951f6f69c78913a8f9a7540fd5", size = 302856, upload-time = "2026-04-03T19:14:45.118Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/98/78/01c019cdb5d6498122777c1a43056ebb3ebfeef2076d9d026bfe15583b2b/click-8.3.1-py3-none-any.whl", hash = "sha256:981153a64e25f12d547d3426c367a4857371575ee7ad18df2a6183ab0545b2a6", size = 108274, upload-time = "2025-11-15T20:45:41.139Z" }, + { url = "https://files.pythonhosted.org/packages/e4/20/71885d8b97d4f3dde17b1fdb92dbd4908b00541c5a3379787137285f602e/click-8.3.2-py3-none-any.whl", hash = "sha256:1924d2c27c5653561cd2cae4548d1406039cb79b858b747cfea24924bbc1616d", size = 108379, upload-time = "2026-04-03T19:14:43.505Z" }, ] [[package]] @@ -666,7 +666,7 @@ wheels = [ [[package]] name = "google-api-core" -version = "2.30.1" +version = "2.30.2" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "google-auth" }, @@ -675,9 +675,9 @@ dependencies = [ { name = "protobuf" }, { name = "requests" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f7/0b/b6e296aff70bef900766934cf4e83eaacc3f244adb61936b66d24b204080/google_api_core-2.30.1.tar.gz", hash = "sha256:7304ef3bd7e77fd26320a36eeb75868f9339532bfea21694964f4765b37574ee", size = 176742, upload-time = "2026-03-30T22:50:52.637Z" } +sdist = { url = "https://files.pythonhosted.org/packages/1a/2e/83ca41eb400eb228f9279ec14ed66f6475218b59af4c6daec2d5a509fe83/google_api_core-2.30.2.tar.gz", hash = "sha256:9a8113e1a88bdc09a7ff629707f2214d98d61c7f6ceb0ea38c42a095d02dc0f9", size = 176862, upload-time = "2026-04-02T21:23:44.876Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/43/86/a00ea4596780ef3f0721c1f073c0c5ae992da4f35cf12f0d8c92d19267a6/google_api_core-2.30.1-py3-none-any.whl", hash = "sha256:3be893babbb54a89c6807b598383ddf212112130e3d24d06c681b5d18f082e08", size = 173238, upload-time = "2026-03-30T22:48:50.586Z" }, + { url = "https://files.pythonhosted.org/packages/84/e1/ebd5100cbb202e561c0c8b59e485ef3bd63fa9beb610f3fdcaea443f0288/google_api_core-2.30.2-py3-none-any.whl", hash = "sha256:a4c226766d6af2580577db1f1a51bf53cd262f722b49731ce7414c43068a9594", size = 173236, upload-time = "2026-04-02T21:23:06.395Z" }, ] [[package]] @@ -737,14 +737,14 @@ wheels = [ [[package]] name = "googleapis-common-protos" -version = "1.73.1" +version = "1.74.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "protobuf" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/a1/c0/4a54c386282c13449eca8bbe2ddb518181dc113e78d240458a68856b4d69/googleapis_common_protos-1.73.1.tar.gz", hash = "sha256:13114f0e9d2391756a0194c3a8131974ed7bffb06086569ba193364af59163b6", size = 147506, upload-time = "2026-03-26T22:17:38.451Z" } +sdist = { url = "https://files.pythonhosted.org/packages/20/18/a746c8344152d368a5aac738d4c857012f2c5d1fd2eac7e17b647a7861bd/googleapis_common_protos-1.74.0.tar.gz", hash = "sha256:57971e4eeeba6aad1163c1f0fc88543f965bb49129b8bb55b2b7b26ecab084f1", size = 151254, upload-time = "2026-04-02T21:23:26.679Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/dc/82/fcb6520612bec0c39b973a6c0954b6a0d948aadfe8f7e9487f60ceb8bfa6/googleapis_common_protos-1.73.1-py3-none-any.whl", hash = "sha256:e51f09eb0a43a8602f5a915870972e6b4a394088415c79d79605a46d8e826ee8", size = 297556, upload-time = "2026-03-26T22:15:58.455Z" }, + { url = "https://files.pythonhosted.org/packages/b6/b0/be5d3329badb9230b765de6eea66b73abd5944bdeb5afb3562ddcd80ae84/googleapis_common_protos-1.74.0-py3-none-any.whl", hash = "sha256:702216f78610bb510e3f12ac3cafd281b7ac45cc5d86e90ad87e4d301a3426b5", size = 300743, upload-time = "2026-04-02T21:22:49.108Z" }, ] [[package]] @@ -1276,17 +1276,17 @@ wheels = [ [[package]] name = "protobuf" -version = "6.33.6" +version = "7.34.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/66/70/e908e9c5e52ef7c3a6c7902c9dfbb34c7e29c25d2f81ade3856445fd5c94/protobuf-6.33.6.tar.gz", hash = "sha256:a6768d25248312c297558af96a9f9c929e8c4cee0659cb07e780731095f38135", size = 444531, upload-time = "2026-03-18T19:05:00.988Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6b/6b/a0e95cad1ad7cc3f2c6821fcab91671bd5b78bd42afb357bb4765f29bc41/protobuf-7.34.1.tar.gz", hash = "sha256:9ce42245e704cc5027be797c1db1eb93184d44d1cdd71811fb2d9b25ad541280", size = 454708, upload-time = "2026-03-20T17:34:47.036Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/fc/9f/2f509339e89cfa6f6a4c4ff50438db9ca488dec341f7e454adad60150b00/protobuf-6.33.6-cp310-abi3-win32.whl", hash = "sha256:7d29d9b65f8afef196f8334e80d6bc1d5d4adedb449971fefd3723824e6e77d3", size = 425739, upload-time = "2026-03-18T19:04:48.373Z" }, - { url = "https://files.pythonhosted.org/packages/76/5d/683efcd4798e0030c1bab27374fd13a89f7c2515fb1f3123efdfaa5eab57/protobuf-6.33.6-cp310-abi3-win_amd64.whl", hash = "sha256:0cd27b587afca21b7cfa59a74dcbd48a50f0a6400cfb59391340ad729d91d326", size = 437089, upload-time = "2026-03-18T19:04:50.381Z" }, - { url = "https://files.pythonhosted.org/packages/5c/01/a3c3ed5cd186f39e7880f8303cc51385a198a81469d53d0fdecf1f64d929/protobuf-6.33.6-cp39-abi3-macosx_10_9_universal2.whl", hash = "sha256:9720e6961b251bde64edfdab7d500725a2af5280f3f4c87e57c0208376aa8c3a", size = 427737, upload-time = "2026-03-18T19:04:51.866Z" }, - { url = "https://files.pythonhosted.org/packages/ee/90/b3c01fdec7d2f627b3a6884243ba328c1217ed2d978def5c12dc50d328a3/protobuf-6.33.6-cp39-abi3-manylinux2014_aarch64.whl", hash = "sha256:e2afbae9b8e1825e3529f88d514754e094278bb95eadc0e199751cdd9a2e82a2", size = 324610, upload-time = "2026-03-18T19:04:53.096Z" }, - { url = "https://files.pythonhosted.org/packages/9b/ca/25afc144934014700c52e05103c2421997482d561f3101ff352e1292fb81/protobuf-6.33.6-cp39-abi3-manylinux2014_s390x.whl", hash = "sha256:c96c37eec15086b79762ed265d59ab204dabc53056e3443e702d2681f4b39ce3", size = 339381, upload-time = "2026-03-18T19:04:54.616Z" }, - { url = "https://files.pythonhosted.org/packages/16/92/d1e32e3e0d894fe00b15ce28ad4944ab692713f2e7f0a99787405e43533a/protobuf-6.33.6-cp39-abi3-manylinux2014_x86_64.whl", hash = "sha256:e9db7e292e0ab79dd108d7f1a94fe31601ce1ee3f7b79e0692043423020b0593", size = 323436, upload-time = "2026-03-18T19:04:55.768Z" }, - { url = "https://files.pythonhosted.org/packages/c4/72/02445137af02769918a93807b2b7890047c32bfb9f90371cbc12688819eb/protobuf-6.33.6-py3-none-any.whl", hash = "sha256:77179e006c476e69bf8e8ce866640091ec42e1beb80b213c3900006ecfba6901", size = 170656, upload-time = "2026-03-18T19:04:59.826Z" }, + { url = "https://files.pythonhosted.org/packages/ec/11/3325d41e6ee15bf1125654301211247b042563bcc898784351252549a8ad/protobuf-7.34.1-cp310-abi3-macosx_10_9_universal2.whl", hash = "sha256:d8b2cc79c4d8f62b293ad9b11ec3aebce9af481fa73e64556969f7345ebf9fc7", size = 429247, upload-time = "2026-03-20T17:34:37.024Z" }, + { url = "https://files.pythonhosted.org/packages/eb/9d/aa69df2724ff63efa6f72307b483ce0827f4347cc6d6df24b59e26659fef/protobuf-7.34.1-cp310-abi3-manylinux2014_aarch64.whl", hash = "sha256:5185e0e948d07abe94bb76ec9b8416b604cfe5da6f871d67aad30cbf24c3110b", size = 325753, upload-time = "2026-03-20T17:34:38.751Z" }, + { url = "https://files.pythonhosted.org/packages/92/e8/d174c91fd48e50101943f042b09af9029064810b734e4160bbe282fa1caa/protobuf-7.34.1-cp310-abi3-manylinux2014_s390x.whl", hash = "sha256:403b093a6e28a960372b44e5eb081775c9b056e816a8029c61231743d63f881a", size = 340198, upload-time = "2026-03-20T17:34:39.871Z" }, + { url = "https://files.pythonhosted.org/packages/53/1b/3b431694a4dc6d37b9f653f0c64b0a0d9ec074ee810710c0c3da21d67ba7/protobuf-7.34.1-cp310-abi3-manylinux2014_x86_64.whl", hash = "sha256:8ff40ce8cd688f7265326b38d5a1bed9bfdf5e6723d49961432f83e21d5713e4", size = 324267, upload-time = "2026-03-20T17:34:41.1Z" }, + { url = "https://files.pythonhosted.org/packages/85/29/64de04a0ac142fb685fd09999bc3d337943fb386f3a0ec57f92fd8203f97/protobuf-7.34.1-cp310-abi3-win32.whl", hash = "sha256:34b84ce27680df7cca9f231043ada0daa55d0c44a2ddfaa58ec1d0d89d8bf60a", size = 426628, upload-time = "2026-03-20T17:34:42.536Z" }, + { url = "https://files.pythonhosted.org/packages/4d/87/cb5e585192a22b8bd457df5a2c16a75ea0db9674c3a0a39fc9347d84e075/protobuf-7.34.1-cp310-abi3-win_amd64.whl", hash = "sha256:e97b55646e6ce5cbb0954a8c28cd39a5869b59090dfaa7df4598a7fba869468c", size = 437901, upload-time = "2026-03-20T17:34:44.112Z" }, + { url = "https://files.pythonhosted.org/packages/88/95/608f665226bca68b736b79e457fded9a2a38c4f4379a4a7614303d9db3bc/protobuf-7.34.1-py3-none-any.whl", hash = "sha256:bb3812cd53aefea2b028ef42bd780f5b96407247f20c6ef7c679807e9d188f11", size = 170715, upload-time = "2026-03-20T17:34:45.384Z" }, ] [[package]] diff --git a/devel-common/pyproject.toml b/devel-common/pyproject.toml index 9460824e68e46..a841183a5f98b 100644 --- a/devel-common/pyproject.toml +++ b/devel-common/pyproject.toml @@ -35,7 +35,7 @@ dependencies = [ "kgb>=7.2.0", "requests_mock>=1.11.0", "rich>=13.6.0", - "ruff==0.15.8", + "ruff==0.15.9", "semver>=3.0.2", "typer-slim>=0.15.1", "time-machine[dateutil]>=3.0.0", diff --git a/docker-stack-docs/build-arg-ref.rst b/docker-stack-docs/build-arg-ref.rst index cfbed5b423d00..692b43036f3f3 100644 --- a/docker-stack-docs/build-arg-ref.rst +++ b/docker-stack-docs/build-arg-ref.rst @@ -32,7 +32,7 @@ Those are the most common arguments that you use when you want to build a custom +==========================================+===========================================+=============================================+ | ``AIRFLOW_VERSION`` | :subst-code:`|airflow-version|` | Version of Airflow. | +------------------------------------------+-------------------------------------------+---------------------------------------------+ -| ``AIRFLOW_PYTHON_VERSION`` | ``3.12.13`` | Version of Python. | +| ``AIRFLOW_PYTHON_VERSION`` | ``3.13.13`` | Version of Python. | +------------------------------------------+-------------------------------------------+---------------------------------------------+ | ``AIRFLOW_EXTRAS`` | (see below the table) | Default extras with which Airflow is | | | | installed. | diff --git a/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh b/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh index 0812218650566..712ec11d39e94 100755 --- a/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh +++ b/docker-stack-docs/docker-examples/customizing/add-build-essential-custom.sh @@ -33,7 +33,7 @@ docker build . \ --pull \ --build-arg BASE_IMAGE="debian:bookworm-slim" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ - --build-arg AIRFLOW_PYTHON_VERSION="3.12.13" \ + --build-arg AIRFLOW_PYTHON_VERSION="3.13.13" \ --build-arg ADDITIONAL_PYTHON_DEPS="mpi4py==4.1.1" \ --build-arg ADDITIONAL_DEV_APT_DEPS="libopenmpi-dev" \ --build-arg ADDITIONAL_RUNTIME_APT_DEPS="openmpi-common" \ diff --git a/docker-stack-docs/docker-examples/customizing/custom-sources.sh b/docker-stack-docs/docker-examples/customizing/custom-sources.sh index 23adb16ca1c9e..185fc7a5250f0 100755 --- a/docker-stack-docs/docker-examples/customizing/custom-sources.sh +++ b/docker-stack-docs/docker-examples/customizing/custom-sources.sh @@ -33,7 +33,7 @@ docker build . -f Dockerfile \ --pull \ --platform 'linux/amd64' \ --build-arg BASE_IMAGE="debian:bookworm-slim" \ - --build-arg AIRFLOW_PYTHON_VERSION="3.12.13" \ + --build-arg AIRFLOW_PYTHON_VERSION="3.13.13" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="slack,odbc" \ --build-arg ADDITIONAL_PYTHON_DEPS=" \ diff --git a/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh b/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh index aa92f40cdf9c8..b9104baeef806 100755 --- a/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh +++ b/docker-stack-docs/docker-examples/customizing/pypi-dev-runtime-deps.sh @@ -33,7 +33,7 @@ export DOCKER_BUILDKIT=1 docker build . \ --pull \ --build-arg BASE_IMAGE="debian:bookworm-slim" \ - --build-arg AIRFLOW_PYTHON_VERSION="3.12.13" \ + --build-arg AIRFLOW_PYTHON_VERSION="3.13.13" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="jdbc" \ --build-arg ADDITIONAL_PYTHON_DEPS="pandas==2.1.2" \ diff --git a/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh b/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh index f5a090ebec6e7..622efcfeaffa8 100755 --- a/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh +++ b/docker-stack-docs/docker-examples/customizing/pypi-extras-and-deps.sh @@ -32,7 +32,7 @@ export DOCKER_BUILDKIT=1 docker build . \ --pull \ --build-arg BASE_IMAGE="debian:bookworm-slim" \ - --build-arg AIRFLOW_PYTHON_VERSION="3.12.13" \ + --build-arg AIRFLOW_PYTHON_VERSION="3.13.13" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg ADDITIONAL_AIRFLOW_EXTRAS="mssql,hdfs" \ --build-arg ADDITIONAL_PYTHON_DEPS="oauth2client" \ diff --git a/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh b/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh index 4043291dab965..a68810e79b282 100755 --- a/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh +++ b/docker-stack-docs/docker-examples/customizing/pypi-selected-version.sh @@ -31,7 +31,7 @@ export DOCKER_BUILDKIT=1 docker build . \ --build-arg BASE_IMAGE="debian:bookworm-slim" \ - --build-arg AIRFLOW_PYTHON_VERSION="3.12.13" \ + --build-arg AIRFLOW_PYTHON_VERSION="3.13.13" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --tag "my-pypi-selected-version:0.0.1" # [END build] diff --git a/docker-stack-docs/docker-examples/restricted/restricted_environments.sh b/docker-stack-docs/docker-examples/restricted/restricted_environments.sh index 443789917a908..43908fb0fd216 100755 --- a/docker-stack-docs/docker-examples/restricted/restricted_environments.sh +++ b/docker-stack-docs/docker-examples/restricted/restricted_environments.sh @@ -48,7 +48,7 @@ export DOCKER_BUILDKIT=1 docker build . \ --pull \ --build-arg BASE_IMAGE="debian:bookworm-slim" \ - --build-arg AIRFLOW_PYTHON_VERSION="3.12.13" \ + --build-arg AIRFLOW_PYTHON_VERSION="3.13.13" \ --build-arg AIRFLOW_INSTALLATION_METHOD="apache-airflow" \ --build-arg AIRFLOW_VERSION="${AIRFLOW_VERSION}" \ --build-arg INSTALL_MYSQL_CLIENT="false" \ diff --git a/docs/spelling_wordlist.txt b/docs/spelling_wordlist.txt index 4465ada3dfe3f..c8981a00bd83f 100644 --- a/docs/spelling_wordlist.txt +++ b/docs/spelling_wordlist.txt @@ -159,6 +159,7 @@ balancers Bas BaseClient BaseHook +basename BaseOperator baseOperator BaseOperatorLink @@ -512,6 +513,7 @@ dttm dtypes du duckdb +dumpable dunder dup durable @@ -1134,6 +1136,7 @@ os OSS oss ot +OTel otel PaaS Pagerduty @@ -1195,6 +1198,7 @@ pooler Popen positionally POSIX +posix postfix Postgres postgres @@ -1219,6 +1223,7 @@ prefetches preflight prek Preload +preload prepend prepended preprocess @@ -1280,6 +1285,7 @@ pytest pythonic PythonOperator pythonpath +pyvespa pywinrm Qdrant qdrant @@ -1385,6 +1391,7 @@ salesforce samesite saml sandboxed +sandboxing sanitization sas Sasl @@ -1728,6 +1735,7 @@ unpause unpaused unpausing unpredicted +unsanitized untestable untransformed untrusted @@ -1775,6 +1783,8 @@ venv venvs Vertica vertica +Vespa +vespa videointelligence views virtualenv @@ -1832,6 +1842,7 @@ Xiaodong xlarge xml xpath +XSS xyz yaml Yandex diff --git a/generated/PYPI_README.md b/generated/PYPI_README.md index 2bfd0d13dc68a..191997bd26046 100644 --- a/generated/PYPI_README.md +++ b/generated/PYPI_README.md @@ -55,14 +55,14 @@ Use Airflow to author workflows (Dags) that orchestrate tasks. The Airflow sched Apache Airflow is tested with: -| | Main version (dev) | Stable version (3.1.8) | Stable version (2.11.2) | -|------------|------------------------------------|------------------------|------------------------------| -| Python | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12, 3.13 | 3.10, 3.11, 3.12 | -| Platform | AMD64/ARM64 | AMD64/ARM64 | AMD64/ARM64(\*) | -| Kubernetes | 1.30, 1.31, 1.32, 1.33, 1.34, 1.35 | 1.30, 1.31, 1.32, 1.33 | 1.26, 1.27, 1.28, 1.29, 1.30 | -| PostgreSQL | 14, 15, 16, 17, 18 | 13, 14, 15, 16, 17 | 12, 13, 14, 15, 16 | -| MySQL | 8.0, 8.4, Innovation | 8.0, 8.4, Innovation | 8.0, Innovation | -| SQLite | 3.15.0+ | 3.15.0+ | 3.15.0+ | +| | Main version (dev) | Stable version (3.2.0) | Stable version (2.11.2) | +|------------|------------------------------------|-------------------------------------|------------------------------| +| Python | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12, 3.13, 3.14 | 3.10, 3.11, 3.12 | +| Platform | AMD64/ARM64 | AMD64/ARM64 | AMD64/ARM64(\*) | +| Kubernetes | 1.30, 1.31, 1.32, 1.33, 1.34, 1.35 | 1.30, 1.31, 1.32, 1.33, 1.34, 1.35 | 1.26, 1.27, 1.28, 1.29, 1.30 | +| PostgreSQL | 14, 15, 16, 17, 18 | 14, 15, 16, 17, 18 | 12, 13, 14, 15, 16 | +| MySQL | 8.0, 8.4, Innovation | 8.0, 8.4, Innovation | 8.0, Innovation | +| SQLite | 3.15.0+ | 3.15.0+ | 3.15.0+ | \* Experimental @@ -124,15 +124,15 @@ them to the appropriate format and workflow that your tool requires. ```bash -pip install 'apache-airflow==3.1.8' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.1.8/constraints-3.10.txt" +pip install 'apache-airflow==3.2.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.2.0/constraints-3.10.txt" ``` 2. Installing with extras (i.e., postgres, google) ```bash -pip install 'apache-airflow[postgres,google]==3.1.8' \ - --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.1.8/constraints-3.10.txt" +pip install 'apache-airflow[postgres,google]==3.2.0' \ + --constraint "https://raw.githubusercontent.com/apache/airflow/constraints-3.2.0/constraints-3.10.txt" ``` For information on installing provider distributions, check diff --git a/generated/provider_metadata.json b/generated/provider_metadata.json index 44aa1013bd5ae..d401969609991 100644 --- a/generated/provider_metadata.json +++ b/generated/provider_metadata.json @@ -145,12 +145,16 @@ "date_released": "2026-01-17T10:59:32Z" }, "5.3.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:33Z" }, "5.4.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "5.4.1": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:40Z" } }, "alibaba": { @@ -315,12 +319,16 @@ "date_released": "2026-02-02T15:39:05Z" }, "3.3.5": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:33Z" }, "3.3.6": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.3.7": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:15Z" } }, "amazon": { @@ -677,11 +685,11 @@ "date_released": "2026-03-04T17:35:54Z" }, "9.23.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "9.24.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" } }, @@ -1021,8 +1029,12 @@ "date_released": "2026-02-14T13:25:37Z" }, "3.9.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.9.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:30Z" } }, "apache.drill": { @@ -1171,8 +1183,12 @@ "date_released": "2026-03-02T21:08:47Z" }, "3.3.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.3.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:50Z" } }, "apache.druid": { @@ -1353,8 +1369,12 @@ "date_released": "2026-03-02T21:08:55Z" }, "4.5.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.5.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:57Z" } }, "apache.flink": { @@ -1455,11 +1475,11 @@ "date_released": "2026-01-17T10:59:32Z" }, "1.8.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:35Z" }, "1.8.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" } }, @@ -1637,8 +1657,12 @@ "date_released": "2026-02-02T15:39:05Z" }, "4.11.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.11.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:32Z" } }, "apache.hive": { @@ -1899,12 +1923,16 @@ "date_released": "2026-03-02T21:08:22Z" }, "9.4.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "9.4.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "9.4.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:33Z" } }, "apache.iceberg": { @@ -1953,12 +1981,16 @@ "date_released": "2026-01-17T10:59:32Z" }, "2.0.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:33Z" }, "2.0.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "2.0.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:06Z" } }, "apache.impala": { @@ -2059,8 +2091,12 @@ "date_released": "2026-03-02T21:08:35Z" }, "1.9.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "1.9.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:29Z" } }, "apache.kafka": { @@ -2177,12 +2213,16 @@ "date_released": "2026-02-14T13:25:37Z" }, "1.13.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "1.13.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "1.13.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:23Z" } }, "apache.kylin": { @@ -2303,8 +2343,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.10.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.10.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:45Z" } }, "apache.livy": { @@ -2489,8 +2533,12 @@ "date_released": "2026-02-02T15:39:05Z" }, "4.5.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.5.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:25Z" } }, "apache.pig": { @@ -2603,8 +2651,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "4.8.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.8.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:37Z" } }, "apache.pinot": { @@ -2757,8 +2809,12 @@ "date_released": "2026-03-02T21:08:49Z" }, "4.10.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.10.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:49Z" } }, "apache.spark": { @@ -2975,12 +3031,16 @@ "date_released": "2026-03-02T21:08:27Z" }, "5.6.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "6.0.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "6.0.1": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:37Z" } }, "apache.tinkerpop": { @@ -3017,8 +3077,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "1.1.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "1.1.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:09Z" } }, "apprise": { @@ -3111,8 +3175,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "2.3.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "2.3.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:36Z" } }, "arangodb": { @@ -3217,8 +3285,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "2.9.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "2.9.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:54Z" } }, "asana": { @@ -3331,8 +3403,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "2.11.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "2.11.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:48Z" } }, "atlassian.jira": { @@ -3437,8 +3513,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.3.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.3.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:24Z" } }, "celery": { @@ -3671,12 +3751,16 @@ "date_released": "2026-03-04T17:35:58Z" }, "3.17.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "3.17.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.18.0": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:14Z" } }, "cloudant": { @@ -3805,8 +3889,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "4.3.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.3.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:15Z" } }, "cncf.kubernetes": { @@ -4183,12 +4271,16 @@ "date_released": "2026-03-02T21:08:52Z" }, "10.14.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "10.15.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "10.16.0": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:07Z" } }, "cohere": { @@ -4269,12 +4361,16 @@ "date_released": "2026-02-02T15:39:05Z" }, "1.6.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:33Z" }, "1.6.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "1.6.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:05Z" } }, "common.compat": { @@ -4375,11 +4471,11 @@ "date_released": "2026-03-02T21:08:45Z" }, "1.14.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "1.14.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" } }, @@ -4477,7 +4573,7 @@ "date_released": "2026-01-17T10:59:33Z" }, "1.7.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" } }, @@ -4519,7 +4615,7 @@ "date_released": "2026-01-17T10:59:33Z" }, "2.0.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" } }, @@ -4765,11 +4861,11 @@ "date_released": "2026-03-02T21:08:42Z" }, "1.33.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "1.34.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" } }, @@ -5047,12 +5143,16 @@ "date_released": "2026-03-02T21:08:24Z" }, "7.11.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:35Z" }, "7.12.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "7.12.1": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:11Z" } }, "datadog": { @@ -5177,8 +5277,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.10.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.10.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:34Z" } }, "dbt.cloud": { @@ -5383,12 +5487,16 @@ "date_released": "2026-03-02T21:08:19Z" }, "4.7.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "4.8.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.8.1": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:17Z" } }, "dingding": { @@ -5501,8 +5609,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.9.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.9.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:43Z" } }, "discord": { @@ -5635,8 +5747,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.12.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.12.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:28Z" } }, "docker": { @@ -5881,12 +5997,16 @@ "date_released": "2026-01-17T10:59:33Z" }, "4.5.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:35Z" }, "4.5.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.5.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:31Z" } }, "edge3": { @@ -5963,12 +6083,16 @@ "date_released": "2026-03-04T17:35:56Z" }, "3.2.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:33Z" }, "3.3.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.4.0": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:52Z" } }, "elasticsearch": { @@ -6209,8 +6333,12 @@ "date_released": "2026-03-04T17:35:55Z" }, "6.5.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "6.5.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:38Z" } }, "exasol": { @@ -6411,8 +6539,12 @@ "date_released": "2026-03-02T21:08:37Z" }, "4.10.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.10.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:20Z" } }, "fab": { @@ -6573,12 +6705,16 @@ "date_released": "2026-03-02T21:08:51Z" }, "3.5.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:33Z" }, "3.6.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.6.1": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:46Z" } }, "facebook": { @@ -6711,8 +6847,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.9.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.9.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:09Z" } }, "ftp": { @@ -6869,8 +7009,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.14.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.14.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:27Z" } }, "git": { @@ -6939,8 +7083,12 @@ "date_released": "2026-03-02T21:08:40Z" }, "0.3.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "0.3.1": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:37Z" } }, "github": { @@ -7065,8 +7213,12 @@ "date_released": "2026-02-14T13:25:37Z" }, "2.11.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "2.11.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:34Z" } }, "google": { @@ -7411,8 +7563,12 @@ "date_released": "2026-03-02T21:08:49Z" }, "21.0.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "21.1.0": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:58Z" } }, "grpc": { @@ -7537,8 +7693,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.9.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.9.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:03Z" } }, "hashicorp": { @@ -7715,8 +7875,12 @@ "date_released": "2026-02-14T13:25:37Z" }, "4.5.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.5.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:55Z" } }, "http": { @@ -7933,8 +8097,12 @@ "date_released": "2026-03-02T21:08:33Z" }, "6.0.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "6.0.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:33Z" } }, "imap": { @@ -8083,8 +8251,12 @@ "date_released": "2026-02-14T13:25:37Z" }, "3.11.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.11.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:47Z" } }, "influxdb": { @@ -8209,18 +8381,26 @@ "date_released": "2026-01-17T10:59:33Z" }, "2.10.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "2.10.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:29Z" } }, "informatica": { "0.1.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-04T17:35:57Z" }, "0.1.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "0.1.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:10Z" } }, "jdbc": { @@ -8389,12 +8569,16 @@ "date_released": "2026-03-02T21:08:41Z" }, "5.4.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:35Z" }, "5.4.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "5.4.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:01Z" } }, "jenkins": { @@ -8559,12 +8743,16 @@ "date_released": "2026-01-17T10:59:33Z" }, "4.2.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:33Z" }, "4.2.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.2.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:04Z" } }, "keycloak": { @@ -8609,12 +8797,16 @@ "date_released": "2026-03-02T21:08:26Z" }, "0.6.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "0.7.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "0.7.1": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:18Z" } }, "microsoft.azure": { @@ -8947,12 +9139,16 @@ "date_released": "2026-03-02T21:08:44Z" }, "13.0.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:35Z" }, "13.1.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "13.1.1": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:26Z" } }, "microsoft.mssql": { @@ -9117,8 +9313,12 @@ "date_released": "2026-03-02T21:08:36Z" }, "4.5.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.5.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:23Z" } }, "microsoft.psrp": { @@ -9255,8 +9455,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.2.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.2.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:13Z" } }, "microsoft.winrm": { @@ -9409,8 +9613,12 @@ "date_released": "2026-03-02T21:08:21Z" }, "3.14.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.14.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:02Z" } }, "mongo": { @@ -9567,12 +9775,16 @@ "date_released": "2026-01-17T10:59:33Z" }, "5.3.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "5.3.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "5.3.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:22Z" } }, "mysql": { @@ -9809,8 +10021,12 @@ "date_released": "2026-03-02T21:08:34Z" }, "6.5.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "6.5.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:32Z" } }, "neo4j": { @@ -9959,8 +10175,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.11.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.11.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:19Z" } }, "odbc": { @@ -10121,8 +10341,12 @@ "date_released": "2026-03-02T21:08:46Z" }, "4.12.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.12.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:51Z" } }, "openai": { @@ -10211,8 +10435,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "1.7.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "1.7.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:25Z" } }, "openfaas": { @@ -10321,8 +10549,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.9.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.9.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:14Z" } }, "openlineage": { @@ -10515,11 +10747,11 @@ "date_released": "2026-03-02T21:08:29Z" }, "2.12.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "2.13.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" } }, @@ -10621,8 +10853,12 @@ "date_released": "2026-02-02T15:39:05Z" }, "1.8.5": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "1.9.0": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:08Z" } }, "opsgenie": { @@ -10751,8 +10987,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "5.10.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "5.10.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:27Z" } }, "oracle": { @@ -10949,12 +11189,16 @@ "date_released": "2026-03-02T21:08:28Z" }, "4.5.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:33Z" }, "4.5.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "4.5.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:18Z" } }, "pagerduty": { @@ -11107,8 +11351,12 @@ "date_released": "2026-02-02T15:39:05Z" }, "5.2.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "5.2.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:42Z" } }, "papermill": { @@ -11265,8 +11513,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.12.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" + }, + "3.12.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:26Z" } }, "pgvector": { @@ -11335,7 +11587,7 @@ "date_released": "2026-03-02T21:08:20Z" }, "1.7.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:25Z" } }, @@ -11421,8 +11673,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "2.4.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "2.4.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:56Z" } }, "postgres": { @@ -11659,12 +11915,16 @@ "date_released": "2026-03-02T21:08:48Z" }, "6.6.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "6.6.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "6.6.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:24Z" } }, "presto": { @@ -11869,8 +12129,12 @@ "date_released": "2026-03-02T21:08:23Z" }, "5.11.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "5.11.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:19Z" } }, "qdrant": { @@ -11943,8 +12207,12 @@ "date_released": "2026-03-02T21:08:53Z" }, "1.5.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "1.5.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:44Z" } }, "redis": { @@ -12089,8 +12357,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "4.4.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "4.4.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:21Z" } }, "salesforce": { @@ -12263,12 +12535,16 @@ "date_released": "2026-01-17T10:59:33Z" }, "5.12.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:35Z" }, "5.13.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "5.14.0": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:53Z" } }, "samba": { @@ -12401,12 +12677,16 @@ "date_released": "2026-01-17T10:59:33Z" }, "4.12.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "4.12.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "4.12.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:21Z" } }, "segment": { @@ -12515,8 +12795,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.9.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "3.9.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:00Z" } }, "sendgrid": { @@ -12633,7 +12917,7 @@ "date_released": "2026-01-17T10:59:33Z" }, "4.2.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" } }, @@ -12871,12 +13155,16 @@ "date_released": "2026-02-02T15:39:06Z" }, "5.7.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:35Z" }, "5.7.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "5.7.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:36Z" } }, "singularity": { @@ -12989,7 +13277,7 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.9.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" } }, @@ -13215,12 +13503,16 @@ "date_released": "2026-03-02T21:08:18Z" }, "9.8.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:35Z" }, "9.9.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "9.10.0": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:16Z" } }, "smtp": { @@ -13353,12 +13645,16 @@ "date_released": "2026-01-17T10:59:33Z" }, "2.4.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:33Z" }, "2.4.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "2.4.5": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:17Z" } }, "snowflake": { @@ -13667,12 +13963,16 @@ "date_released": "2026-03-02T21:08:39Z" }, "6.11.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "6.12.0": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "6.12.1": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:41Z" } }, "sqlite": { @@ -13833,8 +14133,12 @@ "date_released": "2026-03-02T21:08:38Z" }, "4.3.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "4.3.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:16Z" } }, "ssh": { @@ -14039,12 +14343,16 @@ "date_released": "2026-02-02T15:39:05Z" }, "4.3.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "4.3.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "5.0.0": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:12Z" } }, "standard": { @@ -14161,12 +14469,16 @@ "date_released": "2026-03-02T21:08:25Z" }, "1.12.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-13T17:32:34Z" }, "1.12.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "1.12.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:31Z" } }, "tableau": { @@ -14327,8 +14639,12 @@ "date_released": "2026-02-14T13:25:37Z" }, "5.3.4": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "5.4.0": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:22Z" } }, "telegram": { @@ -14465,8 +14781,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "4.9.3": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "4.9.4": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:17Z" } }, "teradata": { @@ -14571,8 +14891,12 @@ "date_released": "2026-03-02T21:08:54Z" }, "3.5.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "3.5.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:59Z" } }, "trino": { @@ -14793,8 +15117,12 @@ "date_released": "2026-03-02T21:08:30Z" }, "6.5.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "6.5.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:35Z" } }, "vertica": { @@ -14947,8 +15275,12 @@ "date_released": "2026-03-02T21:08:43Z" }, "4.3.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "4.3.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:35Z" } }, "weaviate": { @@ -15053,8 +15385,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "3.3.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "3.3.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:20Z" } }, "yandex": { @@ -15207,8 +15543,12 @@ "date_released": "2026-02-14T13:25:37Z" }, "4.4.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "4.4.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:24:39Z" } }, "ydb": { @@ -15281,8 +15621,12 @@ "date_released": "2026-03-02T21:08:32Z" }, "2.5.1": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "2.5.2": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:30Z" } }, "zendesk": { @@ -15403,8 +15747,12 @@ "date_released": "2026-01-17T10:59:33Z" }, "4.11.2": { - "associated_airflow_version": "3.1.8", + "associated_airflow_version": "3.2.0", "date_released": "2026-03-28T11:03:26Z" + }, + "4.11.3": { + "associated_airflow_version": "3.2.0", + "date_released": "2026-04-12T14:25:28Z" } } } diff --git a/helm-tests/tests/chart_utils/log_groomer.py b/helm-tests/tests/chart_utils/log_groomer.py index 4d88c7da9472e..36a2c1eb7ae2b 100644 --- a/helm-tests/tests/chart_utils/log_groomer.py +++ b/helm-tests/tests/chart_utils/log_groomer.py @@ -26,18 +26,27 @@ class LogGroomerTestBase: obj_name: str = "" folder: str = "" + def get_show_only(self): + if self.obj_name == "workers-celery": + return [f"templates/{self.folder}/worker-deployment.yaml"] + + return [f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] + def test_log_groomer_collector_default_enabled(self): if self.obj_name == "dag-processor": values = {"dagProcessor": {"enabled": True}} else: values = None - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + if self.obj_name == "workers-celery": + container_name = "worker-log-groomer" + else: + container_name = f"{self.obj_name}-log-groomer" + + docs = render_chart(values=values, show_only=self.get_show_only()) assert len(jmespath.search("spec.template.spec.containers", docs[0])) == 2 - assert f"{self.obj_name}-log-groomer" in [ + assert container_name in [ c["name"] for c in jmespath.search("spec.template.spec.containers", docs[0]) ] @@ -49,14 +58,18 @@ def test_log_groomer_collector_can_be_disabled(self): "logGroomerSidecar": {"enabled": False}, } } + elif self.obj_name == "workers-celery": + values = { + "workers": { + "celery": { + "logGroomerSidecar": {"enabled": False}, + } + } + } else: values = {f"{self.folder}": {"logGroomerSidecar": {"enabled": False}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) - + docs = render_chart(values=values, show_only=self.get_show_only()) actual = jmespath.search("spec.template.spec.containers", docs[0]) assert len(actual) == 1 @@ -67,9 +80,7 @@ def test_log_groomer_collector_default_command_and_args(self): else: values = None - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert jmespath.search("spec.template.spec.containers[1].command", docs[0]) is None assert jmespath.search("spec.template.spec.containers[1].args", docs[0]) == ["bash", "/clean-logs"] @@ -80,9 +91,7 @@ def test_log_groomer_collector_default_retention_days(self): else: values = None - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert {"name": "AIRFLOW__LOG_RETENTION_DAYS", "value": "15"} in jmespath.search( "spec.template.spec.containers[1].env", docs[0] @@ -96,6 +105,8 @@ def test_log_groomer_collector_custom_env(self): if self.obj_name == "dag-processor": values = {"dagProcessor": {"enabled": True, "logGroomerSidecar": {"env": env}}} + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"env": env}}}} else: values = { "workers": {"logGroomerSidecar": {"env": env}}, @@ -103,9 +114,7 @@ def test_log_groomer_collector_custom_env(self): "triggerer": {"logGroomerSidecar": {"env": env}}, } - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert {"name": "APP_RELEASE_NAME", "value": "release-name-airflow"} in jmespath.search( "spec.template.spec.containers[1].env", docs[0] @@ -124,16 +133,22 @@ def test_log_groomer_command_and_args_overrides(self, command, args): "logGroomerSidecar": {"command": command, "args": args}, } } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"command": command, "args": args}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"command": command, "args": args}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert command == jmespath.search("spec.template.spec.containers[1].command", docs[0]) - assert args == jmespath.search("spec.template.spec.containers[1].args", docs[0]) + + if self.obj_name == "workers-celery" and args is None: + assert jmespath.search("spec.template.spec.containers[1].args", docs[0]) == [ + "bash", + "/clean-logs", + ] + else: + assert args == jmespath.search("spec.template.spec.containers[1].args", docs[0]) def test_log_groomer_command_and_args_overrides_are_templated(self): if self.obj_name == "dag-processor": @@ -146,6 +161,17 @@ def test_log_groomer_command_and_args_overrides_are_templated(self): }, } } + elif self.obj_name == "workers-celery": + values = { + "workers": { + "celery": { + "logGroomerSidecar": { + "command": ["{{ .Release.Name }}"], + "args": ["{{ .Release.Service }}"], + } + } + } + } else: values = { f"{self.folder}": { @@ -156,10 +182,7 @@ def test_log_groomer_command_and_args_overrides_are_templated(self): } } - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert jmespath.search("spec.template.spec.containers[1].command", docs[0]) == ["release-name"] assert jmespath.search("spec.template.spec.containers[1].args", docs[0]) == ["Helm"] @@ -170,13 +193,12 @@ def test_log_groomer_retention_days_overrides(self, retention_days, retention_re values = { "dagProcessor": {"enabled": True, "logGroomerSidecar": {"retentionDays": retention_days}} } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"retentionDays": retention_days}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"retentionDays": retention_days}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) if retention_result: assert ( @@ -186,6 +208,15 @@ def test_log_groomer_retention_days_overrides(self, retention_days, retention_re ) == retention_result ) + elif self.obj_name == "workers-celery" and retention_result is None: + # Testing backward compatibility of move from workers to workers.celery + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_DAYS'].value | [0]", + docs[0], + ) + == "15" + ) else: assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2 @@ -198,13 +229,12 @@ def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, freque "logGroomerSidecar": {"frequencyMinutes": frequency_minutes}, } } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"frequencyMinutes": frequency_minutes}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"frequencyMinutes": frequency_minutes}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) if frequency_result: assert ( @@ -214,6 +244,15 @@ def test_log_groomer_frequency_minutes_overrides(self, frequency_minutes, freque ) == frequency_result ) + elif self.obj_name == "workers-celery" and frequency_result is None: + # Testing backward compatibility of move from workers to workers.celery + assert ( + jmespath.search( + "spec.template.spec.containers[1].env[?name=='AIRFLOW__LOG_RETENTION_DAYS'].value | [0]", + docs[0], + ) + == "15" + ) else: assert len(jmespath.search("spec.template.spec.containers[1].env", docs[0])) == 2 @@ -228,13 +267,12 @@ def test_log_groomer_max_size_bytes_overrides(self, max_size_bytes, max_size_res "logGroomerSidecar": {"maxSizeBytes": max_size_bytes}, } } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"maxSizeBytes": max_size_bytes}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizeBytes": max_size_bytes}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) if max_size_result: assert ( @@ -262,13 +300,12 @@ def test_log_groomer_max_size_percent_overrides(self, max_size_percent, max_size "logGroomerSidecar": {"maxSizePercent": max_size_percent}, } } + elif self.obj_name == "workers-celery": + values = {"workers": {"celery": {"logGroomerSidecar": {"maxSizePercent": max_size_percent}}}} else: values = {f"{self.folder}": {"logGroomerSidecar": {"maxSizePercent": max_size_percent}}} - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) if max_size_result: assert ( @@ -300,6 +337,19 @@ def test_log_groomer_resources(self): }, } } + elif self.obj_name == "workers-celery": + values = { + "workers": { + "celery": { + "logGroomerSidecar": { + "resources": { + "requests": {"memory": "2Gi", "cpu": "1"}, + "limits": {"memory": "3Gi", "cpu": "2"}, + } + } + } + } + } else: values = { f"{self.folder}": { @@ -312,10 +362,7 @@ def test_log_groomer_resources(self): } } - docs = render_chart( - values=values, - show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"], - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert jmespath.search("spec.template.spec.containers[1].resources", docs[0]) == { "limits": { @@ -334,9 +381,7 @@ def test_log_groomer_has_airflow_home(self): else: values = None - docs = render_chart( - values=values, show_only=[f"templates/{self.folder}/{self.obj_name}-deployment.yaml"] - ) + docs = render_chart(values=values, show_only=self.get_show_only()) assert ( jmespath.search("spec.template.spec.containers[1].env[?name=='AIRFLOW_HOME'].name | [0]", docs[0]) diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py b/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py index 3df818effa7c2..1bbf800c46af3 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_annotations.py @@ -18,9 +18,7 @@ import copy -import jmespath import pytest -import yaml from chart_utils.helm_template_generator import render_chart @@ -122,6 +120,105 @@ class TestServiceAccountAnnotations: "example": "worker", }, ), + ( + { + "workers": { + "celery": { + "serviceAccount": { + "annotations": { + "example": "worker", + }, + } + }, + }, + }, + "templates/workers/worker-serviceaccount.yaml", + { + "example": "worker", + }, + ), + ( + { + "workers": { + "serviceAccount": { + "annotations": { + "worker": "example", + }, + }, + "celery": { + "serviceAccount": { + "annotations": { + "example": "worker", + }, + } + }, + }, + }, + "templates/workers/worker-serviceaccount.yaml", + { + "example": "worker", + }, + ), + ( + { + "executor": "KubernetesExecutor", + "workers": { + "serviceAccount": { + "annotations": { + "example": "worker", + }, + }, + "kubernetes": {"serviceAccount": {"create": True}}, + }, + }, + "templates/workers/worker-kubernetes-serviceaccount.yaml", + { + "example": "worker", + }, + ), + ( + { + "executor": "KubernetesExecutor", + "workers": { + "kubernetes": { + "serviceAccount": { + "create": True, + "annotations": { + "example": "worker", + }, + } + }, + }, + }, + "templates/workers/worker-kubernetes-serviceaccount.yaml", + { + "example": "worker", + }, + ), + ( + { + "executor": "KubernetesExecutor", + "workers": { + "serviceAccount": { + "annotations": { + "worker": "example", + }, + }, + "kubernetes": { + "serviceAccount": { + "create": True, + "annotations": { + "example": "worker", + }, + } + }, + }, + }, + "templates/workers/worker-kubernetes-serviceaccount.yaml", + { + "example": "worker", + }, + ), ( { "flower": { @@ -264,6 +361,158 @@ def test_annotations_are_added(self, values, show_only, expected_annotations): assert k in obj["metadata"]["annotations"] assert v == obj["metadata"]["annotations"][k] + @pytest.mark.parametrize( + ("values_key", "show_only"), + [ + ("scheduler", "templates/scheduler/scheduler-serviceaccount.yaml"), + ("triggerer", "templates/triggerer/triggerer-serviceaccount.yaml"), + ], + ) + def test_tpl_rendered_annotations_airflow_2(self, values_key, show_only): + """Test SA annotations support tpl rendering for Airflow 2.x components.""" + k8s_objects = render_chart( + values={ + "airflowVersion": "2.11.0", + values_key: { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-sa@project.iam", + }, + }, + }, + }, + show_only=[show_only], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-sa@project.iam" + + def test_tpl_rendered_multiple_annotations(self): + """Test that multiple annotations render correctly with tpl.""" + k8s_objects = render_chart( + values={ + "airflowVersion": "2.11.0", + "scheduler": { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-sa@project.iam", + "another-annotation": "{{ .Release.Name }}-other", + "plain-annotation": "no-template", + }, + }, + }, + }, + show_only=["templates/scheduler/scheduler-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-sa@project.iam" + assert annotations["another-annotation"] == "release-name-other" + assert annotations["plain-annotation"] == "no-template" + + def test_tpl_rendered_annotations_pgbouncer(self): + """Test pgbouncer SA annotations support tpl rendering.""" + k8s_objects = render_chart( + values={ + "airflowVersion": "2.11.0", + "pgbouncer": { + "enabled": True, + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-sa@project.iam", + }, + }, + }, + }, + show_only=["templates/pgbouncer/pgbouncer-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-sa@project.iam" + + @pytest.mark.parametrize( + ("values_key", "show_only"), + [ + ("dagProcessor", "templates/dag-processor/dag-processor-serviceaccount.yaml"), + ("apiServer", "templates/api-server/api-server-serviceaccount.yaml"), + ], + ) + def test_tpl_rendered_annotations_airflow_3(self, values_key, show_only): + """Test SA annotations support tpl rendering for Airflow 3.x components.""" + k8s_objects = render_chart( + values={ + values_key: { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-sa@project.iam", + }, + }, + }, + }, + show_only=[show_only], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-sa@project.iam" + + def test_tpl_rendered_annotations_celery_worker(self): + """Test Celery worker SA annotations support tpl rendering.""" + k8s_objects = render_chart( + values={ + "workers": { + "celery": { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-worker@project.iam", + }, + }, + }, + }, + }, + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-worker@project.iam" + + def test_tpl_rendered_annotations_kubernetes_worker(self): + """Test KubernetesExecutor worker SA annotations support tpl rendering.""" + k8s_objects = render_chart( + values={ + "executor": "KubernetesExecutor", + "workers": { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-worker@project.iam", + }, + }, + }, + }, + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-worker@project.iam" + + def test_tpl_rendered_annotations_webserver(self): + """Test webserver SA annotations support tpl rendering (Airflow 2.x only).""" + k8s_objects = render_chart( + values={ + "airflowVersion": "2.11.0", + "webserver": { + "serviceAccount": { + "annotations": { + "iam.gke.io/gcp-service-account": "{{ .Release.Name }}-web@project.iam", + }, + }, + }, + }, + show_only=["templates/webserver/webserver-serviceaccount.yaml"], + ) + assert len(k8s_objects) == 1 + annotations = k8s_objects[0]["metadata"]["annotations"] + assert annotations["iam.gke.io/gcp-service-account"] == "release-name-web@project.iam" + def test_annotations_on_webserver(self): """Test annotations are added on webserver for Airflow 2""" k8s_objects = render_chart( @@ -293,39 +542,72 @@ def test_annotations_on_webserver(self): { "scheduler": { "podAnnotations": { - "example": "scheduler", + "example": "{{ .Release.Name }}-scheduler", }, }, }, "templates/scheduler/scheduler-deployment.yaml", { - "example": "scheduler", + "example": "release-name-scheduler", }, ), ( { "apiServer": { "podAnnotations": { - "example": "api-server", + "example": "{{ .Release.Name }}-api-server", }, }, }, "templates/api-server/api-server-deployment.yaml", { - "example": "api-server", + "example": "release-name-api-server", + }, + ), + ( + { + "workers": { + "podAnnotations": { + "example": "{{ .Release.Name }}-worker", + }, + }, + }, + "templates/workers/worker-deployment.yaml", + { + "example": "release-name-worker", + }, + ), + ( + { + "workers": { + "celery": { + "podAnnotations": { + "example": "{{ .Release.Name }}-worker", + }, + } + }, + }, + "templates/workers/worker-deployment.yaml", + { + "example": "release-name-worker", }, ), ( { "workers": { "podAnnotations": { - "example": "worker", + "test": "test", + }, + "celery": { + "podAnnotations": { + "example": "{{ .Release.Name }}-worker", + }, }, }, }, "templates/workers/worker-deployment.yaml", { - "example": "worker", + "example": "release-name-worker", }, ), ( @@ -333,26 +615,26 @@ def test_annotations_on_webserver(self): "flower": { "enabled": True, "podAnnotations": { - "example": "flower", + "example": "{{ .Release.Name }}-flower", }, }, }, "templates/flower/flower-deployment.yaml", { - "example": "flower", + "example": "release-name-flower", }, ), ( { "triggerer": { "podAnnotations": { - "example": "triggerer", + "example": "{{ .Release.Name }}-triggerer", }, }, }, "templates/triggerer/triggerer-deployment.yaml", { - "example": "triggerer", + "example": "release-name-triggerer", }, ), ( @@ -360,13 +642,13 @@ def test_annotations_on_webserver(self): "dagProcessor": { "enabled": True, "podAnnotations": { - "example": "dag-processor", + "example": "{{ .Release.Name }}-dag-processor", }, }, }, "templates/dag-processor/dag-processor-deployment.yaml", { - "example": "dag-processor", + "example": "release-name-dag-processor", }, ), ( @@ -375,13 +657,13 @@ def test_annotations_on_webserver(self): "cleanup": { "enabled": True, "podAnnotations": { - "example": "cleanup", + "example": "{{ .Release.Name }}-cleanup", }, }, }, "templates/cleanup/cleanup-cronjob.yaml", { - "example": "cleanup", + "example": "release-name-cleanup", }, ), ( @@ -389,39 +671,39 @@ def test_annotations_on_webserver(self): "databaseCleanup": { "enabled": True, "podAnnotations": { - "example": "database-cleanup", + "example": "{{ .Release.Name }}-database-cleanup", }, } }, "templates/database-cleanup/database-cleanup-cronjob.yaml", { - "example": "database-cleanup", + "example": "release-name-database-cleanup", }, ), ( { "redis": { "podAnnotations": { - "example": "redis", + "example": "{{ .Release.Name }}-redis", }, }, }, "templates/redis/redis-statefulset.yaml", { - "example": "redis", + "example": "release-name-redis", }, ), ( { "statsd": { "podAnnotations": { - "example": "statsd", + "example": "{{ .Release.Name }}-statsd", }, }, }, "templates/statsd/statsd-deployment.yaml", { - "example": "statsd", + "example": "release-name-statsd", }, ), ( @@ -429,13 +711,13 @@ def test_annotations_on_webserver(self): "pgbouncer": { "enabled": True, "podAnnotations": { - "example": "pgbouncer", + "example": "{{ .Release.Name }}-pgbouncer", }, }, }, "templates/pgbouncer/pgbouncer-deployment.yaml", { - "example": "pgbouncer", + "example": "release-name-pgbouncer", }, ), ], @@ -484,19 +766,15 @@ def test_precedence(self, values, show_only, expected_annotations): assert v == annotations[k] def test_pod_annotations_are_templated(self, values, show_only, expected_annotations): - templated_values = copy.deepcopy(values) - for val in templated_values.values(): - if isinstance(val, dict) and "podAnnotations" in val: - val["podAnnotations"] = {"release-name": "{{ .Release.Name }}"} - k8s_objects = render_chart( - values=templated_values, + values=values, show_only=[show_only], ) assert len(k8s_objects) == 1 annotations = get_object_annotations(k8s_objects[0]) - assert annotations["release-name"] == "release-name" + assert annotations["example"] == expected_annotations["example"] + assert "test" not in annotations def test_airflow_pod_annotations_are_templated(self, values, show_only, expected_annotations): templated_values = copy.deepcopy(values) @@ -546,50 +824,6 @@ def test_redis_annotations_are_added(self): assert v == obj["metadata"]["annotations"][k] -class TestPodTemplateFileAnnotationsTemplating: - """Tests that podAnnotations are templated in the pod template file.""" - - def test_pod_template_file_annotations_are_templated(self): - k8s_objects = render_chart( - values={ - "executor": "KubernetesExecutor", - "workers": { - "podAnnotations": { - "release-name": "{{ .Release.Name }}", - }, - }, - }, - show_only=["templates/configmaps/configmap.yaml"], - ) - - assert len(k8s_objects) == 1 - pod_template = k8s_objects[0]["data"]["pod_template_file.yaml"] - annotations = jmespath.search( - "metadata.annotations", - yaml.safe_load(pod_template), - ) - assert annotations["release-name"] == "release-name" - - def test_pod_template_file_global_annotations_are_templated(self): - k8s_objects = render_chart( - values={ - "executor": "KubernetesExecutor", - "airflowPodAnnotations": { - "global-release": "{{ .Release.Name }}", - }, - }, - show_only=["templates/configmaps/configmap.yaml"], - ) - - assert len(k8s_objects) == 1 - pod_template = k8s_objects[0]["data"]["pod_template_file.yaml"] - annotations = jmespath.search( - "metadata.annotations", - yaml.safe_load(pod_template), - ) - assert annotations["global-release"] == "release-name" - - class TestWebserverPodAnnotationsTemplating: """Tests webserver podAnnotations templating (requires airflowVersion < 3.0.0).""" diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py index a6afbd69eabac..046587cc14dff 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_basic_helm_chart.py @@ -738,6 +738,13 @@ def test_postgres_connection_url_name_override(self): == "postgresql://postgres:postgres@overrideName:5432/postgres?sslmode=disable" ) + def test_jwt_secret_has_recommended_length(self): + doc = render_chart( + show_only=["templates/secrets/jwt-secret.yaml"], + )[0] + + assert len(base64.b64decode(doc["data"]["jwt-secret"]).decode("utf-8")) >= 64 + def test_priority_classes(self): pc = [ {"name": "class1", "preemptionPolicy": "PreemptLowerPriority", "value": 1000}, diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py b/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py index 04bef2de7dfef..f701e3f9ba8aa 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_container_lifecycle.py @@ -265,25 +265,78 @@ def test_worker_kerberos_container_setting(self, workers_values, expected_hook_t # Test container lifecycle hooks for log-groomer-sidecar main container @pytest.mark.parametrize("hook_type", ["preStop", "postStart"]) - def test_log_groomer_sidecar_container_setting(self, hook_type): + def test_log_groomer_sidecar_container_setting_scheduler(self, hook_type): docs = render_chart( name=RELEASE_NAME, values={ "scheduler": { "logGroomerSidecar": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}} }, - "workers": { - "logGroomerSidecar": {"containerLifecycleHooks": {hook_type: LIFECYCLE_TEMPLATE}} - }, }, show_only=[ "templates/scheduler/scheduler-deployment.yaml", + ], + ) + + assert ( + jmespath.search(f"spec.template.spec.containers[1].lifecycle.{hook_type}", docs[0]) + == LIFECYCLE_PARSED + ) + + @pytest.mark.parametrize( + ("workers_values", "expected"), + [ + ( + {"logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}}}, + {"preStop": LIFECYCLE_PARSED}, + ), + ( + {"logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}}}, + {"postStart": LIFECYCLE_PARSED}, + ), + ( + { + "celery": { + "logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}} + } + }, + {"preStop": LIFECYCLE_PARSED}, + ), + ( + { + "celery": { + "logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}} + } + }, + {"postStart": LIFECYCLE_PARSED}, + ), + ( + { + "logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}}, + "celery": { + "logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}} + }, + }, + {"preStop": LIFECYCLE_PARSED}, + ), + ( + { + "logGroomerSidecar": {"containerLifecycleHooks": {"preStop": LIFECYCLE_TEMPLATE}}, + "celery": { + "logGroomerSidecar": {"containerLifecycleHooks": {"postStart": LIFECYCLE_TEMPLATE}} + }, + }, + {"postStart": LIFECYCLE_PARSED}, + ), + ], + ) + def test_log_groomer_sidecar_container_setting(self, workers_values, expected): + docs = render_chart( + name=RELEASE_NAME, + values={"workers": workers_values}, + show_only=[ "templates/workers/worker-deployment.yaml", ], ) - for doc in docs: - assert ( - jmespath.search(f"spec.template.spec.containers[1].lifecycle.{hook_type}", doc) - == LIFECYCLE_PARSED - ) + assert jmespath.search("spec.template.spec.containers[1].lifecycle", docs[0]) == expected diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup.py b/helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup.py index 4ad2e990d6b95..9c4db3a0dabbb 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_database_cleanup.py @@ -72,6 +72,32 @@ def test_should_work_with_custom_schedule_string(self, release_name, schedule_va class TestDatabaseCleanup: """Tests database cleanup.""" + def test_ttl_seconds_after_finished_default_behavior(self): + values = {"databaseCleanup": {"enabled": True}} + docs = render_chart( + values=values, + show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"], + ) + + assert "ttlSecondsAfterFinished" not in docs[0]["spec"]["jobTemplate"]["spec"] + + @pytest.mark.parametrize( + ("ttl_value", "expected_rendered"), + [ + (300, 300), + (0, 0), + ], + ) + def test_ttl_seconds_after_finished_rendering(self, ttl_value, expected_rendered): + values = {"databaseCleanup": {"enabled": True, "ttlSecondsAfterFinished": ttl_value}} + docs = render_chart( + values=values, + show_only=["templates/database-cleanup/database-cleanup-cronjob.yaml"], + ) + + actual_ttl = jmespath.search("spec.jobTemplate.spec.ttlSecondsAfterFinished", docs[0]) + assert actual_ttl == expected_rendered + def test_should_create_cronjob_for_enabled_cleanup(self): docs = render_chart( values={ diff --git a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py index a963638459cb9..45b4570047152 100644 --- a/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py +++ b/helm-tests/tests/helm_tests/airflow_aux/test_pod_template_file.py @@ -462,11 +462,26 @@ def test_global_node_selector(self): assert jmespath.search("kind", docs[0]) == "Pod" assert jmespath.search("spec.nodeSelector", docs[0]) == {"diskType": "ssd"} - def test_workers_affinity(self): - docs = render_chart( - values={ - "executor": "KubernetesExecutor", - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": "foo", "operator": "In", "values": ["true"]}, + ] + } + ] + } + } + }, + }, + { + "kubernetes": { "affinity": { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -480,7 +495,45 @@ def test_workers_affinity(self): } } }, + } + }, + { + "affinity": { + "podAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "topologyKey": "foo", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + }, + "weight": 1, + } + ] + } }, + "kubernetes": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": "foo", "operator": "In", "values": ["true"]}, + ] + } + ] + } + } + }, + }, + }, + ], + ) + def test_workers_affinity(self, workers_values): + docs = render_chart( + values={ + "executor": "KubernetesExecutor", + "workers": workers_values, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, @@ -501,16 +554,37 @@ def test_workers_affinity(self): } } - def test_workers_tolerations(self): - docs = render_chart( - values={ - "executor": "KubernetesExecutor", - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "tolerations": [ + {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ], + }, + { + "kubernetes": { "tolerations": [ {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ], + ] + }, + }, + { + "tolerations": [{"key": "pods", "operator": "Exists", "effect": "PreferNoSchedule"}], + "kubernetes": { + "tolerations": [ + {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ] }, }, + ], + ) + def test_workers_tolerations(self, workers_values): + docs = render_chart( + values={ + "executor": "KubernetesExecutor", + "workers": workers_values, + }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -520,11 +594,41 @@ def test_workers_tolerations(self): {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} ] - def test_workers_topology_spread_constraints(self): - docs = render_chart( - values={ - "executor": "KubernetesExecutor", - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ], + }, + { + "kubernetes": { + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ], + } + }, + { + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "not-me", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ], + "kubernetes": { "topologySpreadConstraints": [ { "maxSkew": 1, @@ -535,6 +639,11 @@ def test_workers_topology_spread_constraints(self): ], }, }, + ], + ) + def test_workers_topology_spread_constraints(self, workers_values): + docs = render_chart( + values={"executor": "KubernetesExecutor", "workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -871,28 +980,60 @@ def test_should_add_uid_to_the_pod_template(self): assert jmespath.search("spec.securityContext.runAsUser", docs[0]) == 1 - def test_should_create_valid_volume_mount_and_volume(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}, + {"kubernetes": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}}, + { + "extraVolumes": [{"name": "test", "emptyDir": {}}], + "kubernetes": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}, + }, + ], + ) + def test_should_create_valid_volume(self, workers_values): docs = render_chart( - values={ - "workers": { - "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], + values={"workers": workers_values}, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + # [2:] -> Skipping logs and config volumes + assert jmespath.search("spec.volumes[2:].name", docs[0]) == ["test-volume-airflow"] + + @pytest.mark.parametrize( + "workers_values", + [ + { + "extraVolumeMounts": [{"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}], + }, + { + "kubernetes": { "extraVolumeMounts": [ {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} ], } }, + { + "extraVolumeMounts": [{"name": "test", "mountPath": "test"}], + "kubernetes": { + "extraVolumeMounts": [ + {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + }, + }, + ], + ) + def test_should_create_valid_volume_mount(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - assert "test-volume-airflow" in jmespath.search( - "spec.volumes[*].name", - docs[0], - ) - assert "test-volume-airflow" in jmespath.search( - "spec.containers[0].volumeMounts[*].name", - docs[0], - ) + volume_mounts = jmespath.search("spec.containers[0].volumeMounts", docs[0]) + assert {"name": "test-volume-airflow", "mountPath": "/opt/test"} in volume_mounts + assert {"name": "test", "mountPath": "test"} not in volume_mounts def test_should_add_env_for_gitsync(self): docs = render_chart( @@ -990,95 +1131,198 @@ def test_safe_to_evict_annotation_other_services(self, workers_values): annotations = jmespath.search("spec.template.metadata.annotations", doc) assert annotations.get("cluster-autoscaler.kubernetes.io/safe-to-evict") == "true" - def test_workers_pod_annotations(self): + def test_global_pod_annotations_templated(self): docs = render_chart( - values={"workers": {"podAnnotations": {"my_annotation": "annotated!"}}}, + values={"airflowPodAnnotations": {"global": "{{ .Release.Name }}"}}, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert jmespath.search("metadata.annotations", docs[0])["global"] == "release-name" + + @pytest.mark.parametrize( + "workers_values", + [ + {"podAnnotations": {"my_annotation": "{{ .Release.Name }}"}}, + {"kubernetes": {"podAnnotations": {"my_annotation": "{{ .Release.Name }}"}}}, + { + "podAnnotations": {"test": "test"}, + "kubernetes": {"podAnnotations": {"my_annotation": "{{ .Release.Name }}"}}, + }, + ], + ) + def test_workers_pod_annotations_templated(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - annotations = jmespath.search("metadata.annotations", docs[0]) - assert "my_annotation" in annotations - assert "annotated!" in annotations["my_annotation"] - def test_airflow_and_workers_pod_annotations(self): - # should give preference to workers.podAnnotations + assert jmespath.search("metadata.annotations", docs[0])["my_annotation"] == "release-name" + + @pytest.mark.parametrize( + "workers_values", + [ + {"podAnnotations": {"my_annotation": "workerPodAnnotations"}}, + {"kubernetes": {"podAnnotations": {"my_annotation": "workerPodAnnotations"}}}, + ], + ) + def test_workers_pod_annotations_override(self, workers_values): + # Worker-specific pod annotations should override global airflowPodAnnotations, + # whether they come from workers.podAnnotations or workers.kubernetes.podAnnotations docs = render_chart( values={ "airflowPodAnnotations": {"my_annotation": "airflowPodAnnotations"}, - "workers": {"podAnnotations": {"my_annotation": "workerPodAnnotations"}}, + "workers": workers_values, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - annotations = jmespath.search("metadata.annotations", docs[0]) - assert "my_annotation" in annotations - assert "workerPodAnnotations" in annotations["my_annotation"] - def test_should_add_extra_init_containers(self): + assert jmespath.search("metadata.annotations", docs[0])["my_annotation"] == "workerPodAnnotations" + + @pytest.mark.parametrize( + "workers_values", + [ + {"podAnnotations": {"local": "workerPodAnnotations"}}, + {"kubernetes": {"podAnnotations": {"local": "workerPodAnnotations"}}}, + ], + ) + def test_pod_annotations_merge(self, workers_values): docs = render_chart( values={ - "workers": { + "airflowPodAnnotations": {"global": "airflowPodAnnotations"}, + "workers": workers_values, + }, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + annotations = jmespath.search("metadata.annotations", docs[0]) + assert annotations["global"] == "airflowPodAnnotations" + assert annotations["local"] == "workerPodAnnotations" + + @pytest.mark.parametrize( + "workers_values", + [ + { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] + }, + { + "kubernetes": { "extraInitContainers": [ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ], + ] + } + }, + { + "extraInitContainers": [{"name": "test", "image": "repo:tag"}], + "kubernetes": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] }, }, + ], + ) + def test_should_add_extra_init_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - assert jmespath.search("spec.initContainers[-1]", docs[0]) == { - "name": "test-init-container", - "image": "test-registry/test-repo:test-tag", - } + assert jmespath.search("spec.initContainers", docs[0]) == [ + { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + } + ] - def test_should_template_extra_init_containers(self): - docs = render_chart( - values={ - "workers": { - "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, + {"kubernetes": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}}, + { + "extraInitContainers": [{"name": "container"}], + "kubernetes": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, }, + ], + ) + def test_should_template_extra_init_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - assert jmespath.search("spec.initContainers[-1]", docs[0]) == { - "name": "release-name-test-init-container", - } + assert jmespath.search("spec.initContainers", docs[0]) == [ + { + "name": "release-name-test-init-container", + } + ] - def test_should_add_extra_containers(self): - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + {"extraContainers": [{"name": "test-container", "image": "test-registry/test-repo:test-tag"}]}, + { + "kubernetes": { "extraContainers": [ {"name": "test-container", "image": "test-registry/test-repo:test-tag"} - ], + ] + } + }, + { + "extraContainers": [{"name": "container", "image": "repo:tag"}], + "kubernetes": { + "extraContainers": [ + {"name": "test-container", "image": "test-registry/test-repo:test-tag"} + ] }, }, + ], + ) + def test_should_add_extra_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - assert jmespath.search("spec.containers[-1]", docs[0]) == { - "name": "test-container", - "image": "test-registry/test-repo:test-tag", - } + assert jmespath.search("spec.containers[1:]", docs[0]) == [ + { + "name": "test-container", + "image": "test-registry/test-repo:test-tag", + } + ] - def test_should_template_extra_containers(self): - docs = render_chart( - values={ - "workers": { - "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + {"extraContainers": [{"name": "{{ .Release.Name }}-test-container"}]}, + {"kubernetes": {"extraContainers": [{"name": "{{ .Release.Name }}-test-container"}]}}, + { + "extraContainers": [{"name": "{{ .Release.Name }}-test"}], + "kubernetes": {"extraContainers": [{"name": "{{ .Release.Name }}-test-container"}]}, }, + ], + ) + def test_should_template_extra_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - assert jmespath.search("spec.containers[-1]", docs[0]) == { - "name": "release-name-test-container", - } + assert jmespath.search("spec.containers[1:]", docs[0]) == [ + { + "name": "release-name-test-container", + } + ] def test_should_add_pod_labels(self): docs = render_chart( @@ -1095,10 +1339,24 @@ def test_should_add_pod_labels(self): "tier": "airflow", } - def test_should_add_extraEnvs(self): - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ] + }, + { + "kubernetes": { "env": [ {"name": "TEST_ENV_1", "value": "test_env_1"}, { @@ -1112,6 +1370,27 @@ def test_should_add_extraEnvs(self): ] } }, + { + "env": [{"name": "TEST", "value": "test"}], + "kubernetes": { + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ] + }, + }, + ], + ) + def test_should_add_extraEnvs(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) @@ -1119,6 +1398,7 @@ def test_should_add_extraEnvs(self): assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.containers[0].env", docs[0] ) + assert {"name": "TEST", "value": "test"} not in jmespath.search("spec.containers[0].env", docs[0]) assert { "name": "TEST_ENV_2", "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, @@ -1128,20 +1408,38 @@ def test_should_add_extraEnvs(self): "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, } in jmespath.search("spec.containers[0].env", docs[0]) - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "labels": {"test_label": "test_label_value"}, + }, + { + "kubernetes": { + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "kubernetes": { + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "KubernetesExecutor", - "workers": { - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/pod-template-file.yaml"], chart_dir=self.temp_chart_dir, ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels @pytest.mark.parametrize( "workers_values", @@ -1810,3 +2108,22 @@ def test_kerberos_init_container_lifecycle_hooks(self, workers_values): assert jmespath.search("spec.initContainers[?name=='kerberos-init'] | [0].lifecycle", docs[0]) == { "postStart": {"exec": {"command": ["echo", "test-release"]}} } + + def test_service_account_name_default(self): + docs = render_chart( + name="test-release", + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert jmespath.search("spec.serviceAccountName", docs[0]) == "test-release-airflow-worker" + + def test_dedicated_service_account_name_default(self): + docs = render_chart( + name="test-release", + values={"workers": {"kubernetes": {"serviceAccount": {"create": True}}}}, + show_only=["templates/pod-template-file.yaml"], + chart_dir=self.temp_chart_dir, + ) + + assert jmespath.search("spec.serviceAccountName", docs[0]) == "test-release-airflow-worker-kubernetes" diff --git a/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py index b3ba14679108d..2e18abad536df 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_pdb_worker.py @@ -57,22 +57,35 @@ def test_pod_disruption_budget_name(self, workers_values): @pytest.mark.parametrize( "workers_values", [ - {"podDisruptionBudget": {"enabled": True}}, - {"celery": {"podDisruptionBudget": {"enabled": True}}}, + {"podDisruptionBudget": {"enabled": True}, "labels": {"test_label": "test_label_value"}}, + { + "celery": {"podDisruptionBudget": {"enabled": True}}, + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": { + "podDisruptionBudget": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "celery": { + "podDisruptionBudget": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, ], ) def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( - values={ - "workers": { - **workers_values, - "labels": {"test_label": "test_label_value"}, - }, - }, + values={"workers": workers_values}, show_only=["templates/workers/worker-poddisruptionbudget.yaml"], ) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels @pytest.mark.parametrize( "workers_values", diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker.py b/helm-tests/tests/helm_tests/airflow_core/test_worker.py index 03188cda33074..3e31d2d454efe 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker.py @@ -135,23 +135,53 @@ def test_revision_history_limit_zero(self, worker_values, global_limit, expected assert jmespath.search("spec.revisionHistoryLimit", docs[0]) == expected - def test_should_add_extra_containers(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "extraContainers": [ + {"name": "{{ .Chart.Name }}-test-container", "image": "test-registry/test-repo:test-tag"} + ] + }, + { + "celery": { + "extraContainers": [ + { + "name": "{{ .Chart.Name }}-test-container", + "image": "test-registry/test-repo:test-tag", + } + ] + } + }, + { + "extraContainers": [{"name": "test", "image": "repo:test"}], + "celery": { + "extraContainers": [ + { + "name": "{{ .Chart.Name }}-test-container", + "image": "test-registry/test-repo:test-tag", + } + ] + }, + }, + ], + ) + def test_should_add_extra_containers_with_template(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "extraContainers": [ - {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} - ], - }, + "workers": workers_values, }, show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.containers[-1]", docs[0]) == { - "name": "airflow", - "image": "test-registry/test-repo:test-tag", - } + # [2:] -> Skipping worker and worker-log-groomer containers + assert jmespath.search("spec.template.spec.containers[2:]", docs[0]) == [ + { + "name": "airflow-test-container", + "image": "test-registry/test-repo:test-tag", + } + ] @pytest.mark.parametrize( "workers_values", @@ -188,28 +218,16 @@ def test_persistent_volume_claim_retention_policy(self, workers_values): "whenDeleted": "Delete", } - def test_should_template_extra_containers(self): - docs = render_chart( - values={ - "executor": "CeleryExecutor", - "workers": { - "extraContainers": [{"name": "{{ .Release.Name }}-test-container"}], - }, - }, - show_only=["templates/workers/worker-deployment.yaml"], - ) - - assert jmespath.search("spec.template.spec.containers[-1]", docs[0]) == { - "name": "release-name-test-container" - } - - def test_disable_wait_for_migration(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"waitForMigrations": {"enabled": False}}, + {"celery": {"waitForMigrations": {"enabled": False}}}, + ], + ) + def test_disable_wait_for_migration(self, workers_values): docs = render_chart( - values={ - "workers": { - "waitForMigrations": {"enabled": False}, - }, - }, + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) actual = jmespath.search( @@ -261,60 +279,130 @@ def test_logs_mount_on_wait_for_migrations_initcontainer(self, logs_values, expe for m in mounts ) - def test_should_add_extra_init_containers(self): - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] + }, + { + "celery": { "extraInitContainers": [ {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} - ], + ] + } + }, + { + "extraInitContainers": [{"name": "container", "image": "repo:tag"}], + "celery": { + "extraInitContainers": [ + {"name": "test-init-container", "image": "test-registry/test-repo:test-tag"} + ] }, }, + ], + ) + def test_should_add_extra_init_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.initContainers[-1]", docs[0]) == { - "name": "test-init-container", - "image": "test-registry/test-repo:test-tag", - } + # [1:] -> Skipping wait-for-airflow-migrations init container + assert jmespath.search("spec.template.spec.initContainers[1:]", docs[0]) == [ + { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + } + ] - def test_should_template_extra_init_containers(self): - docs = render_chart( - values={ - "workers": { - "extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, + {"celery": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}}, + { + "extraInitContainers": [{"name": "container"}], + "celery": {"extraInitContainers": [{"name": "{{ .Release.Name }}-test-init-container"}]}, }, + ], + ) + def test_should_template_extra_init_containers(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.initContainers[-1]", docs[0]) == { - "name": "release-name-test-init-container" - } + # [1:] -> Skipping wait-for-airflow-migrations init container + assert jmespath.search("spec.template.spec.initContainers[1:]", docs[0]) == [ + {"name": "release-name-test-init-container"} + ] - def test_should_add_extra_volume_and_extra_volume_mount(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}, + {"celery": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}}, + { + "extraVolumes": [{"name": "test", "emptyDir": {}}], + "celery": {"extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}]}, + }, + ], + ) + def test_should_add_extra_volume(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], + "workers": workers_values, + }, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + # [:-1] -> Skipping config volume + assert jmespath.search("spec.template.spec.volumes[:-1].name", docs[0]) == ["test-volume-airflow"] + + @pytest.mark.parametrize( + "workers_values", + [ + { + "extraVolumeMounts": [{"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"}], + }, + { + "celery": { + "extraVolumeMounts": [ + {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + } + }, + { + "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}], + "celery": { "extraVolumeMounts": [ {"name": "test-volume-{{ .Chart.Name }}", "mountPath": "/opt/test"} ], }, }, + ], + ) + def test_should_add_extra_volume_mount(self, workers_values): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "workers": workers_values, + }, show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.volumes[0].name", docs[0]) == "test-volume-airflow" - assert ( - jmespath.search("spec.template.spec.containers[0].volumeMounts[0].name", docs[0]) - == "test-volume-airflow" - ) - assert ( - jmespath.search("spec.template.spec.initContainers[0].volumeMounts[-1].name", docs[0]) - == "test-volume-airflow" - ) + volume_mounts = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + init_volume_mounts = jmespath.search("spec.template.spec.initContainers[0].volumeMounts", docs[0]) + + assert {"name": "test-volume-airflow", "mountPath": "/opt/test"} in init_volume_mounts + assert {"name": "test", "mountPath": "/opt"} not in init_volume_mounts + + assert {"name": "test-volume-airflow", "mountPath": "/opt/test"} in volume_mounts + assert {"name": "test", "mountPath": "/opt"} not in volume_mounts def test_should_add_global_volume_and_global_volume_mount(self): docs = render_chart( @@ -330,10 +418,40 @@ def test_should_add_global_volume_and_global_volume_mount(self): jmespath.search("spec.template.spec.containers[0].volumeMounts[0].name", docs[0]) == "test-volume" ) - def test_should_add_extraEnvs(self): - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], + }, + { + "celery": { + "env": [ + {"name": "TEST_ENV_1", "value": "test_env_1"}, + { + "name": "TEST_ENV_2", + "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, + }, + { + "name": "TEST_ENV_3", + "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, + }, + ], + } + }, + { + "env": [{"name": "TEST", "value": "test"}], + "celery": { "env": [ {"name": "TEST_ENV_1", "value": "test_env_1"}, { @@ -347,12 +465,20 @@ def test_should_add_extraEnvs(self): ], }, }, + ], + ) + def test_should_add_extraEnvs(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) assert {"name": "TEST_ENV_1", "value": "test_env_1"} in jmespath.search( "spec.template.spec.containers[0].env", docs[0] ) + assert {"name": "TEST", "value": "test"} not in jmespath.search( + "spec.template.spec.containers[0].env", docs[0] + ) assert { "name": "TEST_ENV_2", "valueFrom": {"secretKeyRef": {"name": "my-secret", "key": "my-key"}}, @@ -362,13 +488,20 @@ def test_should_add_extraEnvs(self): "valueFrom": {"configMapKeyRef": {"name": "my-config-map", "key": "my-key"}}, } in jmespath.search("spec.template.spec.containers[0].env", docs[0]) - def test_should_add_extraEnvs_to_wait_for_migration_container(self): - docs = render_chart( - values={ - "workers": { - "waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}, - }, + @pytest.mark.parametrize( + "workers_values", + [ + {"waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}}, + {"celery": {"waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}}}, + { + "waitForMigrations": {"env": [{"name": "TEST", "value": "test"}]}, + "celery": {"waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}}, }, + ], + ) + def test_should_add_extraEnvs_to_wait_for_migration_container(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) @@ -376,19 +509,37 @@ def test_should_add_extraEnvs_to_wait_for_migration_container(self): "spec.template.spec.initContainers[0].env", docs[0] ) - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": { + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "celery": { + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-deployment.yaml"], ) - assert "test_label" in jmespath.search("spec.template.metadata.labels", docs[0]) - assert jmespath.search("spec.template.metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("spec.template.metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels @pytest.mark.parametrize( "workers_values", @@ -500,11 +651,26 @@ def test_workers_strategy(self, workers_values, expected_strategy): assert expected_strategy == jmespath.search("spec.strategy", docs[0]) - def test_affinity(self): - docs = render_chart( - values={ - "executor": "CeleryExecutor", - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": "foo", "operator": "In", "values": ["true"]}, + ] + } + ] + } + } + }, + }, + { + "celery": { "affinity": { "nodeAffinity": { "requiredDuringSchedulingIgnoredDuringExecution": { @@ -518,37 +684,96 @@ def test_affinity(self): } } }, - }, - }, - show_only=["templates/workers/worker-deployment.yaml"], - ) - - assert jmespath.search("kind", docs[0]) == "StatefulSet" - assert jmespath.search("spec.template.spec.affinity", docs[0]) == { - "nodeAffinity": { - "requiredDuringSchedulingIgnoredDuringExecution": { - "nodeSelectorTerms": [ - { - "matchExpressions": [ - {"key": "foo", "operator": "In", "values": ["true"]}, - ] - } - ] } - } - } - - def test_tolerations(self): - docs = render_chart( - values={ - "executor": "CeleryExecutor", - "workers": { - "tolerations": [ - {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} - ], - }, }, - show_only=["templates/workers/worker-deployment.yaml"], + { + "affinity": { + "podAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "topologyKey": "foo", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + }, + "weight": 1, + } + ] + } + }, + "celery": { + "affinity": { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": "foo", "operator": "In", "values": ["true"]}, + ] + } + ] + } + } + }, + }, + }, + ], + ) + def test_affinity(self, workers_values): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "workers": workers_values, + }, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + assert jmespath.search("kind", docs[0]) == "StatefulSet" + assert jmespath.search("spec.template.spec.affinity", docs[0]) == { + "nodeAffinity": { + "requiredDuringSchedulingIgnoredDuringExecution": { + "nodeSelectorTerms": [ + { + "matchExpressions": [ + {"key": "foo", "operator": "In", "values": ["true"]}, + ] + } + ] + } + } + } + + @pytest.mark.parametrize( + "workers_values", + [ + { + "tolerations": [ + {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ], + }, + { + "celery": { + "tolerations": [ + {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ] + }, + }, + { + "tolerations": [{"key": "pods", "operator": "Exists", "effect": "PreferNoSchedule"}], + "celery": { + "tolerations": [ + {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ] + }, + }, + ], + ) + def test_tolerations(self, workers_values): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "workers": workers_values, + }, + show_only=["templates/workers/worker-deployment.yaml"], ) assert jmespath.search("kind", docs[0]) == "StatefulSet" @@ -556,8 +781,60 @@ def test_tolerations(self): {"key": "dynamic-pods", "operator": "Equal", "value": "true", "effect": "NoSchedule"} ] - def test_topology_spread_constraints(self): - expected_topology_spread_constraints = [ + @pytest.mark.parametrize( + "workers_values", + [ + { + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ] + }, + { + "celery": { + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ] + } + }, + { + "topologySpreadConstraints": [ + { + "maxSkew": 2, + "topologyKey": "not-me", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"airflow": "test"}}, + } + ], + "celery": { + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ] + }, + }, + ], + ) + def test_topology_spread_constraints(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, + show_only=["templates/workers/worker-deployment.yaml"], + ) + + assert jmespath.search("spec.template.spec.topologySpreadConstraints", docs[0]) == [ { "maxSkew": 1, "topologyKey": "foo", @@ -565,14 +842,6 @@ def test_topology_spread_constraints(self): "labelSelector": {"matchLabels": {"tier": "airflow"}}, } ] - docs = render_chart( - values={"workers": {"topologySpreadConstraints": expected_topology_spread_constraints}}, - show_only=["templates/workers/worker-deployment.yaml"], - ) - - assert expected_topology_spread_constraints == jmespath.search( - "spec.template.spec.topologySpreadConstraints", docs[0] - ) @pytest.mark.parametrize( "workers_values", @@ -862,19 +1131,24 @@ def test_extra_init_container_restart_policy_is_configurable(self): docs = render_chart( values={ "workers": { - "extraInitContainers": [ - { - "name": "test-init-container", - "image": "test-registry/test-repo:test-tag", - "restartPolicy": "Always", - } - ] + "celery": { + "extraInitContainers": [ + { + "name": "test-init-container", + "image": "test-registry/test-repo:test-tag", + "restartPolicy": "Always", + } + ] + } }, }, show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.initContainers[1].restartPolicy", docs[0]) == "Always" + # [1:] -> Skipping wait-for-airflow-migrations init container + assert jmespath.search("spec.template.spec.initContainers[1:] | [*].restartPolicy", docs[0]) == [ + "Always" + ] @pytest.mark.parametrize( ("log_values", "expected_volume"), @@ -1273,17 +1547,34 @@ def test_persistence_volume_annotations(self, workers_values): ) assert jmespath.search("spec.volumeClaimTemplates[0].metadata.annotations", docs[0]) == {"foo": "bar"} - def test_should_add_component_specific_annotations(self): - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "annotations": {"test_annotation": "test_annotation_value"}, + }, + { + "celery": { + "annotations": {"test_annotation": "test_annotation_value"}, + } + }, + { + "annotations": {"test": "test"}, + "celery": { "annotations": {"test_annotation": "test_annotation_value"}, }, }, + ], + ) + def test_should_add_component_specific_annotations(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) - assert "annotations" in jmespath.search("metadata", docs[0]) - assert jmespath.search("metadata.annotations", docs[0])["test_annotation"] == "test_annotation_value" + + assert jmespath.search("metadata.annotations", docs[0]) == { + "test_annotation": "test_annotation_value" + } @pytest.mark.parametrize( ("globalScope", "localScope", "precedence"), @@ -1298,7 +1589,7 @@ def test_should_add_component_specific_annotations(self): ( {}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, "safeToEvict": True, }, "true", @@ -1306,15 +1597,17 @@ def test_should_add_component_specific_annotations(self): ( {}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - "celery": {"safeToEvict": True}, + "celery": { + "safeToEvict": True, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + }, }, "true", ), ( {}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, "safeToEvict": False, }, "true", @@ -1322,15 +1615,17 @@ def test_should_add_component_specific_annotations(self): ( {}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - "celery": {"safeToEvict": False}, + "celery": { + "safeToEvict": False, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + }, }, "true", ), ( {}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, "safeToEvict": True, }, "false", @@ -1338,15 +1633,17 @@ def test_should_add_component_specific_annotations(self): ( {}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - "celery": {"safeToEvict": True}, + "celery": { + "safeToEvict": True, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + }, }, "false", ), ( {}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, "safeToEvict": False, }, "false", @@ -1354,8 +1651,10 @@ def test_should_add_component_specific_annotations(self): ( {}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - "celery": {"safeToEvict": False}, + "celery": { + "safeToEvict": False, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + }, }, "false", ), @@ -1402,7 +1701,7 @@ def test_should_add_component_specific_annotations(self): ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, "safeToEvict": False, }, "true", @@ -1410,15 +1709,17 @@ def test_should_add_component_specific_annotations(self): ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - "celery": {"safeToEvict": False}, + "celery": { + "safeToEvict": False, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + }, }, "true", ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, "safeToEvict": False, }, "false", @@ -1426,15 +1727,17 @@ def test_should_add_component_specific_annotations(self): ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - "celery": {"safeToEvict": False}, + "celery": { + "safeToEvict": False, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + }, }, "false", ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, "safeToEvict": False, }, "true", @@ -1442,15 +1745,17 @@ def test_should_add_component_specific_annotations(self): ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - "celery": {"safeToEvict": False}, + "celery": { + "safeToEvict": False, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + }, }, "true", ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, "safeToEvict": False, }, "false", @@ -1458,15 +1763,17 @@ def test_should_add_component_specific_annotations(self): ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - "celery": {"safeToEvict": False}, + "celery": { + "safeToEvict": False, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + }, }, "false", ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, "safeToEvict": True, }, "true", @@ -1474,15 +1781,17 @@ def test_should_add_component_specific_annotations(self): ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - "celery": {"safeToEvict": True}, + "celery": { + "safeToEvict": True, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + }, }, "true", ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, "safeToEvict": True, }, "false", @@ -1490,15 +1799,17 @@ def test_should_add_component_specific_annotations(self): ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - "celery": {"safeToEvict": True}, + "celery": { + "safeToEvict": True, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + }, }, "false", ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}}, "safeToEvict": True, }, "true", @@ -1506,15 +1817,17 @@ def test_should_add_component_specific_annotations(self): ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, - "celery": {"safeToEvict": True}, + "celery": { + "safeToEvict": True, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "true"}, + }, }, "true", ), ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + "celery": {"podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}}, "safeToEvict": True, }, "false", @@ -1522,8 +1835,10 @@ def test_should_add_component_specific_annotations(self): ( {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, { - "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, - "celery": {"safeToEvict": True}, + "celery": { + "safeToEvict": True, + "podAnnotations": {"cluster-autoscaler.kubernetes.io/safe-to-evict": "false"}, + }, }, "false", ), @@ -2031,23 +2346,50 @@ class TestWorkerLogGroomer(LogGroomerTestBase): folder = "workers" +class TestWorkerCeleryLogGroomer(LogGroomerTestBase): + """Worker Celery groomer.""" + + obj_name = "workers-celery" + folder = "workers" + + class TestWorkerKedaAutoScaler: """Tests worker keda auto scaler.""" - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": {"keda": {"enabled": True}}, + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": { + "keda": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "celery": { + "keda": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "celery": {"keda": {"enabled": True}}, - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-kedaautoscaler.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels def test_should_remove_replicas_field(self): docs = render_chart( @@ -2181,45 +2523,59 @@ class TestWorkerHPAAutoScaler: """Tests worker HPA auto scaler.""" @pytest.mark.parametrize( - "workers_keda_values", + "workers_values", [ - {"keda": {"enabled": True}}, - {"celery": {"keda": {"enabled": True}}}, + {"keda": {"enabled": True}, "hpa": {"enabled": True}}, + {"celery": {"keda": {"enabled": True}}, "hpa": {"enabled": True}}, + {"celery": {"keda": {"enabled": True}, "hpa": {"enabled": True}}}, + {"keda": {"enabled": True}, "celery": {"hpa": {"enabled": True}}}, ], ) - def test_should_be_disabled_on_keda_enabled(self, workers_keda_values): + def test_should_be_disabled_on_keda_enabled(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - **workers_keda_values, - "hpa": {"enabled": True}, - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=[ "templates/workers/worker-kedaautoscaler.yaml", "templates/workers/worker-hpa.yaml", ], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + assert len(docs) == 1 - def test_should_add_component_specific_labels(self): - docs = render_chart( - values={ - "executor": "CeleryExecutor", - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + {"celery": {"hpa": {"enabled": True}}, "labels": {"test_label": "test_label_value"}}, + { + "celery": { + "hpa": {"enabled": True}, + "labels": {"test_label": "test_label_value"}, + } + }, + { + "labels": {"key": "value"}, + "celery": { "hpa": {"enabled": True}, "labels": {"test_label": "test_label_value"}, }, }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "workers": workers_values, + }, show_only=["templates/workers/worker-hpa.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels def test_should_remove_replicas_field(self): docs = render_chart( @@ -2233,157 +2589,513 @@ def test_should_remove_replicas_field(self): ) assert "replicas" not in jmespath.search("spec", docs[0]) + @pytest.mark.parametrize("executor", ["CeleryExecutor", "CeleryKubernetesExecutor"]) @pytest.mark.parametrize( - ("metrics", "executor", "expected_metrics"), + "workers_values", [ - # default metrics - ( - None, - "CeleryExecutor", - { - "type": "Resource", - "resource": {"name": "cpu", "target": {"type": "Utilization", "averageUtilization": 80}}, - }, - ), - # custom metric - ( - [ - { - "type": "Pods", - "pods": { - "metric": {"name": "custom"}, - "target": {"type": "Utilization", "averageUtilization": 80}, - }, - } - ], - "CeleryKubernetesExecutor", - { - "type": "Pods", - "pods": { - "metric": {"name": "custom"}, - "target": {"type": "Utilization", "averageUtilization": 80}, - }, - }, - ), + {"hpa": {"enabled": True}}, + {"celery": {"hpa": {"enabled": True}}}, ], ) - def test_should_use_hpa_metrics(self, metrics, executor, expected_metrics): + def test_hpa_metrics_default(self, executor, workers_values): docs = render_chart( values={ "executor": executor, - "workers": { - "hpa": {"enabled": True, **({"metrics": metrics} if metrics else {})}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-hpa.yaml"], ) - assert expected_metrics == jmespath.search("spec.metrics[0]", docs[0]) + + assert jmespath.search("spec.metrics", docs[0]) == [ + { + "type": "Resource", + "resource": {"name": "cpu", "target": {"type": "Utilization", "averageUtilization": 80}}, + } + ] + + @pytest.mark.parametrize("executor", ["CeleryExecutor", "CeleryKubernetesExecutor"]) + @pytest.mark.parametrize( + "workers_values", + [ + { + "hpa": { + "enabled": True, + "metrics": [ + { + "type": "Pods", + "pods": { + "metric": {"name": "custom"}, + "target": {"type": "Utilization", "averageUtilization": 80}, + }, + } + ], + } + }, + { + "celery": { + "hpa": { + "enabled": True, + "metrics": [ + { + "type": "Pods", + "pods": { + "metric": {"name": "custom"}, + "target": {"type": "Utilization", "averageUtilization": 80}, + }, + } + ], + } + } + }, + { + "hpa": { + "enabled": True, + "metrics": [ + { + "type": "Resource", + "resource": { + "name": "memory", + "target": {"type": "Utilization", "averageUtilization": 1}, + }, + } + ], + }, + "celery": { + "hpa": { + "enabled": True, + "metrics": [ + { + "type": "Pods", + "pods": { + "metric": {"name": "custom"}, + "target": {"type": "Utilization", "averageUtilization": 80}, + }, + } + ], + } + }, + }, + ], + ) + def test_hpa_metrics_override(self, executor, workers_values): + docs = render_chart( + values={ + "executor": executor, + "workers": workers_values, + }, + show_only=["templates/workers/worker-hpa.yaml"], + ) + + assert jmespath.search("spec.metrics", docs[0]) == [ + { + "type": "Pods", + "pods": { + "metric": {"name": "custom"}, + "target": {"type": "Utilization", "averageUtilization": 80}, + }, + } + ] class TestWorkerNetworkPolicy: """Tests worker network policy.""" - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": {"labels": {"test_label": "test_label_value"}}, + }, + { + "labels": {"key": "value"}, + "celery": {"labels": {"test_label": "test_label_value"}}, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "networkPolicies": {"enabled": True}, "executor": "CeleryExecutor", - "workers": { - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-networkpolicy.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels class TestWorkerService: """Tests worker service.""" - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + { + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": {"labels": {"test_label": "test_label_value"}}, + }, + { + "labels": {"key": "value"}, + "celery": {"labels": {"test_label": "test_label_value"}}, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): docs = render_chart( values={ "executor": "CeleryExecutor", - "workers": { - "labels": {"test_label": "test_label_value"}, - }, + "workers": workers_values, }, show_only=["templates/workers/worker-service.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels -class TestWorkerServiceAccount: - """Tests worker service account.""" +class TestWorkerCeleryServiceAccount: + @pytest.mark.parametrize( + "workers_values", + [ + {"serviceAccount": {"create": False}}, + {"celery": {"serviceAccount": {"create": False}}}, + {"serviceAccount": {"create": True}, "celery": {"serviceAccount": {"create": False}}}, + ], + ) + def test_should_not_create_service_account_when_disabled(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + + assert len(docs) == 0 + + def test_should_create_service_account_by_default(self): + docs = render_chart( + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + + assert len(docs) == 1 - def test_should_add_component_specific_labels(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"serviceAccount": {"create": True}}, + {"celery": {"serviceAccount": {"create": True}}}, + {"serviceAccount": {"create": False}, "celery": {"serviceAccount": {"create": True}}}, + ], + ) + def test_should_not_create_service_account_for_local_executor(self, workers_values): docs = render_chart( values={ - "executor": "CeleryExecutor", - "workers": { - "serviceAccount": {"create": True}, - "labels": {"test_label": "test_label_value"}, - }, + "executor": "LocalExecutor", + "workers": workers_values, }, show_only=["templates/workers/worker-serviceaccount.yaml"], ) - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" + assert len(docs) == 0 @pytest.mark.parametrize( - ("executor", "creates_service_account"), + "executor", [ - ("LocalExecutor", False), - ("CeleryExecutor", True), - ("CeleryKubernetesExecutor", True), - ("CeleryExecutor,KubernetesExecutor", True), - ("KubernetesExecutor", True), - ("LocalKubernetesExecutor", True), + "CeleryExecutor", + "CeleryKubernetesExecutor", + "CeleryExecutor,KubernetesExecutor", + "KubernetesExecutor", + "LocalKubernetesExecutor", ], ) - def test_should_create_worker_service_account_for_specific_executors( - self, executor, creates_service_account - ): + @pytest.mark.parametrize( + "workers_values", + [ + {"serviceAccount": {"create": True}}, + {"celery": {"serviceAccount": {"create": True}}}, + {"serviceAccount": {"create": False}, "celery": {"serviceAccount": {"create": True}}}, + ], + ) + def test_should_create_service_account_for_specific_executors(self, executor, workers_values): docs = render_chart( values={ "executor": executor, - "workers": { + "workers": workers_values, + }, + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + + assert len(docs) == 1 + assert jmespath.search("kind", docs[0]) == "ServiceAccount" + + @pytest.mark.parametrize( + "workers_values", + [ + {"serviceAccount": {"create": True}}, + {"celery": {"serviceAccount": {"create": True}}}, + { + "serviceAccount": {"automountServiceAccountToken": False}, + "celery": {"serviceAccount": {"create": True, "automountServiceAccountToken": True}}, + }, + ], + ) + def test_automount_service_account_token_true(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + assert jmespath.search("automountServiceAccountToken", docs[0]) is True + + @pytest.mark.parametrize( + "workers_values", + [ + {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}, + {"celery": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}}, + { + "serviceAccount": {"automountServiceAccountToken": True}, + "celery": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}, + }, + ], + ) + def test_automount_service_account_token_false(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + + assert jmespath.search("automountServiceAccountToken", docs[0]) is False + + @pytest.mark.parametrize( + "workers_values", + [ + {"serviceAccount": {"create": True, "name": "test"}}, + {"celery": {"serviceAccount": {"create": True, "name": "test"}}}, + { + "serviceAccount": {"name": "none"}, + "celery": {"serviceAccount": {"create": True, "name": "test"}}, + }, + ], + ) + def test_overwrite_name(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, + show_only=["templates/workers/worker-serviceaccount.yaml"], + ) + + assert jmespath.search("metadata.name", docs[0]) == "test" + + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": {"serviceAccount": {"create": True}}, + "labels": {"test_label": "test_label_value"}, + }, + { + "celery": { "serviceAccount": {"create": True}, "labels": {"test_label": "test_label_value"}, }, }, + { + "labels": {"key": "value"}, + "celery": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): + docs = render_chart( + values={ + "executor": "CeleryExecutor", + "workers": workers_values, + }, show_only=["templates/workers/worker-serviceaccount.yaml"], ) - if creates_service_account: - assert jmespath.search("kind", docs[0]) == "ServiceAccount" - assert "test_label" in jmespath.search("metadata.labels", docs[0]) - assert jmespath.search("metadata.labels", docs[0])["test_label"] == "test_label_value" - else: - assert docs == [] - def test_default_automount_service_account_token(self): + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels + + +class TestWorkerKubernetesServiceAccount: + def test_should_not_create_service_account_by_default(self): + docs = render_chart( + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], + ) + + assert len(docs) == 0 + + @pytest.mark.parametrize( + "workers_values", + [ + {"serviceAccount": {"create": True}}, # Should not have effect + {"kubernetes": {"serviceAccount": {"create": False}}}, + {"serviceAccount": {"create": True}, "kubernetes": {"serviceAccount": {"create": False}}}, + ], + ) + def test_should_not_create_service_account_when_disabled(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], + ) + + assert len(docs) == 0 + + def test_should_create_service_account_when_enabled(self): docs = render_chart( values={ - "workers": { - "serviceAccount": {"create": True}, - }, + "executor": "KubernetesExecutor", + "workers": {"kubernetes": {"serviceAccount": {"create": True}}}, }, - show_only=["templates/workers/worker-serviceaccount.yaml"], + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], + ) + + assert len(docs) == 1 + + @pytest.mark.parametrize( + "executor", + [ + "CeleryExecutor", + "LocalExecutor", + "LocalExecutor,CeleryExecutor", + ], + ) + def test_should_not_create_service_account_non_k8s_executors(self, executor): + docs = render_chart( + values={"executor": executor, "workers": {"kubernetes": {"serviceAccount": {"create": True}}}}, + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], + ) + + assert len(docs) == 0 + + @pytest.mark.parametrize( + "executor", + [ + "CeleryKubernetesExecutor", + "CeleryExecutor,KubernetesExecutor", + "KubernetesExecutor", + "LocalKubernetesExecutor", + ], + ) + def test_should_create_service_account_when_k8s_executors(self, executor): + docs = render_chart( + values={"executor": executor, "workers": {"kubernetes": {"serviceAccount": {"create": True}}}}, + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], + ) + + assert len(docs) == 1 + + @pytest.mark.parametrize( + "workers_values", + [ + {"kubernetes": {"serviceAccount": {"create": True}}}, + {"kubernetes": {"serviceAccount": {"create": True, "automountServiceAccountToken": True}}}, + { + "serviceAccount": {"automountServiceAccountToken": False}, + "kubernetes": {"serviceAccount": {"create": True, "automountServiceAccountToken": True}}, + }, + ], + ) + def test_automount_service_account_token_true(self, workers_values): + docs = render_chart( + values={"executor": "KubernetesExecutor", "workers": workers_values}, + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], ) assert jmespath.search("automountServiceAccountToken", docs[0]) is True - def test_overridden_automount_service_account_token(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"kubernetes": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}}, + { + "serviceAccount": {"automountServiceAccountToken": True}, + "kubernetes": {"serviceAccount": {"create": True, "automountServiceAccountToken": False}}, + }, + ], + ) + def test_automount_service_account_token_false(self, workers_values): docs = render_chart( - values={ - "workers": { - "serviceAccount": {"create": True, "automountServiceAccountToken": False}, + values={"executor": "KubernetesExecutor", "workers": workers_values}, + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], + ) + + assert jmespath.search("automountServiceAccountToken", docs[0]) is False + + @pytest.mark.parametrize( + "workers_values", + [ + {"kubernetes": {"serviceAccount": {"create": True}}}, + {"serviceAccount": {"name": "test"}, "kubernetes": {"serviceAccount": {"create": True}}}, + ], + ) + def test_default_name(self, workers_values): + docs = render_chart( + name="test", + values={"executor": "KubernetesExecutor", "workers": workers_values}, + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], + ) + + assert jmespath.search("metadata.name", docs[0]) == "test-airflow-worker-kubernetes" + + @pytest.mark.parametrize( + "workers_values", + [ + {"kubernetes": {"serviceAccount": {"create": True, "name": "test"}}}, + { + "serviceAccount": {"name": "none"}, + "kubernetes": {"serviceAccount": {"create": True, "name": "test"}}, + }, + ], + ) + def test_overwrite_name(self, workers_values): + docs = render_chart( + values={"executor": "KubernetesExecutor", "workers": workers_values}, + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], + ) + + assert jmespath.search("metadata.name", docs[0]) == "test" + + @pytest.mark.parametrize( + "workers_values", + [ + { + "kubernetes": {"serviceAccount": {"create": True}}, + "labels": {"test_label": "test_label_value"}, + }, + { + "kubernetes": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, }, }, - show_only=["templates/workers/worker-serviceaccount.yaml"], + { + "labels": {"key": "value"}, + "kubernetes": { + "serviceAccount": {"create": True}, + "labels": {"test_label": "test_label_value"}, + }, + }, + ], + ) + def test_should_add_component_specific_labels(self, workers_values): + docs = render_chart( + values={ + "executor": "KubernetesExecutor", + "workers": workers_values, + }, + show_only=["templates/workers/worker-kubernetes-serviceaccount.yaml"], ) - assert jmespath.search("automountServiceAccountToken", docs[0]) is False + + labels = jmespath.search("metadata.labels", docs[0]) + assert labels["test_label"] == "test_label_value" + assert "key" not in labels diff --git a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py index b706c61153bfa..a2695fe39e419 100644 --- a/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py +++ b/helm-tests/tests/helm_tests/airflow_core/test_worker_sets.py @@ -1205,46 +1205,100 @@ def test_create_service_account_sets(self, enable_default, expected): assert jmespath.search("[*].metadata.name", docs) == expected - def test_overwrite_service_account_automount_service_account_token_disable(self): - docs = render_chart( - values={ - "workers": { - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "serviceAccount": {"automountServiceAccountToken": False}}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "serviceAccount": {"create": False}}], } }, + { + "celery": { + "enableDefault": False, + "serviceAccount": {"create": True}, + "sets": [{"name": "test", "serviceAccount": {"create": False}}], + } + }, + { + "serviceAccount": {"create": True}, + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "serviceAccount": {"create": False}}], + }, + }, + ], + ) + def test_overwrite_service_account_create_disable(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-serviceaccount.yaml"], ) - assert jmespath.search("automountServiceAccountToken", docs[0]) is False + assert len(docs) == 0 - def test_overwrite_service_account_create_disable(self): - docs = render_chart( - values={ - "workers": { - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "serviceAccount": {"create": False}}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "serviceAccount": {"automountServiceAccountToken": False}}], } }, + { + "celery": { + "enableDefault": False, + "serviceAccount": {"automountServiceAccountToken": True}, + "sets": [{"name": "test", "serviceAccount": {"automountServiceAccountToken": False}}], + } + }, + { + "serviceAccount": {"automountServiceAccountToken": True}, + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "serviceAccount": {"automountServiceAccountToken": False}}], + }, + }, + ], + ) + def test_overwrite_service_account_automount_service_account_token_disable(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-serviceaccount.yaml"], ) - assert len(docs) == 0 + assert jmespath.search("automountServiceAccountToken", docs[0]) is False - def test_overwrite_service_account_name(self): - docs = render_chart( - values={ - "workers": { - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "serviceAccount": {"name": "test"}}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "serviceAccount": {"name": "test"}}], } }, + { + "celery": { + "enableDefault": False, + "serviceAccount": {"name": "nontest"}, + "sets": [{"name": "test", "serviceAccount": {"name": "test"}}], + } + }, + { + "serviceAccount": {"name": "nontest"}, + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "serviceAccount": {"name": "test"}}], + }, + }, + ], + ) + def test_overwrite_service_account_name(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-serviceaccount.yaml"], ) @@ -1264,6 +1318,13 @@ def test_overwrite_service_account_name(self): "sets": [{"name": "test", "serviceAccount": {"annotations": {"test": "echo"}}}], }, }, + { + "celery": { + "enableDefault": False, + "serviceAccount": {"annotations": {"echo": "test"}}, + "sets": [{"name": "test", "serviceAccount": {"annotations": {"test": "echo"}}}], + }, + }, ], ) def test_overwrite_service_account_annotations(self, workers_values): @@ -1673,8 +1734,8 @@ def test_create_hpa_sets(self, enable_default, expected): name="test", values={ "workers": { - "hpa": {"enabled": True}, "celery": { + "hpa": {"enabled": True}, "enableDefault": enable_default, "sets": [ {"name": "set1"}, @@ -1688,56 +1749,116 @@ def test_create_hpa_sets(self, enable_default, expected): assert jmespath.search("[*].metadata.name", docs) == expected - def test_overwrite_hpa_enabled(self): - docs = render_chart( - values={ - "workers": { - "celery": {"enableDefault": False, "sets": [{"name": "test", "hpa": {"enabled": True}}]}, + @pytest.mark.parametrize( + "workers_values", + [ + {"celery": {"enableDefault": False, "sets": [{"name": "test", "hpa": {"enabled": True}}]}}, + { + "celery": { + "enableDefault": False, + "hpa": {"enabled": False}, + "sets": [{"name": "test", "hpa": {"enabled": True}}], } }, + { + "hpa": {"enabled": False}, + "celery": {"enableDefault": False, "sets": [{"name": "test", "hpa": {"enabled": True}}]}, + }, + ], + ) + def test_overwrite_hpa_enabled(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-hpa.yaml"], ) assert len(docs) == 1 - def test_overwrite_hpa_disable(self): - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": { + "enableDefault": False, "hpa": {"enabled": True}, - "celery": {"enableDefault": False, "sets": [{"name": "test", "hpa": {"enabled": False}}]}, + "sets": [{"name": "test", "hpa": {"enabled": False}}], } }, + { + "hpa": {"enabled": True}, + "celery": {"enableDefault": False, "sets": [{"name": "test", "hpa": {"enabled": False}}]}, + }, + ], + ) + def test_overwrite_hpa_disable(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-hpa.yaml"], ) assert len(docs) == 0 - def test_overwrite_hpa_min_replica_count(self): - docs = render_chart( - values={ - "workers": { - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "hpa": {"enabled": True, "minReplicaCount": 10}}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "hpa": {"enabled": True, "minReplicaCount": 10}}], + } + }, + { + "celery": { + "enableDefault": False, + "hpa": {"minReplicaCount": 7}, + "sets": [{"name": "test", "hpa": {"enabled": True, "minReplicaCount": 10}}], } }, + { + "hpa": {"minReplicaCount": 7}, + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "hpa": {"enabled": True, "minReplicaCount": 10}}], + }, + }, + ], + ) + def test_overwrite_hpa_min_replica_count(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-hpa.yaml"], ) assert jmespath.search("spec.minReplicas", docs[0]) == 10 - def test_overwrite_hpa_max_replica_count(self): - docs = render_chart( - values={ - "workers": { - "celery": { - "enableDefault": False, - "sets": [{"name": "test", "hpa": {"enabled": True, "maxReplicaCount": 10}}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "hpa": {"enabled": True, "maxReplicaCount": 10}}], + } + }, + { + "celery": { + "enableDefault": False, + "hpa": {"maxReplicaCount": 7}, + "sets": [{"name": "test", "hpa": {"enabled": True, "maxReplicaCount": 10}}], } }, + { + "hpa": {"maxReplicaCount": 7}, + "celery": { + "enableDefault": False, + "sets": [{"name": "test", "hpa": {"enabled": True, "maxReplicaCount": 10}}], + }, + }, + ], + ) + def test_overwrite_hpa_max_replica_count(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-hpa.yaml"], ) @@ -1801,6 +1922,39 @@ def test_overwrite_hpa_max_replica_count(self): ], }, }, + { + "celery": { + "enableDefault": False, + "hpa": { + "metrics": [ + { + "type": "Resource", + "resource": { + "name": "memory", + "target": {"type": "Utilization", "averageUtilization": 1}, + }, + } + ], + }, + "sets": [ + { + "name": "test", + "hpa": { + "enabled": True, + "metrics": [ + { + "type": "Resource", + "resource": { + "name": "cpu", + "target": {"type": "Utilization", "averageUtilization": 80}, + }, + } + ], + }, + } + ], + }, + }, ], ) def test_overwrite_hpa_metrics(self, workers_values): @@ -1842,6 +1996,18 @@ def test_overwrite_hpa_metrics(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "hpa": {"behavior": {"scaleUp": {"selectPolicy": "Min"}}}, + "sets": [ + { + "name": "test", + "hpa": {"enabled": True, "behavior": {"scaleDown": {"selectPolicy": "Max"}}}, + } + ], + }, + }, ], ) def test_overwrite_hpa_behavior(self, workers_values): @@ -2292,6 +2458,20 @@ def test_overwrite_safe_to_evict_disable(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "extraContainers": [{"name": "test", "image": "test"}], + "sets": [ + { + "name": "set1", + "extraContainers": [ + {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} + ], + } + ], + }, + }, ], ) def test_overwrite_extra_containers(self, workers_values): @@ -2302,13 +2482,13 @@ def test_overwrite_extra_containers(self, workers_values): show_only=["templates/workers/worker-deployment.yaml"], ) - containers = jmespath.search("spec.template.spec.containers", docs[0]) - - assert len(containers) == 3 # worker, worker-log-groomer, extra - assert containers[-1] == { - "name": "airflow", - "image": "test-registry/test-repo:test-tag", - } + # [2:] -> Skipping worker and worker-log-groomer containers + assert jmespath.search("spec.template.spec.containers[2:]", docs[0]) == [ + { + "name": "airflow", + "image": "test-registry/test-repo:test-tag", + } + ] @pytest.mark.parametrize( "workers_values", @@ -2333,6 +2513,20 @@ def test_overwrite_extra_containers(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "extraInitContainers": [{"name": "test", "image": "test"}], + "sets": [ + { + "name": "set1", + "extraInitContainers": [ + {"name": "{{ .Chart.Name }}", "image": "test-registry/test-repo:test-tag"} + ], + } + ], + }, + }, ], ) def test_overwrite_extra_init_containers(self, workers_values): @@ -2370,6 +2564,18 @@ def test_overwrite_extra_init_containers(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "extraVolumes": [{"name": "test", "emptyDir": {}}], + "sets": [ + { + "name": "set1", + "extraVolumes": [{"name": "test-volume-{{ .Chart.Name }}", "emptyDir": {}}], + } + ], + }, + }, ], ) def test_overwrite_extra_volumes(self, workers_values): @@ -2408,6 +2614,20 @@ def test_overwrite_extra_volumes(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "extraVolumeMounts": [{"name": "test", "mountPath": "/opt"}], + "sets": [ + { + "name": "set1", + "extraVolumeMounts": [ + {"name": "test-volume-mount-{{ .Chart.Name }}", "mountPath": "/opt/test"} + ], + } + ], + }, + }, ], ) def test_overwrite_extra_volume_mounts(self, workers_values): @@ -2418,10 +2638,10 @@ def test_overwrite_extra_volume_mounts(self, workers_values): show_only=["templates/workers/worker-deployment.yaml"], ) - assert jmespath.search("spec.template.spec.containers[0].volumeMounts[0]", docs[0]) == { - "name": "test-volume-mount-airflow", - "mountPath": "/opt/test", - } + volume_mounts = jmespath.search("spec.template.spec.containers[0].volumeMounts", docs[0]) + + assert {"name": "test-volume-mount-airflow", "mountPath": "/opt/test"} in volume_mounts + assert {"name": "test", "mountPath": "/opt"} not in volume_mounts @pytest.mark.parametrize( "workers_values", @@ -2625,6 +2845,43 @@ def test_overwrite_priority_class_name(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "affinity": { + "podAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "podAffinityTerm": { + "topologyKey": "foo", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + }, + "weight": 1, + } + ] + } + }, + "sets": [ + { + "name": "set1", + "affinity": { + "nodeAffinity": { + "preferredDuringSchedulingIgnoredDuringExecution": [ + { + "weight": 1, + "preference": { + "matchExpressions": [ + {"key": "not-me", "operator": "In", "values": ["true"]}, + ] + }, + } + ] + } + }, + } + ], + }, + }, ], ) def test_overwrite_affinity(self, workers_values): @@ -2680,6 +2937,27 @@ def test_overwrite_affinity(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "tolerations": [ + {"key": "not-me", "operator": "Equal", "value": "true", "effect": "NoSchedule"} + ], + "sets": [ + { + "name": "set1", + "tolerations": [ + { + "key": "dynamic-pods", + "operator": "Equal", + "value": "true", + "effect": "NoSchedule", + } + ], + } + ], + }, + }, ], ) def test_overwrite_tolerations(self, workers_values): @@ -2734,6 +3012,32 @@ def test_overwrite_tolerations(self, workers_values): ], }, }, + { + "celery": { + "enableDefault": False, + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "not-me", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ], + "sets": [ + { + "name": "set1", + "topologySpreadConstraints": [ + { + "maxSkew": 1, + "topologyKey": "foo", + "whenUnsatisfiable": "ScheduleAnyway", + "labelSelector": {"matchLabels": {"tier": "airflow"}}, + } + ], + } + ], + }, + }, ], ) def test_overwrite_topology_spread_constraints(self, workers_values): @@ -2803,6 +3107,13 @@ def test_overwrite_host_aliases(self, workers_values): "sets": [{"name": "set1", "annotations": {"test": "echo"}}], }, }, + { + "celery": { + "enableDefault": False, + "annotations": {"echo": "test"}, + "sets": [{"name": "set1", "annotations": {"test": "echo"}}], + }, + }, ], ) def test_overwrite_annotations(self, workers_values): @@ -2829,6 +3140,13 @@ def test_overwrite_annotations(self, workers_values): "sets": [{"name": "set1", "podAnnotations": {"test": "echo"}}], }, }, + { + "celery": { + "enableDefault": False, + "podAnnotations": {"echo": "test"}, + "sets": [{"name": "set1", "podAnnotations": {"test": "echo"}}], + }, + }, ], ) def test_overwrite_pod_annotations(self, workers_values): @@ -2849,6 +3167,13 @@ def test_overwrite_pod_annotations(self, workers_values): "labels": {"echo": "test"}, "celery": {"enableDefault": False, "sets": [{"name": "set1", "labels": {"test": "echo"}}]}, }, + { + "celery": { + "enableDefault": False, + "labels": {"echo": "test"}, + "sets": [{"name": "set1", "labels": {"test": "echo"}}], + }, + }, ], ) def test_overwrite_labels(self, workers_values): @@ -2866,16 +3191,27 @@ def test_overwrite_labels(self, workers_values): assert labels["test"] == "echo" assert labels.get("echo") is None - def test_overwrite_wait_for_migration_disable(self): - docs = render_chart( - values={ - "workers": { - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "waitForMigrations": {"enabled": False}}], - }, - }, + @pytest.mark.parametrize( + "workers_values", + [ + { + "celery": { + "enableDefault": False, + "sets": [{"name": "set1", "waitForMigrations": {"enabled": False}}], + } }, + { + "celery": { + "waitForMigrations": {"enabled": True}, + "enableDefault": False, + "sets": [{"name": "set1", "waitForMigrations": {"enabled": False}}], + } + }, + ], + ) + def test_overwrite_wait_for_migration_disable(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) assert ( @@ -2885,17 +3221,28 @@ def test_overwrite_wait_for_migration_disable(self): is None ) - def test_overwrite_wait_for_migration_enable(self): - docs = render_chart( - values={ - "workers": { - "waitForMigrations": {"enabled": False}, - "celery": { - "enableDefault": False, - "sets": [{"name": "set1", "waitForMigrations": {"enabled": True}}], - }, + @pytest.mark.parametrize( + "workers_values", + [ + { + "waitForMigrations": {"enabled": False}, + "celery": { + "enableDefault": False, + "sets": [{"name": "set1", "waitForMigrations": {"enabled": True}}], }, }, + { + "celery": { + "waitForMigrations": {"enabled": False}, + "enableDefault": False, + "sets": [{"name": "set1", "waitForMigrations": {"enabled": True}}], + } + }, + ], + ) + def test_overwrite_wait_for_migration_enable(self, workers_values): + docs = render_chart( + values={"workers": workers_values}, show_only=["templates/workers/worker-deployment.yaml"], ) assert ( @@ -2924,6 +3271,18 @@ def test_overwrite_wait_for_migration_enable(self): ], }, }, + { + "celery": { + "waitForMigrations": {"env": [{"name": "TEST", "value": "test"}]}, + "enableDefault": False, + "sets": [ + { + "name": "set1", + "waitForMigrations": {"env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}, + } + ], + }, + }, ], ) def test_overwrite_wait_for_migration_env(self, workers_values): @@ -2960,6 +3319,20 @@ def test_overwrite_wait_for_migration_env(self, workers_values): ], }, }, + { + "celery": { + "waitForMigrations": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + }, + "enableDefault": False, + "sets": [ + { + "name": "set1", + "waitForMigrations": {"securityContexts": {"container": {"runAsUser": 10}}}, + } + ], + }, + }, ], ) def test_overwrite_wait_for_migration_security_context_container(self, workers_values): @@ -2989,6 +3362,13 @@ def test_overwrite_wait_for_migration_security_context_container(self, workers_v "sets": [{"name": "set1", "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}], }, }, + { + "celery": { + "enableDefault": False, + "env": [{"name": "TEST", "value": "test"}], + "sets": [{"name": "set1", "env": [{"name": "TEST_ENV_1", "value": "test_env_1"}]}], + }, + }, ], ) def test_overwrite_env(self, workers_values): diff --git a/helm-tests/tests/helm_tests/other/test_hpa.py b/helm-tests/tests/helm_tests/other/test_hpa.py index 83bcf68b12c8a..1f2f632e6a19d 100644 --- a/helm-tests/tests/helm_tests/other/test_hpa.py +++ b/helm-tests/tests/helm_tests/other/test_hpa.py @@ -25,7 +25,6 @@ class TestHPA: """Tests HPA.""" def test_hpa_disabled_by_default(self): - """Disabled by default.""" docs = render_chart( values={}, show_only=["templates/workers/worker-hpa.yaml"], @@ -40,11 +39,21 @@ def test_hpa_disabled_by_default(self): "CeleryExecutor,KubernetesExecutor", ], ) - def test_hpa_enabled(self, executor): - """HPA should only be created when enabled and executor is Celery or CeleryKubernetes.""" + @pytest.mark.parametrize( + "workers_values", + [ + {"hpa": {"enabled": True}, "celery": {"persistence": {"enabled": False}}}, + {"celery": {"hpa": {"enabled": True}, "persistence": {"enabled": False}}}, + { + "hpa": {"enabled": False}, + "celery": {"hpa": {"enabled": True}, "persistence": {"enabled": False}}, + }, + ], + ) + def test_hpa_enabled(self, executor, workers_values): docs = render_chart( values={ - "workers": {"hpa": {"enabled": True}, "celery": {"persistence": {"enabled": False}}}, + "workers": workers_values, "executor": executor, }, show_only=["templates/workers/worker-hpa.yaml"], @@ -52,69 +61,118 @@ def test_hpa_enabled(self, executor): assert jmespath.search("metadata.name", docs[0]) == "release-name-worker" + def test_min_max_replicas_default(self): + docs = render_chart( + values={"workers": {"celery": {"hpa": {"enabled": True}}}}, + show_only=["templates/workers/worker-hpa.yaml"], + ) + + assert jmespath.search("spec.minReplicas", docs[0]) == 0 + assert jmespath.search("spec.maxReplicas", docs[0]) == 5 + @pytest.mark.parametrize( - ("min_replicas", "max_replicas"), + "workers_values", [ - (None, None), - (2, 8), + {"hpa": {"enabled": True, "minReplicaCount": 2, "maxReplicaCount": 8}}, + {"celery": {"hpa": {"enabled": True, "minReplicaCount": 2, "maxReplicaCount": 8}}}, + { + "hpa": {"enabled": True, "minReplicaCount": 1, "maxReplicaCount": 10}, + "celery": {"hpa": {"enabled": True, "minReplicaCount": 2, "maxReplicaCount": 8}}, + }, ], ) - def test_min_max_replicas(self, min_replicas, max_replicas): - """Verify minimum and maximum replicas.""" + def test_min_max_replicas(self, workers_values): docs = render_chart( - values={ - "workers": { - "hpa": { - "enabled": True, - **({"minReplicaCount": min_replicas} if min_replicas else {}), - **({"maxReplicaCount": max_replicas} if max_replicas else {}), - } - }, - }, + values={"workers": workers_values}, show_only=["templates/workers/worker-hpa.yaml"], ) - assert jmespath.search("spec.minReplicas", docs[0]) == 0 if min_replicas is None else min_replicas - assert jmespath.search("spec.maxReplicas", docs[0]) == 5 if max_replicas is None else max_replicas + + assert jmespath.search("spec.minReplicas", docs[0]) == 2 + assert jmespath.search("spec.maxReplicas", docs[0]) == 8 @pytest.mark.parametrize( "executor", ["CeleryExecutor", "CeleryKubernetesExecutor", "CeleryExecutor,KubernetesExecutor"] ) - def test_hpa_behavior(self, executor): - """Verify HPA behavior.""" - expected_behavior = { - "scaleDown": { - "stabilizationWindowSeconds": 300, - "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], - } - } - docs = render_chart( - values={ - "workers": { + @pytest.mark.parametrize( + "workers_values", + [ + { + "hpa": { + "enabled": True, + "behavior": { + "scaleDown": { + "stabilizationWindowSeconds": 300, + "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], + } + }, + } + }, + { + "celery": { "hpa": { "enabled": True, - "behavior": expected_behavior, - }, + "behavior": { + "scaleDown": { + "stabilizationWindowSeconds": 300, + "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], + } + }, + } + } + }, + { + "hpa": { + "behavior": { + "scaleUp": { + "stabilizationWindowSeconds": 300, + "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], + } + } + }, + "celery": { + "hpa": { + "enabled": True, + "behavior": { + "scaleDown": { + "stabilizationWindowSeconds": 300, + "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], + } + }, + } }, + }, + ], + ) + def test_hpa_behavior(self, executor, workers_values): + """Verify HPA behavior.""" + docs = render_chart( + values={ + "workers": workers_values, "executor": executor, }, show_only=["templates/workers/worker-hpa.yaml"], ) - assert jmespath.search("spec.behavior", docs[0]) == expected_behavior + assert jmespath.search("spec.behavior", docs[0]) == { + "scaleDown": { + "stabilizationWindowSeconds": 300, + "policies": [{"type": "Percent", "value": 100, "periodSeconds": 15}], + } + } @pytest.mark.parametrize( - ("workers_persistence_values", "kind"), + ("workers_values", "kind"), [ - ({"celery": {"persistence": {"enabled": True}}}, "StatefulSet"), - ({"celery": {"persistence": {"enabled": False}}}, "Deployment"), - ({"persistence": {"enabled": True}}, "StatefulSet"), - ({"persistence": {"enabled": False}}, "Deployment"), + ({"celery": {"hpa": {"enabled": True}, "persistence": {"enabled": True}}}, "StatefulSet"), + ({"celery": {"hpa": {"enabled": True}, "persistence": {"enabled": False}}}, "Deployment"), + ({"persistence": {"enabled": True}, "celery": {"hpa": {"enabled": True}}}, "StatefulSet"), + ({"persistence": {"enabled": False}, "celery": {"hpa": {"enabled": True}}}, "Deployment"), ], ) - def test_persistence(self, workers_persistence_values, kind): + def test_persistence(self, workers_values, kind): """If worker persistence is enabled, scaleTargetRef should be StatefulSet else Deployment.""" docs = render_chart( values={ - "workers": {"hpa": {"enabled": True}, **workers_persistence_values}, + "workers": workers_values, "executor": "CeleryExecutor", }, show_only=["templates/workers/worker-hpa.yaml"], diff --git a/helm-tests/tests/helm_tests/security/test_metadata_connection_secret.py b/helm-tests/tests/helm_tests/security/test_metadata_connection_secret.py index d2479835faff1..d5727db485efd 100644 --- a/helm-tests/tests/helm_tests/security/test_metadata_connection_secret.py +++ b/helm-tests/tests/helm_tests/security/test_metadata_connection_secret.py @@ -125,6 +125,46 @@ def test_should_correctly_handle_password_with_special_characters(self): "somedb?sslmode=disable" ) + def test_tpl_rendered_user_and_db(self): + """Test that metadataConnection.user and .db support tpl rendering.""" + connection = self._get_connection( + { + "data": { + "metadataConnection": { + "user": "{{ .Release.Name }}-dbuser", + "pass": "", + "host": "localhost", + "port": 5432, + "db": "{{ .Release.Name }}-mydb", + "protocol": "postgresql", + "sslmode": "disable", + } + } + } + ) + assert "release-name-dbuser" in connection + assert "release-name-mydb" in connection + + def test_tpl_rendered_user_and_db_plain_values(self): + """Test that plain (non-template) user and db still work after tpl rendering.""" + connection = self._get_connection( + { + "data": { + "metadataConnection": { + "user": "plainuser", + "pass": "plainpass", + "host": "localhost", + "port": 5432, + "db": "plaindb", + "protocol": "postgresql", + "sslmode": "disable", + } + } + } + ) + assert "plainuser" in connection + assert "plaindb" in connection + def test_should_add_annotations_to_metadata_connection_secret(self): docs = render_chart( values={ diff --git a/helm-tests/tests/helm_tests/security/test_security_context.py b/helm-tests/tests/helm_tests/security/test_security_context.py index dd9da798f5630..39e9281d04b40 100644 --- a/helm-tests/tests/helm_tests/security/test_security_context.py +++ b/helm-tests/tests/helm_tests/security/test_security_context.py @@ -554,13 +554,34 @@ def test_main_container_setting(self, workers_values): assert ctx_value == jmespath.search("spec.template.spec.containers[0].securityContext", doc) # Test securityContexts for log-groomer-sidecar main container - def test_log_groomer_sidecar_container_setting(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"logGroomerSidecar": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}}, + { + "celery": { + "logGroomerSidecar": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + } + } + }, + { + "logGroomerSidecar": {"securityContexts": {"container": {"runAsUser": 20}}}, + "celery": { + "logGroomerSidecar": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + } + }, + }, + ], + ) + def test_log_groomer_sidecar_container_setting(self, workers_values): ctx_value = {"allowPrivilegeEscalation": False} spec = {"logGroomerSidecar": {"securityContexts": {"container": ctx_value}}} docs = render_chart( values={ "scheduler": spec, - "workers": spec, + "workers": workers_values, "dagProcessor": spec, "triggerer": spec, }, @@ -671,7 +692,28 @@ def test_worker_kerberos_init_container_security_contexts(self, workers_values): ) == {"runAsUser": 2000} # Test securityContexts for the wait-for-migrations init containers - def test_wait_for_migrations_init_container_setting_airflow_2(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"waitForMigrations": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}}, + { + "celery": { + "waitForMigrations": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + } + } + }, + { + "waitForMigrations": {"securityContexts": {"container": {"runAsUser": 0}}}, + "celery": { + "waitForMigrations": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + } + }, + }, + ], + ) + def test_wait_for_migrations_init_container_setting_airflow_2(self, workers_values): ctx_value = {"allowPrivilegeEscalation": False} spec = { "waitForMigrations": { @@ -684,7 +726,7 @@ def test_wait_for_migrations_init_container_setting_airflow_2(self): "scheduler": spec, "webserver": spec, "triggerer": spec, - "workers": {"waitForMigrations": {"securityContexts": {"container": ctx_value}}}, + "workers": workers_values, "airflowVersion": "2.11.0", }, show_only=[ @@ -698,7 +740,28 @@ def test_wait_for_migrations_init_container_setting_airflow_2(self): for doc in docs: assert ctx_value == jmespath.search("spec.template.spec.initContainers[0].securityContext", doc) - def test_wait_for_migrations_init_container_setting(self): + @pytest.mark.parametrize( + "workers_values", + [ + {"waitForMigrations": {"securityContexts": {"container": {"allowPrivilegeEscalation": False}}}}, + { + "celery": { + "waitForMigrations": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + } + } + }, + { + "waitForMigrations": {"securityContexts": {"container": {"runAsUser": 0}}}, + "celery": { + "waitForMigrations": { + "securityContexts": {"container": {"allowPrivilegeEscalation": False}} + } + }, + }, + ], + ) + def test_wait_for_migrations_init_container_setting(self, workers_values): ctx_value = {"allowPrivilegeEscalation": False} spec = { "waitForMigrations": { @@ -712,7 +775,7 @@ def test_wait_for_migrations_init_container_setting(self): "apiServer": spec, "triggerer": spec, "dagProcessor": spec, - "workers": {"waitForMigrations": {"securityContexts": {"container": ctx_value}}}, + "workers": workers_values, }, show_only=[ "templates/scheduler/scheduler-deployment.yaml", diff --git a/providers/.last_release_date.txt b/providers/.last_release_date.txt index 2261237fe8327..5c4e706044a16 100644 --- a/providers/.last_release_date.txt +++ b/providers/.last_release_date.txt @@ -1 +1 @@ -2026-03-24 +2026-04-08 diff --git a/providers/ACCEPTING_PROVIDERS.rst b/providers/ACCEPTING_PROVIDERS.rst index a17dc397f4ff0..f79bc1420c79c 100644 --- a/providers/ACCEPTING_PROVIDERS.rst +++ b/providers/ACCEPTING_PROVIDERS.rst @@ -72,8 +72,15 @@ provides visibility for 3rd-party providers, and this approach allows service pr There is no difference in technical capabilities between community and 3rd-party providers. See `3rd-party providers `_ for more details. -Historical examples --------------------- +Examples +-------- + +* Vespa.ai - `[PROPOSAL] New Provider: Vespa.ai - AIP-95 `_, `Missing Vote `_ +* Informatica - `[PROPOSAL] New Provider: Informatica - AIP-95 `_, `Missing Vote `_ +* Stripe - `[DISCUSS] Interest in adding a Stripe provider to Airflow `_, `Missing Vote `_ + +Historical examples (Before AIP-95) +----------------------------------- * Huawei Cloud provider - `Discussion `_ * Cloudera provider - `Discussion `_, `Vote `_ diff --git a/providers/airbyte/README.rst b/providers/airbyte/README.rst index f3b02aca44508..0c9ef06691cc1 100644 --- a/providers/airbyte/README.rst +++ b/providers/airbyte/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-airbyte`` -Release: ``5.4.0`` +Release: ``5.4.1`` `Airbyte `__ @@ -36,7 +36,7 @@ This is a provider package for ``airbyte`` provider. All classes for this provid are in ``airflow.providers.airbyte`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -79,4 +79,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/airbyte/docs/changelog.rst b/providers/airbyte/docs/changelog.rst index bc3b078ec4c27..622493ace1f9a 100644 --- a/providers/airbyte/docs/changelog.rst +++ b/providers/airbyte/docs/changelog.rst @@ -27,6 +27,17 @@ Changelog --------- +5.4.1 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 5.4.0 ..... diff --git a/providers/airbyte/docs/index.rst b/providers/airbyte/docs/index.rst index bc95fe0e1a6a3..2080c8160c22f 100644 --- a/providers/airbyte/docs/index.rst +++ b/providers/airbyte/docs/index.rst @@ -76,7 +76,7 @@ apache-airflow-providers-airbyte package `Airbyte `__ -Release: 5.4.0 +Release: 5.4.1 Provider package ---------------- @@ -130,5 +130,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-airbyte 5.4.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-airbyte 5.4.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-airbyte 5.4.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-airbyte 5.4.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/airbyte/provider.yaml b/providers/airbyte/provider.yaml index c12e15e064bb1..82b5778f4ca19 100644 --- a/providers/airbyte/provider.yaml +++ b/providers/airbyte/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298099 +source-date-epoch: 1775591506 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 5.4.1 - 5.4.0 - 5.3.3 - 5.3.2 diff --git a/providers/airbyte/pyproject.toml b/providers/airbyte/pyproject.toml index 812f548c44432..bd613bf7dafe1 100644 --- a/providers/airbyte/pyproject.toml +++ b/providers/airbyte/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-airbyte" -version = "5.4.0" +version = "5.4.1" description = "Provider package apache-airflow-providers-airbyte for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -100,8 +100,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-airbyte/5.4.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-airbyte/5.4.0/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-airbyte/5.4.1" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-airbyte/5.4.1/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/airbyte/src/airflow/providers/airbyte/__init__.py b/providers/airbyte/src/airflow/providers/airbyte/__init__.py index 1a202d9985b5b..c8d18ceaa336c 100644 --- a/providers/airbyte/src/airflow/providers/airbyte/__init__.py +++ b/providers/airbyte/src/airflow/providers/airbyte/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "5.4.0" +__version__ = "5.4.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/airbyte/tests/system/airbyte/example_airbyte_trigger_job.py b/providers/airbyte/tests/system/airbyte/example_airbyte_trigger_job.py index 240a293bbd71b..68f4585f2797e 100644 --- a/providers/airbyte/tests/system/airbyte/example_airbyte_trigger_job.py +++ b/providers/airbyte/tests/system/airbyte/example_airbyte_trigger_job.py @@ -63,5 +63,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/alibaba/README.rst b/providers/alibaba/README.rst index af1d885c6b9a3..1d4b6bfe7c8e1 100644 --- a/providers/alibaba/README.rst +++ b/providers/alibaba/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-alibaba`` -Release: ``3.3.6`` +Release: ``3.3.7`` Alibaba Cloud integration (including `Alibaba Cloud `__). @@ -36,7 +36,7 @@ This is a provider package for ``alibaba`` provider. All classes for this provid are in ``airflow.providers.alibaba`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -50,16 +50,17 @@ The package supports the following python versions: 3.10,3.11,3.12,3.13,3.14 Requirements ------------ -========================================== ================== +========================================== ======================================== PIP package Version required -========================================== ================== +========================================== ======================================== ``apache-airflow`` ``>=2.11.0`` ``apache-airflow-providers-common-compat`` ``>=1.13.0`` -``oss2`` ``>=2.14.0`` +``alibabacloud-oss-v2`` ``>=1.2.0`` ``alibabacloud_adb20211201`` ``>=1.0.0`` ``alibabacloud_tea_openapi`` ``>=0.3.7`` -``pyodps`` ``>=0.12.2.2`` -========================================== ================== +``pyodps`` ``>=0.12.2.2; python_version < "3.13"`` +``pyodps`` ``>=0.12.5.1; python_version >= "3.13"`` +========================================== ======================================== Cross provider package dependencies ----------------------------------- @@ -81,4 +82,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/alibaba/docs/changelog.rst b/providers/alibaba/docs/changelog.rst index 6a5ed288792b7..6e088a506cfad 100644 --- a/providers/alibaba/docs/changelog.rst +++ b/providers/alibaba/docs/changelog.rst @@ -26,6 +26,20 @@ Changelog --------- +3.3.7 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` +* ``Replace 'oss2' with 'alibabacloud-oss-v2' (#64361)`` +* ``Bump pyodps for python>=3.13 (#64210)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Add 4-day cooldown for uv dependency resolution (#64249)`` + 3.3.6 ..... diff --git a/providers/alibaba/docs/index.rst b/providers/alibaba/docs/index.rst index 474e795dab143..b01df6423a1d7 100644 --- a/providers/alibaba/docs/index.rst +++ b/providers/alibaba/docs/index.rst @@ -77,7 +77,7 @@ apache-airflow-providers-alibaba package Alibaba Cloud integration (including `Alibaba Cloud `__). -Release: 3.3.6 +Release: 3.3.7 Provider package ---------------- @@ -134,5 +134,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-alibaba 3.3.6 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-alibaba 3.3.6 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-alibaba 3.3.7 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-alibaba 3.3.7 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/alibaba/provider.yaml b/providers/alibaba/provider.yaml index f5bed7b7429b0..46d43d4a62095 100644 --- a/providers/alibaba/provider.yaml +++ b/providers/alibaba/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298167 +source-date-epoch: 1775591532 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.3.7 - 3.3.6 - 3.3.5 - 3.3.4 diff --git a/providers/alibaba/pyproject.toml b/providers/alibaba/pyproject.toml index 8bec1f952d75f..c63f4ee3745e7 100644 --- a/providers/alibaba/pyproject.toml +++ b/providers/alibaba/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-alibaba" -version = "3.3.6" +version = "3.3.7" description = "Provider package apache-airflow-providers-alibaba for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -103,8 +103,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-alibaba/3.3.6" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-alibaba/3.3.6/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-alibaba/3.3.7" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-alibaba/3.3.7/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/alibaba/src/airflow/providers/alibaba/__init__.py b/providers/alibaba/src/airflow/providers/alibaba/__init__.py index 4c06dceb522ac..d32951d60397f 100644 --- a/providers/alibaba/src/airflow/providers/alibaba/__init__.py +++ b/providers/alibaba/src/airflow/providers/alibaba/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.3.6" +__version__ = "3.3.7" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/alibaba/tests/system/alibaba/example_adb_spark_batch.py b/providers/alibaba/tests/system/alibaba/example_adb_spark_batch.py index 785a352734c9b..ca663b8c7371b 100644 --- a/providers/alibaba/tests/system/alibaba/example_adb_spark_batch.py +++ b/providers/alibaba/tests/system/alibaba/example_adb_spark_batch.py @@ -65,5 +65,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/alibaba/tests/system/alibaba/example_adb_spark_sql.py b/providers/alibaba/tests/system/alibaba/example_adb_spark_sql.py index dfd2f55d47acb..135e4e42e0e08 100644 --- a/providers/alibaba/tests/system/alibaba/example_adb_spark_sql.py +++ b/providers/alibaba/tests/system/alibaba/example_adb_spark_sql.py @@ -63,5 +63,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/alibaba/tests/system/alibaba/example_maxcompute_sql.py b/providers/alibaba/tests/system/alibaba/example_maxcompute_sql.py index c9fb6bb3b22fd..bb1fb2a06166e 100644 --- a/providers/alibaba/tests/system/alibaba/example_maxcompute_sql.py +++ b/providers/alibaba/tests/system/alibaba/example_maxcompute_sql.py @@ -48,5 +48,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/alibaba/tests/system/alibaba/example_oss_bucket.py b/providers/alibaba/tests/system/alibaba/example_oss_bucket.py index e8dae7290a6ca..3d590b61fe6a4 100644 --- a/providers/alibaba/tests/system/alibaba/example_oss_bucket.py +++ b/providers/alibaba/tests/system/alibaba/example_oss_bucket.py @@ -50,5 +50,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/alibaba/tests/system/alibaba/example_oss_object.py b/providers/alibaba/tests/system/alibaba/example_oss_object.py index ca7364ecf92d0..9ddc0fe03bc73 100644 --- a/providers/alibaba/tests/system/alibaba/example_oss_object.py +++ b/providers/alibaba/tests/system/alibaba/example_oss_object.py @@ -80,5 +80,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/README.rst b/providers/amazon/README.rst index 11103430a0ef1..b2e0d7d183025 100644 --- a/providers/amazon/README.rst +++ b/providers/amazon/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-amazon`` -Release: ``9.24.0`` +Release: ``9.25.0`` Amazon integration (including `Amazon Web Services (AWS) `__). @@ -36,7 +36,7 @@ This is a provider package for ``amazon`` provider. All classes for this provide are in ``airflow.providers.amazon`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -54,7 +54,7 @@ Requirements PIP package Version required ========================================== ====================================== ``apache-airflow`` ``>=2.11.0`` -``apache-airflow-providers-common-compat`` ``>=1.13.0`` +``apache-airflow-providers-common-compat`` ``>=1.14.3`` ``apache-airflow-providers-common-sql`` ``>=1.32.0`` ``apache-airflow-providers-http`` ``boto3`` ``>=1.41.0`` @@ -63,6 +63,7 @@ PIP package Version required ``watchtower`` ``>=3.3.1,<4`` ``jsonpath_ng`` ``>=1.5.3`` ``redshift_connector`` ``>=2.1.3`` +``redshift_connector`` ``>=2.1.13; python_version >= "3.14"`` ``asgiref`` ``>=2.3.0; python_version < "3.14"`` ``asgiref`` ``>=3.11.1; python_version >= "3.14"`` ``PyAthena`` ``>=3.10.0`` @@ -110,7 +111,7 @@ Optional dependencies ==================== ============================================================================================================================================================ Extra Dependencies ==================== ============================================================================================================================================================ -``aiobotocore`` ``aiobotocore[boto3]>=2.26.0`` +``aiobotocore`` ``aiobotocore>=3.0.0`` ``cncf.kubernetes`` ``apache-airflow-providers-cncf-kubernetes>=7.2.0`` ``s3fs`` ``s3fs>=2023.10.0`` ``python3-saml`` ``python3-saml>=1.16.0; python_version < '3.13'``, ``xmlsec>=1.3.14; python_version < '3.13'``, ``lxml>=6.0.0; python_version < '3.13'`` @@ -132,4 +133,4 @@ Extra Dependencies ==================== ============================================================================================================================================================ The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/amazon/docs/changelog.rst b/providers/amazon/docs/changelog.rst index 5a7f725feaef9..811cdbbd93b32 100644 --- a/providers/amazon/docs/changelog.rst +++ b/providers/amazon/docs/changelog.rst @@ -26,6 +26,44 @@ Changelog --------- +9.25.0 +...... + +Features +~~~~~~~~ + +* ``Add OpenLineage parent and transport info injection to 'EmrServerlessStartJobOperator' (#64807)`` +* ``Add 'EksPodTrigger' (#64187)`` +* ``Add SageMakerConditionOperator and SageMakerFailOperator (#64545)`` + +Bug Fixes +~~~~~~~~~ + +* ``Fix AwsBaseWaiterTrigger losing error details on deferred task failure (#64085)`` +* ``Fix assume_role_with_web_identity not using botocore config for STS calls (#64216)`` +* ``Fix GlueJobOperator verbose logs not showing in deferrable mode (#64342)`` + +Misc +~~~~ + +* ``Improve debuggability of SQS, Lambda, EC2, and RDS hooks (#64661)`` +* ``Load hook metadata from YAML without importing Hook class (#63826)`` +* ``Bump the min aibotocore version to 3.0.0 (#64631)`` +* ``Remove the lxml workaround (#64554)`` +* ``Add debug logging and fix exception handling in DynamoDB hook (#64629)`` +* ``Add OpenLineage parent info injection to GlueJobOperator (#64513)`` +* ``Remove obsolete boto3 extra from aiobotocore dependency (#64330)`` +* ``Replace AWS keys with placeholder text in documentation and code examples (#63577)`` +* ``Fix stale system test documentation links (#65071)`` +* ``add more debugging logs when emr_eks system tests fail (#64817)`` +* ``Compat sdk conf follow-up for multiple providers (#64161)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Revert "fix(glue): Fix GlueJobOperator verbose logs not showing in deferrable mode (#63086)" (#64340)`` + * ``fix(glue): Fix GlueJobOperator verbose logs not showing in deferrable mode (#63086)`` + * ``Prepare providers release 2026-04-07 (#64864)`` + 9.24.0 ...... diff --git a/providers/amazon/docs/index.rst b/providers/amazon/docs/index.rst index f97d8883da137..0b8bcfba67f6a 100644 --- a/providers/amazon/docs/index.rst +++ b/providers/amazon/docs/index.rst @@ -87,7 +87,7 @@ apache-airflow-providers-amazon package Amazon integration (including `Amazon Web Services (AWS) `__). -Release: 9.24.0 +Release: 9.25.0 Provider package ---------------- @@ -111,7 +111,7 @@ The minimum Apache Airflow version supported by this provider distribution is `` PIP package Version required ========================================== ====================================== ``apache-airflow`` ``>=2.11.0`` -``apache-airflow-providers-common-compat`` ``>=1.13.0`` +``apache-airflow-providers-common-compat`` ``>=1.14.3`` ``apache-airflow-providers-common-sql`` ``>=1.32.0`` ``apache-airflow-providers-http`` ``boto3`` ``>=1.41.0`` @@ -168,5 +168,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-amazon 9.24.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-amazon 9.24.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-amazon 9.25.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-amazon 9.25.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/amazon/provider.yaml b/providers/amazon/provider.yaml index fa59480e5c471..2dc4a94a42faf 100644 --- a/providers/amazon/provider.yaml +++ b/providers/amazon/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298274 +source-date-epoch: 1775591628 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 9.25.0 - 9.24.0 - 9.23.0 - 9.22.0 diff --git a/providers/amazon/pyproject.toml b/providers/amazon/pyproject.toml index 1bfdaef9ebe40..579e015a3f627 100644 --- a/providers/amazon/pyproject.toml +++ b/providers/amazon/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-amazon" -version = "9.24.0" +version = "9.25.0" description = "Provider package apache-airflow-providers-amazon for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -60,7 +60,7 @@ requires-python = ">=3.10" # After you modify the dependencies, and rebuild your Breeze CI image with ``breeze ci-image build`` dependencies = [ "apache-airflow>=2.11.0", - "apache-airflow-providers-common-compat>=1.13.0", # use next version + "apache-airflow-providers-common-compat>=1.14.3", "apache-airflow-providers-common-sql>=1.32.0", "apache-airflow-providers-http", # We should update minimum version of boto3 and here regularly to avoid `pip` backtracking with the number @@ -217,8 +217,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.24.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.24.0/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.25.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-amazon/9.25.0/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/amazon/src/airflow/providers/amazon/__init__.py b/providers/amazon/src/airflow/providers/amazon/__init__.py index 6fe0646f1912d..3aff3c7801842 100644 --- a/providers/amazon/src/airflow/providers/amazon/__init__.py +++ b/providers/amazon/src/airflow/providers/amazon/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "9.24.0" +__version__ = "9.25.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/amazon/src/airflow/providers/amazon/aws/hooks/glue.py b/providers/amazon/src/airflow/providers/amazon/aws/hooks/glue.py index cc7cfd2849ca9..0e58d60568921 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/hooks/glue.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/hooks/glue.py @@ -42,6 +42,28 @@ ERROR_LOG_SUFFIX = "error" +def get_glue_log_group_names(job_run: dict[str, Any]) -> tuple[str, str]: + """Extract the output and error CloudWatch log group names from a Glue job run response.""" + log_group_prefix = job_run["LogGroupName"] + return ( + f"{log_group_prefix}/{DEFAULT_LOG_SUFFIX}", + f"{log_group_prefix}/{ERROR_LOG_SUFFIX}", + ) + + +def format_glue_logs(fetched_logs: list[str], log_group: str) -> str: + """ + Format fetched CloudWatch log messages for display. + + Shared between ``GlueJobHook.print_job_logs`` and ``GlueJobCompleteTrigger._forward_logs`` + so that both the sync and async paths produce identical output. + """ + if fetched_logs: + messages = "\t".join(line.rstrip() + "\n" for line in fetched_logs) + return f"Glue Job Run {log_group} Logs:\n\t{messages}" + return f"No new log from the Glue Job in {log_group}" + + class GlueJobHook(AwsBaseHook): """ Interact with AWS Glue. @@ -350,22 +372,14 @@ def display_logs_from(log_group: str, continuation_token: str | None) -> str | N else: raise - if len(fetched_logs): - # Add a tab to indent those logs and distinguish them from airflow logs. - # Log lines returned already contain a newline character at the end. - messages = "\t".join(fetched_logs) - self.log.info("Glue Job Run %s Logs:\n\t%s", log_group, messages) - else: - self.log.info("No new log from the Glue Job in %s", log_group) + self.log.info(format_glue_logs(fetched_logs, log_group)) return next_token - log_group_prefix = job_run["LogGroupName"] - log_group_default = f"{log_group_prefix}/{DEFAULT_LOG_SUFFIX}" - log_group_error = f"{log_group_prefix}/{ERROR_LOG_SUFFIX}" + log_group_output, log_group_error = get_glue_log_group_names(job_run) # one would think that the error log group would contain only errors, but it actually contains # a lot of interesting logs too, so it's valuable to have both continuation_tokens.output_stream_continuation = display_logs_from( - log_group_default, continuation_tokens.output_stream_continuation + log_group_output, continuation_tokens.output_stream_continuation ) continuation_tokens.error_stream_continuation = display_logs_from( log_group_error, continuation_tokens.error_stream_continuation diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/eks.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/eks.py index f024ab056d2ea..cf6d147bdcfc6 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/eks.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/eks.py @@ -39,6 +39,7 @@ EksDeleteClusterTrigger, EksDeleteFargateProfileTrigger, EksDeleteNodegroupTrigger, + EksPodTrigger, ) from airflow.providers.amazon.aws.utils import validate_execute_complete_event from airflow.providers.amazon.aws.utils.mixins import aws_template_fields @@ -1119,6 +1120,82 @@ def __init__( if self.config_file: raise AirflowException("The config_file is not an allowed parameter for the EksPodOperator.") + def invoke_defer_method(self, last_log_time=None, context=None) -> None: + """Override to use EksPodTrigger which regenerates kubeconfig with fresh credentials.""" + import datetime + + from airflow.providers.cncf.kubernetes.triggers.pod import ContainerState + from airflow.providers.common.compat.sdk import AirflowNotFoundException + + self.convert_config_file_to_dict() + + connection_extras = None + if self.kubernetes_conn_id: + try: + try: + from airflow.sdk import BaseHook + except ImportError: + from airflow.hooks.base import BaseHook # type: ignore[attr-defined, no-redef] + + conn = BaseHook.get_connection(self.kubernetes_conn_id) + except AirflowNotFoundException: + self.log.warning( + "Could not resolve connection extras for deferral: connection `%s` not found. " + "Triggerer will try to resolve it from its own environment.", + self.kubernetes_conn_id, + ) + else: + connection_extras = conn.extra_dejson + self.log.info("Successfully resolved connection extras for deferral.") + + trigger_start_time = datetime.datetime.now(tz=datetime.timezone.utc) + + if self.pod is None or self.pod.metadata is None: + raise RuntimeError("Pod must be created with metadata before deferring") + + trigger = EksPodTrigger( + eks_cluster_name=self.cluster_name, + aws_conn_id=self.aws_conn_id, + region=self.region, + pod_name=self.pod.metadata.name, + pod_namespace=self.pod.metadata.namespace, + trigger_start_time=trigger_start_time, + kubernetes_conn_id=self.kubernetes_conn_id, + connection_extras=connection_extras, + cluster_context=self.cluster_context, + config_dict=self._config_dict, + in_cluster=self.in_cluster, + poll_interval=self.poll_interval, + get_logs=self.get_logs, + startup_timeout=self.startup_timeout_seconds, + startup_check_interval=self.startup_check_interval_seconds, + schedule_timeout=self.schedule_timeout_seconds, + base_container_name=self.base_container_name, + on_finish_action=self.on_finish_action.value, + on_kill_action=self.on_kill_action.value, + termination_grace_period=self.termination_grace_period, + last_log_time=last_log_time, + logging_interval=self.logging_interval, + trigger_kwargs=self.trigger_kwargs, + ) + container_state = trigger.define_container_state(self.pod) if self.pod else None + if context and ( + container_state == ContainerState.TERMINATED or container_state == ContainerState.FAILED + ): + self.log.info("Skipping deferral as pod is already in a terminal state") + self.trigger_reentry( + context=context, + event={ + "status": "success" if container_state == ContainerState.TERMINATED else "failed", + "namespace": self.pod.metadata.namespace, + "name": self.pod.metadata.name, + "last_log_time": last_log_time, + **(self.trigger_kwargs or {}), + }, + ) + else: + self.defer(trigger=trigger, method_name="trigger_reentry") + def execute(self, context: Context): eks_hook = EksHook( aws_conn_id=self.aws_conn_id, diff --git a/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py b/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py index 313cc6494899e..8efc71289e457 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/operators/emr.py @@ -58,6 +58,10 @@ ) from airflow.providers.amazon.aws.utils.waiter_with_logging import wait from airflow.providers.amazon.version_compat import NOTSET, ArgNotSet +from airflow.providers.common.compat.openlineage.utils.spark import ( + inject_parent_job_information_into_emr_serverless_properties, + inject_transport_information_into_emr_serverless_properties, +) from airflow.providers.common.compat.sdk import AirflowException, conf from airflow.utils.helpers import exactly_one, prune_dict @@ -1197,6 +1201,14 @@ class EmrServerlessStartJobOperator(AwsBaseOperator[EmrServerlessHook]): :param cancel_on_kill: If True, the EMR Serverless job will be cancelled when the task is killed while in deferrable mode. This ensures that orphan jobs are not left running in EMR Serverless when an Airflow task is cancelled. Defaults to True. + :param openlineage_inject_parent_job_info: If True, injects OpenLineage parent job information + into the EMR Serverless ``spark-defaults`` configuration so the Spark job emits a + ``parentRunFacet`` linking back to the Airflow task. Defaults to the + ``openlineage.spark_inject_parent_job_info`` config value. + :param openlineage_inject_transport_info: If True, injects OpenLineage transport configuration + into the EMR Serverless ``spark-defaults`` configuration so the Spark job sends OL events + to the same backend as Airflow. Defaults to the + ``openlineage.spark_inject_transport_info`` config value. """ aws_hook_class = EmrServerlessHook @@ -1236,6 +1248,12 @@ def __init__( deferrable: bool = conf.getboolean("operators", "default_deferrable", fallback=False), enable_application_ui_links: bool = False, cancel_on_kill: bool = True, + openlineage_inject_parent_job_info: bool = conf.getboolean( + "openlineage", "spark_inject_parent_job_info", fallback=False + ), + openlineage_inject_transport_info: bool = conf.getboolean( + "openlineage", "spark_inject_transport_info", fallback=False + ), **kwargs, ): waiter_delay = 60 if waiter_delay is NOTSET else waiter_delay @@ -1254,6 +1272,8 @@ def __init__( self.deferrable = deferrable self.enable_application_ui_links = enable_application_ui_links self.cancel_on_kill = cancel_on_kill + self.openlineage_inject_parent_job_info = openlineage_inject_parent_job_info + self.openlineage_inject_transport_info = openlineage_inject_transport_info super().__init__(**kwargs) self.client_request_token = client_request_token or str(uuid4()) @@ -1287,6 +1307,19 @@ def execute(self, context: Context, event: dict[str, Any] | None = None) -> str ) self.log.info("Starting job on Application: %s", self.application_id) self.name = self.name or self.config.pop("name", f"emr_serverless_job_airflow_{uuid4()}") + + configuration_overrides = self.configuration_overrides + if self.openlineage_inject_parent_job_info: + self.log.info("Injecting OpenLineage parent job information into EMR Serverless configuration.") + configuration_overrides = inject_parent_job_information_into_emr_serverless_properties( + configuration_overrides, context + ) + if self.openlineage_inject_transport_info: + self.log.info("Injecting OpenLineage transport information into EMR Serverless configuration.") + configuration_overrides = inject_transport_information_into_emr_serverless_properties( + configuration_overrides, context + ) + args = { "clientToken": self.client_request_token, "applicationId": self.application_id, @@ -1295,8 +1328,8 @@ def execute(self, context: Context, event: dict[str, Any] | None = None) -> str "name": self.name, **self.config, } - if self.configuration_overrides is not None: - args["configurationOverrides"] = self.configuration_overrides + if configuration_overrides is not None: + args["configurationOverrides"] = configuration_overrides response = self.hook.conn.start_job_run( **args, ) diff --git a/providers/amazon/src/airflow/providers/amazon/aws/triggers/eks.py b/providers/amazon/src/airflow/providers/amazon/aws/triggers/eks.py index 3ce4c7f5fb0b3..a386f96da1b26 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/triggers/eks.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/triggers/eks.py @@ -16,6 +16,7 @@ # under the License. from __future__ import annotations +import datetime from typing import TYPE_CHECKING, Any from botocore.exceptions import ClientError @@ -23,10 +24,13 @@ from airflow.providers.amazon.aws.hooks.eks import EksHook from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger from airflow.providers.amazon.aws.utils.waiter_with_logging import async_wait +from airflow.providers.cncf.kubernetes.triggers.pod import KubernetesPodTrigger from airflow.providers.common.compat.sdk import AirflowException from airflow.triggers.base import TriggerEvent if TYPE_CHECKING: + from pendulum import DateTime + from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook @@ -89,6 +93,132 @@ async def run(self): yield TriggerEvent({"status": "success"}) +class EksPodTrigger(KubernetesPodTrigger): + """ + KubernetesPodTrigger for EKS that generates fresh kubeconfig with new credentials. + + When ``EksPodOperator`` defers, the kubeconfig stored in ``config_dict`` contains + an exec command that references a temporary credentials file. That file is cleaned + up when the operator's context managers exit (on deferral). By the time the trigger + runs — whether in a real triggerer process or inline via ``dag.test()`` — the file + is gone. + + This trigger solves the problem by regenerating the kubeconfig with fresh AWS + credentials before executing. The temporary files are kept alive for the entire + duration of the trigger's ``run()`` method. + + :param eks_cluster_name: The name of the Amazon EKS Cluster. + :param aws_conn_id: The Airflow connection used for AWS credentials. + :param region: Which AWS region the connection should use. + """ + + def __init__( + self, + *, + eks_cluster_name: str, + aws_conn_id: str | None = None, + region: str | None = None, + pod_name: str, + pod_namespace: str, + trigger_start_time: datetime.datetime, + base_container_name: str, + kubernetes_conn_id: str | None = None, + connection_extras: dict | None = None, + poll_interval: float = 2, + cluster_context: str | None = None, + config_dict: dict | None = None, + in_cluster: bool | None = None, + get_logs: bool = True, + startup_timeout: int = 120, + startup_check_interval: float = 5, + schedule_timeout: int = 120, + on_finish_action: str = "delete_pod", + on_kill_action: str = "delete_pod", + termination_grace_period: int | None = None, + last_log_time: DateTime | None = None, + logging_interval: int | None = None, + trigger_kwargs: dict | None = None, + ): + super().__init__( + pod_name=pod_name, + pod_namespace=pod_namespace, + trigger_start_time=trigger_start_time, + base_container_name=base_container_name, + kubernetes_conn_id=kubernetes_conn_id, + connection_extras=connection_extras, + poll_interval=poll_interval, + cluster_context=cluster_context, + config_dict=config_dict, + in_cluster=in_cluster, + get_logs=get_logs, + startup_timeout=startup_timeout, + startup_check_interval=startup_check_interval, + schedule_timeout=schedule_timeout, + on_finish_action=on_finish_action, + on_kill_action=on_kill_action, + termination_grace_period=termination_grace_period, + last_log_time=last_log_time, + logging_interval=logging_interval, + trigger_kwargs=trigger_kwargs, + ) + self.eks_cluster_name = eks_cluster_name + self._aws_conn_id = aws_conn_id + self.region = region + + def serialize(self) -> tuple[str, dict[str, Any]]: + """Serialize EksPodTrigger arguments and classpath.""" + _, kwargs = super().serialize() + kwargs["eks_cluster_name"] = self.eks_cluster_name + kwargs["aws_conn_id"] = self._aws_conn_id + kwargs["region"] = self.region + return ( + "airflow.providers.amazon.aws.triggers.eks.EksPodTrigger", + kwargs, + ) + + async def run(self): + """Generate fresh kubeconfig, then delegate to the parent trigger.""" + from airflow.utils import yaml + + eks_hook = EksHook( + aws_conn_id=self._aws_conn_id, + region_name=self.region, + ) + session = eks_hook.get_session() + credentials_obj = session.get_credentials() + if credentials_obj is None: + raise RuntimeError( + "Unable to retrieve AWS credentials for EKS trigger. " + "Credentials may have expired or not been configured." + ) + credentials = credentials_obj.get_frozen_credentials() + + # Create fresh credential and kubeconfig files. The context managers + # keep the temp files alive for the entire duration of the trigger. + with eks_hook._secure_credential_context( + credentials.access_key, credentials.secret_key, credentials.token + ) as credentials_file: + with eks_hook.generate_config_file( + eks_cluster_name=self.eks_cluster_name, + pod_namespace=self.pod_namespace, + credentials_file=credentials_file, + ) as config_file_path: + # Reading a small local temp file created by the context manager above. + # Blocking I/O is acceptable here as the file is tiny and local. + from pathlib import Path + + self.config_dict = yaml.safe_load( + Path(config_file_path).read_text() # noqa: ASYNC240 + ) + + # Invalidate any previously cached hook so the new config_dict + # is picked up when the parent creates the AsyncKubernetesHook. + self.__dict__.pop("hook", None) + + async for event in super().run(): + yield event + + class EksDeleteClusterTrigger(AwsBaseWaiterTrigger): """ Trigger for EksDeleteClusterOperator. diff --git a/providers/amazon/src/airflow/providers/amazon/aws/triggers/glue.py b/providers/amazon/src/airflow/providers/amazon/aws/triggers/glue.py index f54f761825eac..e499763aafaba 100644 --- a/providers/amazon/src/airflow/providers/amazon/aws/triggers/glue.py +++ b/providers/amazon/src/airflow/providers/amazon/aws/triggers/glue.py @@ -22,11 +22,19 @@ from functools import cached_property from typing import TYPE_CHECKING, Any +from botocore.exceptions import ClientError + if TYPE_CHECKING: from airflow.providers.amazon.aws.hooks.base_aws import AwsGenericHook -from airflow.providers.amazon.aws.hooks.glue import GlueDataQualityHook, GlueJobHook +from airflow.providers.amazon.aws.hooks.glue import ( + GlueDataQualityHook, + GlueJobHook, + format_glue_logs, + get_glue_log_group_names, +) from airflow.providers.amazon.aws.hooks.glue_catalog import GlueCatalogHook +from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook from airflow.providers.amazon.aws.triggers.base import AwsBaseWaiterTrigger from airflow.triggers.base import BaseTrigger, TriggerEvent @@ -87,6 +95,131 @@ def hook(self) -> AwsGenericHook: config=self.botocore_config, ) + async def run(self) -> AsyncIterator[TriggerEvent]: + if not self.verbose: + async for event in super().run(): + yield event + return + + hook = self.hook() + async with ( + await hook.get_async_conn() as glue_client, + await AwsLogsHook( + aws_conn_id=self.aws_conn_id, region_name=self.region_name + ).get_async_conn() as logs_client, + ): + # Get log group names from job run metadata + job_run_resp = await glue_client.get_job_run(JobName=self.job_name, RunId=self.run_id) + log_group_output, log_group_error = get_glue_log_group_names(job_run_resp["JobRun"]) + + output_token: str | None = None + error_token: str | None = None + + for _attempt in range(self.attempts): + # Fetch current job state + resp = await glue_client.get_job_run(JobName=self.job_name, RunId=self.run_id) + job_run_state = resp["JobRun"]["JobRunState"] + + # Fetch and print logs from both output and error streams + try: + output_token = await self._forward_logs( + logs_client, log_group_output, self.run_id, output_token + ) + error_token = await self._forward_logs( + logs_client, log_group_error, self.run_id, error_token + ) + except ClientError as e: + self.log.error( + "Failed to fetch logs for Glue Job %s Run %s: %s", + self.job_name, + self.run_id, + e, + ) + yield TriggerEvent( + { + "status": "error", + "message": f"Failed to fetch logs for Glue Job {self.job_name} Run {self.run_id}: {e}", + self.return_key: self.return_value, + } + ) + return + + if job_run_state in ("FAILED", "TIMEOUT"): + yield TriggerEvent( + { + "status": "error", + "message": f"Glue Job {self.job_name} Run {self.run_id}" + f" exited with state: {job_run_state}", + self.return_key: self.return_value, + } + ) + return + if job_run_state in ("SUCCEEDED", "STOPPED"): + self.log.info( + "Exiting Job %s Run %s State: %s", + self.job_name, + self.run_id, + job_run_state, + ) + yield TriggerEvent({"status": "success", self.return_key: self.return_value}) + return + + self.log.info( + "Polling for AWS Glue Job %s current run state: %s", + self.job_name, + job_run_state, + ) + await asyncio.sleep(self.waiter_delay) + + yield TriggerEvent( + { + "status": "error", + "message": f"Glue Job {self.job_name} Run {self.run_id}" + f" waiter exceeded max attempts ({self.attempts})", + self.return_key: self.return_value, + } + ) + + async def _forward_logs( + self, + logs_client: Any, + log_group: str, + log_stream: str, + next_token: str | None, + ) -> str | None: + # Matches the format used by the synchronous GlueJobHook.print_job_logs. + fetched_logs: list[str] = [] + while True: + token_arg: dict[str, str] = {"nextToken": next_token} if next_token else {} + try: + response = await logs_client.get_log_events( + logGroupName=log_group, + logStreamName=log_stream, + startFromHead=True, + **token_arg, + ) + except ClientError as e: + if e.response["Error"]["Code"] == "ResourceNotFoundException": + region = logs_client.meta.region_name + self.log.warning( + "No new Glue driver logs so far.\n" + "If this persists, check the CloudWatch dashboard at: %r.", + f"https://{region}.console.aws.amazon.com/cloudwatch/home", + ) + return None + raise + + events = response["events"] + fetched_logs.extend(event["message"] for event in events) + + if not events or next_token == response["nextForwardToken"]: + break + next_token = response["nextForwardToken"] + + self.log.info(format_glue_logs(fetched_logs, log_group)) + + return response.get("nextForwardToken") + class GlueCatalogPartitionTrigger(BaseTrigger): """ diff --git a/providers/amazon/tests/system/amazon/CONTRIBUTING.md b/providers/amazon/tests/system/amazon/CONTRIBUTING.md index ebdc23c462353..22b27a05835d3 100644 --- a/providers/amazon/tests/system/amazon/CONTRIBUTING.md +++ b/providers/amazon/tests/system/amazon/CONTRIBUTING.md @@ -30,7 +30,7 @@ will always be given. For example, I stole that line from @potiuk! # Scope This guide is meant to be used in addition to the [community system test design -guide](/tests/system/README.md) and only applies to system tests within the Amazon +guide](/contributing-docs/testing/system_tests.rst) and only applies to system tests within the Amazon provider package, though other providers are welcome to adopt them as well. In any cases of conflicting information or confusion, this document should take precedence within the Amazon provider package. diff --git a/providers/amazon/tests/system/amazon/aws/example_appflow.py b/providers/amazon/tests/system/amazon/aws/example_appflow.py index c739b6d987b38..db8b6dbc5ce90 100644 --- a/providers/amazon/tests/system/amazon/aws/example_appflow.py +++ b/providers/amazon/tests/system/amazon/aws/example_appflow.py @@ -122,5 +122,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_appflow_run.py b/providers/amazon/tests/system/amazon/aws/example_appflow_run.py index a5024848e7716..f764830cd5ab0 100644 --- a/providers/amazon/tests/system/amazon/aws/example_appflow_run.py +++ b/providers/amazon/tests/system/amazon/aws/example_appflow_run.py @@ -212,5 +212,5 @@ def delete_flow(flow_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_athena.py b/providers/amazon/tests/system/amazon/aws/example_athena.py index 8c3b8ea2c931a..36a37a8e1c654 100644 --- a/providers/amazon/tests/system/amazon/aws/example_athena.py +++ b/providers/amazon/tests/system/amazon/aws/example_athena.py @@ -193,5 +193,5 @@ def read_results_from_s3(bucket_name, query_execution_id): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_azure_blob_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_azure_blob_to_s3.py index e8dba9674126d..6efb2e1f3cf83 100644 --- a/providers/amazon/tests/system/amazon/aws/example_azure_blob_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_azure_blob_to_s3.py @@ -83,5 +83,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_batch.py b/providers/amazon/tests/system/amazon/aws/example_batch.py index 1ea4c820c4b94..8354db8721369 100644 --- a/providers/amazon/tests/system/amazon/aws/example_batch.py +++ b/providers/amazon/tests/system/amazon/aws/example_batch.py @@ -302,5 +302,5 @@ def delete_job_queue(job_queue_name): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_bedrock.py b/providers/amazon/tests/system/amazon/aws/example_bedrock.py index 135a33dd78a61..409f7e22980ef 100644 --- a/providers/amazon/tests/system/amazon/aws/example_bedrock.py +++ b/providers/amazon/tests/system/amazon/aws/example_bedrock.py @@ -253,5 +253,5 @@ def should_run_provision_throughput(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_bedrock_batch_inference.py b/providers/amazon/tests/system/amazon/aws/example_bedrock_batch_inference.py index e1c8e1ddd135f..0aa3b6b22bd4c 100644 --- a/providers/amazon/tests/system/amazon/aws/example_bedrock_batch_inference.py +++ b/providers/amazon/tests/system/amazon/aws/example_bedrock_batch_inference.py @@ -183,5 +183,5 @@ def stop_batch_inference(job_arn: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py b/providers/amazon/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py index 8eb8046a96a5b..16b150d1a854f 100644 --- a/providers/amazon/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py +++ b/providers/amazon/tests/system/amazon/aws/example_bedrock_retrieve_and_generate.py @@ -603,5 +603,5 @@ def delete_opensearch_policies(collection_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_cloudformation.py b/providers/amazon/tests/system/amazon/aws/example_cloudformation.py index 6630304ee5616..d5bd8a810676e 100644 --- a/providers/amazon/tests/system/amazon/aws/example_cloudformation.py +++ b/providers/amazon/tests/system/amazon/aws/example_cloudformation.py @@ -120,5 +120,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_comprehend.py b/providers/amazon/tests/system/amazon/aws/example_comprehend.py index cb0333b6dcd7e..de99b3d77adbd 100644 --- a/providers/amazon/tests/system/amazon/aws/example_comprehend.py +++ b/providers/amazon/tests/system/amazon/aws/example_comprehend.py @@ -145,5 +145,5 @@ def pii_entities_detection_job_workflow(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_comprehend_document_classifier.py b/providers/amazon/tests/system/amazon/aws/example_comprehend_document_classifier.py index bf14444005069..748db647eeccd 100644 --- a/providers/amazon/tests/system/amazon/aws/example_comprehend_document_classifier.py +++ b/providers/amazon/tests/system/amazon/aws/example_comprehend_document_classifier.py @@ -226,5 +226,5 @@ def create_kwargs_doctors_notes(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_datasync.py b/providers/amazon/tests/system/amazon/aws/example_datasync.py index 8cbcb4fa79a83..540f6fadbbb3a 100644 --- a/providers/amazon/tests/system/amazon/aws/example_datasync.py +++ b/providers/amazon/tests/system/amazon/aws/example_datasync.py @@ -254,5 +254,5 @@ def delete_locations(locations): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_dms.py b/providers/amazon/tests/system/amazon/aws/example_dms.py index f6035d54e358a..eb8e4ed19ef9b 100644 --- a/providers/amazon/tests/system/amazon/aws/example_dms.py +++ b/providers/amazon/tests/system/amazon/aws/example_dms.py @@ -448,5 +448,5 @@ def delete_security_group(security_group_id: str, security_group_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_dms_serverless.py b/providers/amazon/tests/system/amazon/aws/example_dms_serverless.py index 9c814a39fb478..770cc43b6fe54 100644 --- a/providers/amazon/tests/system/amazon/aws/example_dms_serverless.py +++ b/providers/amazon/tests/system/amazon/aws/example_dms_serverless.py @@ -380,5 +380,5 @@ def delete_dms_assets( from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_dynamodb.py b/providers/amazon/tests/system/amazon/aws/example_dynamodb.py index 988c51f3b3062..9b452d06083bc 100644 --- a/providers/amazon/tests/system/amazon/aws/example_dynamodb.py +++ b/providers/amazon/tests/system/amazon/aws/example_dynamodb.py @@ -136,5 +136,5 @@ def delete_table(table_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_dynamodb_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_dynamodb_to_s3.py index cf2d57b2a554b..0de6169a3da5d 100644 --- a/providers/amazon/tests/system/amazon/aws/example_dynamodb_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_dynamodb_to_s3.py @@ -269,5 +269,5 @@ def stop_execution(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_ec2.py b/providers/amazon/tests/system/amazon/aws/example_ec2.py index 760a369524fda..1d714ae0aee02 100644 --- a/providers/amazon/tests/system/amazon/aws/example_ec2.py +++ b/providers/amazon/tests/system/amazon/aws/example_ec2.py @@ -191,5 +191,5 @@ def parse_response(instance_ids: list): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_ecs.py b/providers/amazon/tests/system/amazon/aws/example_ecs.py index abd066b36a018..8936639a2d76e 100644 --- a/providers/amazon/tests/system/amazon/aws/example_ecs.py +++ b/providers/amazon/tests/system/amazon/aws/example_ecs.py @@ -238,5 +238,5 @@ def clean_logs(group_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_ecs_fargate.py b/providers/amazon/tests/system/amazon/aws/example_ecs_fargate.py index 391038fa8f27a..adabe9e2be33a 100644 --- a/providers/amazon/tests/system/amazon/aws/example_ecs_fargate.py +++ b/providers/amazon/tests/system/amazon/aws/example_ecs_fargate.py @@ -177,5 +177,5 @@ def delete_cluster(cluster_name: str) -> None: from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_eks_templated.py b/providers/amazon/tests/system/amazon/aws/example_eks_templated.py index 709a2f9415da6..e8ce5889ee78e 100644 --- a/providers/amazon/tests/system/amazon/aws/example_eks_templated.py +++ b/providers/amazon/tests/system/amazon/aws/example_eks_templated.py @@ -154,5 +154,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py b/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py index 70382efbff857..e22fd0ec2bb7f 100644 --- a/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py +++ b/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_in_one_step.py @@ -168,5 +168,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_profile.py b/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_profile.py index 4b173e3dd56f7..8d2a3987f26e8 100644 --- a/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_profile.py +++ b/providers/amazon/tests/system/amazon/aws/example_eks_with_fargate_profile.py @@ -210,5 +210,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py b/providers/amazon/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py index 43e8f4a4fdbf0..2cdc633f4b70a 100644 --- a/providers/amazon/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py +++ b/providers/amazon/tests/system/amazon/aws/example_eks_with_nodegroup_in_one_step.py @@ -189,5 +189,5 @@ def delete_launch_template(template_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_eks_with_nodegroups.py b/providers/amazon/tests/system/amazon/aws/example_eks_with_nodegroups.py index e48243e81a262..79abe2b39199b 100644 --- a/providers/amazon/tests/system/amazon/aws/example_eks_with_nodegroups.py +++ b/providers/amazon/tests/system/amazon/aws/example_eks_with_nodegroups.py @@ -241,5 +241,5 @@ def delete_launch_template(template_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_emr.py b/providers/amazon/tests/system/amazon/aws/example_emr.py index fdf85c88b016d..bdea79c75240f 100644 --- a/providers/amazon/tests/system/amazon/aws/example_emr.py +++ b/providers/amazon/tests/system/amazon/aws/example_emr.py @@ -238,5 +238,5 @@ def get_step_id(step_ids: list): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_emr_eks.py b/providers/amazon/tests/system/amazon/aws/example_emr_eks.py index 2a672bc4657a7..f73335b5f1e94 100644 --- a/providers/amazon/tests/system/amazon/aws/example_emr_eks.py +++ b/providers/amazon/tests/system/amazon/aws/example_emr_eks.py @@ -50,6 +50,7 @@ from airflow.utils.trigger_rule import TriggerRule # type: ignore[no-redef,attr-defined] from system.amazon.aws.utils import ENV_ID_KEY, SystemTestContextBuilder +from system.amazon.aws.utils.k8s import get_describe_pod_operator DAG_ID = "example_emr_eks" @@ -292,6 +293,10 @@ def delete_virtual_cluster(virtual_cluster_id): ) # [END howto_sensor_emr_container] + # Describe pods only on failure to help diagnose EMR on EKS job issues. + describe_pod = get_describe_pod_operator(cluster_name=eks_cluster_name, namespace=eks_namespace) + describe_pod.trigger_rule = TriggerRule.ONE_FAILED + delete_eks_cluster = EksDeleteClusterOperator( task_id="delete_eks_cluster", cluster_name=eks_cluster_name, @@ -330,6 +335,7 @@ def delete_virtual_cluster(virtual_cluster_id): create_emr_eks_cluster, job_starter, job_waiter, + describe_pod, # TEST TEARDOWN delete_iam_oidc_identity_provider(eks_cluster_name), delete_virtual_cluster(str(create_emr_eks_cluster.output)), @@ -347,5 +353,5 @@ def delete_virtual_cluster(virtual_cluster_id): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_emr_notebook_execution.py b/providers/amazon/tests/system/amazon/aws/example_emr_notebook_execution.py index 24d950bc41b02..c8f529c9b94c8 100644 --- a/providers/amazon/tests/system/amazon/aws/example_emr_notebook_execution.py +++ b/providers/amazon/tests/system/amazon/aws/example_emr_notebook_execution.py @@ -118,5 +118,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_emr_serverless.py b/providers/amazon/tests/system/amazon/aws/example_emr_serverless.py index 9ead279420d69..95ada682a8afc 100644 --- a/providers/amazon/tests/system/amazon/aws/example_emr_serverless.py +++ b/providers/amazon/tests/system/amazon/aws/example_emr_serverless.py @@ -166,5 +166,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_eventbridge.py b/providers/amazon/tests/system/amazon/aws/example_eventbridge.py index cce99476c7e8b..74345cbc6bc96 100644 --- a/providers/amazon/tests/system/amazon/aws/example_eventbridge.py +++ b/providers/amazon/tests/system/amazon/aws/example_eventbridge.py @@ -81,5 +81,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_ftp_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_ftp_to_s3.py index b2550e1a3b1b9..5bde32bfc52e1 100644 --- a/providers/amazon/tests/system/amazon/aws/example_ftp_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_ftp_to_s3.py @@ -83,5 +83,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_gcs_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_gcs_to_s3.py index 51a2bc4d0ebef..b2e0a38e362d5 100644 --- a/providers/amazon/tests/system/amazon/aws/example_gcs_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_gcs_to_s3.py @@ -134,5 +134,5 @@ def upload_gcs_file(bucket_name: str, object_name: str, user_project: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_glacier_to_gcs.py b/providers/amazon/tests/system/amazon/aws/example_glacier_to_gcs.py index ebe763bb4721e..814fb77126836 100644 --- a/providers/amazon/tests/system/amazon/aws/example_glacier_to_gcs.py +++ b/providers/amazon/tests/system/amazon/aws/example_glacier_to_gcs.py @@ -127,5 +127,5 @@ def delete_vault(vault_name): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_glue.py b/providers/amazon/tests/system/amazon/aws/example_glue.py index 4d0d2adb7bc1f..d62bcf3b30883 100644 --- a/providers/amazon/tests/system/amazon/aws/example_glue.py +++ b/providers/amazon/tests/system/amazon/aws/example_glue.py @@ -223,5 +223,5 @@ def glue_cleanup(crawler_name: str, job_name: str, db_name: str) -> None: from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_glue_data_quality.py b/providers/amazon/tests/system/amazon/aws/example_glue_data_quality.py index b23de1ee63458..42b0696a63efd 100644 --- a/providers/amazon/tests/system/amazon/aws/example_glue_data_quality.py +++ b/providers/amazon/tests/system/amazon/aws/example_glue_data_quality.py @@ -217,5 +217,5 @@ def delete_ruleset(ruleset_name): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py b/providers/amazon/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py index 09763af40e124..97ec9a9d4bfc7 100644 --- a/providers/amazon/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py +++ b/providers/amazon/tests/system/amazon/aws/example_glue_data_quality_with_recommendation.py @@ -216,5 +216,5 @@ def delete_ruleset(ruleset_name): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_glue_databrew.py b/providers/amazon/tests/system/amazon/aws/example_glue_databrew.py index 1453f67f0b2b1..3e9b33aa3edd4 100644 --- a/providers/amazon/tests/system/amazon/aws/example_glue_databrew.py +++ b/providers/amazon/tests/system/amazon/aws/example_glue_databrew.py @@ -175,5 +175,5 @@ def delete_job(job_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_google_api_sheets_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_google_api_sheets_to_s3.py index 08f5ac4c4e5af..f52e738495940 100644 --- a/providers/amazon/tests/system/amazon/aws/example_google_api_sheets_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_google_api_sheets_to_s3.py @@ -94,5 +94,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_google_api_youtube_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_google_api_youtube_to_s3.py index 52111fc576917..a8e2482dbbc1c 100644 --- a/providers/amazon/tests/system/amazon/aws/example_google_api_youtube_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_google_api_youtube_to_s3.py @@ -214,5 +214,5 @@ def transform_video_ids(**kwargs): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_hive_to_dynamodb.py b/providers/amazon/tests/system/amazon/aws/example_hive_to_dynamodb.py index 0410ff37414ca..56a0cb677d715 100644 --- a/providers/amazon/tests/system/amazon/aws/example_hive_to_dynamodb.py +++ b/providers/amazon/tests/system/amazon/aws/example_hive_to_dynamodb.py @@ -173,5 +173,5 @@ def configure_hive_connection(connection_id: str, hostname: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_http_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_http_to_s3.py index 57c9fc6349409..9389c7f06738e 100644 --- a/providers/amazon/tests/system/amazon/aws/example_http_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_http_to_s3.py @@ -142,5 +142,5 @@ def create_connection(conn_id_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_imap_attachment_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_imap_attachment_to_s3.py index 04d621ae2efa8..edac64b0468c0 100644 --- a/providers/amazon/tests/system/amazon/aws/example_imap_attachment_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_imap_attachment_to_s3.py @@ -101,5 +101,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_kinesis_analytics.py b/providers/amazon/tests/system/amazon/aws/example_kinesis_analytics.py index 28d0517c9c969..a0c728128bcb1 100644 --- a/providers/amazon/tests/system/amazon/aws/example_kinesis_analytics.py +++ b/providers/amazon/tests/system/amazon/aws/example_kinesis_analytics.py @@ -288,5 +288,5 @@ def delete_kinesis_stream(stream: str, region: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_lambda.py b/providers/amazon/tests/system/amazon/aws/example_lambda.py index b478062ac718b..dc6dfd3e4d426 100644 --- a/providers/amazon/tests/system/amazon/aws/example_lambda.py +++ b/providers/amazon/tests/system/amazon/aws/example_lambda.py @@ -148,5 +148,5 @@ def delete_lambda(function_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_local_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_local_to_s3.py index 6c120b695f516..cda56757da7bb 100644 --- a/providers/amazon/tests/system/amazon/aws/example_local_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_local_to_s3.py @@ -102,5 +102,5 @@ def delete_temp_file(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_mongo_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_mongo_to_s3.py index 0610334e3b1e6..b3b8d612b96cb 100644 --- a/providers/amazon/tests/system/amazon/aws/example_mongo_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_mongo_to_s3.py @@ -94,5 +94,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_mwaa.py b/providers/amazon/tests/system/amazon/aws/example_mwaa.py index 454fe192c491d..ba79b5afa0a48 100644 --- a/providers/amazon/tests/system/amazon/aws/example_mwaa.py +++ b/providers/amazon/tests/system/amazon/aws/example_mwaa.py @@ -179,5 +179,5 @@ def test_iam_fallback(role_to_assume_arn, mwaa_env_name): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_mwaa_airflow2.py b/providers/amazon/tests/system/amazon/aws/example_mwaa_airflow2.py index ccece095e3af4..d1882cf80df08 100644 --- a/providers/amazon/tests/system/amazon/aws/example_mwaa_airflow2.py +++ b/providers/amazon/tests/system/amazon/aws/example_mwaa_airflow2.py @@ -172,5 +172,5 @@ def test_iam_fallback(role_to_assume_arn, mwaa_env_name): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_neptune.py b/providers/amazon/tests/system/amazon/aws/example_neptune.py index c4798421c8206..f554056021bd9 100644 --- a/providers/amazon/tests/system/amazon/aws/example_neptune.py +++ b/providers/amazon/tests/system/amazon/aws/example_neptune.py @@ -92,5 +92,5 @@ def delete_cluster(cluster_id): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_quicksight.py b/providers/amazon/tests/system/amazon/aws/example_quicksight.py index 33b90cbec0c78..cad531e60e0bd 100644 --- a/providers/amazon/tests/system/amazon/aws/example_quicksight.py +++ b/providers/amazon/tests/system/amazon/aws/example_quicksight.py @@ -240,5 +240,5 @@ def delete_ingestion(aws_account_id: str, dataset_name: str, ingestion_name: str from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_rds_event.py b/providers/amazon/tests/system/amazon/aws/example_rds_event.py index 47d67fe326511..e073c9c58c8e7 100644 --- a/providers/amazon/tests/system/amazon/aws/example_rds_event.py +++ b/providers/amazon/tests/system/amazon/aws/example_rds_event.py @@ -138,5 +138,5 @@ def delete_sns_topic(topic_arn) -> None: from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_rds_export.py b/providers/amazon/tests/system/amazon/aws/example_rds_export.py index 45067e3a49d46..d803e01644671 100644 --- a/providers/amazon/tests/system/amazon/aws/example_rds_export.py +++ b/providers/amazon/tests/system/amazon/aws/example_rds_export.py @@ -197,5 +197,5 @@ def get_snapshot_arn(snapshot_name: str) -> str: from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_rds_instance.py b/providers/amazon/tests/system/amazon/aws/example_rds_instance.py index a610e6ea00b8b..8751987f46511 100644 --- a/providers/amazon/tests/system/amazon/aws/example_rds_instance.py +++ b/providers/amazon/tests/system/amazon/aws/example_rds_instance.py @@ -123,5 +123,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_rds_snapshot.py b/providers/amazon/tests/system/amazon/aws/example_rds_snapshot.py index 985edbcecb6a2..23720d0a5ae47 100644 --- a/providers/amazon/tests/system/amazon/aws/example_rds_snapshot.py +++ b/providers/amazon/tests/system/amazon/aws/example_rds_snapshot.py @@ -150,5 +150,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_redshift.py b/providers/amazon/tests/system/amazon/aws/example_redshift.py index f54e360930502..5970d93b2c189 100644 --- a/providers/amazon/tests/system/amazon/aws/example_redshift.py +++ b/providers/amazon/tests/system/amazon/aws/example_redshift.py @@ -264,5 +264,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_redshift_s3_transfers.py b/providers/amazon/tests/system/amazon/aws/example_redshift_s3_transfers.py index 46373164ebb98..e8400ff69d252 100644 --- a/providers/amazon/tests/system/amazon/aws/example_redshift_s3_transfers.py +++ b/providers/amazon/tests/system/amazon/aws/example_redshift_s3_transfers.py @@ -330,5 +330,5 @@ def _insert_data(table_name: str) -> str: from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_s3.py b/providers/amazon/tests/system/amazon/aws/example_s3.py index b0b7dbbaee13a..6ddf6d95bbd38 100644 --- a/providers/amazon/tests/system/amazon/aws/example_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_s3.py @@ -326,5 +326,5 @@ def check_fn(files: list, **kwargs) -> bool: from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_s3_to_dynamodb.py b/providers/amazon/tests/system/amazon/aws/example_s3_to_dynamodb.py index 19b504d9b47a0..14749e7d9ac99 100644 --- a/providers/amazon/tests/system/amazon/aws/example_s3_to_dynamodb.py +++ b/providers/amazon/tests/system/amazon/aws/example_s3_to_dynamodb.py @@ -200,5 +200,5 @@ def delete_dynamodb_table(table_name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_s3_to_ftp.py b/providers/amazon/tests/system/amazon/aws/example_s3_to_ftp.py index a824635d332cf..83f92a68cd8dc 100644 --- a/providers/amazon/tests/system/amazon/aws/example_s3_to_ftp.py +++ b/providers/amazon/tests/system/amazon/aws/example_s3_to_ftp.py @@ -82,5 +82,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_s3_to_sftp.py b/providers/amazon/tests/system/amazon/aws/example_s3_to_sftp.py index cc36b8b911f0b..f19a514138aed 100644 --- a/providers/amazon/tests/system/amazon/aws/example_s3_to_sftp.py +++ b/providers/amazon/tests/system/amazon/aws/example_s3_to_sftp.py @@ -82,5 +82,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_s3_to_sql.py b/providers/amazon/tests/system/amazon/aws/example_s3_to_sql.py index 117de53376b99..e53c5657e0271 100644 --- a/providers/amazon/tests/system/amazon/aws/example_s3_to_sql.py +++ b/providers/amazon/tests/system/amazon/aws/example_s3_to_sql.py @@ -277,5 +277,5 @@ def parse_csv_to_generator(filepath): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sagemaker.py b/providers/amazon/tests/system/amazon/aws/example_sagemaker.py index c6eed2a329723..57975ca2ec2b9 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sagemaker.py +++ b/providers/amazon/tests/system/amazon/aws/example_sagemaker.py @@ -737,5 +737,5 @@ def delete_docker_image(image_name): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sagemaker_condition.py b/providers/amazon/tests/system/amazon/aws/example_sagemaker_condition.py index 90a392fa0c1cb..42b0cdf350baf 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sagemaker_condition.py +++ b/providers/amazon/tests/system/amazon/aws/example_sagemaker_condition.py @@ -175,5 +175,5 @@ def logical_fail(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sagemaker_endpoint.py b/providers/amazon/tests/system/amazon/aws/example_sagemaker_endpoint.py index 14433cb3b6502..fce190b5e7c2b 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sagemaker_endpoint.py +++ b/providers/amazon/tests/system/amazon/aws/example_sagemaker_endpoint.py @@ -310,5 +310,5 @@ def set_up(env_id, role_arn, ti=None): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sagemaker_notebook.py b/providers/amazon/tests/system/amazon/aws/example_sagemaker_notebook.py index 19adc9a18c723..29b3967379778 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sagemaker_notebook.py +++ b/providers/amazon/tests/system/amazon/aws/example_sagemaker_notebook.py @@ -103,5 +103,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sagemaker_pipeline.py b/providers/amazon/tests/system/amazon/aws/example_sagemaker_pipeline.py index c03551e27f375..9b28e89972424 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sagemaker_pipeline.py +++ b/providers/amazon/tests/system/amazon/aws/example_sagemaker_pipeline.py @@ -136,5 +136,5 @@ def delete_pipeline(name: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sagemaker_unified_studio.py b/providers/amazon/tests/system/amazon/aws/example_sagemaker_unified_studio.py index 5019de24b2aa1..1eeac449bdcdb 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sagemaker_unified_studio.py +++ b/providers/amazon/tests/system/amazon/aws/example_sagemaker_unified_studio.py @@ -208,5 +208,5 @@ def mock_mwaa_environment(parameters: dict): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_salesforce_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_salesforce_to_s3.py index 2ec5be9d4ac21..f948b14b12cb4 100644 --- a/providers/amazon/tests/system/amazon/aws/example_salesforce_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_salesforce_to_s3.py @@ -89,5 +89,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_ses.py b/providers/amazon/tests/system/amazon/aws/example_ses.py index df1d0d92ee1f6..773bc8bee1bbf 100644 --- a/providers/amazon/tests/system/amazon/aws/example_ses.py +++ b/providers/amazon/tests/system/amazon/aws/example_ses.py @@ -114,5 +114,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sftp_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_sftp_to_s3.py index f29a651c9bfe9..189f21d8b292b 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sftp_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_sftp_to_s3.py @@ -82,5 +82,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sns.py b/providers/amazon/tests/system/amazon/aws/example_sns.py index a9c2b049416cb..45228de30ed1d 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sns.py +++ b/providers/amazon/tests/system/amazon/aws/example_sns.py @@ -94,5 +94,5 @@ def delete_topic(topic_arn) -> None: from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sql_to_s3.py b/providers/amazon/tests/system/amazon/aws/example_sql_to_s3.py index 7d364da46e93e..657f36be007b3 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sql_to_s3.py +++ b/providers/amazon/tests/system/amazon/aws/example_sql_to_s3.py @@ -227,5 +227,5 @@ def create_connection(conn_id_name: str, cluster_id: str): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_sqs.py b/providers/amazon/tests/system/amazon/aws/example_sqs.py index 8d51191d11105..02db97c4b7968 100644 --- a/providers/amazon/tests/system/amazon/aws/example_sqs.py +++ b/providers/amazon/tests/system/amazon/aws/example_sqs.py @@ -118,5 +118,5 @@ def delete_queue(queue_url): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_ssm.py b/providers/amazon/tests/system/amazon/aws/example_ssm.py index 98d7f3fe7d5f1..c7fe319e5118b 100644 --- a/providers/amazon/tests/system/amazon/aws/example_ssm.py +++ b/providers/amazon/tests/system/amazon/aws/example_ssm.py @@ -338,5 +338,5 @@ def handle_exit_code(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/example_step_functions.py b/providers/amazon/tests/system/amazon/aws/example_step_functions.py index 36f4eb45aa064..2a33c6edd6a75 100644 --- a/providers/amazon/tests/system/amazon/aws/example_step_functions.py +++ b/providers/amazon/tests/system/amazon/aws/example_step_functions.py @@ -127,5 +127,5 @@ def delete_state_machine(state_machine_arn): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/amazon/tests/system/amazon/aws/utils/k8s.py b/providers/amazon/tests/system/amazon/aws/utils/k8s.py index 6954d0f5d013e..8aa8e38ce3e92 100644 --- a/providers/amazon/tests/system/amazon/aws/utils/k8s.py +++ b/providers/amazon/tests/system/amazon/aws/utils/k8s.py @@ -16,6 +16,8 @@ # under the License. from __future__ import annotations +from airflow.utils.helpers import exactly_one + try: from airflow.providers.standard.operators.bash import BashOperator except ImportError: @@ -23,8 +25,34 @@ from airflow.operators.bash import BashOperator # type: ignore[no-redef] -def get_describe_pod_operator(cluster_name: str, pod_name: str) -> BashOperator: - """Returns an operator that'll print the output of a `k describe pod` in the airflow logs.""" +def get_describe_pod_operator( + cluster_name: str, + *, + pod_name: str | None = None, + namespace: str | None = None, +) -> BashOperator: + """Return an operator that prints ``kubectl describe pod(s)`` output in the Airflow logs. + + Exactly one of *pod_name* or *namespace* must be provided. + + :param cluster_name: Name of the EKS cluster + :param pod_name: Describe a single pod by name + :param namespace: List and describe *all* pods in the given namespace + """ + if not exactly_one(pod_name, namespace): + raise ValueError("Exactly one of 'pod_name' or 'namespace' must be provided.") + + if pod_name: + kubectl_commands = f""" + echo "***** pod description *****"; + kubectl describe pod {pod_name};""" + else: + kubectl_commands = f""" + echo "***** pods in namespace {namespace} *****"; + kubectl get pods -n {namespace} -o wide; + echo "***** pod descriptions *****"; + kubectl describe pods -n {namespace};""" + return BashOperator( task_id="describe_pod", bash_command=f""" @@ -32,8 +60,6 @@ def get_describe_pod_operator(cluster_name: str, pod_name: str) -> BashOperator: install_kubectl.sh; # configure kubectl to hit the right cluster aws eks update-kubeconfig --name {cluster_name}; - # once all this setup is done, actually describe the pod - echo "vvv pod description below vvv"; - kubectl describe pod {pod_name}; - echo "^^^ pod description above ^^^" """, + {kubectl_commands} + """, ) diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_eks.py b/providers/amazon/tests/unit/amazon/aws/operators/test_eks.py index 2e1c850ba9435..5003e56d76a0b 100644 --- a/providers/amazon/tests/unit/amazon/aws/operators/test_eks.py +++ b/providers/amazon/tests/unit/amazon/aws/operators/test_eks.py @@ -39,6 +39,7 @@ EksCreateFargateProfileTrigger, EksCreateNodegroupTrigger, EksDeleteFargateProfileTrigger, + EksPodTrigger, ) from airflow.providers.cncf.kubernetes.utils.pod_manager import OnFinishAction from airflow.providers.common.compat.sdk import TaskDeferred @@ -1116,3 +1117,60 @@ def test_refresh_cached_properties_raises_when_no_credentials( # Verify super()._refresh_cached_properties() was NOT called since we raised mock_super_refresh.assert_not_called() + + @mock.patch( + "airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator.convert_config_file_to_dict" + ) + def test_invoke_defer_method_uses_eks_trigger(self, mock_convert_config): + """invoke_defer_method should create an EksPodTrigger and call defer.""" + op = EksPodOperator( + task_id="run_pod", + pod_name="run_pod", + cluster_name=CLUSTER_NAME, + image="amazon/aws-cli:latest", + cmds=["sh", "-c", "ls"], + labels={"demo": "hello_world"}, + get_logs=True, + on_finish_action="delete_pod", + ) + + # Set up pod metadata as it would be after execute creates the pod + mock_pod = mock.MagicMock() + mock_pod.metadata.name = "test-pod-abc123" + mock_pod.metadata.namespace = "default" + # Set status to None so define_container_state returns UNDEFINED (not terminal) + mock_pod.status = None + op.pod = mock_pod + + with pytest.raises(TaskDeferred) as exc: + op.invoke_defer_method() + + # Verify the trigger is an EksPodTrigger (not the base KubernetesPodTrigger) + trigger = exc.value.trigger + assert isinstance(trigger, EksPodTrigger) + assert trigger.eks_cluster_name == CLUSTER_NAME + assert trigger._aws_conn_id == "aws_default" + assert trigger.pod_name == "test-pod-abc123" + assert trigger.pod_namespace == "default" + + @mock.patch( + "airflow.providers.cncf.kubernetes.operators.pod.KubernetesPodOperator.convert_config_file_to_dict" + ) + def test_invoke_defer_method_raises_when_pod_is_none(self, mock_convert_config): + """invoke_defer_method should raise RuntimeError when pod is None.""" + op = EksPodOperator( + task_id="run_pod", + pod_name="run_pod", + cluster_name=CLUSTER_NAME, + image="amazon/aws-cli:latest", + cmds=["sh", "-c", "ls"], + labels={"demo": "hello_world"}, + get_logs=True, + on_finish_action="delete_pod", + ) + + # pod is None by default + op.pod = None + + with pytest.raises(RuntimeError, match="Pod must be created with metadata before deferring"): + op.invoke_defer_method() diff --git a/providers/amazon/tests/unit/amazon/aws/operators/test_emr_serverless.py b/providers/amazon/tests/unit/amazon/aws/operators/test_emr_serverless.py index 62cccc528d0d2..c851114e665ce 100644 --- a/providers/amazon/tests/unit/amazon/aws/operators/test_emr_serverless.py +++ b/providers/amazon/tests/unit/amazon/aws/operators/test_emr_serverless.py @@ -1513,3 +1513,224 @@ def test_template_fields(self): ) validate_template_fields(operator) + + +class TestEmrServerlessStartJobOperatorOpenLineageInjection: + """Tests for OpenLineage parent job info and transport info injection in EmrServerlessStartJobOperator.""" + + @mock.patch.object(EmrServerlessHook, "get_waiter") + @mock.patch.object(EmrServerlessHook, "conn") + @mock.patch( + "airflow.providers.amazon.aws.operators.emr" + ".inject_parent_job_information_into_emr_serverless_properties" + ) + def test_inject_parent_job_info_called_when_enabled(self, mock_inject_parent, mock_conn, mock_get_waiter): + mock_inject_parent.side_effect = lambda overrides, ctx: { + "applicationConfiguration": [ + { + "classification": "spark-defaults", + "properties": {"spark.openlineage.parentJobNamespace": "ns"}, + } + ] + } + mock_conn.get_application.return_value = {"application": {"state": "STARTED"}} + mock_conn.start_job_run.return_value = { + "jobRunId": job_run_id, + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + + operator = EmrServerlessStartJobOperator( + task_id=task_id, + application_id=application_id, + execution_role_arn=execution_role_arn, + job_driver=job_driver, + wait_for_completion=False, + openlineage_inject_parent_job_info=True, + ) + operator.execute(mock.MagicMock()) + + mock_inject_parent.assert_called_once() + call_kwargs = mock_conn.start_job_run.call_args.kwargs + config_overrides = call_kwargs["configurationOverrides"] + assert ( + config_overrides["applicationConfiguration"][0]["properties"][ + "spark.openlineage.parentJobNamespace" + ] + == "ns" + ) + + @mock.patch.object(EmrServerlessHook, "get_waiter") + @mock.patch.object(EmrServerlessHook, "conn") + @mock.patch( + "airflow.providers.amazon.aws.operators.emr" + ".inject_parent_job_information_into_emr_serverless_properties" + ) + def test_inject_parent_job_info_not_called_when_disabled( + self, mock_inject_parent, mock_conn, mock_get_waiter + ): + mock_conn.get_application.return_value = {"application": {"state": "STARTED"}} + mock_conn.start_job_run.return_value = { + "jobRunId": job_run_id, + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + + operator = EmrServerlessStartJobOperator( + task_id=task_id, + application_id=application_id, + execution_role_arn=execution_role_arn, + job_driver=job_driver, + wait_for_completion=False, + openlineage_inject_parent_job_info=False, + ) + operator.execute(mock.MagicMock()) + + mock_inject_parent.assert_not_called() + + @mock.patch.object(EmrServerlessHook, "get_waiter") + @mock.patch.object(EmrServerlessHook, "conn") + @mock.patch( + "airflow.providers.amazon.aws.operators.emr" + ".inject_transport_information_into_emr_serverless_properties" + ) + def test_inject_transport_info_called_when_enabled( + self, mock_inject_transport, mock_conn, mock_get_waiter + ): + mock_inject_transport.side_effect = lambda overrides, ctx: { + "applicationConfiguration": [ + { + "classification": "spark-defaults", + "properties": {"spark.openlineage.transport.type": "http"}, + } + ] + } + mock_conn.get_application.return_value = {"application": {"state": "STARTED"}} + mock_conn.start_job_run.return_value = { + "jobRunId": job_run_id, + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + + operator = EmrServerlessStartJobOperator( + task_id=task_id, + application_id=application_id, + execution_role_arn=execution_role_arn, + job_driver=job_driver, + wait_for_completion=False, + openlineage_inject_transport_info=True, + ) + operator.execute(mock.MagicMock()) + + mock_inject_transport.assert_called_once() + call_kwargs = mock_conn.start_job_run.call_args.kwargs + config_overrides = call_kwargs["configurationOverrides"] + assert ( + config_overrides["applicationConfiguration"][0]["properties"]["spark.openlineage.transport.type"] + == "http" + ) + + @mock.patch.object(EmrServerlessHook, "get_waiter") + @mock.patch.object(EmrServerlessHook, "conn") + @mock.patch( + "airflow.providers.amazon.aws.operators.emr" + ".inject_parent_job_information_into_emr_serverless_properties" + ) + @mock.patch( + "airflow.providers.amazon.aws.operators.emr" + ".inject_transport_information_into_emr_serverless_properties" + ) + def test_inject_both_parent_and_transport_info( + self, mock_inject_transport, mock_inject_parent, mock_conn, mock_get_waiter + ): + mock_inject_parent.side_effect = lambda overrides, ctx: { + "applicationConfiguration": [ + { + "classification": "spark-defaults", + "properties": {"spark.openlineage.parentJobNamespace": "ns"}, + } + ] + } + mock_inject_transport.side_effect = lambda overrides, ctx: { + "applicationConfiguration": [ + { + "classification": "spark-defaults", + "properties": { + **overrides.get("applicationConfiguration", [{}])[0].get("properties", {}), + "spark.openlineage.transport.type": "http", + }, + } + ] + } + mock_conn.get_application.return_value = {"application": {"state": "STARTED"}} + mock_conn.start_job_run.return_value = { + "jobRunId": job_run_id, + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + + operator = EmrServerlessStartJobOperator( + task_id=task_id, + application_id=application_id, + execution_role_arn=execution_role_arn, + job_driver=job_driver, + wait_for_completion=False, + openlineage_inject_parent_job_info=True, + openlineage_inject_transport_info=True, + ) + operator.execute(mock.MagicMock()) + + mock_inject_parent.assert_called_once() + mock_inject_transport.assert_called_once() + + @mock.patch.object(EmrServerlessHook, "get_waiter") + @mock.patch.object(EmrServerlessHook, "conn") + @mock.patch( + "airflow.providers.amazon.aws.operators.emr" + ".inject_parent_job_information_into_emr_serverless_properties" + ) + def test_inject_parent_job_info_preserves_existing_config( + self, mock_inject_parent, mock_conn, mock_get_waiter + ): + """Existing configuration_overrides (e.g. monitoringConfiguration) are preserved.""" + existing_config = { + "monitoringConfiguration": {"s3MonitoringConfiguration": {"logUri": "s3://bucket/logs"}}, + "applicationConfiguration": [ + {"classification": "spark-defaults", "properties": {"spark.driver.memory": "8G"}} + ], + } + mock_inject_parent.side_effect = lambda overrides, ctx: { + **overrides, + "applicationConfiguration": [ + { + "classification": "spark-defaults", + "properties": { + **overrides["applicationConfiguration"][0]["properties"], + "spark.openlineage.parentJobNamespace": "ns", + }, + } + ], + } + mock_conn.get_application.return_value = {"application": {"state": "STARTED"}} + mock_conn.start_job_run.return_value = { + "jobRunId": job_run_id, + "ResponseMetadata": {"HTTPStatusCode": 200}, + } + + operator = EmrServerlessStartJobOperator( + task_id=task_id, + application_id=application_id, + execution_role_arn=execution_role_arn, + job_driver=job_driver, + configuration_overrides=existing_config, + wait_for_completion=False, + openlineage_inject_parent_job_info=True, + ) + operator.execute(mock.MagicMock()) + + call_kwargs = mock_conn.start_job_run.call_args.kwargs + config_overrides = call_kwargs["configurationOverrides"] + # Monitoring config preserved + assert config_overrides["monitoringConfiguration"]["s3MonitoringConfiguration"]["logUri"] == ( + "s3://bucket/logs" + ) + # OL parent info injected + props = config_overrides["applicationConfiguration"][0]["properties"] + assert props["spark.openlineage.parentJobNamespace"] == "ns" + assert props["spark.driver.memory"] == "8G" diff --git a/providers/amazon/tests/unit/amazon/aws/triggers/test_eks.py b/providers/amazon/tests/unit/amazon/aws/triggers/test_eks.py index e8fe934be8fe4..fcdd712cc562d 100644 --- a/providers/amazon/tests/unit/amazon/aws/triggers/test_eks.py +++ b/providers/amazon/tests/unit/amazon/aws/triggers/test_eks.py @@ -16,7 +16,8 @@ # under the License. from __future__ import annotations -from unittest.mock import AsyncMock, Mock, call, patch +import datetime +from unittest.mock import AsyncMock, MagicMock, Mock, call, patch import pytest from botocore.exceptions import ClientError @@ -24,6 +25,7 @@ from airflow.providers.amazon.aws.triggers.eks import ( EksCreateClusterTrigger, EksDeleteClusterTrigger, + EksPodTrigger, ) from airflow.providers.common.compat.sdk import AirflowException from airflow.triggers.base import TriggerEvent @@ -318,3 +320,127 @@ async def test_when_there_are_no_fargate_profiles_it_should_only_log_message(sel self.trigger.log.info.assert_called_once_with( "No Fargate profiles associated with cluster %s", CLUSTER_NAME ) + + +class TestEksPodTrigger: + """Tests for EksPodTrigger.""" + + TRIGGER_START_TIME = datetime.datetime(2026, 1, 1, tzinfo=datetime.timezone.utc) + + def _create_trigger(self, **overrides): + """Create an EksPodTrigger with sensible defaults.""" + defaults = { + "eks_cluster_name": CLUSTER_NAME, + "aws_conn_id": AWS_CONN_ID, + "region": REGION_NAME, + "pod_name": "test-pod", + "pod_namespace": "default", + "trigger_start_time": self.TRIGGER_START_TIME, + "base_container_name": "base", + "config_dict": {"old": "stale-config"}, + } + defaults.update(overrides) + return EksPodTrigger(**defaults) + + def test_serialize_includes_eks_fields(self): + """serialize() should include eks_cluster_name, aws_conn_id, and region.""" + trigger = self._create_trigger() + classpath, kwargs = trigger.serialize() + + assert classpath == "airflow.providers.amazon.aws.triggers.eks.EksPodTrigger" + assert kwargs["eks_cluster_name"] == CLUSTER_NAME + assert kwargs["aws_conn_id"] == AWS_CONN_ID + assert kwargs["region"] == REGION_NAME + # Also verify parent fields are present + assert kwargs["pod_name"] == "test-pod" + assert kwargs["pod_namespace"] == "default" + + def test_serialize_roundtrip(self): + """A trigger created from serialized kwargs should serialize identically.""" + trigger = self._create_trigger() + classpath, kwargs = trigger.serialize() + + trigger2 = EksPodTrigger(**kwargs) + classpath2, kwargs2 = trigger2.serialize() + + assert classpath == classpath2 + assert kwargs == kwargs2 + + @pytest.mark.asyncio + @patch("airflow.providers.cncf.kubernetes.triggers.pod.KubernetesPodTrigger.run") + @patch("airflow.providers.amazon.aws.hooks.eks.EksHook.generate_config_file") + @patch("airflow.providers.amazon.aws.hooks.eks.EksHook._secure_credential_context") + @patch("airflow.providers.amazon.aws.hooks.eks.EksHook.get_session") + @patch("airflow.providers.amazon.aws.hooks.eks.EksHook.__init__", return_value=None) + async def test_run_generates_fresh_kubeconfig( + self, + mock_eks_hook_init, + mock_get_session, + mock_secure_credential_context, + mock_generate_config_file, + mock_parent_run, + ): + """run() should get fresh credentials, generate kubeconfig, and delegate to parent.""" + # Set up credential mocks + mock_session = MagicMock() + mock_credentials = MagicMock() + mock_frozen = MagicMock() + mock_frozen.access_key = "AKIATEST" + mock_frozen.secret_key = "secret123" + mock_frozen.token = "token456" + mock_get_session.return_value = mock_session + mock_session.get_credentials.return_value = mock_credentials + mock_credentials.get_frozen_credentials.return_value = mock_frozen + + # Set up context manager mocks + mock_secure_credential_context.return_value.__enter__.return_value = "/tmp/test.aws_creds" + mock_generate_config_file.return_value.__enter__.return_value = "/tmp/test_kubeconfig" + + # Mock reading the kubeconfig file + with patch("pathlib.Path.read_text", return_value="apiVersion: v1\nkind: Config\nclusters: []"): + + async def mock_gen(): + yield TriggerEvent({"status": "success"}) + + mock_parent_run.return_value = mock_gen() + + trigger = self._create_trigger() + events = [] + async for event in trigger.run(): + events.append(event) + + assert len(events) == 1 + assert events[0] == TriggerEvent({"status": "success"}) + + # Verify credentials were fetched + mock_eks_hook_init.assert_called_once_with(aws_conn_id=AWS_CONN_ID, region_name=REGION_NAME) + mock_get_session.assert_called_once() + mock_session.get_credentials.assert_called_once() + mock_credentials.get_frozen_credentials.assert_called_once() + + # Verify credential context and config generation + mock_secure_credential_context.assert_called_once_with("AKIATEST", "secret123", "token456") + mock_generate_config_file.assert_called_once_with( + eks_cluster_name=CLUSTER_NAME, + pod_namespace="default", + credentials_file="/tmp/test.aws_creds", + ) + + @pytest.mark.asyncio + @patch("airflow.providers.amazon.aws.hooks.eks.EksHook.get_session") + @patch("airflow.providers.amazon.aws.hooks.eks.EksHook.__init__", return_value=None) + async def test_run_raises_when_credentials_unavailable( + self, + mock_eks_hook_init, + mock_get_session, + ): + """run() should raise RuntimeError when credentials cannot be retrieved.""" + mock_session = MagicMock() + mock_get_session.return_value = mock_session + mock_session.get_credentials.return_value = None + + trigger = self._create_trigger() + + with pytest.raises(RuntimeError, match="Unable to retrieve AWS credentials"): + async for _ in trigger.run(): + pass diff --git a/providers/amazon/tests/unit/amazon/aws/triggers/test_glue.py b/providers/amazon/tests/unit/amazon/aws/triggers/test_glue.py index 4339d36ce3893..61a1f8385e6b4 100644 --- a/providers/amazon/tests/unit/amazon/aws/triggers/test_glue.py +++ b/providers/amazon/tests/unit/amazon/aws/triggers/test_glue.py @@ -21,9 +21,11 @@ from unittest.mock import AsyncMock import pytest +from botocore.exceptions import ClientError from airflow.providers.amazon.aws.hooks.glue import GlueDataQualityHook, GlueJobHook from airflow.providers.amazon.aws.hooks.glue_catalog import GlueCatalogHook +from airflow.providers.amazon.aws.hooks.logs import AwsLogsHook from airflow.providers.amazon.aws.triggers.glue import ( GlueCatalogPartitionTrigger, GlueDataQualityRuleRecommendationRunCompleteTrigger, @@ -111,6 +113,249 @@ def test_serialization(self): "waiter_delay": 10, } + def test_serialization_verbose(self): + trigger = GlueJobCompleteTrigger( + job_name="job_name", + run_id="JobRunId", + verbose=True, + aws_conn_id="aws_conn_id", + waiter_max_attempts=3, + waiter_delay=10, + ) + classpath, kwargs = trigger.serialize() + assert kwargs["verbose"] is True + + @pytest.mark.asyncio + @mock.patch.object(AwsLogsHook, "get_async_conn") + @mock.patch.object(GlueJobHook, "get_async_conn") + async def test_verbose_run_success(self, mock_glue_conn, mock_logs_conn): + """When verbose=True, the trigger polls job state and fetches CloudWatch logs.""" + glue_client = AsyncMock() + glue_client.get_job_run = AsyncMock( + side_effect=[ + # First call: log group metadata + {"JobRun": {"JobRunState": "RUNNING", "LogGroupName": "/aws-glue/python-jobs"}}, + # Second call: state check at top of iteration 1 (RUNNING) + {"JobRun": {"JobRunState": "RUNNING", "LogGroupName": "/aws-glue/python-jobs"}}, + # Third call: state check at top of iteration 2 (SUCCEEDED) + {"JobRun": {"JobRunState": "SUCCEEDED", "LogGroupName": "/aws-glue/python-jobs"}}, + ] + ) + mock_glue_conn.return_value.__aenter__ = AsyncMock(return_value=glue_client) + mock_glue_conn.return_value.__aexit__ = AsyncMock(return_value=False) + + logs_client = AsyncMock() + logs_client.get_log_events = AsyncMock( + return_value={ + "events": [{"timestamp": 1234, "message": "Processing step 1\n"}], + "nextForwardToken": "token_1", + } + ) + mock_logs_conn.return_value.__aenter__ = AsyncMock(return_value=logs_client) + mock_logs_conn.return_value.__aexit__ = AsyncMock(return_value=False) + + trigger = GlueJobCompleteTrigger( + job_name="job_name", + run_id="jr_123", + verbose=True, + aws_conn_id="aws_conn_id", + waiter_delay=0, + waiter_max_attempts=5, + ) + generator = trigger.run() + event = await generator.asend(None) + + assert event.payload["status"] == "success" + assert event.payload["run_id"] == "jr_123" + # Logs client was called for both output and error streams + assert logs_client.get_log_events.call_count >= 2 + + @pytest.mark.asyncio + @mock.patch.object(AwsLogsHook, "get_async_conn") + @mock.patch.object(GlueJobHook, "get_async_conn") + async def test_verbose_run_job_failed(self, mock_glue_conn, mock_logs_conn): + """When verbose=True and the job fails, the trigger yields an error event.""" + glue_client = AsyncMock() + glue_client.get_job_run = AsyncMock( + return_value={"JobRun": {"JobRunState": "FAILED", "LogGroupName": "/aws-glue/python-jobs"}} + ) + mock_glue_conn.return_value.__aenter__ = AsyncMock(return_value=glue_client) + mock_glue_conn.return_value.__aexit__ = AsyncMock(return_value=False) + + logs_client = AsyncMock() + logs_client.get_log_events = AsyncMock(return_value={"events": [], "nextForwardToken": "token_1"}) + mock_logs_conn.return_value.__aenter__ = AsyncMock(return_value=logs_client) + mock_logs_conn.return_value.__aexit__ = AsyncMock(return_value=False) + + trigger = GlueJobCompleteTrigger( + job_name="job_name", + run_id="jr_123", + verbose=True, + aws_conn_id="aws_conn_id", + waiter_delay=0, + waiter_max_attempts=5, + ) + generator = trigger.run() + event = await generator.asend(None) + assert event.payload["status"] == "error" + assert "FAILED" in event.payload["message"] + assert event.payload["run_id"] == "jr_123" + + @pytest.mark.asyncio + @mock.patch.object(AwsLogsHook, "get_async_conn") + @mock.patch.object(GlueJobHook, "get_async_conn") + async def test_verbose_run_max_attempts(self, mock_glue_conn, mock_logs_conn): + """When verbose=True and the job stays RUNNING past max attempts, yields an error event.""" + glue_client = AsyncMock() + glue_client.get_job_run = AsyncMock( + return_value={"JobRun": {"JobRunState": "RUNNING", "LogGroupName": "/aws-glue/python-jobs"}} + ) + mock_glue_conn.return_value.__aenter__ = AsyncMock(return_value=glue_client) + mock_glue_conn.return_value.__aexit__ = AsyncMock(return_value=False) + + logs_client = AsyncMock() + logs_client.get_log_events = AsyncMock(return_value={"events": [], "nextForwardToken": "token_1"}) + mock_logs_conn.return_value.__aenter__ = AsyncMock(return_value=logs_client) + mock_logs_conn.return_value.__aexit__ = AsyncMock(return_value=False) + + trigger = GlueJobCompleteTrigger( + job_name="job_name", + run_id="jr_123", + verbose=True, + aws_conn_id="aws_conn_id", + waiter_delay=0, + waiter_max_attempts=2, + ) + generator = trigger.run() + event = await generator.asend(None) + assert event.payload["status"] == "error" + assert "max attempts" in event.payload["message"] + assert event.payload["run_id"] == "jr_123" + + @pytest.mark.asyncio + @mock.patch.object(AwsLogsHook, "get_async_conn") + @mock.patch.object(GlueJobHook, "get_async_conn") + async def test_verbose_run_cloudwatch_client_error(self, mock_glue_conn, mock_logs_conn): + """When verbose=True and CloudWatch returns an unexpected ClientError, yields error event.""" + glue_client = AsyncMock() + glue_client.get_job_run = AsyncMock( + return_value={"JobRun": {"JobRunState": "RUNNING", "LogGroupName": "/aws-glue/python-jobs"}} + ) + mock_glue_conn.return_value.__aenter__ = AsyncMock(return_value=glue_client) + mock_glue_conn.return_value.__aexit__ = AsyncMock(return_value=False) + + logs_client = AsyncMock() + logs_client.get_log_events = AsyncMock( + side_effect=ClientError( + {"Error": {"Code": "AccessDeniedException", "Message": "not authorized"}}, + "GetLogEvents", + ) + ) + mock_logs_conn.return_value.__aenter__ = AsyncMock(return_value=logs_client) + mock_logs_conn.return_value.__aexit__ = AsyncMock(return_value=False) + + trigger = GlueJobCompleteTrigger( + job_name="job_name", + run_id="jr_123", + verbose=True, + aws_conn_id="aws_conn_id", + waiter_delay=0, + waiter_max_attempts=5, + ) + generator = trigger.run() + event = await generator.asend(None) + assert event.payload["status"] == "error" + assert "Failed to fetch logs" in event.payload["message"] + assert "AccessDeniedException" in event.payload["message"] + assert event.payload["run_id"] == "jr_123" + + @pytest.mark.asyncio + async def test_forward_logs_resource_not_found(self): + """_forward_logs handles ResourceNotFoundException gracefully and uses resolved region in URL.""" + logs_client = AsyncMock() + logs_client.get_log_events = AsyncMock( + side_effect=ClientError( + {"Error": {"Code": "ResourceNotFoundException", "Message": "not found"}}, + "GetLogEvents", + ) + ) + logs_client.meta.region_name = "eu-west-1" + + trigger = GlueJobCompleteTrigger( + job_name="job_name", + run_id="jr_123", + verbose=True, + aws_conn_id="aws_conn_id", + region_name=None, + waiter_delay=0, + waiter_max_attempts=5, + ) + with mock.patch.object(trigger.log, "warning") as mock_log_warning: + result = await trigger._forward_logs(logs_client, "/aws-glue/python-jobs/output", "jr_123", None) + assert result is None + # Verify the URL uses the resolved region from the client, not self.region_name (which is None) + url_arg = mock_log_warning.call_args[0][1] + assert "eu-west-1" in url_arg + + @pytest.mark.asyncio + async def test_forward_logs_pagination(self): + """_forward_logs follows nextForwardToken and formats logs like the sync path.""" + logs_client = AsyncMock() + logs_client.get_log_events = AsyncMock( + side_effect=[ + { + "events": [{"timestamp": 1, "message": "line 1\n"}], + "nextForwardToken": "token_2", + }, + { + "events": [{"timestamp": 2, "message": "line 2\n"}], + "nextForwardToken": "token_3", + }, + { + "events": [], + "nextForwardToken": "token_3", + }, + ] + ) + + trigger = GlueJobCompleteTrigger( + job_name="job_name", + run_id="jr_123", + verbose=True, + aws_conn_id="aws_conn_id", + waiter_delay=0, + waiter_max_attempts=5, + ) + with mock.patch.object(trigger.log, "info") as mock_log_info: + result = await trigger._forward_logs(logs_client, "/aws-glue/python-jobs/output", "jr_123", None) + assert result == "token_3" + assert logs_client.get_log_events.call_count == 3 + # Verify log format matches sync path: "Glue Job Run Logs:" with tab-indented lines + log_output = mock_log_info.call_args[0][0] + assert "Glue Job Run /aws-glue/python-jobs/output Logs:" in log_output + assert "\tline 1" in log_output + assert "\tline 2" in log_output + + @pytest.mark.asyncio + async def test_forward_logs_no_new_events(self): + """_forward_logs logs 'No new log' when there are no events, matching sync path.""" + logs_client = AsyncMock() + logs_client.get_log_events = AsyncMock(return_value={"events": [], "nextForwardToken": "token_1"}) + + trigger = GlueJobCompleteTrigger( + job_name="job_name", + run_id="jr_123", + verbose=True, + aws_conn_id="aws_conn_id", + waiter_delay=0, + waiter_max_attempts=5, + ) + with mock.patch.object(trigger.log, "info") as mock_log_info: + result = await trigger._forward_logs(logs_client, "/aws-glue/python-jobs/output", "jr_123", None) + assert result == "token_1" + log_output = mock_log_info.call_args[0][0] + assert "No new log from the Glue Job in /aws-glue/python-jobs/output" in log_output + class TestGlueCatalogPartitionSensorTrigger: @pytest.mark.asyncio diff --git a/providers/apache/beam/tests/system/apache/beam/example_beam.py b/providers/apache/beam/tests/system/apache/beam/example_beam.py index 56b44e1649d1b..fb911505d3a7e 100644 --- a/providers/apache/beam/tests/system/apache/beam/example_beam.py +++ b/providers/apache/beam/tests/system/apache/beam/example_beam.py @@ -63,5 +63,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/beam/tests/system/apache/beam/example_beam_java_flink.py b/providers/apache/beam/tests/system/apache/beam/example_beam_java_flink.py index 06bb224204478..2756d7a94cc0e 100644 --- a/providers/apache/beam/tests/system/apache/beam/example_beam_java_flink.py +++ b/providers/apache/beam/tests/system/apache/beam/example_beam_java_flink.py @@ -62,5 +62,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/beam/tests/system/apache/beam/example_beam_java_spark.py b/providers/apache/beam/tests/system/apache/beam/example_beam_java_spark.py index 14e8b85a971cd..8225afb391933 100644 --- a/providers/apache/beam/tests/system/apache/beam/example_beam_java_spark.py +++ b/providers/apache/beam/tests/system/apache/beam/example_beam_java_spark.py @@ -62,5 +62,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/beam/tests/system/apache/beam/example_go.py b/providers/apache/beam/tests/system/apache/beam/example_go.py index d5a3290e2acfd..9230ee532a329 100644 --- a/providers/apache/beam/tests/system/apache/beam/example_go.py +++ b/providers/apache/beam/tests/system/apache/beam/example_go.py @@ -105,5 +105,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/beam/tests/system/apache/beam/example_go_dataflow.py b/providers/apache/beam/tests/system/apache/beam/example_go_dataflow.py index 50532be91c6ae..5870365eb9bd5 100644 --- a/providers/apache/beam/tests/system/apache/beam/example_go_dataflow.py +++ b/providers/apache/beam/tests/system/apache/beam/example_go_dataflow.py @@ -78,5 +78,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/beam/tests/system/apache/beam/example_java_dataflow.py b/providers/apache/beam/tests/system/apache/beam/example_java_dataflow.py index 849587de31d63..21a43aac3d462 100644 --- a/providers/apache/beam/tests/system/apache/beam/example_java_dataflow.py +++ b/providers/apache/beam/tests/system/apache/beam/example_java_dataflow.py @@ -68,5 +68,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/beam/tests/system/apache/beam/example_python.py b/providers/apache/beam/tests/system/apache/beam/example_python.py index 41db93f3d0d1c..fd115377fef23 100644 --- a/providers/apache/beam/tests/system/apache/beam/example_python.py +++ b/providers/apache/beam/tests/system/apache/beam/example_python.py @@ -122,5 +122,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/beam/tests/system/apache/beam/example_python_async.py b/providers/apache/beam/tests/system/apache/beam/example_python_async.py index fa237c1eecde7..4ed0a655fe477 100644 --- a/providers/apache/beam/tests/system/apache/beam/example_python_async.py +++ b/providers/apache/beam/tests/system/apache/beam/example_python_async.py @@ -131,5 +131,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/beam/tests/system/apache/beam/example_python_dataflow.py b/providers/apache/beam/tests/system/apache/beam/example_python_dataflow.py index 32391eb29a38d..ce594aff03a81 100644 --- a/providers/apache/beam/tests/system/apache/beam/example_python_dataflow.py +++ b/providers/apache/beam/tests/system/apache/beam/example_python_dataflow.py @@ -81,5 +81,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/cassandra/README.rst b/providers/apache/cassandra/README.rst index 7e70e9de1afbe..20415a3e3e250 100644 --- a/providers/apache/cassandra/README.rst +++ b/providers/apache/cassandra/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-cassandra`` -Release: ``3.9.3`` +Release: ``3.9.4`` `Apache Cassandra `__. @@ -36,7 +36,7 @@ This is a provider package for ``apache.cassandra`` provider. All classes for th are in ``airflow.providers.apache.cassandra`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -80,4 +80,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/cassandra/docs/changelog.rst b/providers/apache/cassandra/docs/changelog.rst index a2cfaf75e5f39..f020e29443a9e 100644 --- a/providers/apache/cassandra/docs/changelog.rst +++ b/providers/apache/cassandra/docs/changelog.rst @@ -26,6 +26,17 @@ Changelog --------- +3.9.4 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 3.9.3 ..... diff --git a/providers/apache/cassandra/docs/index.rst b/providers/apache/cassandra/docs/index.rst index 0827731918fb5..add665b7acc6e 100644 --- a/providers/apache/cassandra/docs/index.rst +++ b/providers/apache/cassandra/docs/index.rst @@ -76,7 +76,7 @@ apache-airflow-providers-apache-cassandra package `Apache Cassandra `__. -Release: 3.9.3 +Release: 3.9.4 Provider package ---------------- @@ -131,5 +131,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-cassandra 3.9.3 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-cassandra 3.9.3 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-cassandra 3.9.4 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-cassandra 3.9.4 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/cassandra/provider.yaml b/providers/apache/cassandra/provider.yaml index 2567553f833e2..b1aecd4ad6ac5 100644 --- a/providers/apache/cassandra/provider.yaml +++ b/providers/apache/cassandra/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298312 +source-date-epoch: 1775591642 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.9.4 - 3.9.3 - 3.9.2 - 3.9.1 diff --git a/providers/apache/cassandra/pyproject.toml b/providers/apache/cassandra/pyproject.toml index 195016ea3cec3..a3314d5fce8da 100644 --- a/providers/apache/cassandra/pyproject.toml +++ b/providers/apache/cassandra/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-cassandra" -version = "3.9.3" +version = "3.9.4" description = "Provider package apache-airflow-providers-apache-cassandra for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -100,8 +100,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-cassandra/3.9.3" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-cassandra/3.9.3/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-cassandra/3.9.4" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-cassandra/3.9.4/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/cassandra/src/airflow/providers/apache/cassandra/__init__.py b/providers/apache/cassandra/src/airflow/providers/apache/cassandra/__init__.py index 2557d61134024..650e4a2386571 100644 --- a/providers/apache/cassandra/src/airflow/providers/apache/cassandra/__init__.py +++ b/providers/apache/cassandra/src/airflow/providers/apache/cassandra/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.9.3" +__version__ = "3.9.4" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/cassandra/tests/system/apache/cassandra/example_cassandra_dag.py b/providers/apache/cassandra/tests/system/apache/cassandra/example_cassandra_dag.py index b2d98d88df433..a995b92dc13d0 100644 --- a/providers/apache/cassandra/tests/system/apache/cassandra/example_cassandra_dag.py +++ b/providers/apache/cassandra/tests/system/apache/cassandra/example_cassandra_dag.py @@ -57,5 +57,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/drill/README.rst b/providers/apache/drill/README.rst index 3a98698a4d637..393a18d8c7987 100644 --- a/providers/apache/drill/README.rst +++ b/providers/apache/drill/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-drill`` -Release: ``3.3.1`` +Release: ``3.3.2`` `Apache Drill `__. @@ -36,7 +36,7 @@ This is a provider package for ``apache.drill`` provider. All classes for this p are in ``airflow.providers.apache.drill`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -79,4 +79,4 @@ Dependent package ============================================================================================================ ============== The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/drill/docs/changelog.rst b/providers/apache/drill/docs/changelog.rst index 82a0ac1760ac8..7113e22ae2924 100644 --- a/providers/apache/drill/docs/changelog.rst +++ b/providers/apache/drill/docs/changelog.rst @@ -26,6 +26,17 @@ Changelog --------- +3.3.2 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 3.3.1 ..... diff --git a/providers/apache/drill/docs/index.rst b/providers/apache/drill/docs/index.rst index 928ab2683722b..6150d9b75461c 100644 --- a/providers/apache/drill/docs/index.rst +++ b/providers/apache/drill/docs/index.rst @@ -76,7 +76,7 @@ apache-airflow-providers-apache-drill package `Apache Drill `__. -Release: 3.3.1 +Release: 3.3.2 Provider package ---------------- @@ -130,5 +130,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-drill 3.3.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-drill 3.3.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-drill 3.3.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-drill 3.3.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/drill/provider.yaml b/providers/apache/drill/provider.yaml index cf3168ae89077..365fb76f1048d 100644 --- a/providers/apache/drill/provider.yaml +++ b/providers/apache/drill/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298329 +source-date-epoch: 1775591653 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.3.2 - 3.3.1 - 3.3.0 - 3.2.1 diff --git a/providers/apache/drill/pyproject.toml b/providers/apache/drill/pyproject.toml index fa37d712cdce5..d223bccfd7893 100644 --- a/providers/apache/drill/pyproject.toml +++ b/providers/apache/drill/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-drill" -version = "3.3.1" +version = "3.3.2" description = "Provider package apache-airflow-providers-apache-drill for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -102,8 +102,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-drill/3.3.1" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-drill/3.3.1/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-drill/3.3.2" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-drill/3.3.2/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/drill/src/airflow/providers/apache/drill/__init__.py b/providers/apache/drill/src/airflow/providers/apache/drill/__init__.py index 2018388730f4f..99e873d761511 100644 --- a/providers/apache/drill/src/airflow/providers/apache/drill/__init__.py +++ b/providers/apache/drill/src/airflow/providers/apache/drill/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.3.1" +__version__ = "3.3.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/drill/tests/system/apache/drill/example_drill_dag.py b/providers/apache/drill/tests/system/apache/drill/example_drill_dag.py index 7386f3b59cb1a..9ea5c60ac6c9c 100644 --- a/providers/apache/drill/tests/system/apache/drill/example_drill_dag.py +++ b/providers/apache/drill/tests/system/apache/drill/example_drill_dag.py @@ -49,5 +49,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/druid/README.rst b/providers/apache/druid/README.rst index a89371203880b..dc845105939e3 100644 --- a/providers/apache/druid/README.rst +++ b/providers/apache/druid/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-druid`` -Release: ``4.5.1`` +Release: ``4.5.2`` `Apache Druid `__. @@ -36,7 +36,7 @@ This is a provider package for ``apache.druid`` provider. All classes for this p are in ``airflow.providers.apache.druid`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -90,4 +90,4 @@ Extra Dependencies =============== ======================================== The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/druid/docs/changelog.rst b/providers/apache/druid/docs/changelog.rst index 5a62c0b4e9292..c31ee1ea701e8 100644 --- a/providers/apache/druid/docs/changelog.rst +++ b/providers/apache/druid/docs/changelog.rst @@ -27,6 +27,17 @@ Changelog --------- +4.5.2 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.5.1 ..... diff --git a/providers/apache/druid/docs/index.rst b/providers/apache/druid/docs/index.rst index 6970ac304b034..539166d885a91 100644 --- a/providers/apache/druid/docs/index.rst +++ b/providers/apache/druid/docs/index.rst @@ -76,7 +76,7 @@ apache-airflow-providers-apache-druid package `Apache Druid `__. -Release: 4.5.1 +Release: 4.5.2 Provider package ---------------- @@ -132,5 +132,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-druid 4.5.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-druid 4.5.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-druid 4.5.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-druid 4.5.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/druid/provider.yaml b/providers/apache/druid/provider.yaml index 55e8764cb1fdb..3af222d4b1ba5 100644 --- a/providers/apache/druid/provider.yaml +++ b/providers/apache/druid/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298342 +source-date-epoch: 1775591663 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 4.5.2 - 4.5.1 - 4.5.0 - 4.4.2 diff --git a/providers/apache/druid/pyproject.toml b/providers/apache/druid/pyproject.toml index 63c770d1aeb52..b186ae47bcfe0 100644 --- a/providers/apache/druid/pyproject.toml +++ b/providers/apache/druid/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-druid" -version = "4.5.1" +version = "4.5.2" description = "Provider package apache-airflow-providers-apache-druid for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -110,8 +110,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-druid/4.5.1" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-druid/4.5.1/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-druid/4.5.2" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-druid/4.5.2/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py index 40b2dd86c9d86..c70e51aed99e1 100644 --- a/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py +++ b/providers/apache/druid/src/airflow/providers/apache/druid/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.5.1" +__version__ = "4.5.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/druid/tests/system/apache/druid/example_druid_dag.py b/providers/apache/druid/tests/system/apache/druid/example_druid_dag.py index eac9af6d19c57..04766e56e1fac 100644 --- a/providers/apache/druid/tests/system/apache/druid/example_druid_dag.py +++ b/providers/apache/druid/tests/system/apache/druid/example_druid_dag.py @@ -57,5 +57,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/hdfs/README.rst b/providers/apache/hdfs/README.rst index d2e52e59946e5..07067d56a1b2f 100644 --- a/providers/apache/hdfs/README.rst +++ b/providers/apache/hdfs/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-hdfs`` -Release: ``4.11.4`` +Release: ``4.11.5`` `Hadoop Distributed File System (HDFS) `__ @@ -37,7 +37,7 @@ This is a provider package for ``apache.hdfs`` provider. All classes for this pr are in ``airflow.providers.apache.hdfs`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -85,4 +85,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/hdfs/docs/changelog.rst b/providers/apache/hdfs/docs/changelog.rst index fefea44e30606..75405ed0eed80 100644 --- a/providers/apache/hdfs/docs/changelog.rst +++ b/providers/apache/hdfs/docs/changelog.rst @@ -27,6 +27,18 @@ Changelog --------- +4.11.5 +...... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` +* ``Fix advertising some of the missing provider capabilities via provider info (#64127)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.11.4 ...... diff --git a/providers/apache/hdfs/docs/index.rst b/providers/apache/hdfs/docs/index.rst index 7cd10598b7bbb..81807de3505c2 100644 --- a/providers/apache/hdfs/docs/index.rst +++ b/providers/apache/hdfs/docs/index.rst @@ -65,7 +65,7 @@ apache-airflow-providers-apache-hdfs package and `WebHDFS `__. -Release: 4.11.4 +Release: 4.11.5 Provider package ---------------- @@ -124,5 +124,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-hdfs 4.11.4 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-hdfs 4.11.4 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-hdfs 4.11.5 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-hdfs 4.11.5 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/hdfs/provider.yaml b/providers/apache/hdfs/provider.yaml index 63a602aa1b805..1f9cc0e6d9f48 100644 --- a/providers/apache/hdfs/provider.yaml +++ b/providers/apache/hdfs/provider.yaml @@ -24,12 +24,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298372 +source-date-epoch: 1775591680 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 4.11.5 - 4.11.4 - 4.11.3 - 4.11.2 diff --git a/providers/apache/hdfs/pyproject.toml b/providers/apache/hdfs/pyproject.toml index 3101d616e9226..5d407e00ace9d 100644 --- a/providers/apache/hdfs/pyproject.toml +++ b/providers/apache/hdfs/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-hdfs" -version = "4.11.4" +version = "4.11.5" description = "Provider package apache-airflow-providers-apache-hdfs for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -105,8 +105,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hdfs/4.11.4" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hdfs/4.11.4/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hdfs/4.11.5" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hdfs/4.11.5/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/hdfs/src/airflow/providers/apache/hdfs/__init__.py b/providers/apache/hdfs/src/airflow/providers/apache/hdfs/__init__.py index 6a527853b40e8..c8c087ebcf78d 100644 --- a/providers/apache/hdfs/src/airflow/providers/apache/hdfs/__init__.py +++ b/providers/apache/hdfs/src/airflow/providers/apache/hdfs/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.11.4" +__version__ = "4.11.5" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/hive/README.rst b/providers/apache/hive/README.rst index 11966a5f0da76..41e3d0ff20422 100644 --- a/providers/apache/hive/README.rst +++ b/providers/apache/hive/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-hive`` -Release: ``9.4.1`` +Release: ``9.4.2`` `Apache Hive `__ @@ -36,7 +36,7 @@ This is a provider package for ``apache.hive`` provider. All classes for this pr are in ``airflow.providers.apache.hive`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -107,4 +107,4 @@ Extra Dependencies =================== ============================================================================================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/hive/docs/changelog.rst b/providers/apache/hive/docs/changelog.rst index 7099b5f1b1983..460f84a4a8409 100644 --- a/providers/apache/hive/docs/changelog.rst +++ b/providers/apache/hive/docs/changelog.rst @@ -27,6 +27,17 @@ Changelog --------- +9.4.2 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 9.4.1 ..... diff --git a/providers/apache/hive/docs/index.rst b/providers/apache/hive/docs/index.rst index b8280fb3ae79e..7557efe665fe8 100644 --- a/providers/apache/hive/docs/index.rst +++ b/providers/apache/hive/docs/index.rst @@ -79,7 +79,7 @@ apache-airflow-providers-apache-hive package `Apache Hive `__ -Release: 9.4.1 +Release: 9.4.2 Provider package ---------------- @@ -145,5 +145,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-hive 9.4.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-hive 9.4.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-hive 9.4.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-hive 9.4.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/hive/provider.yaml b/providers/apache/hive/provider.yaml index 84dcd4e8b7082..0bb4498ebafaa 100644 --- a/providers/apache/hive/provider.yaml +++ b/providers/apache/hive/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298386 +source-date-epoch: 1775591691 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 9.4.2 - 9.4.1 - 9.4.0 - 9.3.0 diff --git a/providers/apache/hive/pyproject.toml b/providers/apache/hive/pyproject.toml index 3ad8cb7a93791..be07e0a32acaf 100644 --- a/providers/apache/hive/pyproject.toml +++ b/providers/apache/hive/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-hive" -version = "9.4.1" +version = "9.4.2" description = "Provider package apache-airflow-providers-apache-hive for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -144,8 +144,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/9.4.1" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/9.4.1/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/9.4.2" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-hive/9.4.2/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/hive/src/airflow/providers/apache/hive/__init__.py b/providers/apache/hive/src/airflow/providers/apache/hive/__init__.py index f5671f3e49270..ec6789a4afe11 100644 --- a/providers/apache/hive/src/airflow/providers/apache/hive/__init__.py +++ b/providers/apache/hive/src/airflow/providers/apache/hive/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "9.4.1" +__version__ = "9.4.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/hive/tests/system/apache/hive/example_hive.py b/providers/apache/hive/tests/system/apache/hive/example_hive.py index 4faf8c3487c6e..bdd4eda21134d 100644 --- a/providers/apache/hive/tests/system/apache/hive/example_hive.py +++ b/providers/apache/hive/tests/system/apache/hive/example_hive.py @@ -79,5 +79,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/hive/tests/system/apache/hive/example_twitter_dag.py b/providers/apache/hive/tests/system/apache/hive/example_twitter_dag.py index 71fd15d43c1da..8d66a832cc512 100644 --- a/providers/apache/hive/tests/system/apache/hive/example_twitter_dag.py +++ b/providers/apache/hive/tests/system/apache/hive/example_twitter_dag.py @@ -159,5 +159,5 @@ def transfer_to_db(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/iceberg/README.rst b/providers/apache/iceberg/README.rst index 96984e1f29e7c..3814bb2f0c2fa 100644 --- a/providers/apache/iceberg/README.rst +++ b/providers/apache/iceberg/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-iceberg`` -Release: ``2.0.1`` +Release: ``2.0.2`` `Iceberg `__ @@ -36,7 +36,7 @@ This is a provider package for ``apache.iceberg`` provider. All classes for this are in ``airflow.providers.apache.iceberg`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -78,4 +78,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/iceberg/docs/changelog.rst b/providers/apache/iceberg/docs/changelog.rst index c3d3d16dd9fea..b18ba160d7db0 100644 --- a/providers/apache/iceberg/docs/changelog.rst +++ b/providers/apache/iceberg/docs/changelog.rst @@ -26,6 +26,17 @@ Changelog --------- +2.0.2 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 2.0.1 ..... diff --git a/providers/apache/iceberg/docs/index.rst b/providers/apache/iceberg/docs/index.rst index 289cf375018be..f98184d8e7bef 100644 --- a/providers/apache/iceberg/docs/index.rst +++ b/providers/apache/iceberg/docs/index.rst @@ -73,7 +73,7 @@ apache-airflow-providers-apache-iceberg package `Iceberg `__ -Release: 2.0.1 +Release: 2.0.2 Provider package ---------------- @@ -126,5 +126,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-iceberg 2.0.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-iceberg 2.0.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-iceberg 2.0.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-iceberg 2.0.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/iceberg/provider.yaml b/providers/apache/iceberg/provider.yaml index 8ba84038777a1..2cff85b65dd29 100644 --- a/providers/apache/iceberg/provider.yaml +++ b/providers/apache/iceberg/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298402 +source-date-epoch: 1775591708 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 2.0.2 - 2.0.1 - 2.0.0 - 1.4.1 diff --git a/providers/apache/iceberg/pyproject.toml b/providers/apache/iceberg/pyproject.toml index a9e496feba58a..5e158b92eef54 100644 --- a/providers/apache/iceberg/pyproject.toml +++ b/providers/apache/iceberg/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-iceberg" -version = "2.0.1" +version = "2.0.2" description = "Provider package apache-airflow-providers-apache-iceberg for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -100,8 +100,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-iceberg/2.0.1" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-iceberg/2.0.1/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-iceberg/2.0.2" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-iceberg/2.0.2/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/iceberg/src/airflow/providers/apache/iceberg/__init__.py b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/__init__.py index 7324e33c66f75..eb7f2bea204d5 100644 --- a/providers/apache/iceberg/src/airflow/providers/apache/iceberg/__init__.py +++ b/providers/apache/iceberg/src/airflow/providers/apache/iceberg/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "2.0.1" +__version__ = "2.0.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/iceberg/tests/system/apache/iceberg/example_iceberg.py b/providers/apache/iceberg/tests/system/apache/iceberg/example_iceberg.py index e87fe74f95645..daf21db42dd37 100644 --- a/providers/apache/iceberg/tests/system/apache/iceberg/example_iceberg.py +++ b/providers/apache/iceberg/tests/system/apache/iceberg/example_iceberg.py @@ -50,5 +50,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/impala/README.rst b/providers/apache/impala/README.rst index cd174351eff8f..01e532c280371 100644 --- a/providers/apache/impala/README.rst +++ b/providers/apache/impala/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-impala`` -Release: ``1.9.1`` +Release: ``1.9.2`` `Apache Impala `__. @@ -36,7 +36,7 @@ This is a provider package for ``apache.impala`` provider. All classes for this are in ``airflow.providers.apache.impala`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -90,4 +90,4 @@ Extra Dependencies ============== ====================== The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/impala/docs/changelog.rst b/providers/apache/impala/docs/changelog.rst index eb4a4228af838..a9dcc527ef4b3 100644 --- a/providers/apache/impala/docs/changelog.rst +++ b/providers/apache/impala/docs/changelog.rst @@ -26,6 +26,17 @@ Changelog --------- +1.9.2 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.9.1 ..... diff --git a/providers/apache/impala/docs/index.rst b/providers/apache/impala/docs/index.rst index d53c801981eb0..815ed24d18977 100644 --- a/providers/apache/impala/docs/index.rst +++ b/providers/apache/impala/docs/index.rst @@ -84,7 +84,7 @@ apache-airflow-providers-apache-impala package `Apache Impala `__. -Release: 1.9.1 +Release: 1.9.2 Provider package ---------------- @@ -139,5 +139,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-impala 1.9.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-impala 1.9.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-impala 1.9.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-impala 1.9.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/impala/provider.yaml b/providers/apache/impala/provider.yaml index 7dd8636f8730a..b04a0adbdf758 100644 --- a/providers/apache/impala/provider.yaml +++ b/providers/apache/impala/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298417 +source-date-epoch: 1775591713 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 1.9.2 - 1.9.1 - 1.9.0 - 1.8.1 diff --git a/providers/apache/impala/pyproject.toml b/providers/apache/impala/pyproject.toml index 7234622d33d08..00dfdf8c27f66 100644 --- a/providers/apache/impala/pyproject.toml +++ b/providers/apache/impala/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-impala" -version = "1.9.1" +version = "1.9.2" description = "Provider package apache-airflow-providers-apache-impala for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -114,8 +114,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-impala/1.9.1" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-impala/1.9.1/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-impala/1.9.2" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-impala/1.9.2/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/impala/src/airflow/providers/apache/impala/__init__.py b/providers/apache/impala/src/airflow/providers/apache/impala/__init__.py index 57658e09bf0b5..bbcc2fdbe8c3e 100644 --- a/providers/apache/impala/src/airflow/providers/apache/impala/__init__.py +++ b/providers/apache/impala/src/airflow/providers/apache/impala/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.9.1" +__version__ = "1.9.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/impala/tests/system/apache/impala/example_impala.py b/providers/apache/impala/tests/system/apache/impala/example_impala.py index 702402c8e3d1e..d83edbf1289c3 100644 --- a/providers/apache/impala/tests/system/apache/impala/example_impala.py +++ b/providers/apache/impala/tests/system/apache/impala/example_impala.py @@ -85,5 +85,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/kafka/README.rst b/providers/apache/kafka/README.rst index 2937d42210973..08fa80a9a9712 100644 --- a/providers/apache/kafka/README.rst +++ b/providers/apache/kafka/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-kafka`` -Release: ``1.13.1`` +Release: ``1.13.2`` `Apache Kafka `__ @@ -36,7 +36,7 @@ This is a provider package for ``apache.kafka`` provider. All classes for this p are in ``airflow.providers.apache.kafka`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -93,4 +93,4 @@ Extra Dependencies ==================== ==================================================== The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/kafka/docs/changelog.rst b/providers/apache/kafka/docs/changelog.rst index afc87f4221381..68ff7b74d15e6 100644 --- a/providers/apache/kafka/docs/changelog.rst +++ b/providers/apache/kafka/docs/changelog.rst @@ -27,6 +27,17 @@ Changelog --------- +1.13.2 +...... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.13.1 ...... diff --git a/providers/apache/kafka/docs/index.rst b/providers/apache/kafka/docs/index.rst index 708cb8a02b7b4..2a3296b0a8c79 100644 --- a/providers/apache/kafka/docs/index.rst +++ b/providers/apache/kafka/docs/index.rst @@ -83,7 +83,7 @@ apache-airflow-providers-apache-kafka package `Apache Kafka `__ -Release: 1.13.1 +Release: 1.13.2 Provider package ---------------- @@ -141,5 +141,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-kafka 1.13.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-kafka 1.13.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-kafka 1.13.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-kafka 1.13.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/kafka/provider.yaml b/providers/apache/kafka/provider.yaml index fb682f73ba02d..aa102950e2917 100644 --- a/providers/apache/kafka/provider.yaml +++ b/providers/apache/kafka/provider.yaml @@ -21,7 +21,7 @@ name: Apache Kafka state: ready lifecycle: production -source-date-epoch: 1774298448 +source-date-epoch: 1775591721 description: | `Apache Kafka `__ # Note that those versions are maintained by release manager - do not update them manually @@ -29,6 +29,7 @@ description: | # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 1.13.2 - 1.13.1 - 1.13.0 - 1.12.0 diff --git a/providers/apache/kafka/pyproject.toml b/providers/apache/kafka/pyproject.toml index cfbb870eaa1e0..7f1adff1bb2cb 100644 --- a/providers/apache/kafka/pyproject.toml +++ b/providers/apache/kafka/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-kafka" -version = "1.13.1" +version = "1.13.2" description = "Provider package apache-airflow-providers-apache-kafka for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -115,8 +115,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-kafka/1.13.1" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-kafka/1.13.1/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-kafka/1.13.2" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-kafka/1.13.2/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/kafka/src/airflow/providers/apache/kafka/__init__.py b/providers/apache/kafka/src/airflow/providers/apache/kafka/__init__.py index 1a4d956dc207e..b7599c61a1f23 100644 --- a/providers/apache/kafka/src/airflow/providers/apache/kafka/__init__.py +++ b/providers/apache/kafka/src/airflow/providers/apache/kafka/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.13.1" +__version__ = "1.13.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/kafka/tests/system/apache/kafka/example_dag_event_listener.py b/providers/apache/kafka/tests/system/apache/kafka/example_dag_event_listener.py index f120574209364..5adbc739a596c 100644 --- a/providers/apache/kafka/tests/system/apache/kafka/example_dag_event_listener.py +++ b/providers/apache/kafka/tests/system/apache/kafka/example_dag_event_listener.py @@ -122,5 +122,5 @@ def wait_for_event(message, **context): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/kafka/tests/system/apache/kafka/example_dag_hello_kafka.py b/providers/apache/kafka/tests/system/apache/kafka/example_dag_hello_kafka.py index 377e37a7157a7..e52ec067a5157 100644 --- a/providers/apache/kafka/tests/system/apache/kafka/example_dag_hello_kafka.py +++ b/providers/apache/kafka/tests/system/apache/kafka/example_dag_hello_kafka.py @@ -242,5 +242,5 @@ def hello_kafka(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/kafka/tests/system/apache/kafka/example_dag_kafka_message_queue_trigger.py b/providers/apache/kafka/tests/system/apache/kafka/example_dag_kafka_message_queue_trigger.py index 7e2016f96b0e8..6ef90087dd761 100644 --- a/providers/apache/kafka/tests/system/apache/kafka/example_dag_kafka_message_queue_trigger.py +++ b/providers/apache/kafka/tests/system/apache/kafka/example_dag_kafka_message_queue_trigger.py @@ -51,5 +51,5 @@ def apply_function(message): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/kafka/tests/system/apache/kafka/example_dag_message_queue_trigger.py b/providers/apache/kafka/tests/system/apache/kafka/example_dag_message_queue_trigger.py index e07911b72983a..2ed791367da5f 100644 --- a/providers/apache/kafka/tests/system/apache/kafka/example_dag_message_queue_trigger.py +++ b/providers/apache/kafka/tests/system/apache/kafka/example_dag_message_queue_trigger.py @@ -47,5 +47,5 @@ def apply_function(message): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/kylin/README.rst b/providers/apache/kylin/README.rst index 35839b1590286..548625c4645a2 100644 --- a/providers/apache/kylin/README.rst +++ b/providers/apache/kylin/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-kylin`` -Release: ``3.10.3`` +Release: ``3.10.4`` `Apache Kylin `__ @@ -36,7 +36,7 @@ This is a provider package for ``apache.kylin`` provider. All classes for this p are in ``airflow.providers.apache.kylin`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -78,4 +78,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/kylin/docs/changelog.rst b/providers/apache/kylin/docs/changelog.rst index cb725ba1663be..24e2ce025b0ff 100644 --- a/providers/apache/kylin/docs/changelog.rst +++ b/providers/apache/kylin/docs/changelog.rst @@ -29,6 +29,17 @@ Changelog --------- +3.10.4 +...... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 3.10.3 ...... diff --git a/providers/apache/kylin/docs/index.rst b/providers/apache/kylin/docs/index.rst index e619421031ebc..1862d29dc1052 100644 --- a/providers/apache/kylin/docs/index.rst +++ b/providers/apache/kylin/docs/index.rst @@ -77,7 +77,7 @@ apache-airflow-providers-apache-kylin package `Apache Kylin `__ -Release: 3.10.3 +Release: 3.10.4 Provider package ---------------- @@ -130,5 +130,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-kylin 3.10.3 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-kylin 3.10.3 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-kylin 3.10.4 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-kylin 3.10.4 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/kylin/provider.yaml b/providers/apache/kylin/provider.yaml index fa7a110482c05..ba5e9d2e77312 100644 --- a/providers/apache/kylin/provider.yaml +++ b/providers/apache/kylin/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298459 +source-date-epoch: 1775591727 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.10.4 - 3.10.3 - 3.10.2 - 3.10.1 diff --git a/providers/apache/kylin/pyproject.toml b/providers/apache/kylin/pyproject.toml index 0cfa7e9e38c3a..5d88ec096230a 100644 --- a/providers/apache/kylin/pyproject.toml +++ b/providers/apache/kylin/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-kylin" -version = "3.10.3" +version = "3.10.4" description = "Provider package apache-airflow-providers-apache-kylin for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -99,8 +99,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-kylin/3.10.3" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-kylin/3.10.3/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-kylin/3.10.4" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-kylin/3.10.4/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/kylin/src/airflow/providers/apache/kylin/__init__.py b/providers/apache/kylin/src/airflow/providers/apache/kylin/__init__.py index 2aa8a5110c760..fe1634013dac8 100644 --- a/providers/apache/kylin/src/airflow/providers/apache/kylin/__init__.py +++ b/providers/apache/kylin/src/airflow/providers/apache/kylin/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.10.3" +__version__ = "3.10.4" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/kylin/tests/system/apache/kylin/example_kylin_dag.py b/providers/apache/kylin/tests/system/apache/kylin/example_kylin_dag.py index cf5833654b9ad..f6493a1d10840 100644 --- a/providers/apache/kylin/tests/system/apache/kylin/example_kylin_dag.py +++ b/providers/apache/kylin/tests/system/apache/kylin/example_kylin_dag.py @@ -129,5 +129,5 @@ def gen_build_time(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/livy/README.rst b/providers/apache/livy/README.rst index 056a82c983b86..8acf141c4d3f1 100644 --- a/providers/apache/livy/README.rst +++ b/providers/apache/livy/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-livy`` -Release: ``4.5.4`` +Release: ``4.5.5`` `Apache Livy `__ @@ -36,7 +36,7 @@ This is a provider package for ``apache.livy`` provider. All classes for this pr are in ``airflow.providers.apache.livy`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -80,4 +80,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/livy/docs/changelog.rst b/providers/apache/livy/docs/changelog.rst index ba5dcbdab8584..89c9bee0b0a69 100644 --- a/providers/apache/livy/docs/changelog.rst +++ b/providers/apache/livy/docs/changelog.rst @@ -28,6 +28,17 @@ Changelog --------- +4.5.5 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.5.4 ..... diff --git a/providers/apache/livy/docs/index.rst b/providers/apache/livy/docs/index.rst index 7ba5c962f44e9..94ed0c72e439d 100644 --- a/providers/apache/livy/docs/index.rst +++ b/providers/apache/livy/docs/index.rst @@ -76,7 +76,7 @@ apache-airflow-providers-apache-livy package `Apache Livy `__ -Release: 4.5.4 +Release: 4.5.5 Provider package ---------------- @@ -131,5 +131,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-livy 4.5.4 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-livy 4.5.4 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-livy 4.5.5 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-livy 4.5.5 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/livy/provider.yaml b/providers/apache/livy/provider.yaml index 1844bbaddfba8..7435d67265d08 100644 --- a/providers/apache/livy/provider.yaml +++ b/providers/apache/livy/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298473 +source-date-epoch: 1775591733 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 4.5.5 - 4.5.4 - 4.5.3 - 4.5.2 diff --git a/providers/apache/livy/pyproject.toml b/providers/apache/livy/pyproject.toml index 5d8cf5bc33beb..44971a2ad772b 100644 --- a/providers/apache/livy/pyproject.toml +++ b/providers/apache/livy/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-livy" -version = "4.5.4" +version = "4.5.5" description = "Provider package apache-airflow-providers-apache-livy for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -102,8 +102,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-livy/4.5.4" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-livy/4.5.4/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-livy/4.5.5" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-livy/4.5.5/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/livy/src/airflow/providers/apache/livy/__init__.py b/providers/apache/livy/src/airflow/providers/apache/livy/__init__.py index 52a98a7a783a1..506d0f395da05 100644 --- a/providers/apache/livy/src/airflow/providers/apache/livy/__init__.py +++ b/providers/apache/livy/src/airflow/providers/apache/livy/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.5.4" +__version__ = "4.5.5" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/livy/tests/system/apache/livy/example_livy.py b/providers/apache/livy/tests/system/apache/livy/example_livy.py index 8f3ea04a00867..727597b91988d 100644 --- a/providers/apache/livy/tests/system/apache/livy/example_livy.py +++ b/providers/apache/livy/tests/system/apache/livy/example_livy.py @@ -81,5 +81,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/pig/README.rst b/providers/apache/pig/README.rst index 480d78738f97b..73407b309c041 100644 --- a/providers/apache/pig/README.rst +++ b/providers/apache/pig/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-pig`` -Release: ``4.8.3`` +Release: ``4.8.4`` `Apache Pig `__ @@ -36,7 +36,7 @@ This is a provider package for ``apache.pig`` provider. All classes for this pro are in ``airflow.providers.apache.pig`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -77,4 +77,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/pig/docs/changelog.rst b/providers/apache/pig/docs/changelog.rst index 2beb4a535f06c..773b54f841c93 100644 --- a/providers/apache/pig/docs/changelog.rst +++ b/providers/apache/pig/docs/changelog.rst @@ -29,6 +29,17 @@ Changelog --------- +4.8.4 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.8.3 ..... diff --git a/providers/apache/pig/docs/index.rst b/providers/apache/pig/docs/index.rst index 3a3a2bbc1858a..39d2f65eb29a5 100644 --- a/providers/apache/pig/docs/index.rst +++ b/providers/apache/pig/docs/index.rst @@ -75,7 +75,7 @@ apache-airflow-providers-apache-pig package `Apache Pig `__ -Release: 4.8.3 +Release: 4.8.4 Provider package ---------------- @@ -127,5 +127,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-pig 4.8.3 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-pig 4.8.3 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-pig 4.8.4 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-pig 4.8.4 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/pig/provider.yaml b/providers/apache/pig/provider.yaml index 2992c6009b476..fda1239b73825 100644 --- a/providers/apache/pig/provider.yaml +++ b/providers/apache/pig/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298489 +source-date-epoch: 1775591739 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 4.8.4 - 4.8.3 - 4.8.2 - 4.8.1 diff --git a/providers/apache/pig/pyproject.toml b/providers/apache/pig/pyproject.toml index 1d543a930d05a..ad8b88ac1bb6a 100644 --- a/providers/apache/pig/pyproject.toml +++ b/providers/apache/pig/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-pig" -version = "4.8.3" +version = "4.8.4" description = "Provider package apache-airflow-providers-apache-pig for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -98,8 +98,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-pig/4.8.3" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-pig/4.8.3/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-pig/4.8.4" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-pig/4.8.4/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/pig/src/airflow/providers/apache/pig/__init__.py b/providers/apache/pig/src/airflow/providers/apache/pig/__init__.py index 959403b01f003..03231be4547f3 100644 --- a/providers/apache/pig/src/airflow/providers/apache/pig/__init__.py +++ b/providers/apache/pig/src/airflow/providers/apache/pig/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.8.3" +__version__ = "4.8.4" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/pig/tests/system/apache/pig/example_pig.py b/providers/apache/pig/tests/system/apache/pig/example_pig.py index 4d47bf5e6570c..bd9e1c824851a 100644 --- a/providers/apache/pig/tests/system/apache/pig/example_pig.py +++ b/providers/apache/pig/tests/system/apache/pig/example_pig.py @@ -46,5 +46,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/pinot/README.rst b/providers/apache/pinot/README.rst index dc5c30611c887..53b4cce5f33c2 100644 --- a/providers/apache/pinot/README.rst +++ b/providers/apache/pinot/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-pinot`` -Release: ``4.10.1`` +Release: ``4.10.2`` `Apache Pinot `__ @@ -36,7 +36,7 @@ This is a provider package for ``apache.pinot`` provider. All classes for this p are in ``airflow.providers.apache.pinot`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -80,4 +80,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/pinot/docs/changelog.rst b/providers/apache/pinot/docs/changelog.rst index 85e3a8e68b893..e467743794c58 100644 --- a/providers/apache/pinot/docs/changelog.rst +++ b/providers/apache/pinot/docs/changelog.rst @@ -29,6 +29,17 @@ Changelog --------- +4.10.2 +...... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.10.1 ...... diff --git a/providers/apache/pinot/docs/index.rst b/providers/apache/pinot/docs/index.rst index 3b363c066e091..fa7235187eac7 100644 --- a/providers/apache/pinot/docs/index.rst +++ b/providers/apache/pinot/docs/index.rst @@ -70,7 +70,7 @@ apache-airflow-providers-apache-pinot package `Apache Pinot `__ -Release: 4.10.1 +Release: 4.10.2 Provider package ---------------- @@ -125,5 +125,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-pinot 4.10.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-pinot 4.10.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-pinot 4.10.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-pinot 4.10.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/pinot/provider.yaml b/providers/apache/pinot/provider.yaml index 3c50bf17032c3..acc0a87ec2417 100644 --- a/providers/apache/pinot/provider.yaml +++ b/providers/apache/pinot/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298497 +source-date-epoch: 1775591745 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 4.10.2 - 4.10.1 - 4.10.0 - 4.9.2 diff --git a/providers/apache/pinot/pyproject.toml b/providers/apache/pinot/pyproject.toml index 57123e0bda142..4d844cd81a108 100644 --- a/providers/apache/pinot/pyproject.toml +++ b/providers/apache/pinot/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-pinot" -version = "4.10.1" +version = "4.10.2" description = "Provider package apache-airflow-providers-apache-pinot for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -102,8 +102,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-pinot/4.10.1" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-pinot/4.10.1/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-pinot/4.10.2" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-pinot/4.10.2/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/pinot/src/airflow/providers/apache/pinot/__init__.py b/providers/apache/pinot/src/airflow/providers/apache/pinot/__init__.py index 6acb85dac29be..340c3ab45066d 100644 --- a/providers/apache/pinot/src/airflow/providers/apache/pinot/__init__.py +++ b/providers/apache/pinot/src/airflow/providers/apache/pinot/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.10.1" +__version__ = "4.10.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/pinot/tests/system/apache/pinot/example_pinot_dag.py b/providers/apache/pinot/tests/system/apache/pinot/example_pinot_dag.py index 83021c4c51b9f..965b7415f9e8b 100644 --- a/providers/apache/pinot/tests/system/apache/pinot/example_pinot_dag.py +++ b/providers/apache/pinot/tests/system/apache/pinot/example_pinot_dag.py @@ -61,5 +61,5 @@ def pinot_dbi_api(): from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/spark/README.rst b/providers/apache/spark/README.rst index 465c6851d0eb0..562370cc3b44e 100644 --- a/providers/apache/spark/README.rst +++ b/providers/apache/spark/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-spark`` -Release: ``6.0.0`` +Release: ``6.0.1`` `Apache Spark `__ @@ -36,7 +36,7 @@ This is a provider package for ``apache.spark`` provider. All classes for this p are in ``airflow.providers.apache.spark`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -91,4 +91,4 @@ Extra Dependencies =================== =================================================== The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/spark/docs/changelog.rst b/providers/apache/spark/docs/changelog.rst index a63473230a5f8..a1b4b139367ea 100644 --- a/providers/apache/spark/docs/changelog.rst +++ b/providers/apache/spark/docs/changelog.rst @@ -29,6 +29,18 @@ Changelog --------- +6.0.1 +..... + +Misc +~~~~ + +* ``Add post_submit_commands to SparkSubmitHook for sidecar lifecycle management (#64391)`` +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 6.0.0 ..... diff --git a/providers/apache/spark/docs/index.rst b/providers/apache/spark/docs/index.rst index 75dd706f5ae8a..90edff72e243f 100644 --- a/providers/apache/spark/docs/index.rst +++ b/providers/apache/spark/docs/index.rst @@ -77,7 +77,7 @@ apache-airflow-providers-apache-spark package `Apache Spark `__ -Release: 6.0.0 +Release: 6.0.1 Provider package ---------------- @@ -132,5 +132,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-spark 6.0.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-spark 6.0.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-spark 6.0.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-spark 6.0.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/spark/provider.yaml b/providers/apache/spark/provider.yaml index 15bfe3b7ee2d2..eeaa8cc13e219 100644 --- a/providers/apache/spark/provider.yaml +++ b/providers/apache/spark/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298523 +source-date-epoch: 1775591754 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 6.0.1 - 6.0.0 - 5.6.0 - 5.5.1 diff --git a/providers/apache/spark/pyproject.toml b/providers/apache/spark/pyproject.toml index 118a49823cb32..c579b249b8c46 100644 --- a/providers/apache/spark/pyproject.toml +++ b/providers/apache/spark/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-spark" -version = "6.0.0" +version = "6.0.1" description = "Provider package apache-airflow-providers-apache-spark for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -116,8 +116,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-spark/6.0.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-spark/6.0.0/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-spark/6.0.1" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-spark/6.0.1/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/spark/src/airflow/providers/apache/spark/__init__.py b/providers/apache/spark/src/airflow/providers/apache/spark/__init__.py index 5031fd67e8c94..b914a2892b15f 100644 --- a/providers/apache/spark/src/airflow/providers/apache/spark/__init__.py +++ b/providers/apache/spark/src/airflow/providers/apache/spark/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "6.0.0" +__version__ = "6.0.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/spark/tests/system/apache/spark/example_pyspark.py b/providers/apache/spark/tests/system/apache/spark/example_pyspark.py index a855b401d3ffd..46e84d53b97ef 100644 --- a/providers/apache/spark/tests/system/apache/spark/example_pyspark.py +++ b/providers/apache/spark/tests/system/apache/spark/example_pyspark.py @@ -75,5 +75,5 @@ def print_df(df: pd.DataFrame): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/spark/tests/system/apache/spark/example_spark_dag.py b/providers/apache/spark/tests/system/apache/spark/example_spark_dag.py index 8e8b0f959aa71..dd3b3a43f2daf 100644 --- a/providers/apache/spark/tests/system/apache/spark/example_spark_dag.py +++ b/providers/apache/spark/tests/system/apache/spark/example_spark_dag.py @@ -88,5 +88,5 @@ def my_pyspark_job(spark): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apache/tinkerpop/README.rst b/providers/apache/tinkerpop/README.rst index 4cd60891c7350..e41c68fb2449e 100644 --- a/providers/apache/tinkerpop/README.rst +++ b/providers/apache/tinkerpop/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apache-tinkerpop`` -Release: ``1.1.2`` +Release: ``1.1.3`` `Apache TinkerPop `__. @@ -38,7 +38,7 @@ This is a provider package for ``apache.tinkerpop`` provider. All classes for th are in ``airflow.providers.apache.tinkerpop`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -80,4 +80,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apache/tinkerpop/docs/changelog.rst b/providers/apache/tinkerpop/docs/changelog.rst index 19bf74b46df13..0839926a36c03 100644 --- a/providers/apache/tinkerpop/docs/changelog.rst +++ b/providers/apache/tinkerpop/docs/changelog.rst @@ -18,6 +18,17 @@ ``apache-airflow-providers-apache-tinkerpop`` +1.1.3 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.1.2 ..... diff --git a/providers/apache/tinkerpop/docs/index.rst b/providers/apache/tinkerpop/docs/index.rst index 2218619665d03..7e86ba085c37f 100644 --- a/providers/apache/tinkerpop/docs/index.rst +++ b/providers/apache/tinkerpop/docs/index.rst @@ -78,7 +78,7 @@ Apache TinkerPop is a graph computing framework for both graph databases (OLTP) systems (OLAP) and Gremlin is its graph traversal language. -Release: 1.1.2 +Release: 1.1.3 Provider package ---------------- @@ -131,5 +131,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apache-tinkerpop 1.1.2 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apache-tinkerpop 1.1.2 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-tinkerpop 1.1.3 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apache-tinkerpop 1.1.3 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apache/tinkerpop/provider.yaml b/providers/apache/tinkerpop/provider.yaml index b98dfeb73fed4..6c07b97b88a29 100644 --- a/providers/apache/tinkerpop/provider.yaml +++ b/providers/apache/tinkerpop/provider.yaml @@ -23,9 +23,10 @@ description: | systems (OLAP) and Gremlin is its graph traversal language. state: ready lifecycle: production -source-date-epoch: 1774298539 +source-date-epoch: 1775591763 # note that these versions are maintained by the release manager - do not update them manually versions: + - 1.1.3 - 1.1.2 - 1.1.1 - 1.1.0 diff --git a/providers/apache/tinkerpop/pyproject.toml b/providers/apache/tinkerpop/pyproject.toml index 96dc56f46f838..a13cdd6950c29 100644 --- a/providers/apache/tinkerpop/pyproject.toml +++ b/providers/apache/tinkerpop/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apache-tinkerpop" -version = "1.1.2" +version = "1.1.3" description = "Provider package apache-airflow-providers-apache-tinkerpop for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -99,8 +99,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-tinkerpop/1.1.2" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-tinkerpop/1.1.2/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-tinkerpop/1.1.3" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apache-tinkerpop/1.1.3/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/__init__.py b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/__init__.py index 842b2339068c8..ddfdf8b0013f0 100644 --- a/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/__init__.py +++ b/providers/apache/tinkerpop/src/airflow/providers/apache/tinkerpop/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.1.2" +__version__ = "1.1.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/apache/tinkerpop/tests/system/apache/tinkerpop/example_gremlin_dag.py b/providers/apache/tinkerpop/tests/system/apache/tinkerpop/example_gremlin_dag.py index 94f3e304125c6..0becc3ee1e39d 100644 --- a/providers/apache/tinkerpop/tests/system/apache/tinkerpop/example_gremlin_dag.py +++ b/providers/apache/tinkerpop/tests/system/apache/tinkerpop/example_gremlin_dag.py @@ -48,5 +48,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/apprise/README.rst b/providers/apprise/README.rst index 1517fd5bc2664..e22b7d71335eb 100644 --- a/providers/apprise/README.rst +++ b/providers/apprise/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-apprise`` -Release: ``2.3.2`` +Release: ``2.3.3`` `Apprise `__ @@ -36,7 +36,7 @@ This is a provider package for ``apprise`` provider. All classes for this provid are in ``airflow.providers.apprise`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -78,4 +78,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/apprise/docs/changelog.rst b/providers/apprise/docs/changelog.rst index 9b77076bbb8e1..a829353b88aaf 100644 --- a/providers/apprise/docs/changelog.rst +++ b/providers/apprise/docs/changelog.rst @@ -27,6 +27,17 @@ Changelog --------- +2.3.3 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 2.3.2 ..... diff --git a/providers/apprise/docs/index.rst b/providers/apprise/docs/index.rst index 1e94f3052e1ca..68f6555a7b5a2 100644 --- a/providers/apprise/docs/index.rst +++ b/providers/apprise/docs/index.rst @@ -64,7 +64,7 @@ apache-airflow-providers-apprise package `Apprise `__ -Release: 2.3.2 +Release: 2.3.3 Provider package ---------------- @@ -117,5 +117,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-apprise 2.3.2 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-apprise 2.3.2 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apprise 2.3.3 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-apprise 2.3.3 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/apprise/provider.yaml b/providers/apprise/provider.yaml index c95c6c7715dca..25a707baf19e4 100644 --- a/providers/apprise/provider.yaml +++ b/providers/apprise/provider.yaml @@ -25,13 +25,14 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298549 +source-date-epoch: 1775591771 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 2.3.3 - 2.3.2 - 2.3.1 - 2.3.0 diff --git a/providers/apprise/pyproject.toml b/providers/apprise/pyproject.toml index 04307539c6792..e4e73c4177ae3 100644 --- a/providers/apprise/pyproject.toml +++ b/providers/apprise/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-apprise" -version = "2.3.2" +version = "2.3.3" description = "Provider package apache-airflow-providers-apprise for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -99,8 +99,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apprise/2.3.2" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apprise/2.3.2/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-apprise/2.3.3" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-apprise/2.3.3/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/apprise/src/airflow/providers/apprise/__init__.py b/providers/apprise/src/airflow/providers/apprise/__init__.py index e2f963e0af649..28a1bf9474bf6 100644 --- a/providers/apprise/src/airflow/providers/apprise/__init__.py +++ b/providers/apprise/src/airflow/providers/apprise/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "2.3.2" +__version__ = "2.3.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/arangodb/README.rst b/providers/arangodb/README.rst index 8c8b8f310eeeb..ffdf0aa7d3a2f 100644 --- a/providers/arangodb/README.rst +++ b/providers/arangodb/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-arangodb`` -Release: ``2.9.3`` +Release: ``2.9.4`` `ArangoDB `__ @@ -36,7 +36,7 @@ This is a provider package for ``arangodb`` provider. All classes for this provi are in ``airflow.providers.arangodb`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -78,4 +78,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/arangodb/docs/changelog.rst b/providers/arangodb/docs/changelog.rst index 64263188828cd..81243f57923ad 100644 --- a/providers/arangodb/docs/changelog.rst +++ b/providers/arangodb/docs/changelog.rst @@ -28,6 +28,17 @@ Changelog --------- +2.9.4 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 2.9.3 ..... diff --git a/providers/arangodb/docs/index.rst b/providers/arangodb/docs/index.rst index 1f43f903af31a..e6aa182be93a0 100644 --- a/providers/arangodb/docs/index.rst +++ b/providers/arangodb/docs/index.rst @@ -70,7 +70,7 @@ apache-airflow-providers-arangodb package `ArangoDB `__ -Release: 2.9.3 +Release: 2.9.4 Provider package ---------------- @@ -123,5 +123,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-arangodb 2.9.3 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-arangodb 2.9.3 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-arangodb 2.9.4 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-arangodb 2.9.4 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/arangodb/provider.yaml b/providers/arangodb/provider.yaml index c2c001832b0b8..9c6a8d8a21b5c 100644 --- a/providers/arangodb/provider.yaml +++ b/providers/arangodb/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298564 +source-date-epoch: 1775591776 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 2.9.4 - 2.9.3 - 2.9.2 - 2.9.1 diff --git a/providers/arangodb/pyproject.toml b/providers/arangodb/pyproject.toml index 37a65f5d9c564..f8ff120b5da2e 100644 --- a/providers/arangodb/pyproject.toml +++ b/providers/arangodb/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-arangodb" -version = "2.9.3" +version = "2.9.4" description = "Provider package apache-airflow-providers-arangodb for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -99,8 +99,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-arangodb/2.9.3" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-arangodb/2.9.3/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-arangodb/2.9.4" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-arangodb/2.9.4/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/arangodb/src/airflow/providers/arangodb/__init__.py b/providers/arangodb/src/airflow/providers/arangodb/__init__.py index 72405732f9269..3d3fd52172063 100644 --- a/providers/arangodb/src/airflow/providers/arangodb/__init__.py +++ b/providers/arangodb/src/airflow/providers/arangodb/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "2.9.3" +__version__ = "2.9.4" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/asana/README.rst b/providers/asana/README.rst index 11f1e89cb223d..1a82bcaea495e 100644 --- a/providers/asana/README.rst +++ b/providers/asana/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-asana`` -Release: ``2.11.2`` +Release: ``2.11.3`` `Asana `__ @@ -36,7 +36,7 @@ This is a provider package for ``asana`` provider. All classes for this provider are in ``airflow.providers.asana`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -78,4 +78,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/asana/docs/changelog.rst b/providers/asana/docs/changelog.rst index 293190b92d87b..ba9d0b3f0f76b 100644 --- a/providers/asana/docs/changelog.rst +++ b/providers/asana/docs/changelog.rst @@ -26,6 +26,17 @@ Changelog --------- +2.11.3 +...... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 2.11.2 ...... diff --git a/providers/asana/docs/index.rst b/providers/asana/docs/index.rst index 5300e13af463c..2d5c99980613d 100644 --- a/providers/asana/docs/index.rst +++ b/providers/asana/docs/index.rst @@ -77,7 +77,7 @@ apache-airflow-providers-asana package `Asana `__ -Release: 2.11.2 +Release: 2.11.3 Provider package ---------------- @@ -130,5 +130,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-asana 2.11.2 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-asana 2.11.2 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-asana 2.11.3 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-asana 2.11.3 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/asana/provider.yaml b/providers/asana/provider.yaml index 3bf54f515b497..fc4da6080dc49 100644 --- a/providers/asana/provider.yaml +++ b/providers/asana/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298575 +source-date-epoch: 1775591780 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 2.11.3 - 2.11.2 - 2.11.1 - 2.11.0 diff --git a/providers/asana/pyproject.toml b/providers/asana/pyproject.toml index 062269322f664..03f10757b7d11 100644 --- a/providers/asana/pyproject.toml +++ b/providers/asana/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-asana" -version = "2.11.2" +version = "2.11.3" description = "Provider package apache-airflow-providers-asana for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -99,8 +99,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-asana/2.11.2" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-asana/2.11.2/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-asana/2.11.3" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-asana/2.11.3/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/asana/src/airflow/providers/asana/__init__.py b/providers/asana/src/airflow/providers/asana/__init__.py index 58f19c334e666..029932becaa10 100644 --- a/providers/asana/src/airflow/providers/asana/__init__.py +++ b/providers/asana/src/airflow/providers/asana/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "2.11.2" +__version__ = "2.11.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/asana/tests/system/asana/example_asana.py b/providers/asana/tests/system/asana/example_asana.py index 1b52e2101890f..a7f8bf210274f 100644 --- a/providers/asana/tests/system/asana/example_asana.py +++ b/providers/asana/tests/system/asana/example_asana.py @@ -109,5 +109,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/atlassian/jira/README.rst b/providers/atlassian/jira/README.rst index cb6b34974494a..ff94e6886b9b4 100644 --- a/providers/atlassian/jira/README.rst +++ b/providers/atlassian/jira/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-atlassian-jira`` -Release: ``3.3.2`` +Release: ``3.3.3`` `Atlassian Jira `__ @@ -36,7 +36,7 @@ This is a provider package for ``atlassian.jira`` provider. All classes for this are in ``airflow.providers.atlassian.jira`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -80,4 +80,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/atlassian/jira/docs/changelog.rst b/providers/atlassian/jira/docs/changelog.rst index f285ba00489ac..9afbfe79d4d61 100644 --- a/providers/atlassian/jira/docs/changelog.rst +++ b/providers/atlassian/jira/docs/changelog.rst @@ -27,6 +27,18 @@ Changelog --------- +3.3.3 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``CI: Upgrade important CI environment (#64744)`` + 3.3.2 ..... diff --git a/providers/atlassian/jira/docs/index.rst b/providers/atlassian/jira/docs/index.rst index 8ef5dc7cc2e06..882597c047a5e 100644 --- a/providers/atlassian/jira/docs/index.rst +++ b/providers/atlassian/jira/docs/index.rst @@ -69,7 +69,7 @@ apache-airflow-providers-atlassian-jira package `Atlassian Jira `__ -Release: 3.3.2 +Release: 3.3.3 Provider package ---------------- @@ -124,5 +124,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-atlassian-jira 3.3.2 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-atlassian-jira 3.3.2 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-atlassian-jira 3.3.3 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-atlassian-jira 3.3.3 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/atlassian/jira/provider.yaml b/providers/atlassian/jira/provider.yaml index 4c6ce85c0f0c4..bddee25f75cdb 100644 --- a/providers/atlassian/jira/provider.yaml +++ b/providers/atlassian/jira/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298583 +source-date-epoch: 1775591790 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.3.3 - 3.3.2 - 3.3.1 - 3.3.0 diff --git a/providers/atlassian/jira/pyproject.toml b/providers/atlassian/jira/pyproject.toml index 360f014f2fbf1..664921955c243 100644 --- a/providers/atlassian/jira/pyproject.toml +++ b/providers/atlassian/jira/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-atlassian-jira" -version = "3.3.2" +version = "3.3.3" description = "Provider package apache-airflow-providers-atlassian-jira for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -101,8 +101,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-atlassian-jira/3.3.2" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-atlassian-jira/3.3.2/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-atlassian-jira/3.3.3" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-atlassian-jira/3.3.3/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/atlassian/jira/src/airflow/providers/atlassian/jira/__init__.py b/providers/atlassian/jira/src/airflow/providers/atlassian/jira/__init__.py index 3c58e14609994..1ad0b5f33e25c 100644 --- a/providers/atlassian/jira/src/airflow/providers/atlassian/jira/__init__.py +++ b/providers/atlassian/jira/src/airflow/providers/atlassian/jira/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.3.2" +__version__ = "3.3.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/celery/README.rst b/providers/celery/README.rst index 25349344a01f7..1c77e5fa9f0df 100644 --- a/providers/celery/README.rst +++ b/providers/celery/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-celery`` -Release: ``3.17.2`` +Release: ``3.18.0`` `Celery `__ @@ -36,7 +36,7 @@ This is a provider package for ``celery`` provider. All classes for this provide are in ``airflow.providers.celery`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -89,4 +89,4 @@ Extra Dependencies =================== =================================================== The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/celery/docs/changelog.rst b/providers/celery/docs/changelog.rst index 39594cd99024e..b4b325d75e8fb 100644 --- a/providers/celery/docs/changelog.rst +++ b/providers/celery/docs/changelog.rst @@ -27,6 +27,29 @@ Changelog --------- +3.18.0 +...... + +Features +~~~~~~~~ + +* ``Ignore redelivered message for already-running task (#64052)`` + +Bug Fixes +~~~~~~~~~ + +* ``Fix amqps:// SSL config and celery_config_options bypass (#64392)`` + +Misc +~~~~ + +* ``Clean up CeleryExecutor to use workload terminology and typing (#63888)`` +* ``Compat sdk conf follow-up: Celery, Common AI, FAB, Edge3 (#64292)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Fix Celery tests after conf-backcompat merge (#64388)`` + 3.17.2 ...... diff --git a/providers/celery/docs/index.rst b/providers/celery/docs/index.rst index 640afad2d935e..559d64efeb6e7 100644 --- a/providers/celery/docs/index.rst +++ b/providers/celery/docs/index.rst @@ -67,7 +67,7 @@ apache-airflow-providers-celery package `Celery `__ -Release: 3.17.2 +Release: 3.18.0 Provider package ---------------- @@ -122,5 +122,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-celery 3.17.2 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-celery 3.17.2 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-celery 3.18.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-celery 3.18.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/celery/provider.yaml b/providers/celery/provider.yaml index 0da7df9f09a20..b89ea4e16ae50 100644 --- a/providers/celery/provider.yaml +++ b/providers/celery/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298619 +source-date-epoch: 1775591849 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.18.0 - 3.17.2 - 3.17.1 - 3.17.0 diff --git a/providers/celery/pyproject.toml b/providers/celery/pyproject.toml index 703e00a4065a2..2d075e6dbea67 100644 --- a/providers/celery/pyproject.toml +++ b/providers/celery/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-celery" -version = "3.17.2" +version = "3.18.0" description = "Provider package apache-airflow-providers-celery for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -112,8 +112,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-celery/3.17.2" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-celery/3.17.2/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-celery/3.18.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-celery/3.18.0/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/celery/src/airflow/providers/celery/__init__.py b/providers/celery/src/airflow/providers/celery/__init__.py index aa5de35178e48..246882caa33c1 100644 --- a/providers/celery/src/airflow/providers/celery/__init__.py +++ b/providers/celery/src/airflow/providers/celery/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.17.2" +__version__ = "3.18.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/cloudant/README.rst b/providers/cloudant/README.rst index 365059e7e216d..7abcd89a7fb6d 100644 --- a/providers/cloudant/README.rst +++ b/providers/cloudant/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-cloudant`` -Release: ``4.3.3`` +Release: ``4.3.4`` `IBM Cloudant `__ @@ -36,7 +36,7 @@ This is a provider package for ``cloudant`` provider. All classes for this provi are in ``airflow.providers.cloudant`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -78,4 +78,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/cloudant/docs/changelog.rst b/providers/cloudant/docs/changelog.rst index c1e649c2a5df2..bc9945e115ab8 100644 --- a/providers/cloudant/docs/changelog.rst +++ b/providers/cloudant/docs/changelog.rst @@ -27,6 +27,17 @@ Changelog --------- +4.3.4 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.3.3 ..... diff --git a/providers/cloudant/docs/index.rst b/providers/cloudant/docs/index.rst index 7742773deccca..b0254e06ad447 100644 --- a/providers/cloudant/docs/index.rst +++ b/providers/cloudant/docs/index.rst @@ -55,7 +55,7 @@ apache-airflow-providers-cloudant package `IBM Cloudant `__ -Release: 4.3.3 +Release: 4.3.4 Provider package ---------------- @@ -108,5 +108,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-cloudant 4.3.3 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-cloudant 4.3.3 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-cloudant 4.3.4 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-cloudant 4.3.4 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/cloudant/provider.yaml b/providers/cloudant/provider.yaml index 276c602cc7614..105b3471303ba 100644 --- a/providers/cloudant/provider.yaml +++ b/providers/cloudant/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298722 +source-date-epoch: 1775591858 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 4.3.4 - 4.3.3 - 4.3.2 - 4.3.1 diff --git a/providers/cloudant/pyproject.toml b/providers/cloudant/pyproject.toml index bb146ca6d41f8..330485991c708 100644 --- a/providers/cloudant/pyproject.toml +++ b/providers/cloudant/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-cloudant" -version = "4.3.3" +version = "4.3.4" description = "Provider package apache-airflow-providers-cloudant for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -99,8 +99,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cloudant/4.3.3" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cloudant/4.3.3/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cloudant/4.3.4" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cloudant/4.3.4/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/cloudant/src/airflow/providers/cloudant/__init__.py b/providers/cloudant/src/airflow/providers/cloudant/__init__.py index 634cbeff5c89b..67ce7344ac292 100644 --- a/providers/cloudant/src/airflow/providers/cloudant/__init__.py +++ b/providers/cloudant/src/airflow/providers/cloudant/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.3.3" +__version__ = "4.3.4" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/cncf/kubernetes/README.rst b/providers/cncf/kubernetes/README.rst index f116817986dad..90f1b2069af4b 100644 --- a/providers/cncf/kubernetes/README.rst +++ b/providers/cncf/kubernetes/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-cncf-kubernetes`` -Release: ``10.15.0`` +Release: ``10.16.0`` `Kubernetes `__ @@ -36,7 +36,7 @@ This is a provider package for ``cncf.kubernetes`` provider. All classes for thi are in ``airflow.providers.cncf.kubernetes`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -84,4 +84,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/cncf/kubernetes/docs/changelog.rst b/providers/cncf/kubernetes/docs/changelog.rst index b8ab91f7fe09f..fdb53f085b9e5 100644 --- a/providers/cncf/kubernetes/docs/changelog.rst +++ b/providers/cncf/kubernetes/docs/changelog.rst @@ -27,6 +27,32 @@ Changelog --------- +10.16.0 +....... + +Features +~~~~~~~~ + +* ``Add retries for '_write_logs' method in 'KubernetesPodOperator' (#64471)`` + +Bug Fixes +~~~~~~~~~ + +* ``Handle rate limiting of K8s API server in K8s executor (#64504)`` + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` +* ``Log on_kill job deletion in kubernetes spark operator at INFO level (#64633)`` +* ``Update cncf's import conf path to use common compat SDK (#64143)`` +* ``Fix advertising some of the missing provider capabilities via provider info (#64127)`` +* ``Add explicit type annotations to k8s code to fix mypy (#64260)`` +* ``Pass parameters to Kubernetes methods conditionally (#64242)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 10.15.0 ....... diff --git a/providers/cncf/kubernetes/docs/index.rst b/providers/cncf/kubernetes/docs/index.rst index 2cc40d4d73142..d76214c0ed8a5 100644 --- a/providers/cncf/kubernetes/docs/index.rst +++ b/providers/cncf/kubernetes/docs/index.rst @@ -88,7 +88,7 @@ apache-airflow-providers-cncf-kubernetes package `Kubernetes `__ -Release: 10.15.0 +Release: 10.16.0 Provider package ---------------- @@ -147,5 +147,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-cncf-kubernetes 10.15.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-cncf-kubernetes 10.15.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-cncf-kubernetes 10.16.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-cncf-kubernetes 10.16.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/cncf/kubernetes/provider.yaml b/providers/cncf/kubernetes/provider.yaml index 2923ada07c8fc..5d19bfa52417d 100644 --- a/providers/cncf/kubernetes/provider.yaml +++ b/providers/cncf/kubernetes/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298846 +source-date-epoch: 1775591918 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 10.16.0 - 10.15.0 - 10.14.0 - 10.13.0 diff --git a/providers/cncf/kubernetes/pyproject.toml b/providers/cncf/kubernetes/pyproject.toml index bf8fcb170f898..27dc75938d2c0 100644 --- a/providers/cncf/kubernetes/pyproject.toml +++ b/providers/cncf/kubernetes/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-cncf-kubernetes" -version = "10.15.0" +version = "10.16.0" description = "Provider package apache-airflow-providers-cncf-kubernetes for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -121,8 +121,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.15.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.15.0/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.16.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cncf-kubernetes/10.16.0/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/__init__.py b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/__init__.py index 611ce035b08d7..8321b669c2fb9 100644 --- a/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/__init__.py +++ b/providers/cncf/kubernetes/src/airflow/providers/cncf/kubernetes/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "10.15.0" +__version__ = "10.16.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py index dc74895a3e0fd..732831e0e6413 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes.py @@ -175,5 +175,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py index 7eb08442be3ba..e245314386c60 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_async.py @@ -206,5 +206,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_cmd_decorator.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_cmd_decorator.py index 3235dffe0f3c6..ef265d4530eb2 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_cmd_decorator.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_cmd_decorator.py @@ -72,5 +72,5 @@ def apply_templating(message: str) -> list[str]: from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_decorator.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_decorator.py index e1178d37313fa..90de9231c1747 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_decorator.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_decorator.py @@ -70,5 +70,5 @@ def print_pattern(): from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py index c81ff48e792d4..685279e8ef763 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_job.py @@ -102,5 +102,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_kueue.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_kueue.py index 9aac715fdbed9..60d11fdb2d6b8 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_kueue.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_kueue.py @@ -140,5 +140,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_resource.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_resource.py index e7f6a4d42d32e..25a849469c861 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_resource.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_kubernetes_resource.py @@ -80,5 +80,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_spark_kubernetes.py b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_spark_kubernetes.py index b244a9a800bab..1fa022f17ee24 100644 --- a/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_spark_kubernetes.py +++ b/providers/cncf/kubernetes/tests/system/cncf/kubernetes/example_spark_kubernetes.py @@ -84,5 +84,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/cohere/README.rst b/providers/cohere/README.rst index 48a34219ccceb..fa359ee3501f6 100644 --- a/providers/cohere/README.rst +++ b/providers/cohere/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-cohere`` -Release: ``1.6.4`` +Release: ``1.6.5`` `Cohere `__ @@ -36,7 +36,7 @@ This is a provider package for ``cohere`` provider. All classes for this provide are in ``airflow.providers.cohere`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -80,4 +80,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/cohere/docs/changelog.rst b/providers/cohere/docs/changelog.rst index b9528fa0da996..a286f6e6acab0 100644 --- a/providers/cohere/docs/changelog.rst +++ b/providers/cohere/docs/changelog.rst @@ -20,6 +20,17 @@ Changelog --------- +1.6.5 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 1.6.4 ..... diff --git a/providers/cohere/docs/index.rst b/providers/cohere/docs/index.rst index 57018caa51a4d..3bd9e1005ae1d 100644 --- a/providers/cohere/docs/index.rst +++ b/providers/cohere/docs/index.rst @@ -71,7 +71,7 @@ apache-airflow-providers-cohere package `Cohere `__ -Release: 1.6.4 +Release: 1.6.5 Provider package ---------------- @@ -126,5 +126,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-cohere 1.6.4 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-cohere 1.6.4 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-cohere 1.6.5 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-cohere 1.6.5 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/cohere/provider.yaml b/providers/cohere/provider.yaml index bf5189a2a1a06..0d028a1844510 100644 --- a/providers/cohere/provider.yaml +++ b/providers/cohere/provider.yaml @@ -25,13 +25,14 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298858 +source-date-epoch: 1775591929 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 1.6.5 - 1.6.4 - 1.6.3 - 1.6.2 diff --git a/providers/cohere/pyproject.toml b/providers/cohere/pyproject.toml index 4cfb29feb40dc..1ecb90ef3c333 100644 --- a/providers/cohere/pyproject.toml +++ b/providers/cohere/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-cohere" -version = "1.6.4" +version = "1.6.5" description = "Provider package apache-airflow-providers-cohere for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -101,8 +101,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cohere/1.6.4" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cohere/1.6.4/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-cohere/1.6.5" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-cohere/1.6.5/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/cohere/src/airflow/providers/cohere/__init__.py b/providers/cohere/src/airflow/providers/cohere/__init__.py index f24971215052f..0b91dcf5bb757 100644 --- a/providers/cohere/src/airflow/providers/cohere/__init__.py +++ b/providers/cohere/src/airflow/providers/cohere/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.6.4" +__version__ = "1.6.5" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/cohere/tests/system/cohere/example_cohere_embedding_operator.py b/providers/cohere/tests/system/cohere/example_cohere_embedding_operator.py index d8907260d3fd7..af67e1003b927 100644 --- a/providers/cohere/tests/system/cohere/example_cohere_embedding_operator.py +++ b/providers/cohere/tests/system/cohere/example_cohere_embedding_operator.py @@ -36,5 +36,5 @@ from tests_common.test_utils.system_tests import get_test_run -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/common/ai/README.rst b/providers/common/ai/README.rst index 004ef2018a3b1..1181e97391741 100644 --- a/providers/common/ai/README.rst +++ b/providers/common/ai/README.rst @@ -26,13 +26,13 @@ Package ``apache-airflow-providers-common-ai`` Release: ``0.1.0`` -``Common AI Provider`` +AI/LLM hooks and operators for Airflow pipelines using `pydantic-ai `__. Provider package ---------------- -This is a provider package for ``common-ai`` provider. All classes for this provider package +This is a provider package for ``common.ai`` provider. All classes for this provider package are in ``airflow.providers.common.ai`` python package. You can find package information and changelog for the provider @@ -41,20 +41,61 @@ in the `documentation =3.0.0`` -================== ================== +========================================== ================== +PIP package Version required +========================================== ================== +``apache-airflow`` ``>=3.0.0`` +``apache-airflow-providers-common-compat`` ``>=1.14.1`` +``apache-airflow-providers-standard`` ``>=1.12.1`` +``pydantic-ai-slim`` ``>=1.34.0`` +========================================== ================== + +Cross provider package dependencies +----------------------------------- + +Those are dependencies that might be needed in order to use all the features of the package. +You need to install the specified providers in order to use them. + +You can install such cross-provider dependencies when installing from PyPI. For example: + +.. code-block:: bash + + pip install apache-airflow-providers-common-ai[common.compat] + + +================================================================================================================== ================= +Dependent package Extra +================================================================================================================== ================= +`apache-airflow-providers-common-compat `_ ``common.compat`` +`apache-airflow-providers-common-sql `_ ``common.sql`` +`apache-airflow-providers-standard `_ ``standard`` +================================================================================================================== ================= + +Optional dependencies +---------------------- + +============== ============================================================================================= +Extra Dependencies +============== ============================================================================================= +``anthropic`` ``pydantic-ai-slim[anthropic]`` +``bedrock`` ``pydantic-ai-slim[bedrock]`` +``google`` ``pydantic-ai-slim[google]`` +``openai`` ``pydantic-ai-slim[openai]`` +``mcp`` ``pydantic-ai-slim[mcp]`` +``avro`` ``fastavro>=1.10.0; python_version < "3.14"``, ``fastavro>=1.12.1; python_version >= "3.14"`` +``parquet`` ``pyarrow>=18.0.0; python_version < '3.14'``, ``pyarrow>=22.0.0; python_version >= '3.14'`` +``sql`` ``apache-airflow-providers-common-sql``, ``sqlglot>=30.0.0`` +``common.sql`` ``apache-airflow-providers-common-sql`` +============== ============================================================================================= The changelog for the provider package can be found in the `changelog `_. diff --git a/providers/common/ai/docs/commits.rst b/providers/common/ai/docs/commits.rst index 1407450a2287a..5f9d6c77437ee 100644 --- a/providers/common/ai/docs/commits.rst +++ b/providers/common/ai/docs/commits.rst @@ -1,3 +1,4 @@ + .. Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements. See the NOTICE file distributed with this work for additional information @@ -25,10 +26,10 @@ Package apache-airflow-providers-common-ai ------------------------------------------------------ -``Common AI Provider`` +AI/LLM hooks and operators for Airflow pipelines using `pydantic-ai `__. -This is detailed commit list of changes for versions provider package: ``common-ai``. +This is detailed commit list of changes for versions provider package: ``common.ai``. For high-level changelog, see :doc:`package information including changelog `. .. airflow-providers-commits:: diff --git a/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_llm_survey_analysis.py b/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_llm_survey_analysis.py new file mode 100644 index 0000000000000..2c88ae8e6f9b4 --- /dev/null +++ b/providers/common/ai/src/airflow/providers/common/ai/example_dags/example_llm_survey_analysis.py @@ -0,0 +1,387 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +""" +Natural language analysis of a survey CSV — interactive and scheduled variants. + +Both DAGs query the `Airflow Community Survey 2025 +`__ CSV using +:class:`~airflow.providers.common.ai.operators.llm_sql.LLMSQLQueryOperator` +and :class:`~airflow.providers.common.sql.operators.analytics.AnalyticsOperator`. + +``example_llm_survey_interactive`` (five tasks, manual trigger) + Adds human-in-the-loop review at both ends of the pipeline: + + 1. **HITLEntryOperator** — human reviews and optionally edits the question. + 2. **LLMSQLQueryOperator** — translates the confirmed question into SQL. + 3. **AnalyticsOperator** — executes the SQL against the CSV via Apache DataFusion. + 4. A ``@task`` function — extracts the data rows from the JSON payload. + 5. **ApprovalOperator** — human approves or rejects the query result. + +``example_llm_survey_scheduled`` (three tasks, runs on a schedule) + Runs a fixed question end-to-end without human review — suitable for + recurring reporting or dashboards: + + 1. **LLMSQLQueryOperator** — translates the question into SQL. + 2. **AnalyticsOperator** — executes the SQL against the CSV. + 3. A ``@task`` function — extracts the data rows from the JSON payload. + +Before running either DAG: + +1. Create an LLM connection named ``pydanticai_default`` (or the value of + ``LLM_CONN_ID`` below) for your chosen model provider. +2. Place the survey CSV at the path set by the ``SURVEY_CSV_PATH`` + environment variable, or update ``SURVEY_CSV_PATH`` below. + A cleaned copy of the 2025 survey CSV (duplicate columns renamed, embedded + newlines removed) is required — Apache DataFusion is strict about these. +""" + +from __future__ import annotations + +import datetime +import json +import os + +from airflow.providers.common.ai.operators.llm_schema_compare import LLMSchemaCompareOperator +from airflow.providers.common.ai.operators.llm_sql import LLMSQLQueryOperator +from airflow.providers.common.compat.sdk import dag, task +from airflow.providers.common.sql.config import DataSourceConfig +from airflow.providers.common.sql.operators.analytics import AnalyticsOperator +from airflow.providers.standard.operators.hitl import ApprovalOperator, HITLEntryOperator +from airflow.sdk import Param + +try: + from airflow.providers.http.operators.http import HttpOperator + + _has_http_provider = True +except ImportError: + _has_http_provider = False + +# --------------------------------------------------------------------------- +# Configuration +# --------------------------------------------------------------------------- + +# LLM provider connection (OpenAI, Anthropic, Vertex AI, etc.) +LLM_CONN_ID = "pydanticai_default" + +# HTTP connection pointing at https://airflow.apache.org (scheduled DAG only). +# Create a connection with host=https://airflow.apache.org, no auth required. +AIRFLOW_WEBSITE_CONN_ID = "airflow_website" + +# Endpoint path for the survey CSV download, relative to the HTTP connection base URL. +SURVEY_CSV_ENDPOINT = "/survey/airflow-user-survey-2025.csv" + +# Path to the survey CSV. Set the SURVEY_CSV_PATH environment variable to +# override — no code change needed when moving between environments. +SURVEY_CSV_PATH = os.environ.get( + "SURVEY_CSV_PATH", + "/opt/airflow/data/airflow-user-survey-2025.csv", +) +SURVEY_CSV_URI = f"file://{SURVEY_CSV_PATH}" + +# Path where the reference schema CSV is written at runtime (scheduled DAG only). +REFERENCE_CSV_PATH = os.environ.get( + "REFERENCE_CSV_PATH", + "/opt/airflow/data/airflow-user-survey-2025-reference.csv", +) +REFERENCE_CSV_URI = f"file://{REFERENCE_CSV_PATH}" + +# SMTP connection for the result notification step (scheduled DAG only). +# Set to None to skip email and log the result instead. +SMTP_CONN_ID = os.environ.get("SMTP_CONN_ID", None) +NOTIFY_EMAIL = os.environ.get("NOTIFY_EMAIL", None) + +# Default question for the interactive DAG — the human can edit it in the first HITL step. +INTERACTIVE_PROMPT = ( + "How does AI tool usage for writing Airflow code compare between Airflow 3 users and Airflow 2 users?" +) + +# Fixed question for the scheduled DAG — runs unattended on every trigger. +SCHEDULED_PROMPT = "What is the breakdown of respondents by Airflow version currently in use?" + +# Schema context for LLMSQLQueryOperator. +# Lists the analytically relevant columns from the 2025 survey CSV (168 total). +# All column names must be quoted in SQL because they contain spaces and +# punctuation. +SURVEY_SCHEMA = """ +Table: survey +Key columns (quote all names in SQL): + "How important is Airflow to your business?" TEXT + "Which version of Airflow do you currently use?" TEXT + "CeleryExecutor" TEXT + "KubernetesExecutor" TEXT + "LocalExecutor" TEXT + "How do you deploy Airflow?" TEXT + "What best describes your current occupation?" TEXT + "What industry do you currently work in?" TEXT + "What city do you currently reside in?" TEXT + "How many years of experience do you have with Airflow?" TEXT + "Which of the following is your company's primary cloud provider for Airflow?" TEXT + "How many people work at your company?" TEXT + "How many people at your company directly work on data?" TEXT + "How many people at your company use Airflow?" TEXT + "How likely are you to recommend Apache Airflow?" TEXT + "Are you using AI/LLM (ChatGPT/Cursor/Claude etc) to assist you in writing Airflow code?" TEXT +""" + +survey_datasource = DataSourceConfig( + conn_id="", + table_name="survey", + uri=SURVEY_CSV_URI, + format="csv", +) + +reference_datasource = DataSourceConfig( + conn_id="", + table_name="survey_reference", + uri=REFERENCE_CSV_URI, + format="csv", +) + + +# --------------------------------------------------------------------------- +# DAG 1: Interactive survey question example +# --------------------------------------------------------------------------- + + +# [START example_llm_survey_interactive] +@dag(schedule=None) +def example_llm_survey_interactive(): + """ + Ask a natural language question about the survey with human review at each end. + + Task graph:: + + prompt_confirmation (HITLEntryOperator) + → generate_sql (LLMSQLQueryOperator) + → run_query (AnalyticsOperator) + → extract_data (@task) + → result_confirmation (ApprovalOperator) + + The first HITL step lets the analyst review and optionally reword the + question before it reaches the LLM. The final HITL step presents the + query result for approval or rejection. + """ + + # ------------------------------------------------------------------ + # Step 1: Prompt confirmation — review or edit the question. + # ------------------------------------------------------------------ + prompt_confirmation = HITLEntryOperator( + task_id="prompt_confirmation", + subject="Review the survey analysis question", + params={ + "prompt": Param( + INTERACTIVE_PROMPT, + type="string", + description="The natural language question to answer via SQL", + ) + }, + response_timeout=datetime.timedelta(hours=1), + ) + + # ------------------------------------------------------------------ + # Step 2: SQL generation — LLM translates the confirmed question. + # ------------------------------------------------------------------ + generate_sql = LLMSQLQueryOperator( + task_id="generate_sql", + prompt="{{ ti.xcom_pull(task_ids='prompt_confirmation')['params_input']['prompt'] }}", + llm_conn_id=LLM_CONN_ID, + datasource_config=survey_datasource, + schema_context=SURVEY_SCHEMA, + ) + + # ------------------------------------------------------------------ + # Step 3: SQL execution via Apache DataFusion. + # ------------------------------------------------------------------ + run_query = AnalyticsOperator( + task_id="run_query", + datasource_configs=[survey_datasource], + queries=["{{ ti.xcom_pull(task_ids='generate_sql') }}"], + result_output_format="json", + ) + + # ------------------------------------------------------------------ + # Step 4: Extract data rows from the JSON result. + # AnalyticsOperator returns [{"query": "...", "data": [...]}, ...] + # This step strips the query field so only the rows reach the reviewer. + # ------------------------------------------------------------------ + @task + def extract_data(raw: str) -> str: + results = json.loads(raw) + data = [row for item in results for row in item["data"]] + return json.dumps(data, indent=2) + + result_data = extract_data(run_query.output) + + # ------------------------------------------------------------------ + # Step 5: Result confirmation — approve or reject the query result. + # ------------------------------------------------------------------ + result_confirmation = ApprovalOperator( + task_id="result_confirmation", + subject="Review the survey query result", + body="{{ ti.xcom_pull(task_ids='extract_data') }}", + response_timeout=datetime.timedelta(hours=1), + ) + + prompt_confirmation >> generate_sql >> run_query >> result_data >> result_confirmation + + +# [END example_llm_survey_interactive] + +example_llm_survey_interactive() + + +# --------------------------------------------------------------------------- +# DAG 2: Scheduled survey question example +# --------------------------------------------------------------------------- + + +# [START example_llm_survey_scheduled] +@dag(schedule="@monthly", start_date=None) +def example_llm_survey_scheduled(): + """ + Download, validate, query, and report on the survey CSV on a schedule. + + Task graph:: + + download_survey (HttpOperator) + → prepare_csv (@task) + → check_schema (LLMSchemaCompareOperator) + → generate_sql (LLMSQLQueryOperator) + → run_query (AnalyticsOperator) + → extract_data (@task) + → send_result (@task) + + No human review steps — suitable for recurring reporting or dashboards. + Change ``schedule`` to any cron expression or Airflow timetable to adjust + the run frequency. + + Prerequisites: + - HTTP connection ``airflow_website`` pointing at ``https://airflow.apache.org``. + - Set ``SMTP_CONN_ID`` and ``NOTIFY_EMAIL`` environment variables to enable + email delivery of results; otherwise results are logged to the task log. + """ + # ------------------------------------------------------------------ + # Step 1: Download the survey CSV from the Airflow website. + # ------------------------------------------------------------------ + download_survey = HttpOperator( + task_id="download_survey", + http_conn_id=AIRFLOW_WEBSITE_CONN_ID, + endpoint=SURVEY_CSV_ENDPOINT, + method="GET", + response_filter=lambda r: r.text, + log_response=False, + ) + + # ------------------------------------------------------------------ + # Step 2: Write the downloaded CSV to disk and generate a reference + # schema file for the schema comparison step. + # ------------------------------------------------------------------ + @task + def prepare_csv(csv_text: str) -> None: + import csv as csv_mod + import os + + os.makedirs(os.path.dirname(SURVEY_CSV_PATH), exist_ok=True) + with open(SURVEY_CSV_PATH, "w", encoding="utf-8") as f: + f.write(csv_text) + + # Write a single-row reference CSV from the schema context so + # LLMSchemaCompareOperator has a structured baseline to compare against. + os.makedirs(os.path.dirname(REFERENCE_CSV_PATH), exist_ok=True) + columns = [line.split('"')[1] for line in SURVEY_SCHEMA.strip().splitlines() if '"' in line] + with open(REFERENCE_CSV_PATH, "w", newline="", encoding="utf-8") as ref: + csv_mod.writer(ref).writerow(columns) + + csv_ready = prepare_csv(download_survey.output) + + # ------------------------------------------------------------------ + # Step 3: Validate the downloaded CSV schema against the reference. + # Raises if critical columns are missing or renamed. + # ------------------------------------------------------------------ + check_schema = LLMSchemaCompareOperator( + task_id="check_schema", + prompt=( + "Compare the survey CSV schema against the reference schema. " + "Flag any missing or renamed columns that would break the downstream SQL queries." + ), + llm_conn_id=LLM_CONN_ID, + data_sources=[survey_datasource, reference_datasource], + context_strategy="basic", + ) + csv_ready >> check_schema + + # ------------------------------------------------------------------ + # Step 4: SQL generation — LLM translates the fixed question. + # ------------------------------------------------------------------ + generate_sql = LLMSQLQueryOperator( + task_id="generate_sql", + prompt=SCHEDULED_PROMPT, + llm_conn_id=LLM_CONN_ID, + datasource_config=survey_datasource, + schema_context=SURVEY_SCHEMA, + ) + check_schema >> generate_sql + + # ------------------------------------------------------------------ + # Step 5: SQL execution via Apache DataFusion. + # ------------------------------------------------------------------ + run_query = AnalyticsOperator( + task_id="run_query", + datasource_configs=[survey_datasource], + queries=["{{ ti.xcom_pull(task_ids='generate_sql') }}"], + result_output_format="json", + ) + + # ------------------------------------------------------------------ + # Step 6: Extract data rows from the JSON result. + # AnalyticsOperator returns [{"query": "...", "data": [...]}, ...] + # ------------------------------------------------------------------ + @task + def extract_data(raw: str) -> str: + results = json.loads(raw) + data = [row for item in results for row in item["data"]] + return json.dumps(data, indent=2) + + result_data = extract_data(run_query.output) + + # ------------------------------------------------------------------ + # Step 7: Send result via email if SMTP is configured, otherwise log. + # Set the SMTP_CONN_ID and NOTIFY_EMAIL environment variables to enable + # email delivery. + # ------------------------------------------------------------------ + @task + def send_result(data: str) -> None: + if SMTP_CONN_ID and NOTIFY_EMAIL: + from airflow.providers.smtp.operators.smtp import EmailOperator + + EmailOperator( + task_id="_send_email", + smtp_conn_id=SMTP_CONN_ID, + to=NOTIFY_EMAIL, + subject=f"Airflow Survey Analysis: {SCHEDULED_PROMPT}", + html_content=f"
{data}
", + ).execute({}) + else: + print(f"Survey analysis result:\n{data}") + + generate_sql >> run_query >> result_data >> send_result(result_data) + + +# [END example_llm_survey_scheduled] + +if _has_http_provider: + example_llm_survey_scheduled() diff --git a/providers/common/ai/src/airflow/providers/common/ai/plugins/hitl_review.py b/providers/common/ai/src/airflow/providers/common/ai/plugins/hitl_review.py index e870588f79392..911a61d5c4d00 100644 --- a/providers/common/ai/src/airflow/providers/common/ai/plugins/hitl_review.py +++ b/providers/common/ai/src/airflow/providers/common/ai/plugins/hitl_review.py @@ -17,39 +17,12 @@ from __future__ import annotations -import logging -import mimetypes -from pathlib import Path -from types import SimpleNamespace from typing import Annotated, Any from urllib.parse import urlparse -from fastapi import Depends, FastAPI, HTTPException, Query -from fastapi.staticfiles import StaticFiles -from sqlalchemy import select -from sqlalchemy.orm import Session - -from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity -from airflow.api_fastapi.core_api.security import requires_access_dag -from airflow.models.taskinstance import TaskInstance as TI -from airflow.models.xcom import XComModel from airflow.plugins_manager import AirflowPlugin -from airflow.providers.common.ai.utils.hitl_review import ( - XCOM_AGENT_OUTPUT_PREFIX, - XCOM_AGENT_SESSION, - XCOM_HUMAN_ACTION, - XCOM_HUMAN_FEEDBACK_PREFIX, - AgentSessionData, - HITLReviewResponse, - HumanActionData, - HumanFeedbackRequest, - SessionStatus, -) from airflow.providers.common.compat.sdk import conf -from airflow.utils.session import create_session -from airflow.utils.state import TaskInstanceState - -log = logging.getLogger(__name__) +from airflow.providers.common.compat.version_compat import AIRFLOW_V_3_1_PLUS _PLUGIN_PREFIX = "/hitl-review" @@ -81,445 +54,462 @@ def _get_bundle_url() -> str: return path -def _get_session(): - with create_session(scoped=False) as session: - yield session - - -SessionDep = Annotated[Session, Depends(_get_session)] - - -def _get_map_index(q: str = Query("-1", alias="map_index")) -> int: - """Parse map_index query; use -1 when placeholder unreplaced (e.g. ``{MAP_INDEX}``) or invalid.""" - try: - return int(q) - except (ValueError, TypeError): - return -1 - - -MapIndexDep = Annotated[int, Depends(_get_map_index)] - - -def _read_xcom(session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1, key: str): - """Read a single XCom value from the database.""" - row = session.scalars( - XComModel.get_many( - run_id=run_id, +if AIRFLOW_V_3_1_PLUS: + import mimetypes + from pathlib import Path + from types import SimpleNamespace + + from fastapi import Depends, FastAPI, HTTPException, Query + from fastapi.staticfiles import StaticFiles + from sqlalchemy import select + from sqlalchemy.orm import Session + + from airflow.api_fastapi.auth.managers.models.resource_details import DagAccessEntity + from airflow.api_fastapi.core_api.security import requires_access_dag + from airflow.models.taskinstance import TaskInstance as TI + from airflow.models.xcom import XComModel + from airflow.providers.common.ai.utils.hitl_review import ( + XCOM_AGENT_OUTPUT_PREFIX, + XCOM_AGENT_SESSION, + XCOM_HUMAN_ACTION, + XCOM_HUMAN_FEEDBACK_PREFIX, + AgentSessionData, + HITLReviewResponse, + HumanActionData, + HumanFeedbackRequest, + SessionStatus, + ) + from airflow.utils.session import create_session + from airflow.utils.state import TaskInstanceState + + def _get_session(): + with create_session(scoped=False) as session: + yield session + + SessionDep = Annotated[Session, Depends(_get_session)] + + def _read_xcom( + session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1, key: str + ): + """Read a single XCom value from the database.""" + row = session.scalars( + XComModel.get_many( + run_id=run_id, + key=key, + dag_ids=dag_id, + task_ids=task_id, + map_indexes=map_index, + limit=1, + ) + ).first() + if row is None: + return None + return XComModel.deserialize_value(row) + + def _read_xcom_by_prefix( + session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1, prefix: str + ) -> dict[int, Any]: + """Read all iteration-keyed XCom entries matching *prefix* (e.g. ``airflow_hitl_review_agent_output_``).""" + query = select(XComModel.key, XComModel.value).where( + XComModel.dag_id == dag_id, + XComModel.run_id == run_id, + XComModel.task_id == task_id, + XComModel.map_index == map_index, + XComModel.key.like(f"{prefix}%"), + ) + result: dict[int, Any] = {} + for key, value in session.execute(query).all(): + suffix = key[len(prefix) :] + if suffix.isdigit(): + # deserialize_value expects an object with a .value attribute; + # wrap the raw column value so we can reuse the standard deserialization path. + row = SimpleNamespace(value=value) + result[int(suffix)] = XComModel.deserialize_value(row) + return result + + def _write_xcom( + session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1, key: str, value + ): + """Write data to db.""" + XComModel.set( key=key, - dag_ids=dag_id, - task_ids=task_id, - map_indexes=map_index, - limit=1, + value=value, + dag_id=dag_id, + task_id=task_id, + run_id=run_id, + map_index=map_index, + session=session, ) - ).first() - if row is None: - return None - return XComModel.deserialize_value(row) - - -def _read_xcom_by_prefix( - session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1, prefix: str -) -> dict[int, Any]: - """Read all iteration-keyed XCom entries matching *prefix* (e.g. ``airflow_hitl_review_agent_output_``).""" - query = select(XComModel.key, XComModel.value).where( - XComModel.dag_id == dag_id, - XComModel.run_id == run_id, - XComModel.task_id == task_id, - XComModel.map_index == map_index, - XComModel.key.like(f"{prefix}%"), - ) - result: dict[int, Any] = {} - for key, value in session.execute(query).all(): - suffix = key[len(prefix) :] - if suffix.isdigit(): - # deserialize_value expects an object with a .value attribute; - # wrap the raw column value so we can reuse the standard deserialization path. - row = SimpleNamespace(value=value) - result[int(suffix)] = XComModel.deserialize_value(row) - return result - - -def _write_xcom( - session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1, key: str, value -): - """Write data to db.""" - XComModel.set( - key=key, - value=value, - dag_id=dag_id, - task_id=task_id, - run_id=run_id, - map_index=map_index, - session=session, - ) - -_RUNNING_TI_STATES = frozenset( - { - TaskInstanceState.RUNNING, - TaskInstanceState.DEFERRED, - TaskInstanceState.UP_FOR_RETRY, - TaskInstanceState.QUEUED, - TaskInstanceState.SCHEDULED, - } -) - - -def _is_task_completed( - session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1 -) -> bool: - """Return True if the task instance is no longer running.""" - state = session.scalar( - select(TI.state).where( - TI.dag_id == dag_id, - TI.run_id == run_id, - TI.task_id == task_id, - TI.map_index == map_index, - ) - ) - if state is None: - return True - return state not in _RUNNING_TI_STATES - - -def _build_session_response( - session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1 -) -> HITLReviewResponse | None: - """Build `HITLReviewResponse` from XCom entries.""" - raw = _read_xcom( - session, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_AGENT_SESSION, - ) - if raw is None: - return None - sess_data = AgentSessionData.model_validate(raw) - outputs = _read_xcom_by_prefix( - session, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - prefix=XCOM_AGENT_OUTPUT_PREFIX, - ) - human_responses = _read_xcom_by_prefix( - session, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - prefix=XCOM_HUMAN_FEEDBACK_PREFIX, - ) - completed = _is_task_completed( - session, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - ) - return HITLReviewResponse.from_xcom( - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - session=sess_data, - outputs=outputs, - human_entries=human_responses, - task_completed=completed, + _RUNNING_TI_STATES = frozenset( + { + TaskInstanceState.RUNNING, + TaskInstanceState.DEFERRED, + TaskInstanceState.UP_FOR_RETRY, + TaskInstanceState.QUEUED, + TaskInstanceState.SCHEDULED, + } ) - -hitl_review_app = FastAPI( - title="HITL Review", - description=( - "REST API and chat UI for human-in-the-loop LLM feedback sessions. " - "Sessions are stored in XCom entries on the running task instance." - ), -) - - -@hitl_review_app.get("/health") -async def health() -> dict[str, str]: - """Liveness check.""" - return {"status": "ok"} - - -@hitl_review_app.get( - "/sessions/find", - response_model=HITLReviewResponse, - dependencies=[Depends(requires_access_dag(method="GET", access_entity=DagAccessEntity.HITL_DETAIL))], -) -async def find_session( - db: SessionDep, - dag_id: str, - task_id: str, - run_id: str, - map_index: MapIndexDep, -) -> HITLReviewResponse: - """Find the feedback session for a specific task instance.""" - resp = _build_session_response( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - ) - if resp is None: - task_active = not _is_task_completed( - db, + def _is_task_completed( + session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1 + ) -> bool: + """Return True if the task instance is no longer running.""" + state = session.scalar( + select(TI.state).where( + TI.dag_id == dag_id, + TI.run_id == run_id, + TI.task_id == task_id, + TI.map_index == map_index, + ) + ) + if state is None: + return True + return state not in _RUNNING_TI_STATES + + def _build_session_response( + session: Session, *, dag_id: str, run_id: str, task_id: str, map_index: int = -1 + ) -> HITLReviewResponse | None: + """Build `HITLReviewResponse` from XCom entries.""" + raw = _read_xcom( + session, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_AGENT_SESSION, + ) + if raw is None: + return None + sess_data = AgentSessionData.model_validate(raw) + outputs = _read_xcom_by_prefix( + session, dag_id=dag_id, run_id=run_id, task_id=task_id, map_index=map_index, + prefix=XCOM_AGENT_OUTPUT_PREFIX, ) - raise HTTPException( - status_code=404, - detail={"message": "No matching session found.", "task_active": task_active}, + human_responses = _read_xcom_by_prefix( + session, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + prefix=XCOM_HUMAN_FEEDBACK_PREFIX, ) - return resp - - -@hitl_review_app.post( - "/sessions/feedback", - response_model=HITLReviewResponse, - dependencies=[Depends(requires_access_dag(method="PUT", access_entity=DagAccessEntity.HITL_DETAIL))], -) -async def submit_feedback( - body: HumanFeedbackRequest, - db: SessionDep, - dag_id: str, - task_id: str, - run_id: str, - map_index: MapIndexDep, -) -> HITLReviewResponse: - """Request changes — provide human feedback for the LLM.""" - if not (body.feedback and body.feedback.strip()): - raise HTTPException( - status_code=400, - detail="Feedback is required when requesting changes.", + completed = _is_task_completed( + session, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, ) - raw = _read_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_AGENT_SESSION, - ) - if raw is None: - raise HTTPException(status_code=404, detail="No matching session found.") - sess_data = AgentSessionData.model_validate(raw) - if sess_data.status != SessionStatus.PENDING_REVIEW: - raise HTTPException( - status_code=409, - detail=f"Session is '{sess_data.status.value}', expected 'pending_review'.", + return HITLReviewResponse.from_xcom( + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + session=sess_data, + outputs=outputs, + human_entries=human_responses, + task_completed=completed, ) - iteration = sess_data.iteration - _write_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=f"{XCOM_HUMAN_FEEDBACK_PREFIX}{iteration}", - value=body.feedback, + def _get_map_index(q: str = Query("-1", alias="map_index")) -> int: + """Parse map_index query; use -1 when placeholder unreplaced (e.g. ``{MAP_INDEX}``) or invalid.""" + try: + return int(q) + except (ValueError, TypeError): + return -1 + + MapIndexDep = Annotated[int, Depends(_get_map_index)] + + hitl_review_app = FastAPI( + title="HITL Review", + description=( + "REST API and chat UI for human-in-the-loop LLM feedback sessions. " + "Sessions are stored in XCom entries on the running task instance." + ), ) - action = HumanActionData(action="changes_requested", feedback=body.feedback, iteration=iteration) - _write_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_HUMAN_ACTION, - value=action.model_dump(mode="json"), - ) - - sess_data.status = SessionStatus.CHANGES_REQUESTED - _write_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_AGENT_SESSION, - value=sess_data.model_dump(mode="json"), - ) + @hitl_review_app.get("/health") + async def health() -> dict[str, str]: + """Liveness check.""" + return {"status": "ok"} - resp = _build_session_response( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, + @hitl_review_app.get( + "/sessions/find", + response_model=HITLReviewResponse, + dependencies=[Depends(requires_access_dag(method="GET", access_entity=DagAccessEntity.HITL_DETAIL))], ) - if resp is None: - raise HTTPException(status_code=500, detail="Failed to read session after update.") - return resp - - -@hitl_review_app.post( - "/sessions/approve", - response_model=HITLReviewResponse, - dependencies=[Depends(requires_access_dag(method="PUT", access_entity=DagAccessEntity.HITL_DETAIL))], -) -async def approve_session( - db: SessionDep, - dag_id: str, - task_id: str, - run_id: str, - map_index: MapIndexDep, -) -> HITLReviewResponse: - """Approve the current output.""" - raw = _read_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_AGENT_SESSION, - ) - if raw is None: - raise HTTPException(status_code=404, detail="No matching session found.") - - sess_data = AgentSessionData.model_validate(raw) - if sess_data.status != SessionStatus.PENDING_REVIEW: - raise HTTPException( - status_code=409, - detail=f"Session is '{sess_data.status.value}', expected 'pending_review'.", + async def find_session( + db: SessionDep, + dag_id: str, + task_id: str, + run_id: str, + map_index: MapIndexDep, + ) -> HITLReviewResponse: + """Find the feedback session for a specific task instance.""" + resp = _build_session_response( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, ) - - action = HumanActionData(action="approve", iteration=sess_data.iteration) - _write_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_HUMAN_ACTION, - value=action.model_dump(mode="json"), + if resp is None: + task_active = not _is_task_completed( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + ) + raise HTTPException( + status_code=404, + detail={"message": "No matching session found.", "task_active": task_active}, + ) + return resp + + @hitl_review_app.post( + "/sessions/feedback", + response_model=HITLReviewResponse, + dependencies=[Depends(requires_access_dag(method="PUT", access_entity=DagAccessEntity.HITL_DETAIL))], ) + async def submit_feedback( + body: HumanFeedbackRequest, + db: SessionDep, + dag_id: str, + task_id: str, + run_id: str, + map_index: MapIndexDep, + ) -> HITLReviewResponse: + """Request changes — provide human feedback for the LLM.""" + if not (body.feedback and body.feedback.strip()): + raise HTTPException( + status_code=400, + detail="Feedback is required when requesting changes.", + ) + raw = _read_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_AGENT_SESSION, + ) + if raw is None: + raise HTTPException(status_code=404, detail="No matching session found.") + sess_data = AgentSessionData.model_validate(raw) + if sess_data.status != SessionStatus.PENDING_REVIEW: + raise HTTPException( + status_code=409, + detail=f"Session is '{sess_data.status.value}', expected 'pending_review'.", + ) + + iteration = sess_data.iteration + _write_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=f"{XCOM_HUMAN_FEEDBACK_PREFIX}{iteration}", + value=body.feedback, + ) - sess_data.status = SessionStatus.APPROVED - _write_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_AGENT_SESSION, - value=sess_data.model_dump(mode="json"), - ) + action = HumanActionData(action="changes_requested", feedback=body.feedback, iteration=iteration) + _write_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_HUMAN_ACTION, + value=action.model_dump(mode="json"), + ) - resp = _build_session_response( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - ) - if resp is None: - raise HTTPException(status_code=500, detail="Failed to read session after update.") - return resp - - -@hitl_review_app.post( - "/sessions/reject", - response_model=HITLReviewResponse, - dependencies=[Depends(requires_access_dag(method="PUT", access_entity=DagAccessEntity.HITL_DETAIL))], -) -async def reject_session( - db: SessionDep, - dag_id: str, - task_id: str, - run_id: str, - map_index: MapIndexDep, -) -> HITLReviewResponse: - """Reject the output.""" - raw = _read_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_AGENT_SESSION, - ) - if raw is None: - raise HTTPException(status_code=404, detail="No matching session found.") - sess_data = AgentSessionData.model_validate(raw) - if sess_data.status != SessionStatus.PENDING_REVIEW: - raise HTTPException( - status_code=409, - detail=f"Session is '{sess_data.status.value}', expected 'pending_review'.", + sess_data.status = SessionStatus.CHANGES_REQUESTED + _write_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_AGENT_SESSION, + value=sess_data.model_dump(mode="json"), ) - action = HumanActionData(action="reject", iteration=sess_data.iteration) - _write_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_HUMAN_ACTION, - value=action.model_dump(mode="json"), + resp = _build_session_response( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + ) + if resp is None: + raise HTTPException(status_code=500, detail="Failed to read session after update.") + return resp + + @hitl_review_app.post( + "/sessions/approve", + response_model=HITLReviewResponse, + dependencies=[Depends(requires_access_dag(method="PUT", access_entity=DagAccessEntity.HITL_DETAIL))], ) + async def approve_session( + db: SessionDep, + dag_id: str, + task_id: str, + run_id: str, + map_index: MapIndexDep, + ) -> HITLReviewResponse: + """Approve the current output.""" + raw = _read_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_AGENT_SESSION, + ) + if raw is None: + raise HTTPException(status_code=404, detail="No matching session found.") + + sess_data = AgentSessionData.model_validate(raw) + if sess_data.status != SessionStatus.PENDING_REVIEW: + raise HTTPException( + status_code=409, + detail=f"Session is '{sess_data.status.value}', expected 'pending_review'.", + ) + + action = HumanActionData(action="approve", iteration=sess_data.iteration) + _write_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_HUMAN_ACTION, + value=action.model_dump(mode="json"), + ) - sess_data.status = SessionStatus.REJECTED - _write_xcom( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, - key=XCOM_AGENT_SESSION, - value=sess_data.model_dump(mode="json"), - ) + sess_data.status = SessionStatus.APPROVED + _write_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_AGENT_SESSION, + value=sess_data.model_dump(mode="json"), + ) - resp = _build_session_response( - db, - dag_id=dag_id, - run_id=run_id, - task_id=task_id, - map_index=map_index, + resp = _build_session_response( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + ) + if resp is None: + raise HTTPException(status_code=500, detail="Failed to read session after update.") + return resp + + @hitl_review_app.post( + "/sessions/reject", + response_model=HITLReviewResponse, + dependencies=[Depends(requires_access_dag(method="PUT", access_entity=DagAccessEntity.HITL_DETAIL))], ) - if resp is None: - raise HTTPException(status_code=500, detail="Failed to read session after update.") - return resp - + async def reject_session( + db: SessionDep, + dag_id: str, + task_id: str, + run_id: str, + map_index: MapIndexDep, + ) -> HITLReviewResponse: + """Reject the output.""" + raw = _read_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_AGENT_SESSION, + ) + if raw is None: + raise HTTPException(status_code=404, detail="No matching session found.") + sess_data = AgentSessionData.model_validate(raw) + if sess_data.status != SessionStatus.PENDING_REVIEW: + raise HTTPException( + status_code=409, + detail=f"Session is '{sess_data.status.value}', expected 'pending_review'.", + ) + + action = HumanActionData(action="reject", iteration=sess_data.iteration) + _write_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_HUMAN_ACTION, + value=action.model_dump(mode="json"), + ) -# Ensure proper MIME types for plugin bundle (FastAPI serves .cjs as text/plain by default) -mimetypes.add_type("application/javascript", ".cjs") + sess_data.status = SessionStatus.REJECTED + _write_xcom( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + key=XCOM_AGENT_SESSION, + value=sess_data.model_dump(mode="json"), + ) -_WWW_DIR = Path(__file__).parent / "www" -_dist_dir = _WWW_DIR / "dist" -if _dist_dir.is_dir(): - hitl_review_app.mount( - "/static", - StaticFiles(directory=str(_dist_dir.absolute()), html=True), - name="hitl_review_static", - ) + resp = _build_session_response( + db, + dag_id=dag_id, + run_id=run_id, + task_id=task_id, + map_index=map_index, + ) + if resp is None: + raise HTTPException(status_code=500, detail="Failed to read session after update.") + return resp + + # Ensure proper MIME types for plugin bundle (FastAPI serves .cjs as text/plain by default) + mimetypes.add_type("application/javascript", ".cjs") + + _WWW_DIR = Path(__file__).parent / "www" + _dist_dir = _WWW_DIR / "dist" + if _dist_dir.is_dir(): + hitl_review_app.mount( + "/static", + StaticFiles(directory=str(_dist_dir.absolute()), html=True), + name="hitl_review_static", + ) class HITLReviewPlugin(AirflowPlugin): """Register the HITL Review REST API + chat UI on the Airflow API server.""" name = "hitl_review" - fastapi_apps = [ - { - "name": "hitl-review", - "app": hitl_review_app, - "url_prefix": _PLUGIN_PREFIX, - } - ] - react_apps = [ - { - "name": "HITL Review", - "bundle_url": _get_bundle_url(), - "destination": "task_instance", - "url_route": "hitl-review", - } - ] + fastapi_apps: list[dict[str, Any]] = [] + react_apps: list[dict[str, str]] = [] + if AIRFLOW_V_3_1_PLUS: + fastapi_apps = [ + { + "name": "hitl-review", + "app": hitl_review_app, + "url_prefix": _PLUGIN_PREFIX, + } + ] + react_apps = [ + { + "name": "HITL Review", + "bundle_url": _get_bundle_url(), + "destination": "task_instance", + "url_route": "hitl-review", + } + ] diff --git a/providers/common/ai/tests/unit/common/ai/plugins/test_hitl_review_compat.py b/providers/common/ai/tests/unit/common/ai/plugins/test_hitl_review_compat.py new file mode 100644 index 0000000000000..3e99dc99df8a1 --- /dev/null +++ b/providers/common/ai/tests/unit/common/ai/plugins/test_hitl_review_compat.py @@ -0,0 +1,38 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from __future__ import annotations + +import pytest + +from airflow.providers.common.ai.plugins.hitl_review import HITLReviewPlugin + +from tests_common.test_utils.version_compat import AIRFLOW_V_3_1_PLUS + + +def test_hitl_review_plugin_registration_matches_airflow_version(): + if AIRFLOW_V_3_1_PLUS: + assert len(HITLReviewPlugin.fastapi_apps) == 1 + assert len(HITLReviewPlugin.react_apps) == 1 + else: + assert HITLReviewPlugin.fastapi_apps == [] + assert HITLReviewPlugin.react_apps == [] + + +@pytest.mark.skipif(not AIRFLOW_V_3_1_PLUS, reason="Requires Airflow 3.1+") +def test_hitl_review_plugin_registers_expected_app_names(): + assert HITLReviewPlugin.fastapi_apps[0]["name"] == "hitl-review" + assert HITLReviewPlugin.react_apps[0]["name"] == "HITL Review" diff --git a/providers/common/compat/README.rst b/providers/common/compat/README.rst index 5b8a9cebac882..dea11ad6680d6 100644 --- a/providers/common/compat/README.rst +++ b/providers/common/compat/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-common-compat`` -Release: ``1.14.2`` +Release: ``1.14.3`` Common Compatibility Provider - providing compatibility code for previous Airflow versions @@ -36,7 +36,7 @@ This is a provider package for ``common.compat`` provider. All classes for this are in ``airflow.providers.common.compat`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -88,4 +88,4 @@ Extra Dependencies =============== ======================================== The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/common/compat/docs/changelog.rst b/providers/common/compat/docs/changelog.rst index 660dca89073f0..87741173e8720 100644 --- a/providers/common/compat/docs/changelog.rst +++ b/providers/common/compat/docs/changelog.rst @@ -25,6 +25,20 @@ Changelog --------- +1.14.3 +...... + +Misc +~~~~ + +* ``Add OpenLineage parent and transport info injection to 'EmrServerlessStartJobOperator' (#64807)`` +* ``Add OpenLineage parent info injection to GlueJobOperator (#64513)`` +* ``Fix RESOURCE_ASSET compatibility with Airflow 2.x in common-compat (#64933)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``Prepare providers release 2026-04-07 (#64864)`` + 1.14.2 ...... diff --git a/providers/common/compat/docs/index.rst b/providers/common/compat/docs/index.rst index 19aad2668c1b4..3dcdd0eafd602 100644 --- a/providers/common/compat/docs/index.rst +++ b/providers/common/compat/docs/index.rst @@ -62,7 +62,7 @@ apache-airflow-providers-common-compat package Common Compatibility Provider - providing compatibility code for previous Airflow versions -Release: 1.14.2 +Release: 1.14.3 Provider package ---------------- @@ -115,5 +115,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-common-compat 1.14.2 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-common-compat 1.14.2 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-compat 1.14.3 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-common-compat 1.14.3 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/common/compat/provider.yaml b/providers/common/compat/provider.yaml index 8225c66026926..b536ea8c64cd4 100644 --- a/providers/common/compat/provider.yaml +++ b/providers/common/compat/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298879 +source-date-epoch: 1775591951 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 1.14.3 - 1.14.2 - 1.14.1 - 1.14.0 diff --git a/providers/common/compat/pyproject.toml b/providers/common/compat/pyproject.toml index 1f0525c387116..998173aa594a0 100644 --- a/providers/common/compat/pyproject.toml +++ b/providers/common/compat/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-common-compat" -version = "1.14.2" +version = "1.14.3" description = "Provider package apache-airflow-providers-common-compat for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -109,8 +109,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-compat/1.14.2" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-compat/1.14.2/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-common-compat/1.14.3" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-common-compat/1.14.3/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/common/compat/src/airflow/providers/common/compat/__init__.py b/providers/common/compat/src/airflow/providers/common/compat/__init__.py index b72951e217874..3a9efe604ed5b 100644 --- a/providers/common/compat/src/airflow/providers/common/compat/__init__.py +++ b/providers/common/compat/src/airflow/providers/common/compat/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "1.14.2" +__version__ = "1.14.3" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/common/compat/src/airflow/providers/common/compat/openlineage/utils/spark.py b/providers/common/compat/src/airflow/providers/common/compat/openlineage/utils/spark.py index d92dad56dad8f..91b4fae05d50a 100644 --- a/providers/common/compat/src/airflow/providers/common/compat/openlineage/utils/spark.py +++ b/providers/common/compat/src/airflow/providers/common/compat/openlineage/utils/spark.py @@ -24,16 +24,20 @@ if TYPE_CHECKING: from airflow.providers.openlineage.utils.spark import ( + inject_parent_job_information_into_emr_serverless_properties, inject_parent_job_information_into_glue_arguments, inject_parent_job_information_into_spark_properties, + inject_transport_information_into_emr_serverless_properties, inject_transport_information_into_glue_arguments, inject_transport_information_into_spark_properties, ) from airflow.sdk import Context try: from airflow.providers.openlineage.utils.spark import ( + inject_parent_job_information_into_emr_serverless_properties, inject_parent_job_information_into_glue_arguments, inject_parent_job_information_into_spark_properties, + inject_transport_information_into_emr_serverless_properties, inject_transport_information_into_glue_arguments, inject_transport_information_into_spark_properties, ) @@ -67,10 +71,32 @@ def inject_transport_information_into_glue_arguments(script_args: dict, context: ) return script_args + def inject_parent_job_information_into_emr_serverless_properties( + configuration_overrides: dict | None, context: Context + ) -> dict: + log.warning( + "Could not import `airflow.providers.openlineage.plugins.macros`." + "Skipping the injection of OpenLineage parent job information into " + "EMR Serverless configuration." + ) + return configuration_overrides or {} + + def inject_transport_information_into_emr_serverless_properties( + configuration_overrides: dict | None, context: Context + ) -> dict: + log.warning( + "Could not import `airflow.providers.openlineage.plugins.listener`." + "Skipping the injection of OpenLineage transport information into " + "EMR Serverless configuration." + ) + return configuration_overrides or {} + __all__ = [ + "inject_parent_job_information_into_emr_serverless_properties", "inject_parent_job_information_into_glue_arguments", "inject_parent_job_information_into_spark_properties", + "inject_transport_information_into_emr_serverless_properties", "inject_transport_information_into_glue_arguments", "inject_transport_information_into_spark_properties", ] diff --git a/providers/common/compat/src/airflow/providers/common/compat/security/permissions.py b/providers/common/compat/src/airflow/providers/common/compat/security/permissions.py index a1e89802278c6..4f6cba3660f8d 100644 --- a/providers/common/compat/src/airflow/providers/common/compat/security/permissions.py +++ b/providers/common/compat/src/airflow/providers/common/compat/security/permissions.py @@ -16,8 +16,15 @@ # under the License. from __future__ import annotations +from airflow.providers.common.compat.version_compat import AIRFLOW_V_3_0_PLUS + # Resource Constants RESOURCE_BACKFILL = "Backfills" RESOURCE_DAG_VERSION = "DAG Versions" -RESOURCE_ASSET = "Assets" RESOURCE_ASSET_ALIAS = "Asset Aliases" +if AIRFLOW_V_3_0_PLUS: + RESOURCE_ASSET = "Assets" +else: + from airflow.security.permissions import ( # type: ignore[attr-defined, no-redef] + RESOURCE_DATASET as RESOURCE_ASSET, # noqa: F401 + ) diff --git a/providers/common/io/tests/system/common/io/example_file_transfer_local_to_s3.py b/providers/common/io/tests/system/common/io/example_file_transfer_local_to_s3.py index 9c0f2c4744ef2..3276c57f80d77 100644 --- a/providers/common/io/tests/system/common/io/example_file_transfer_local_to_s3.py +++ b/providers/common/io/tests/system/common/io/example_file_transfer_local_to_s3.py @@ -99,5 +99,5 @@ def remove_bucket(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/common/messaging/tests/system/common/messaging/example_message_queue_trigger.py b/providers/common/messaging/tests/system/common/messaging/example_message_queue_trigger.py index 6e2ec6e7aefb8..8c5d9cf68cd03 100644 --- a/providers/common/messaging/tests/system/common/messaging/example_message_queue_trigger.py +++ b/providers/common/messaging/tests/system/common/messaging/example_message_queue_trigger.py @@ -33,5 +33,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/common/sql/tests/system/common/sql/example_generic_transfer.py b/providers/common/sql/tests/system/common/sql/example_generic_transfer.py index 672838d4f42f1..28e64bc961bbc 100644 --- a/providers/common/sql/tests/system/common/sql/example_generic_transfer.py +++ b/providers/common/sql/tests/system/common/sql/example_generic_transfer.py @@ -58,5 +58,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/common/sql/tests/system/common/sql/example_sql_column_table_check.py b/providers/common/sql/tests/system/common/sql/example_sql_column_table_check.py index 49c5d94ce9230..de45dcfb2cc2f 100644 --- a/providers/common/sql/tests/system/common/sql/example_sql_column_table_check.py +++ b/providers/common/sql/tests/system/common/sql/example_sql_column_table_check.py @@ -81,5 +81,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/common/sql/tests/system/common/sql/example_sql_execute_query.py b/providers/common/sql/tests/system/common/sql/example_sql_execute_query.py index f980d4bbc08c2..a018e84d5806e 100644 --- a/providers/common/sql/tests/system/common/sql/example_sql_execute_query.py +++ b/providers/common/sql/tests/system/common/sql/example_sql_execute_query.py @@ -68,5 +68,5 @@ def execute_query_taskflow(): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/common/sql/tests/system/common/sql/example_sql_insert_rows.py b/providers/common/sql/tests/system/common/sql/example_sql_insert_rows.py index ba24082602e6a..2141914ba56da 100644 --- a/providers/common/sql/tests/system/common/sql/example_sql_insert_rows.py +++ b/providers/common/sql/tests/system/common/sql/example_sql_insert_rows.py @@ -86,5 +86,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/common/sql/tests/system/common/sql/example_sql_threshold_check.py b/providers/common/sql/tests/system/common/sql/example_sql_threshold_check.py index ab65a73a789ed..ce1711264a604 100644 --- a/providers/common/sql/tests/system/common/sql/example_sql_threshold_check.py +++ b/providers/common/sql/tests/system/common/sql/example_sql_threshold_check.py @@ -58,5 +58,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/common/sql/tests/system/common/sql/example_sql_value_check.py b/providers/common/sql/tests/system/common/sql/example_sql_value_check.py index a904cdcd11d59..d1f98c30d644b 100644 --- a/providers/common/sql/tests/system/common/sql/example_sql_value_check.py +++ b/providers/common/sql/tests/system/common/sql/example_sql_value_check.py @@ -58,5 +58,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/databricks/README.rst b/providers/databricks/README.rst index 50c53b858ece9..80a7903d945eb 100644 --- a/providers/databricks/README.rst +++ b/providers/databricks/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-databricks`` -Release: ``7.12.0`` +Release: ``7.12.1`` `Databricks `__ @@ -36,7 +36,7 @@ This is a provider package for ``databricks`` provider. All classes for this pro are in ``airflow.providers.databricks`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -107,4 +107,4 @@ Extra Dependencies ================== ================================================================================================================================================================ The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/databricks/docs/changelog.rst b/providers/databricks/docs/changelog.rst index 97445d9a4dc79..a495762f27a82 100644 --- a/providers/databricks/docs/changelog.rst +++ b/providers/databricks/docs/changelog.rst @@ -44,6 +44,17 @@ Changelog mount ``ca.crt`` at ``/var/run/secrets/kubernetes.io/serviceaccount/ca.crt``. If you are affected, please open an issue so support for a configurable CA path can be added. +7.12.1 +...... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 7.12.0 ...... diff --git a/providers/databricks/docs/index.rst b/providers/databricks/docs/index.rst index 6f0799367d5d2..bdc9b6efbbbe0 100644 --- a/providers/databricks/docs/index.rst +++ b/providers/databricks/docs/index.rst @@ -78,7 +78,7 @@ apache-airflow-providers-databricks package `Databricks `__ -Release: 7.12.0 +Release: 7.12.1 Provider package ---------------- @@ -144,5 +144,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-databricks 7.12.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-databricks 7.12.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-databricks 7.12.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-databricks 7.12.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/databricks/provider.yaml b/providers/databricks/provider.yaml index 1a3a3fb18bede..deccacc29c631 100644 --- a/providers/databricks/provider.yaml +++ b/providers/databricks/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298972 +source-date-epoch: 1775591964 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 7.12.1 - 7.12.0 - 7.11.0 - 7.10.0 diff --git a/providers/databricks/pyproject.toml b/providers/databricks/pyproject.toml index ab2d15bb59b2d..729b0389724c7 100644 --- a/providers/databricks/pyproject.toml +++ b/providers/databricks/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-databricks" -version = "7.12.0" +version = "7.12.1" description = "Provider package apache-airflow-providers-databricks for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -150,8 +150,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.12.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.12.0/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.12.1" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-databricks/7.12.1/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/databricks/src/airflow/providers/databricks/__init__.py b/providers/databricks/src/airflow/providers/databricks/__init__.py index 2e00d6e9dcc2e..3a3ea65aa04d5 100644 --- a/providers/databricks/src/airflow/providers/databricks/__init__.py +++ b/providers/databricks/src/airflow/providers/databricks/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "7.12.0" +__version__ = "7.12.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/databricks/tests/system/databricks/example_databricks.py b/providers/databricks/tests/system/databricks/example_databricks.py index 5dca4684eafa3..d2cd6b22cadb0 100644 --- a/providers/databricks/tests/system/databricks/example_databricks.py +++ b/providers/databricks/tests/system/databricks/example_databricks.py @@ -255,5 +255,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/databricks/tests/system/databricks/example_databricks_repos.py b/providers/databricks/tests/system/databricks/example_databricks_repos.py index 33b0d1266f477..6cb4647b1bbf1 100644 --- a/providers/databricks/tests/system/databricks/example_databricks_repos.py +++ b/providers/databricks/tests/system/databricks/example_databricks_repos.py @@ -86,5 +86,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/databricks/tests/system/databricks/example_databricks_sensors.py b/providers/databricks/tests/system/databricks/example_databricks_sensors.py index 177ea8ce2936b..2b41392c763d9 100644 --- a/providers/databricks/tests/system/databricks/example_databricks_sensors.py +++ b/providers/databricks/tests/system/databricks/example_databricks_sensors.py @@ -112,5 +112,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/databricks/tests/system/databricks/example_databricks_sql.py b/providers/databricks/tests/system/databricks/example_databricks_sql.py index 79c2bfa2fca9a..2782525a55ccc 100644 --- a/providers/databricks/tests/system/databricks/example_databricks_sql.py +++ b/providers/databricks/tests/system/databricks/example_databricks_sql.py @@ -121,5 +121,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/databricks/tests/system/databricks/example_databricks_workflow.py b/providers/databricks/tests/system/databricks/example_databricks_workflow.py index 39d7db6b6c771..9ea1a223e91b8 100644 --- a/providers/databricks/tests/system/databricks/example_databricks_workflow.py +++ b/providers/databricks/tests/system/databricks/example_databricks_workflow.py @@ -151,5 +151,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/datadog/README.rst b/providers/datadog/README.rst index cfcb569572453..98850d29e5c16 100644 --- a/providers/datadog/README.rst +++ b/providers/datadog/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-datadog`` -Release: ``3.10.3`` +Release: ``3.10.4`` `Datadog `__ @@ -36,7 +36,7 @@ This is a provider package for ``datadog`` provider. All classes for this provid are in ``airflow.providers.datadog`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -78,4 +78,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/datadog/docs/changelog.rst b/providers/datadog/docs/changelog.rst index 090ee95e3cbf8..e5a70c681bf10 100644 --- a/providers/datadog/docs/changelog.rst +++ b/providers/datadog/docs/changelog.rst @@ -27,6 +27,17 @@ Changelog --------- +3.10.4 +...... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 3.10.3 ...... diff --git a/providers/datadog/docs/index.rst b/providers/datadog/docs/index.rst index 161c9f68120bc..ff721bbeee997 100644 --- a/providers/datadog/docs/index.rst +++ b/providers/datadog/docs/index.rst @@ -68,7 +68,7 @@ apache-airflow-providers-datadog package `Datadog `__ -Release: 3.10.3 +Release: 3.10.4 Provider package ---------------- @@ -121,5 +121,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-datadog 3.10.3 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-datadog 3.10.3 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-datadog 3.10.4 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-datadog 3.10.4 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/datadog/provider.yaml b/providers/datadog/provider.yaml index 1732f099733d6..af55f790ff762 100644 --- a/providers/datadog/provider.yaml +++ b/providers/datadog/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774298983 +source-date-epoch: 1775591970 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.10.4 - 3.10.3 - 3.10.2 - 3.10.1 diff --git a/providers/datadog/pyproject.toml b/providers/datadog/pyproject.toml index 7e42cf85492a1..0b5268d0848be 100644 --- a/providers/datadog/pyproject.toml +++ b/providers/datadog/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-datadog" -version = "3.10.3" +version = "3.10.4" description = "Provider package apache-airflow-providers-datadog for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -99,8 +99,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-datadog/3.10.3" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-datadog/3.10.3/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-datadog/3.10.4" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-datadog/3.10.4/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/datadog/src/airflow/providers/datadog/__init__.py b/providers/datadog/src/airflow/providers/datadog/__init__.py index 39ec13299de58..0d393281b922f 100644 --- a/providers/datadog/src/airflow/providers/datadog/__init__.py +++ b/providers/datadog/src/airflow/providers/datadog/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.10.3" +__version__ = "3.10.4" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/dbt/cloud/README.rst b/providers/dbt/cloud/README.rst index 81c76296ee4f3..ec5fd7ce4b05b 100644 --- a/providers/dbt/cloud/README.rst +++ b/providers/dbt/cloud/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-dbt-cloud`` -Release: ``4.8.0`` +Release: ``4.8.1`` `dbt Cloud `__ @@ -36,7 +36,7 @@ This is a provider package for ``dbt.cloud`` provider. All classes for this prov are in ``airflow.providers.dbt.cloud`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -93,4 +93,4 @@ Extra Dependencies =============== =============================================== The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/dbt/cloud/docs/changelog.rst b/providers/dbt/cloud/docs/changelog.rst index 7ab9ed92a8580..0bf68bb2d36b5 100644 --- a/providers/dbt/cloud/docs/changelog.rst +++ b/providers/dbt/cloud/docs/changelog.rst @@ -28,6 +28,17 @@ Changelog --------- +4.8.1 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.8.0 ..... diff --git a/providers/dbt/cloud/docs/index.rst b/providers/dbt/cloud/docs/index.rst index 6cd15f8414f49..fc9363fe19764 100644 --- a/providers/dbt/cloud/docs/index.rst +++ b/providers/dbt/cloud/docs/index.rst @@ -81,7 +81,7 @@ apache-airflow-providers-dbt-cloud package `dbt Cloud `__ -Release: 4.8.0 +Release: 4.8.1 Provider package ---------------- @@ -140,5 +140,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-dbt-cloud 4.8.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-dbt-cloud 4.8.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dbt-cloud 4.8.1 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dbt-cloud 4.8.1 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/dbt/cloud/provider.yaml b/providers/dbt/cloud/provider.yaml index 30325305fa49e..f39a98769c846 100644 --- a/providers/dbt/cloud/provider.yaml +++ b/providers/dbt/cloud/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774299006 +source-date-epoch: 1775591975 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 4.8.1 - 4.8.0 - 4.7.0 - 4.6.5 diff --git a/providers/dbt/cloud/pyproject.toml b/providers/dbt/cloud/pyproject.toml index 671a612a0ab5d..0544cf8836beb 100644 --- a/providers/dbt/cloud/pyproject.toml +++ b/providers/dbt/cloud/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-dbt-cloud" -version = "4.8.0" +version = "4.8.1" description = "Provider package apache-airflow-providers-dbt-cloud for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -113,8 +113,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.8.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.8.0/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.8.1" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dbt-cloud/4.8.1/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/__init__.py b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/__init__.py index 5274bc29b319f..f82b99ce5894f 100644 --- a/providers/dbt/cloud/src/airflow/providers/dbt/cloud/__init__.py +++ b/providers/dbt/cloud/src/airflow/providers/dbt/cloud/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.8.0" +__version__ = "4.8.1" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/dbt/cloud/tests/system/dbt/cloud/example_dbt_cloud.py b/providers/dbt/cloud/tests/system/dbt/cloud/example_dbt_cloud.py index 1c43ca0eabffb..8a0ebad561166 100644 --- a/providers/dbt/cloud/tests/system/dbt/cloud/example_dbt_cloud.py +++ b/providers/dbt/cloud/tests/system/dbt/cloud/example_dbt_cloud.py @@ -111,5 +111,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/dingding/README.rst b/providers/dingding/README.rst index 9ac14225dfda9..2a7e757467128 100644 --- a/providers/dingding/README.rst +++ b/providers/dingding/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-dingding`` -Release: ``3.9.3`` +Release: ``3.9.4`` `DingTalk `__ @@ -36,7 +36,7 @@ This is a provider package for ``dingding`` provider. All classes for this provi are in ``airflow.providers.dingding`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -79,4 +79,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/dingding/docs/changelog.rst b/providers/dingding/docs/changelog.rst index ea664b359ee0f..d362cbc074ad2 100644 --- a/providers/dingding/docs/changelog.rst +++ b/providers/dingding/docs/changelog.rst @@ -28,6 +28,17 @@ Changelog --------- +3.9.4 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 3.9.3 ..... diff --git a/providers/dingding/docs/index.rst b/providers/dingding/docs/index.rst index 793aa2e855b38..6572b9b4ea130 100644 --- a/providers/dingding/docs/index.rst +++ b/providers/dingding/docs/index.rst @@ -76,7 +76,7 @@ apache-airflow-providers-dingding package `DingTalk `__ -Release: 3.9.3 +Release: 3.9.4 Provider package ---------------- @@ -130,5 +130,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-dingding 3.9.3 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-dingding 3.9.3 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dingding 3.9.4 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-dingding 3.9.4 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/dingding/provider.yaml b/providers/dingding/provider.yaml index 7a0d97ca157e3..9b156dca089b9 100644 --- a/providers/dingding/provider.yaml +++ b/providers/dingding/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774299016 +source-date-epoch: 1775591980 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.9.4 - 3.9.3 - 3.9.2 - 3.9.1 diff --git a/providers/dingding/pyproject.toml b/providers/dingding/pyproject.toml index 4f172229e4031..0221b83e77ed6 100644 --- a/providers/dingding/pyproject.toml +++ b/providers/dingding/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-dingding" -version = "3.9.3" +version = "3.9.4" description = "Provider package apache-airflow-providers-dingding for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -100,8 +100,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dingding/3.9.3" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dingding/3.9.3/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-dingding/3.9.4" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-dingding/3.9.4/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/dingding/src/airflow/providers/dingding/__init__.py b/providers/dingding/src/airflow/providers/dingding/__init__.py index f4882803515a2..faa49d580ba47 100644 --- a/providers/dingding/src/airflow/providers/dingding/__init__.py +++ b/providers/dingding/src/airflow/providers/dingding/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.9.3" +__version__ = "3.9.4" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/dingding/tests/system/dingding/example_dingding.py b/providers/dingding/tests/system/dingding/example_dingding.py index 536f28fcc0d41..850e7eae40679 100644 --- a/providers/dingding/tests/system/dingding/example_dingding.py +++ b/providers/dingding/tests/system/dingding/example_dingding.py @@ -207,5 +207,5 @@ def failure_callback(context): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/discord/README.rst b/providers/discord/README.rst index 9c39dc310708a..97dde4a441a11 100644 --- a/providers/discord/README.rst +++ b/providers/discord/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-discord`` -Release: ``3.12.1`` +Release: ``3.12.2`` `Discord `__ @@ -36,7 +36,7 @@ This is a provider package for ``discord`` provider. All classes for this provid are in ``airflow.providers.discord`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -79,4 +79,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/discord/docs/changelog.rst b/providers/discord/docs/changelog.rst index b62fb9fb126e5..4d9f0b3578d0a 100644 --- a/providers/discord/docs/changelog.rst +++ b/providers/discord/docs/changelog.rst @@ -27,6 +27,17 @@ Changelog --------- +3.12.2 +...... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 3.12.1 ...... diff --git a/providers/discord/docs/index.rst b/providers/discord/docs/index.rst index f15bd5449cfa6..00fd35d30fdc4 100644 --- a/providers/discord/docs/index.rst +++ b/providers/discord/docs/index.rst @@ -63,7 +63,7 @@ apache-airflow-providers-discord package `Discord `__ -Release: 3.12.1 +Release: 3.12.2 Provider package ---------------- @@ -117,5 +117,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-discord 3.12.1 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-discord 3.12.1 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-discord 3.12.2 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-discord 3.12.2 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/discord/provider.yaml b/providers/discord/provider.yaml index 4d8b0a40a8aa1..d0a648b7ea208 100644 --- a/providers/discord/provider.yaml +++ b/providers/discord/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774299031 +source-date-epoch: 1775591984 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.12.2 - 3.12.1 - 3.12.0 - 3.11.1 diff --git a/providers/discord/pyproject.toml b/providers/discord/pyproject.toml index 5e5ea6485cee1..c43d7b2f50e11 100644 --- a/providers/discord/pyproject.toml +++ b/providers/discord/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-discord" -version = "3.12.1" +version = "3.12.2" description = "Provider package apache-airflow-providers-discord for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -100,8 +100,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-discord/3.12.1" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-discord/3.12.1/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-discord/3.12.2" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-discord/3.12.2/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/discord/src/airflow/providers/discord/__init__.py b/providers/discord/src/airflow/providers/discord/__init__.py index 02fbd80602121..e28cefbcf9aa8 100644 --- a/providers/discord/src/airflow/providers/discord/__init__.py +++ b/providers/discord/src/airflow/providers/discord/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.12.1" +__version__ = "3.12.2" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/docker/README.rst b/providers/docker/README.rst index d50377f9df8a1..6ecb393ffad20 100644 --- a/providers/docker/README.rst +++ b/providers/docker/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-docker`` -Release: ``4.5.4`` +Release: ``4.5.5`` `Docker `__ @@ -36,7 +36,7 @@ This is a provider package for ``docker`` provider. All classes for this provide are in ``airflow.providers.docker`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -79,4 +79,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/docker/docs/changelog.rst b/providers/docker/docs/changelog.rst index b40805ea1c3b9..09acd0a22721a 100644 --- a/providers/docker/docs/changelog.rst +++ b/providers/docker/docs/changelog.rst @@ -28,6 +28,17 @@ Changelog --------- +4.5.5 +..... + +Misc +~~~~ + +* ``Load hook metadata from YAML without importing Hook class (#63826)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + 4.5.4 ..... diff --git a/providers/docker/docs/index.rst b/providers/docker/docs/index.rst index 236cc19653ff8..ed3c5fdee36e6 100644 --- a/providers/docker/docs/index.rst +++ b/providers/docker/docs/index.rst @@ -70,7 +70,7 @@ apache-airflow-providers-docker package `Docker `__ -Release: 4.5.4 +Release: 4.5.5 Provider package ---------------- @@ -124,5 +124,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-docker 4.5.4 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-docker 4.5.4 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-docker 4.5.5 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-docker 4.5.5 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/docker/provider.yaml b/providers/docker/provider.yaml index 2043871373e3a..a525b43f0a183 100644 --- a/providers/docker/provider.yaml +++ b/providers/docker/provider.yaml @@ -23,12 +23,13 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774299040 +source-date-epoch: 1775591989 # Note that those versions are maintained by release manager - do not update them manually # with the exception of case where other provider in sources has >= new provider version. # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 4.5.5 - 4.5.4 - 4.5.3 - 4.5.2 diff --git a/providers/docker/pyproject.toml b/providers/docker/pyproject.toml index 61954ed5d7365..aaa4cb826c4f2 100644 --- a/providers/docker/pyproject.toml +++ b/providers/docker/pyproject.toml @@ -25,7 +25,7 @@ build-backend = "flit_core.buildapi" [project] name = "apache-airflow-providers-docker" -version = "4.5.4" +version = "4.5.5" description = "Provider package apache-airflow-providers-docker for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -100,8 +100,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-docker/4.5.4" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-docker/4.5.4/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-docker/4.5.5" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-docker/4.5.5/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/docker/src/airflow/providers/docker/__init__.py b/providers/docker/src/airflow/providers/docker/__init__.py index 01255342e344f..43825d1ef62a8 100644 --- a/providers/docker/src/airflow/providers/docker/__init__.py +++ b/providers/docker/src/airflow/providers/docker/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "4.5.4" +__version__ = "4.5.5" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "2.11.0" diff --git a/providers/docker/tests/system/docker/example_docker.py b/providers/docker/tests/system/docker/example_docker.py index 14fa3fa3203c7..30cc28fba086f 100644 --- a/providers/docker/tests/system/docker/example_docker.py +++ b/providers/docker/tests/system/docker/example_docker.py @@ -59,5 +59,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/docker/tests/system/docker/example_docker_copy_data.py b/providers/docker/tests/system/docker/example_docker_copy_data.py index 85a08e0961d82..cbbf75eb4a91f 100644 --- a/providers/docker/tests/system/docker/example_docker_copy_data.py +++ b/providers/docker/tests/system/docker/example_docker_copy_data.py @@ -106,5 +106,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/docker/tests/system/docker/example_docker_swarm.py b/providers/docker/tests/system/docker/example_docker_swarm.py index a45d1bba351ca..c6417d1d61314 100644 --- a/providers/docker/tests/system/docker/example_docker_swarm.py +++ b/providers/docker/tests/system/docker/example_docker_swarm.py @@ -49,5 +49,5 @@ from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/docker/tests/system/docker/example_taskflow_api_docker_virtualenv.py b/providers/docker/tests/system/docker/example_taskflow_api_docker_virtualenv.py index 7d34688d07f7d..2bb42faf2982f 100644 --- a/providers/docker/tests/system/docker/example_taskflow_api_docker_virtualenv.py +++ b/providers/docker/tests/system/docker/example_taskflow_api_docker_virtualenv.py @@ -123,7 +123,7 @@ def load(total_order_value: float): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) # [END tutorial] @@ -131,5 +131,5 @@ def load(total_order_value: float): from tests_common.test_utils.system_tests import get_test_run # noqa: E402 -# Needed to run the example DAG with pytest (see: tests/system/README.md#run_via_pytest) +# Needed to run the example DAG with pytest (see: contributing-docs/testing/system_tests.rst) test_run = get_test_run(dag) diff --git a/providers/edge3/README.rst b/providers/edge3/README.rst index ee789d3d02087..d734524b9f52e 100644 --- a/providers/edge3/README.rst +++ b/providers/edge3/README.rst @@ -23,7 +23,7 @@ Package ``apache-airflow-providers-edge3`` -Release: ``3.3.0`` +Release: ``3.4.0`` Handle edge workers on remote sites via HTTP(s) connection and orchestrates work over distributed sites. @@ -48,7 +48,7 @@ This is a provider package for ``edge3`` provider. All classes for this provider are in ``airflow.providers.edge3`` python package. You can find package information and changelog for the provider -in the `documentation `_. +in the `documentation `_. Installation ------------ @@ -93,4 +93,4 @@ Dependent package ================================================================================================================== ================= The changelog for the provider package can be found in the -`changelog `_. +`changelog `_. diff --git a/providers/edge3/docs/changelog.rst b/providers/edge3/docs/changelog.rst index 2cf6365c4c54f..6238ab2754e17 100644 --- a/providers/edge3/docs/changelog.rst +++ b/providers/edge3/docs/changelog.rst @@ -27,6 +27,33 @@ Changelog --------- +3.4.0 +..... + +Features +~~~~~~~~ + +* ``AIP 67 - Multi-Team: Update Edge Executor to support multi team (#61646)`` + +Misc +~~~~ + +* ``Remove dependabot alarms in edge provider plugin (#64788)`` +* ``Bump vite in /providers/edge3/src/airflow/providers/edge3/plugins/www (#64800)`` +* ``Add no-op _process_workloads to EdgeExecutor to improve readability (#64236)`` +* ``Add 4-day cooldown for pnpm dependency resolution (#64337)`` +* ``chore(deps-dev): bump happy-dom (#64421)`` +* ``Fix dependabot alarms in Edge provider (#64368)`` +* ``chore(deps-dev): bump happy-dom (#64272)`` + +.. Below changes are excluded from the changelog. Move them to + appropriate section above if needed. Do not delete the lines(!): + * ``CI: Upgrade important CI environment (#64744)`` + * ``CI: Upgrade important CI environment (#64451)`` + * ``Compat sdk conf follow-up: Celery, Common AI, FAB, Edge3 (#64292)`` + * ``[main] Upgrade important CI environment (#64239)`` + * ``Add 4-day cooldown for uv dependency resolution (#64249)`` + 3.3.0 ..... diff --git a/providers/edge3/docs/index.rst b/providers/edge3/docs/index.rst index e12a0c7b84830..20a04faa9c071 100644 --- a/providers/edge3/docs/index.rst +++ b/providers/edge3/docs/index.rst @@ -97,7 +97,7 @@ Additional REST API endpoints are provided to distribute tasks and manage the ed are provided by the API server. -Release: 3.3.0 +Release: 3.4.0 Provider package ---------------- @@ -153,5 +153,5 @@ Downloading official packages You can download officially released packages and verify their checksums and signatures from the `Official Apache Download site `_ -* `The apache-airflow-providers-edge3 3.3.0 sdist package `_ (`asc `__, `sha512 `__) -* `The apache-airflow-providers-edge3 3.3.0 wheel package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-edge3 3.4.0 sdist package `_ (`asc `__, `sha512 `__) +* `The apache-airflow-providers-edge3 3.4.0 wheel package `_ (`asc `__, `sha512 `__) diff --git a/providers/edge3/provider.yaml b/providers/edge3/provider.yaml index 06188d2bbe3a7..1ba8b89067a4c 100644 --- a/providers/edge3/provider.yaml +++ b/providers/edge3/provider.yaml @@ -34,7 +34,7 @@ description: | state: ready lifecycle: production -source-date-epoch: 1774299111 +source-date-epoch: 1775592062 build-system: hatchling # Note that those versions are maintained by release manager - do not update them manually @@ -42,6 +42,7 @@ build-system: hatchling # In such case adding >= NEW_VERSION and bumping to NEW_VERSION in a provider have # to be done in the same PR versions: + - 3.4.0 - 3.3.0 - 3.2.0 - 3.1.0 diff --git a/providers/edge3/pyproject.toml b/providers/edge3/pyproject.toml index 41854615951e2..921544d446805 100644 --- a/providers/edge3/pyproject.toml +++ b/providers/edge3/pyproject.toml @@ -32,7 +32,7 @@ build-backend = "hatchling.build" [project] name = "apache-airflow-providers-edge3" -version = "3.3.0" +version = "3.4.0" description = "Provider package apache-airflow-providers-edge3 for Apache Airflow" readme = "README.rst" license = "Apache-2.0" @@ -109,8 +109,8 @@ apache-airflow-providers-common-sql = {workspace = true} apache-airflow-providers-standard = {workspace = true} [project.urls] -"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-edge3/3.3.0" -"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-edge3/3.3.0/changelog.html" +"Documentation" = "https://airflow.apache.org/docs/apache-airflow-providers-edge3/3.4.0" +"Changelog" = "https://airflow.apache.org/docs/apache-airflow-providers-edge3/3.4.0/changelog.html" "Bug Tracker" = "https://github.com/apache/airflow/issues" "Source Code" = "https://github.com/apache/airflow" "Slack Chat" = "https://s.apache.org/airflow-slack" diff --git a/providers/edge3/src/airflow/providers/edge3/__init__.py b/providers/edge3/src/airflow/providers/edge3/__init__.py index 23a3e8d66ae4e..d7d98d1ff37d4 100644 --- a/providers/edge3/src/airflow/providers/edge3/__init__.py +++ b/providers/edge3/src/airflow/providers/edge3/__init__.py @@ -29,7 +29,7 @@ __all__ = ["__version__"] -__version__ = "3.3.0" +__version__ = "3.4.0" if packaging.version.parse(packaging.version.parse(airflow_version).base_version) < packaging.version.parse( "3.0.0" diff --git a/providers/edge3/src/airflow/providers/edge3/example_dags/win_test.py b/providers/edge3/src/airflow/providers/edge3/example_dags/win_test.py index a3527e4a4e268..0f021bfebc08d 100644 --- a/providers/edge3/src/airflow/providers/edge3/example_dags/win_test.py +++ b/providers/edge3/src/airflow/providers/edge3/example_dags/win_test.py @@ -28,7 +28,7 @@ import os from collections.abc import Callable, Container, Sequence from datetime import datetime -from subprocess import STDOUT, Popen +from subprocess import STDOUT, Popen, SubprocessError from time import sleep from typing import TYPE_CHECKING, Any @@ -36,7 +36,6 @@ from airflow.models.dag import DAG from airflow.models.variable import Variable from airflow.providers.common.compat.sdk import ( - AirflowException, AirflowNotFoundException, AirflowSkipException, ) @@ -125,7 +124,7 @@ class CmdOperator(BaseOperator): * - `skip_on_exit_code` (default: 99) - raise :class:`airflow.exceptions.AirflowSkipException` * - otherwise - - raise :class:`airflow.exceptions.AirflowException` + - raise :class:`subprocess.SubprocessError` .. warning:: @@ -210,9 +209,9 @@ def get_env(self, context): def execute(self, context: Context): if self.cwd is not None: if not os.path.exists(self.cwd): - raise AirflowException(f"Can not find the cwd: {self.cwd}") + raise SubprocessError(f"Can not find the cwd: {self.cwd}") if not os.path.isdir(self.cwd): - raise AirflowException(f"The cwd {self.cwd} must be a directory") + raise SubprocessError(f"The cwd {self.cwd} must be a directory") env = self.get_env(context) # Because the command value is evaluated at runtime using the @task.command decorator, the @@ -237,7 +236,7 @@ def execute(self, context: Context): if exit_code in self.skip_on_exit_code: raise AirflowSkipException(f"Command returned exit code {exit_code}. Skipping.") if exit_code != 0: - raise AirflowException(f"Command failed. The command returned a non-zero exit code {exit_code}.") + raise SubprocessError(f"Command failed. The command returned a non-zero exit code {exit_code}.") return self.output_processor(outs) diff --git a/providers/edge3/src/airflow/providers/edge3/plugins/www/package.json b/providers/edge3/src/airflow/providers/edge3/plugins/www/package.json index 5edc11edfb55f..121eed6f9d0f1 100644 --- a/providers/edge3/src/airflow/providers/edge3/plugins/www/package.json +++ b/providers/edge3/src/airflow/providers/edge3/plugins/www/package.json @@ -36,7 +36,7 @@ "@emotion/react": "^11.14.0", "@tanstack/react-query": "^5.90.21", "@types/semver": "^7.7.1", - "axios": "^1.13.6", + "axios": "^1.15.0", "next-themes": "^0.4.6", "react": "^19.2.4", "react-dom": "^19.2.4", @@ -105,7 +105,12 @@ "brace-expansion@>=4.0.0 <5.0.5": ">=5.0.5", "handlebars@>=4.0.0 <4.7.9": ">=4.7.9", "picomatch@>=4.0.0 <4.0.4": ">=4.0.4", - "yaml@>=1.0.0 <1.10.3": ">=1.10.3" + "yaml@>=1.0.0 <1.10.3": ">=1.10.3", + "lodash-es@>=4.0.0 <=4.17.23": ">=4.18.0", + "lodash@>=4.0.0 <=4.17.23": ">=4.18.0", + "lodash-es@<=4.17.23": ">=4.18.0", + "lodash@<=4.17.23": ">=4.18.0", + "defu@<=6.1.4": ">=6.1.5" } } } diff --git a/providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml b/providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml index 7ab20dbdb6018..1dca521c4dc02 100644 --- a/providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml +++ b/providers/edge3/src/airflow/providers/edge3/plugins/www/pnpm-lock.yaml @@ -22,6 +22,11 @@ overrides: handlebars@>=4.0.0 <4.7.9: '>=4.7.9' picomatch@>=4.0.0 <4.0.4: '>=4.0.4' yaml@>=1.0.0 <1.10.3: '>=1.10.3' + lodash-es@>=4.0.0 <=4.17.23: '>=4.18.0' + lodash@>=4.0.0 <=4.17.23: '>=4.18.0' + lodash-es@<=4.17.23: '>=4.18.0' + lodash@<=4.17.23: '>=4.18.0' + defu@<=6.1.4: '>=6.1.5' importers: @@ -40,8 +45,8 @@ importers: specifier: ^7.7.1 version: 7.7.1 axios: - specifier: ^1.13.6 - version: 1.13.6 + specifier: ^1.15.0 + version: 1.15.0 next-themes: specifier: ^0.4.6 version: 0.4.6(react-dom@19.2.4(react@19.2.4))(react@19.2.4) @@ -1452,8 +1457,8 @@ packages: resolution: {integrity: sha512-BASOg+YwO2C+346x3LZOeoovTIoTrRqEsqMa6fmfAV0P+U9mFr9NsyOEpiYvFjbc64NMrSswhV50WdXzdb/Z5A==} engines: {node: '>=4'} - axios@1.13.6: - resolution: {integrity: sha512-ChTCHMouEe2kn713WHbQGcuYrr6fXTBiu460OTwWrWob16g1bXn4vtz07Ope7ewMozJAnEquLk5lWQWtBig9DQ==} + axios@1.15.0: + resolution: {integrity: sha512-wWyJDlAatxk30ZJer+GeCWS209sA42X+N5jU2jy6oHTp7ufw8uzUTVFBX9+wTfAlhiJXGS0Bq7X6efruWjuK9Q==} axobject-query@4.1.0: resolution: {integrity: sha512-qIj0G9wZbMGNLjLmg1PT6v2mE9AH2zlnADJD/2tC6E00hgmhUOfEB6greHPAfLRSufHqROIUTkw6E+M3lH0PTQ==} @@ -1642,8 +1647,8 @@ packages: resolution: {integrity: sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==} engines: {node: '>= 0.4'} - defu@6.1.4: - resolution: {integrity: sha512-mEQCMmwJu317oSz8CwdIOdwf3xMif1ttiM8LTufzc3g6kR+9Pe236twL8j3IYT1F7GfRgGcW6MWxzZjLIkuHIg==} + defu@6.1.6: + resolution: {integrity: sha512-f8mefEW4WIVg4LckePx3mALjQSPQgFlg9U8yaPdlsbdYcHQyj9n2zL2LJEA52smeYxOvmd/nB7TpMtHGMTHcug==} delayed-stream@1.0.0: resolution: {integrity: sha512-ZySD7Nf91aLB0RxL4KGrKHBXl7Eds1DAmEdcoVawXnLD7SDhpNgtuII2aAkg7a7QS41jxPSZ17p4VdGnMHk3MQ==} @@ -2292,11 +2297,11 @@ packages: resolution: {integrity: sha512-iPZK6eYjbxRu3uB4/WZ3EsEIMJFMqAoopl3R+zuq0UjcAm/MO6KCweDgPfP3elTztoKP3KtnVHxTn2NHBSDVUw==} engines: {node: '>=10'} - lodash-es@4.17.23: - resolution: {integrity: sha512-kVI48u3PZr38HdYz98UmfPnXl2DXrpdctLrFLCd3kOx1xUkOmpFPx7gCWWM5MPkL/fD8zb+Ph0QzjGFs4+hHWg==} + lodash-es@4.18.1: + resolution: {integrity: sha512-J8xewKD/Gk22OZbhpOVSwcs60zhd95ESDwezOFuA3/099925PdHJ7OFHNTGtajL3AlZkykD32HykiMo+BIBI8A==} - lodash@4.17.23: - resolution: {integrity: sha512-LgVTMpQtIopCi79SJeDiP0TfWi5CNEc/L/aRdTh3yIvmZXTnheWpKjSZhnvMl8iXbC1tFg9gdHHDMLoV7CnG+w==} + lodash@4.18.1: + resolution: {integrity: sha512-dMInicTPVE8d1e5otfwmmjlxkZoUpiVLwyeTdUsi/Caj/gfzzblBcCE5sRHV/AsjuCmxWrte2TNGSYuCeCq+0Q==} loose-envify@1.4.0: resolution: {integrity: sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q==} @@ -2565,8 +2570,9 @@ packages: proxy-compare@3.0.1: resolution: {integrity: sha512-V9plBAt3qjMlS1+nC8771KNf6oJ12gExvaxnNzN/9yVRLdTv/lc+oJlnSzrdYDAvBfTStPCoiaCOTmTs0adv7Q==} - proxy-from-env@1.1.0: - resolution: {integrity: sha512-D+zkORCbA9f1tdWRK0RaCR3GPv50cMxcrz4X8k5LTSUD1Dkw47mKJEZQNunItRTkWwgtaUSo1RVFRIG9ZXiFYg==} + proxy-from-env@2.1.0: + resolution: {integrity: sha512-cJ+oHTW1VAEa8cJslgmUZrc+sjRKgAKl3Zyse6+PV38hZe/V6Z14TbCuXcan9F9ghlz4QrFr2c92TNF82UkYHA==} + engines: {node: '>=10'} proxy-memoize@3.0.1: resolution: {integrity: sha512-VDdG/VYtOgdGkWJx7y0o7p+zArSf2383Isci8C+BP3YXgMYDoPd3cCBjw0JdWb6YBb9sFiOPbAADDVTPJnh+9g==} @@ -3599,7 +3605,7 @@ snapshots: '@rushstack/terminal': 0.22.3(@types/node@25.3.3) '@rushstack/ts-command-line': 5.3.3(@types/node@25.3.3) diff: 8.0.3 - lodash: 4.17.23 + lodash: 4.18.1 minimatch: 10.2.4 resolve: 1.22.11 semver: 7.5.4 @@ -3870,7 +3876,7 @@ snapshots: '@babel/traverse': 7.29.0 '@babel/types': 7.29.0 javascript-natural-sort: 0.7.1 - lodash-es: 4.17.23 + lodash-es: 4.18.1 minimatch: 10.2.4 parse-imports-exports: 0.2.4 prettier: 3.8.1 @@ -4796,11 +4802,11 @@ snapshots: axe-core@4.11.1: {} - axios@1.13.6: + axios@1.15.0: dependencies: follow-redirects: 1.15.11 form-data: 4.0.5 - proxy-from-env: 1.1.0 + proxy-from-env: 2.1.0 transitivePeerDependencies: - debug @@ -4838,7 +4844,7 @@ snapshots: dependencies: chokidar: 4.0.3 confbox: 0.1.8 - defu: 6.1.4 + defu: 6.1.6 dotenv: 16.6.1 giget: 1.2.5 jiti: 2.6.1 @@ -4987,7 +4993,7 @@ snapshots: has-property-descriptors: 1.0.2 object-keys: 1.1.1 - defu@6.1.4: {} + defu@6.1.6: {} delayed-stream@1.0.0: {} @@ -5459,7 +5465,7 @@ snapshots: dependencies: citty: 0.1.6 consola: 3.4.2 - defu: 6.1.4 + defu: 6.1.6 node-fetch-native: 1.6.7 nypm: 0.5.4 pathe: 2.0.3 @@ -5774,9 +5780,9 @@ snapshots: dependencies: p-locate: 5.0.0 - lodash-es@4.17.23: {} + lodash-es@4.18.1: {} - lodash@4.17.23: {} + lodash@4.18.1: {} loose-envify@1.4.0: dependencies: @@ -6040,7 +6046,7 @@ snapshots: proxy-compare@3.0.1: {} - proxy-from-env@1.1.0: {} + proxy-from-env@2.1.0: {} proxy-memoize@3.0.1: dependencies: @@ -6052,7 +6058,7 @@ snapshots: rc9@2.1.2: dependencies: - defu: 6.1.4 + defu: 6.1.6 destr: 2.0.5 react-dom@19.2.4(react@19.2.4): diff --git a/providers/edge3/src/airflow/providers/edge3/plugins/www/src/components/BulkWorkerOperations.tsx b/providers/edge3/src/airflow/providers/edge3/plugins/www/src/components/BulkWorkerOperations.tsx index a65ad262609d9..4778d08f827b1 100644 --- a/providers/edge3/src/airflow/providers/edge3/plugins/www/src/components/BulkWorkerOperations.tsx +++ b/providers/edge3/src/airflow/providers/edge3/plugins/www/src/components/BulkWorkerOperations.tsx @@ -16,10 +16,13 @@ * specific language governing permissions and limitations * under the License. */ -import { Button, Dialog, HStack, List, Portal, Text, useDisclosure } from "@chakra-ui/react"; +import { Button, Dialog, HStack, List, Portal, Text, Textarea, useDisclosure } from "@chakra-ui/react"; import type { Worker } from "openapi/requests/types.gen"; +import { useState } from "react"; import { FaPowerOff } from "react-icons/fa"; import { FaRegTrashCan } from "react-icons/fa6"; +import { HiOutlineWrenchScrewdriver } from "react-icons/hi2"; +import { IoMdExit } from "react-icons/io"; import { useBulkWorkerActions } from "src/hooks/useBulkWorkerActions"; @@ -44,12 +47,29 @@ export const BulkWorkerOperations = ({ onOpen: onOpenDeleteDialog, open: isDeleteDialogOpen, } = useDisclosure(); + const { + onClose: onCloseMaintenanceEnterDialog, + onOpen: onOpenMaintenanceEnterDialog, + open: isMaintenanceEnterDialogOpen, + } = useDisclosure(); + const { + onClose: onCloseMaintenanceExitDialog, + onOpen: onOpenMaintenanceExitDialog, + open: isMaintenanceExitDialogOpen, + } = useDisclosure(); + const [maintenanceComment, setMaintenanceComment] = useState(""); const { deleteWorkers, handleBulkDelete, + handleBulkMaintenanceEnter, + handleBulkMaintenanceExit, handleBulkShutdown, isBulkDeletePending, + isBulkMaintenanceEnterPending, + isBulkMaintenanceExitPending, isBulkShutdownPending, + maintenanceEnterWorkers, + maintenanceExitWorkers, shutdownWorkers, } = useBulkWorkerActions({ onClearSelection, @@ -58,24 +78,53 @@ export const BulkWorkerOperations = ({ }); const onBulkShutdown = async () => { - try { - await handleBulkShutdown(); - } finally { - onCloseShutdownDialog(); - } + await handleBulkShutdown(); + onCloseShutdownDialog(); }; const onBulkDelete = async () => { - try { - await handleBulkDelete(); - } finally { - onCloseDeleteDialog(); - } + await handleBulkDelete(); + onCloseDeleteDialog(); + }; + + const onMaintenanceEnterDialogClose = () => { + onCloseMaintenanceEnterDialog(); + setMaintenanceComment(""); + }; + + const onBulkMaintenanceEnter = async () => { + await handleBulkMaintenanceEnter(maintenanceComment); + onMaintenanceEnterDialogClose(); + }; + + const onBulkMaintenanceExit = async () => { + await handleBulkMaintenanceExit(); + onCloseMaintenanceExitDialog(); }; return ( <> + +