From 217e02fcee02424782ccfc9167a67f8dab77e7d1 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Wed, 23 Jul 2025 13:31:56 +0200 Subject: [PATCH] Add possibility to apply a commit with patch to docs when publishing You can specify commit hash to apply when you build documentation. This allows to regenerate past version of the documentation by checking out the exact version tag that was used back then and applying the commit hash with fixes. This might help in fixing issues like https://github.com/apache/airflow/issues/53646 --- .github/actions/breeze/action.yml | 3 - .../prepare_breeze_and_image/action.yml | 2 - .../workflows/additional-ci-image-checks.yml | 2 - .../workflows/airflow-distributions-tests.yml | 2 - .github/workflows/backport-cli.yml | 4 +- .github/workflows/basic-tests.yml | 14 +---- .github/workflows/ci-amd.yml | 2 - .github/workflows/ci-arm.yml | 2 - .github/workflows/ci-image-build.yml | 2 - .github/workflows/ci-image-checks.yml | 2 - .github/workflows/helm-tests.yml | 6 +- .github/workflows/prod-image-build.yml | 4 -- .github/workflows/publish-docs-to-s3.yml | 39 +++++++++++-- .github/workflows/push-image-cache.yml | 4 -- .github/workflows/release_dockerhub_image.yml | 2 - .../release_single_dockerhub_image.yml | 4 -- dev/README_RELEASE_AIRFLOW.md | 19 ++++++ dev/README_RELEASE_PROVIDERS.md | 14 +++++ .../output_workflow-run_publish-docs.svg | 58 +++++++++++-------- .../output_workflow-run_publish-docs.txt | 2 +- .../commands/workflow_commands.py | 9 +++ .../commands/workflow_commands_config.py | 1 + 22 files changed, 120 insertions(+), 77 deletions(-) diff --git a/.github/actions/breeze/action.yml b/.github/actions/breeze/action.yml index 3fef664b337fa..a7fff7dc397fe 100644 --- a/.github/actions/breeze/action.yml +++ b/.github/actions/breeze/action.yml @@ -22,9 +22,6 @@ inputs: python-version: description: 'Python version to use' default: "3.10" - use-uv: - description: 'Whether to use uv tool' - required: true outputs: host-python-version: description: Python version used in host diff --git a/.github/actions/prepare_breeze_and_image/action.yml b/.github/actions/prepare_breeze_and_image/action.yml index 4a2685cc4ebd8..0724167cda519 100644 --- a/.github/actions/prepare_breeze_and_image/action.yml +++ b/.github/actions/prepare_breeze_and_image/action.yml @@ -43,8 +43,6 @@ runs: shell: bash - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} id: breeze - name: "Check free space" shell: bash diff --git a/.github/workflows/additional-ci-image-checks.yml b/.github/workflows/additional-ci-image-checks.yml index 86a75b1d31fb4..f3982389301e6 100644 --- a/.github/workflows/additional-ci-image-checks.yml +++ b/.github/workflows/additional-ci-image-checks.yml @@ -142,7 +142,5 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} - name: "Check that image builds quickly" run: breeze shell --max-time 900 --platform "${PLATFORM}" diff --git a/.github/workflows/airflow-distributions-tests.yml b/.github/workflows/airflow-distributions-tests.yml index 62d4ee1f6b420..98203e47b3ebc 100644 --- a/.github/workflows/airflow-distributions-tests.yml +++ b/.github/workflows/airflow-distributions-tests.yml @@ -105,8 +105,6 @@ jobs: if: ${{ inputs.use-local-venv == 'true' }} - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} if: ${{ inputs.use-local-venv == 'true' }} - name: "Cleanup dist files" run: rm -fv ./dist/* diff --git a/.github/workflows/backport-cli.yml b/.github/workflows/backport-cli.yml index 0ecdfb8e63e04..42f8178868267 100644 --- a/.github/workflows/backport-cli.yml +++ b/.github/workflows/backport-cli.yml @@ -70,8 +70,8 @@ jobs: TARGET_BRANCH: ${{ inputs.target-branch }} COMMIT_SHA: ${{ inputs.commit-sha }} run: | - git config --global user.email "name@example.com" - git config --global user.name "Your Name" + git config --global user.email "bot@airflow.apache.org" + git config --global user.name "Your friendly bot" set +e { echo 'cherry_picker_output< - breeze ci-image build + breeze ci-image build || + docker buildx build --load --builder default --progress=auto --pull + --build-arg AIRFLOW_EXTRAS=devel-ci --build-arg AIRFLOW_PRE_CACHED_PIP_PACKAGES=false + --build-arg AIRFLOW_USE_UV=true --build-arg UV_HTTP_TIMEOUT=300 + --build-arg BUILD_PROGRESS=auto --build-arg INSTALL_MYSQL_CLIENT_TYPE=mariadb + --build-arg VERSION_SUFFIX_FOR_PYPI=dev0 + -t ghcr.io/apache/airflow/main/ci/python3.9:latest --target main . + -f Dockerfile.ci --platform linux/amd64 - name: "Building docs with --docs-only flag using ${{ inputs.ref }} reference breeze" env: INCLUDE_DOCS: ${{ needs.build-info.outputs.include-docs }} @@ -291,8 +324,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} - name: "Download docs prepared as artifacts" uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0 with: diff --git a/.github/workflows/push-image-cache.yml b/.github/workflows/push-image-cache.yml index b523577b46c99..eccd176096137 100644 --- a/.github/workflows/push-image-cache.yml +++ b/.github/workflows/push-image-cache.yml @@ -120,8 +120,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} - name: Login to ghcr.io env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} @@ -191,8 +189,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: ${{ inputs.use-uv }} - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* - name: "Download packages prepared as artifacts" diff --git a/.github/workflows/release_dockerhub_image.yml b/.github/workflows/release_dockerhub_image.yml index 1ec9ec8f91be4..dbe38567c5cb0 100644 --- a/.github/workflows/release_dockerhub_image.yml +++ b/.github/workflows/release_dockerhub_image.yml @@ -96,8 +96,6 @@ jobs: run: uv run scripts/ci/airflow_version_check.py "${AIRFLOW_VERSION}" >> "${GITHUB_OUTPUT}" - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: "true" - name: Selective checks id: selective-checks env: diff --git a/.github/workflows/release_single_dockerhub_image.yml b/.github/workflows/release_single_dockerhub_image.yml index b308de93e72ee..991ac35b13924 100644 --- a/.github/workflows/release_single_dockerhub_image.yml +++ b/.github/workflows/release_single_dockerhub_image.yml @@ -83,8 +83,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: "false" - name: Free space run: breeze ci free-space --answer yes - name: "Cleanup dist and context file" @@ -181,8 +179,6 @@ jobs: run: ./scripts/ci/prepare_and_cleanup_runner.sh - name: "Install Breeze" uses: ./.github/actions/breeze - with: - use-uv: "false" - name: Free space run: breeze ci free-space --answer yes - name: "Cleanup dist and context file" diff --git a/dev/README_RELEASE_AIRFLOW.md b/dev/README_RELEASE_AIRFLOW.md index de201c44f093c..5f6a603de2f22 100644 --- a/dev/README_RELEASE_AIRFLOW.md +++ b/dev/README_RELEASE_AIRFLOW.md @@ -411,6 +411,25 @@ Other available parameters can be found with: breeze workflow-run publish-docs --help ``` +In case you publish the documentation from branch, you can specify `--airflow-version` and `--airflow-base-version` +parameters to specify which version of airflow you want to build the documentation for - as it cannot +be automatically derived from tag name. Normally both are automatically derived from the tag name. + +One of the interesting features of publishing this way is that you can also rebuild historical version of +the documentation with patches applied to the documentation (if they can be applied cleanly). + +Yoy should specify the `--apply-commits` parameter with the list of commits you want to apply +separated by commas and the workflow will apply those commits to the documentation before +building it. (don't forget to add --skip-write-to-stable-folder if you are publishing +previous version of the distribution). Example: + +```shell script +breeze workflow-run publish-docs --ref 3.0.3 --site-env staging \ + --apply-commits 4ae273cbedec66c87dc40218c7a94863390a380d,e61e9618bdd6be8213d277b1427f67079fcb1d9b \ + --skip-write-to-stable-folder \ + apache-airflow docker-stack task-sdk +``` + ### Manually using GitHub Actions There are two steps to publish the documentation: diff --git a/dev/README_RELEASE_PROVIDERS.md b/dev/README_RELEASE_PROVIDERS.md index c5938bd16bd60..c3bda6b64a54e 100644 --- a/dev/README_RELEASE_PROVIDERS.md +++ b/dev/README_RELEASE_PROVIDERS.md @@ -522,6 +522,20 @@ The `--ref` parameter should be the tag of the release candidate you are publish The `--site-env` parameter should be set to `staging` for pre-release versions or `live` for final releases. the default option is `auto` if the tag is rc it publishes to `staging` bucket, otherwise it publishes to `live` bucket. +One of the interesting features of publishing this way is that you can also rebuild historical version of +the documentation with patches applied to the documentation (if they can be applied cleanly). + +Yoy should specify the `--apply-commits` parameter with the list of commits you want to apply +separated by commas and the workflow will apply those commits to the documentation before +building it (don't forget to add --skip-write-to-stable-folder if you are publishing +previous version of the distribution). Example: + +```shell script +breeze workflow-run publish-docs --ref providers-apache-hive/9.0.0 --site-env live \ + --apply-commits 4ae273cbedec66c87dc40218c7a94863390a380d --skip-write-to-stable-folder \ + apache.hive +``` + Other available parameters can be found with: ```shell diff --git a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg index 23f375ca723ff..fb3016cb50db7 100644 --- a/dev/breeze/doc/images/output_workflow-run_publish-docs.svg +++ b/dev/breeze/doc/images/output_workflow-run_publish-docs.svg @@ -1,4 +1,4 @@ - +