From 8b847ed7e7953927a17738f6b68ad2102c63dd57 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 21 Jun 2022 12:20:02 +0200 Subject: [PATCH] Switch to building images in parallell In the new Breeze, switching to using parallelism is a ... breeze. This PR adds the capability of building the images in parallel in Breeze locally - for breeze command, but also uses this capability to build the images in parallel in our CI. Our builds are always executed on powerful, big machines with lots of CPU and docker run in memory filesystem with 32GB RAM, so it should be possible to run all builds in parallel on a single machine rather then spin off parallel machines to run the builds using the matrix strategy of Github Actions. Generally speaking - this will either speed up or get 4x cost saving for the build steps for all the "full test needed" PRs as well as all the main builds. There are a number of savings and improvements we can achieve this way: 1) less overhead for starting and runnning the machines 2) seems that with the new buildkit, the parallel builds are not suffering from some sequential locks (as it used to be, so we are basically do the same job using 25% resources for building the images. 3) we will stop having random "one image failed to build" cases - they will all either fail or succeed. 4) Less checks in the output 5) Production builds will additionally gain from single CI image pulled in order to perform the preparation of the packages and single package preparation step - it will save 4-5 minutes per image. The disadvantage is a less clear output of such parallel build where outputs from multiple builds will be interleaved in one CI output. --- .github/workflows/build-images.yml | 42 +-- .github/workflows/ci.yml | 42 ++- dev/REFRESHING_CI_CACHE.md | 2 +- .../commands/ci_image_commands.py | 84 +++-- .../configuration_and_maintenance_commands.py | 9 +- .../commands/production_image_commands.py | 90 +++-- .../airflow_breeze/utils/common_options.py | 6 - dev/refresh_images.sh | 4 +- images/breeze/output-build-image.svg | 276 +++++++------- images/breeze/output-build-prod-image.svg | 344 +++++++++--------- images/breeze/output-commands-hash.txt | 2 +- scripts/ci/selective_ci_checks.sh | 2 + 12 files changed, 496 insertions(+), 407 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 9970a82e6c213..541c72165997b 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -57,6 +57,7 @@ jobs: pythonVersions: "${{ steps.selective-checks.python-versions }}" upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} allPythonVersions: ${{ steps.selective-checks.outputs.all-python-versions }} + allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} run-tests: ${{ steps.selective-checks.outputs.run-tests }} run-kubernetes-tests: ${{ steps.selective-checks.outputs.run-kubernetes-tests }} @@ -155,19 +156,14 @@ jobs: permissions: packages: write timeout-minutes: 80 - name: "Build CI image ${{matrix.python-version}}" + name: "Build CI images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}" runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true if: | needs.build-info.outputs.image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} BACKEND: sqlite outputs: ${{toJSON(needs.build-info.outputs) }} steps: @@ -226,20 +222,25 @@ jobs: - run: ./scripts/ci/install_breeze.sh - name: "Free space" run: breeze free-space - - name: Build & Push CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze build-image --push-image --tag-as-latest + - name: > + Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + run: breeze build-image --push-image --tag-as-latest --run-in-parallel env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} - name: Push empty CI image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} if: failure() || cancelled() - run: breeze build-image --push-image --empty-image + run: breeze build-image --push-image --empty-image --run-in-parallel env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}" + - name: "Candidates for pip resolver backtrack triggers" if: failure() || cancelled() - run: breeze find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}" + run: > + breeze find-newer-dependencies --max-age 1 + --python "${{ needs.build-info.outputs.defaultPythonVersion }}" - name: "Fix ownership" run: breeze fix-ownership if: always() @@ -248,19 +249,14 @@ jobs: permissions: packages: write timeout-minutes: 80 - name: "Build PROD image ${{matrix.python-version}}" + name: "Build PROD images ${{ needs.build-info.outputs.allPythonVersionsListAsString }}" runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info, build-ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true if: | needs.build-info.outputs.image-build == 'true' && github.event.pull_request.head.repo.full_name != 'apache/airflow' env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} BACKEND: sqlite steps: - name: Cleanup repo @@ -320,7 +316,7 @@ jobs: run: breeze free-space - name: > Pull CI image for PROD build: - ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.defaultPythonVersion }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} run: breeze pull-image --tag-as-latest env: # Always use default Python version of CI image for preparing packages @@ -338,9 +334,12 @@ jobs: run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files - - name: Build & Push PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: > + Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} run: > breeze build-prod-image + --run-in-parallel --tag-as-latest --push-image --install-packages-from-context @@ -350,9 +349,10 @@ jobs: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - - name: Push empty PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + - name: Push empty PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} if: failure() || cancelled() - run: breeze build-prod-image --cleanup-context --push-image --empty-image + run: breeze build-prod-image --cleanup-context --push-image --empty-image --run-in-parallel env: IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Fix ownership" diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c112a6da727a7..5450833f23b63 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -132,6 +132,7 @@ jobs: upgradeToNewerDependencies: ${{ steps.selective-checks.outputs.upgrade-to-newer-dependencies }} pythonVersions: ${{ steps.selective-checks.outputs.python-versions }} pythonVersionsListAsString: ${{ steps.selective-checks.outputs.python-versions-list-as-string }} + allPythonVersionsListAsString: ${{ steps.selective-checks.outputs.all-python-versions-list-as-string }} defaultPythonVersion: ${{ steps.selective-checks.outputs.default-python-version }} kubernetesVersions: ${{ steps.selective-checks.outputs.kubernetes-versions }} kubernetesVersionsListAsString: ${{ steps.selective-checks.outputs.kubernetes-versions-list-as-string }} @@ -285,13 +286,11 @@ jobs: permissions: packages: write timeout-minutes: 80 - name: "${{needs.build-info.outputs.buildJobDescription}} CI image ${{matrix.python-version}}" + name: > + ${{needs.build-info.outputs.buildJobDescription}} CI images + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} steps: @@ -311,7 +310,7 @@ jobs: if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: "Retrieve DEFAULTS from the _initialization.sh" # We cannot "source" the script here because that would be a security problem (we cannot run - # any code that comes from the sources coming from the PR. Therefore we extract the + # any code that comes from the sources coming from the PR. Therefore, we extract the # DEFAULT_BRANCH and DEFAULT_CONSTRAINTS_BRANCH and DEBIAN_VERSION via custom grep/awk/sed commands id: defaults run: | @@ -331,17 +330,21 @@ jobs: - name: "Free space" run: breeze free-space if: needs.build-info.outputs.inWorkflowBuild == 'true' - - name: Build & Push CI image ${{ matrix.python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} - run: breeze build-image --push-image --tag-as-latest + - name: > + Build & Push CI images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} + run: breeze build-image --push-image --tag-as-latest --run-in-parallel env: - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} if: needs.build-info.outputs.inWorkflowBuild == 'true' - - name: "Candidates for pip resolver backtrack triggers: ${{ matrix.python-version }}" + - name: "Candidates for pip resolver backtrack triggers" if: failure() || cancelled() - run: breeze find-newer-dependencies --max-age 1 --python "${{ matrix.python-version }}" + run: > + breeze find-newer-dependencies --max-age 1 + --python "${{ needs.build-info.outputs.defaultPythonVersion }}" - name: "Fix ownership" run: breeze fix-ownership if: always() && needs.build-info.outputs.inWorkflowBuild == 'true' @@ -350,17 +353,14 @@ jobs: permissions: packages: write timeout-minutes: 80 - name: "${{needs.build-info.outputs.buildJobDescription}} PROD image ${{matrix.python-version}}" + name: > + ${{needs.build-info.outputs.buildJobDescription}} PROD images + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} runs-on: ${{ fromJson(needs.build-info.outputs.runsOn) }} needs: [build-info, build-ci-images] - strategy: - matrix: - python-version: ${{ fromJson(needs.build-info.outputs.allPythonVersions) }} - fail-fast: true env: RUNS_ON: ${{ fromJson(needs.build-info.outputs.runsOn)[0] }} BACKEND: sqlite - PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} VERSION_SUFFIX_FOR_PYPI: "dev0" steps: @@ -424,10 +424,13 @@ jobs: - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files if: needs.build-info.outputs.inWorkflowBuild == 'true' - - name: Build & Push PROD image ${{ env.PYTHON_MAJOR_MINOR_VERSION }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} + - name: > + Build & Push PROD images ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + ${{ needs.build-info.outputs.allPythonVersionsListAsString }} run: > breeze build-prod-image --tag-as-latest + --run-in-parallel --push-image --install-packages-from-context --disable-airflow-repo-cache @@ -436,6 +439,7 @@ jobs: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgradeToNewerDependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cacheDirective }} IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} + PYTHON_VERSIONS: ${{ needs.build-info.outputs.allPythonVersionsListAsString }} if: needs.build-info.outputs.inWorkflowBuild == 'true' - name: "Fix ownership" run: breeze fix-ownership @@ -1385,7 +1389,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" key: ${{ runner.os }}-docker-venv-${{ hashFiles('scripts/ci/images/ci_run_docker_tests.py') }} - name: Wait for PROD images ${{ env.PYTHON_VERSIONS }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} # We wait for the images to be available either from "build-images.yml' run as pull_request_target - # or from build-prod-image above. + # or from build-prod-images above. # We are utilising single job to wait for all images because this job merely waits # For the images to be available and test them. run: breeze pull-prod-image --verify-image --wait-for-image --run-in-parallel diff --git a/dev/REFRESHING_CI_CACHE.md b/dev/REFRESHING_CI_CACHE.md index 85194845dc9b2..5fd458bf34a43 100644 --- a/dev/REFRESHING_CI_CACHE.md +++ b/dev/REFRESHING_CI_CACHE.md @@ -51,7 +51,7 @@ manual refresh might be needed. # Manually generating constraint files ```bash -breeze build-image --build-multiple-images --upgrade-to-newer-dependencies --answer yes +breeze build-image --run-in-parallel --upgrade-to-newer-dependencies --answer yes breeze generate-constraints --airflow-constraints-mode constraints --run-in-parallel --answer yes breeze generate-constraints --airflow-constraints-mode constraints-source-providers --run-in-parallel --answer yes breeze generate-constraints --airflow-constraints-mode constraints-no-providers --run-in-parallel --answer yes diff --git a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py index 9deac3a276817..859bb2e905b51 100644 --- a/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/ci_image_commands.py @@ -37,7 +37,6 @@ option_airflow_constraints_mode_ci, option_airflow_constraints_reference_build, option_answer, - option_build_multiple_images, option_debian_version, option_dev_apt_command, option_dev_apt_deps, @@ -115,9 +114,10 @@ ], }, { - "name": "Building multiple images", + "name": "Building images in parallel", "options": [ - "--build-multiple-images", + "--run-in-parallel", + "--parallelism", "--python-versions", ], }, @@ -189,13 +189,46 @@ } +def start_building(ci_image_params: BuildCiParams, dry_run: bool, verbose: bool) -> bool: + """Starts building attempt. Returns false if we should not continue""" + if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies: + if not should_we_run_the_build(build_ci_params=ci_image_params): + return False + if ci_image_params.prepare_buildx_cache or ci_image_params.push_image: + login_to_github_docker_registry(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) + return True + + +def run_build_in_parallel( + image_params_list: List[BuildCiParams], + python_version_list: List[str], + parallelism: int, + dry_run: bool, + verbose: bool, +) -> None: + get_console().print( + f"\n[info]Building with parallelism = {parallelism} for the images: {python_version_list}:" + ) + pool = mp.Pool(parallelism) + results = [ + pool.apply_async( + run_build_ci_image, + args=(verbose, dry_run, image_param, True), + ) + for image_param in image_params_list + ] + check_async_run_results(results) + pool.close() + + @main.command(name='build-image') @option_github_repository @option_verbose @option_dry_run @option_answer @option_python -@option_build_multiple_images +@option_run_in_parallel +@option_parallelism @option_python_versions @option_upgrade_to_newer_dependencies @option_platform @@ -228,7 +261,8 @@ def build_image( verbose: bool, dry_run: bool, - build_multiple_images: bool, + run_in_parallel: bool, + parallelism: int, python_versions: str, answer: str, **kwargs, @@ -236,7 +270,7 @@ def build_image( """Build CI image. Include building multiple images for all python versions (sequentially).""" def run_build(ci_image_params: BuildCiParams) -> None: - return_code, info = build_ci_image( + return_code, info = run_build_ci_image( verbose=verbose, dry_run=dry_run, ci_image_params=ci_image_params, parallel=False ) if return_code != 0: @@ -246,15 +280,26 @@ def run_build(ci_image_params: BuildCiParams) -> None: perform_environment_checks(verbose=verbose) parameters_passed = filter_out_none(**kwargs) parameters_passed['force_build'] = True - if build_multiple_images: + fix_group_permissions(verbose=verbose) + if run_in_parallel: python_version_list = get_python_version_list(python_versions) + params_list: List[BuildCiParams] = [] for python in python_version_list: params = BuildCiParams(**parameters_passed) params.python = python params.answer = answer - run_build(ci_image_params=params) + params_list.append(params) + start_building(params_list[0], dry_run=dry_run, verbose=verbose) + run_build_in_parallel( + image_params_list=params_list, + python_version_list=python_version_list, + parallelism=parallelism, + dry_run=dry_run, + verbose=verbose, + ) else: params = BuildCiParams(**parameters_passed) + start_building(params, dry_run=dry_run, verbose=verbose) run_build(ci_image_params=params) @@ -434,7 +479,7 @@ def should_we_run_the_build(build_ci_params: BuildCiParams) -> bool: sys.exit(1) -def build_ci_image( +def run_build_ci_image( verbose: bool, dry_run: bool, ci_image_params: BuildCiParams, parallel: bool ) -> Tuple[int, str]: """ @@ -465,17 +510,11 @@ def build_ci_image( "preparing buildx cache![/]\n" ) return 1, "Error: building multi-platform image without --push-image." - fix_group_permissions(verbose=verbose) if verbose or dry_run: get_console().print( f"\n[info]Building CI image of airflow from {AIRFLOW_SOURCES_ROOT} " f"python version: {ci_image_params.python}[/]\n" ) - if not ci_image_params.force_build and not ci_image_params.upgrade_to_newer_dependencies: - if not should_we_run_the_build(build_ci_params=ci_image_params): - return 0, f"Image build: {ci_image_params.python}" - if ci_image_params.prepare_buildx_cache or ci_image_params.push_image: - login_to_github_docker_registry(image_params=ci_image_params, dry_run=dry_run, verbose=verbose) if ci_image_params.prepare_buildx_cache: build_command_result = build_cache( image_params=ci_image_params, dry_run=dry_run, verbose=verbose, parallel=parallel @@ -522,19 +561,6 @@ def build_ci_image( return build_command_result.returncode, f"Image build: {ci_image_params.python}" -def build_ci_image_in_parallel( - verbose: bool, dry_run: bool, parallelism: int, python_version_list: List[str], **kwargs -): - """Run CI image builds in parallel.""" - get_console().print( - f"\n[info]Running with parallelism = {parallelism} for the images: {python_version_list}:" - ) - pool = mp.Pool(parallelism) - results = [pool.apply_async(build_ci_image, args=(verbose, dry_run, False), kwds=kwargs)] - check_async_run_results(results) - pool.close() - - def rebuild_or_pull_ci_image_if_needed( command_params: Union[ShellParams, BuildCiParams], dry_run: bool, verbose: bool ) -> None: @@ -573,4 +599,4 @@ def rebuild_or_pull_ci_image_if_needed( 'Forcing build.[/]' ) ci_image_params.force_build = True - build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params, parallel=False) + run_build_ci_image(verbose, dry_run=dry_run, ci_image_params=ci_image_params, parallel=False) diff --git a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py index 116319a2efca3..490a43c5f52a2 100644 --- a/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/configuration_and_maintenance_commands.py @@ -547,7 +547,14 @@ def regenerate_command_images(verbose: bool, dry_run: bool): except FileNotFoundError: # when we go to Python 3.8+ we can add missing_ok = True instead of try/except pass - command_to_execute = [sys.executable, "-m", "pre_commit", 'run', 'update-breeze-file', '--all-files'] + command_to_execute = [ + sys.executable, + "-m", + "pre_commit", + 'run', + 'update-breeze-cmd-output', + '--all-files', + ] env = os.environ.copy() run_command( command_to_execute, diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 0b5ed2e68b06d..9ae3791e31264 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -15,9 +15,10 @@ # specific language governing permissions and limitations # under the License. import contextlib +import multiprocessing as mp import os import sys -from typing import Optional, Tuple +from typing import List, Optional, Tuple import click @@ -36,7 +37,6 @@ option_airflow_constraints_mode_prod, option_airflow_constraints_reference_build, option_answer, - option_build_multiple_images, option_debian_version, option_dev_apt_command, option_dev_apt_deps, @@ -74,6 +74,7 @@ prepare_docker_build_from_input, ) from airflow_breeze.utils.image import run_pull_image, run_pull_in_parallel, tag_image_as_latest +from airflow_breeze.utils.parallel import check_async_run_results from airflow_breeze.utils.path_utils import AIRFLOW_SOURCES_ROOT, DOCKER_CONTEXT_DIR from airflow_breeze.utils.python_versions import get_python_version_list from airflow_breeze.utils.registry import login_to_github_docker_registry @@ -103,9 +104,10 @@ ], }, { - "name": "Building multiple images", + "name": "Building images in parallel", "options": [ - "--build-multiple-images", + "--run-in-parallel", + "--parallelism", "--python-versions", ], }, @@ -192,12 +194,48 @@ } +def start_building(prod_image_params: BuildProdParams, dry_run: bool, verbose: bool): + if prod_image_params.cleanup_context: + clean_docker_context_files(verbose=verbose, dry_run=dry_run) + check_docker_context_files(prod_image_params.install_packages_from_context) + if prod_image_params.prepare_buildx_cache or prod_image_params.push_image: + login_to_github_docker_registry(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) + + +def run_build_in_parallel( + image_params_list: List[BuildProdParams], + python_version_list: List[str], + parallelism: int, + dry_run: bool, + verbose: bool, +) -> None: + get_console().print( + f"\n[info]Building with parallelism = {parallelism} for the images: {python_version_list}:" + ) + pool = mp.Pool(parallelism) + results = [ + pool.apply_async( + run_build_production_image, + args=( + verbose, + dry_run, + image_param, + True, + ), + ) + for image_param in image_params_list + ] + check_async_run_results(results) + pool.close() + + @option_verbose @option_dry_run @option_answer @main.command(name='build-prod-image') @option_python -@option_build_multiple_images +@option_run_in_parallel +@option_parallelism @option_python_versions @option_upgrade_to_newer_dependencies @option_platform @@ -269,7 +307,8 @@ def build_prod_image( verbose: bool, dry_run: bool, - build_multiple_images: bool, + run_in_parallel: bool, + parallelism: int, python_versions: str, answer: Optional[str], **kwargs, @@ -279,8 +318,8 @@ def build_prod_image( """ def run_build(prod_image_params: BuildProdParams) -> None: - return_code, info = build_production_image( - verbose=verbose, dry_run=dry_run, prod_image_params=prod_image_params + return_code, info = run_build_production_image( + verbose=verbose, dry_run=dry_run, prod_image_params=prod_image_params, parallel=False ) if return_code != 0: get_console().print(f"[error]Error when building image! {info}") @@ -288,15 +327,27 @@ def run_build(prod_image_params: BuildProdParams) -> None: perform_environment_checks(verbose=verbose) parameters_passed = filter_out_none(**kwargs) - if build_multiple_images: + + fix_group_permissions(verbose=verbose) + if run_in_parallel: python_version_list = get_python_version_list(python_versions) + params_list: List[BuildProdParams] = [] for python in python_version_list: params = BuildProdParams(**parameters_passed) params.python = python params.answer = answer - run_build(prod_image_params=params) + params_list.append(params) + start_building(prod_image_params=params_list[0], dry_run=dry_run, verbose=verbose) + run_build_in_parallel( + image_params_list=params_list, + python_version_list=python_version_list, + parallelism=parallelism, + dry_run=dry_run, + verbose=verbose, + ) else: params = BuildProdParams(**parameters_passed) + start_building(prod_image_params=params, dry_run=dry_run, verbose=verbose) run_build(prod_image_params=params) @@ -473,8 +524,8 @@ def check_docker_context_files(install_packages_from_context: bool): sys.exit(1) -def build_production_image( - verbose: bool, dry_run: bool, prod_image_params: BuildProdParams +def run_build_production_image( + verbose: bool, dry_run: bool, prod_image_params: BuildProdParams, parallel: bool ) -> Tuple[int, str]: """ Builds PROD image: @@ -505,21 +556,10 @@ def build_production_image( " or preparing buildx cache![/]\n" ) return 1, "Error: building multi-platform image without --push-image." - fix_group_permissions(verbose=verbose) - if verbose or dry_run: - get_console().print( - f"\n[info]Building PROD image of airflow from {AIRFLOW_SOURCES_ROOT} " - f"python version: {prod_image_params.python}[/]\n" - ) - if prod_image_params.cleanup_context: - clean_docker_context_files(verbose=verbose, dry_run=dry_run) - check_docker_context_files(prod_image_params.install_packages_from_context) - if prod_image_params.prepare_buildx_cache or prod_image_params.push_image: - login_to_github_docker_registry(image_params=prod_image_params, dry_run=dry_run, verbose=verbose) get_console().print(f"\n[info]Building PROD Image for Python {prod_image_params.python}\n") if prod_image_params.prepare_buildx_cache: build_command_result = build_cache( - image_params=prod_image_params, dry_run=dry_run, verbose=verbose, parallel=False + image_params=prod_image_params, dry_run=dry_run, verbose=verbose, parallel=parallel ) else: if prod_image_params.empty_image: @@ -547,7 +587,7 @@ def build_production_image( cwd=AIRFLOW_SOURCES_ROOT, check=False, text=True, - enabled_output_group=True, + enabled_output_group=not parallel, ) if build_command_result.returncode == 0: if prod_image_params.tag_as_latest: diff --git a/dev/breeze/src/airflow_breeze/utils/common_options.py b/dev/breeze/src/airflow_breeze/utils/common_options.py index 0bc4d5af9fc86..292e74891dbfa 100644 --- a/dev/breeze/src/airflow_breeze/utils/common_options.py +++ b/dev/breeze/src/airflow_breeze/utils/common_options.py @@ -377,12 +377,6 @@ envvar='PARALLELISM', show_default=True, ) -option_build_multiple_images = click.option( - '--build-multiple-images', - help="Run the operation sequentially on all or selected subset of Python versions.", - is_flag=True, - envvar='BUILD_MULTIPLE_IMAGES', -) argument_packages = click.argument( "packages", nargs=-1, diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index ed5591585aeb0..4a11212b2c82c 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -26,7 +26,7 @@ export GITHUB_TOKEN="" breeze self-upgrade --force breeze build-image \ - --build-multiple-images \ + --run-in-parallel \ --prepare-buildx-cache \ --force-build \ --platform linux/amd64,linux/arm64 \ @@ -44,7 +44,7 @@ breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi mv -v ./dist/*.whl ./docker-context-files breeze build-prod-image \ - --build-multiple-images \ + --run-in-parallel \ --airflow-is-in-context \ --install-packages-from-context \ --prepare-buildx-cache \ diff --git a/images/breeze/output-build-image.svg b/images/breeze/output-build-image.svg index 81703735d0155..389c973d0e273 100644 --- a/images/breeze/output-build-image.svg +++ b/images/breeze/output-build-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: build-image + Command: build-image - + - - -Usage: breeze build-image [OPTIONS] - -Build CI image. Include building multiple images for all python versions (sequentially). - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag of the image which is used to pull or run the image (implies               │ -│--mount-sources=skip when using to run shell or tests)                         │ -│(TEXT)                                                                         │ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -│--force-buildForce image build no matter if it is determined as needed.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-multiple-imagesRun the operation sequentially on all or selected subset of Python versions.│ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-constraints-modeMode of constraints for CI image building                              │ -│(constraints-source-providers | constraints | constraints-no-providers)│ -│[default: constraints-source-providers]                                │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--verbose-vPrint verbose information about performed steps.│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze build-image [OPTIONS] + +Build CI image. Include building multiple images for all python versions (sequentially). + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag of the image which is used to pull or run the image (implies               │ +│--mount-sources=skip when using to run shell or tests)                         │ +│(TEXT)                                                                         │ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +│--force-buildForce image build no matter if it is determined as needed.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Advanced options (for power users) ─────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-constraints-modeMode of constraints for CI image building                              │ +│(constraints-source-providers | constraints | constraints-no-providers)│ +│[default: constraints-source-providers]                                │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--push-imagePush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--verbose-vPrint verbose information about performed steps.│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg index dcbd32a96662c..331cd3808ba34 100644 --- a/images/breeze/output-build-prod-image.svg +++ b/images/breeze/output-build-prod-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + + + + + + + - Command: build-prod-image + Command: build-prod-image - + - - -Usage: breeze build-prod-image [OPTIONS] - -Build Production image. Include building multiple images for all or selected Python versions sequentially. - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag of the image which is used to pull or run the image (implies               │ -│--mount-sources=skip when using to run shell or tests)                         │ -│(TEXT)                                                                         │ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building multiple images ───────────────────────────────────────────────────────────────────────────────────────────╮ -│--build-multiple-imagesRun the operation sequentially on all or selected subset of Python versions.│ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ -│--airflow-constraints-modeMode of constraints for PROD image building                            │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.│ -│--airflow-is-in-contextIf set Airflow is installed from docker-context-files only rather than    │ -│from PyPI or sources.                                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze build-prod-image [OPTIONS] + +Build Production image. Include building multiple images for all or selected Python versions sequentially. + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag of the image which is used to pull or run the image (implies               │ +│--mount-sources=skip when using to run shell or tests)                         │ +│(TEXT)                                                                         │ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│--airflow-constraints-modeMode of constraints for PROD image building                            │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.│ +│--airflow-is-in-contextIf set Airflow is installed from docker-context-files only rather than    │ +│from PyPI or sources.                                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--push-imagePush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index ef6122b57ad37..bf36da39c3388 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -2,4 +2,4 @@ # This file is automatically generated by pre-commit. If you have a conflict with this file # Please do not solve it but run `breeze regenerate-command-images`. # This command should fix the conflict and regenerate help images that you have conflict with. -82cd0f28c2c53f699a5b571275f8f674 +4051e9cd6b50474e5d2c0fb0ec515f33 diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index c9f215c53c251..26fbf13f66ed9 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -55,6 +55,7 @@ function output_all_basic_variables() { "$(initialization::parameters_to_json "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" initialization::ga_output all-python-versions \ "$(initialization::parameters_to_json "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[@]}")" + initialization::ga_output all-python-versions-list-as-string "${ALL_PYTHON_MAJOR_MINOR_VERSIONS[*]}" initialization::ga_output python-versions-list-as-string "${CURRENT_PYTHON_MAJOR_MINOR_VERSIONS[*]}" initialization::ga_output kubernetes-versions-list-as-string "${CURRENT_KUBERNETES_VERSIONS[*]}" else @@ -64,6 +65,7 @@ function output_all_basic_variables() { # all-python-versions are used in BuildImage Workflow initialization::ga_output all-python-versions \ "$(initialization::parameters_to_json "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}")" + initialization::ga_output all-python-versions-list-as-string "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" initialization::ga_output python-versions-list-as-string "${DEFAULT_PYTHON_MAJOR_MINOR_VERSION}" initialization::ga_output kubernetes-versions-list-as-string "${DEFAULT_KUBERNETES_VERSION}" fi