From af19551157cce0b6adcb33c15f1dae2495d8e825 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 14 Jul 2022 11:40:34 +0200 Subject: [PATCH 1/2] Disable provider packages building for non-main branch builds When running the builds from non-main, we do not want to use provider packages built locally, but we want to install those from PyPI. This is achieved by skipping the step of building providers in case default branch is not main. During this change it was also discovered that we do not need AIRFLOW_FROM_CONTEXT flag to indicate whether airflow is among those being installed from docker-context files. This is now detected automatically and airflow is installed from Pypi if not installed from docker context files. Providers and airflow are automatically detected and installed if they are present. Installing pre-cached dependencies from GitHub is skipped in case docker-context-files are used. --- .github/workflows/build-images.yml | 5 +- .github/workflows/ci.yml | 19 +- Dockerfile | 29 +- IMAGES.rst | 3 +- .../commands/production_image_commands.py | 9 +- .../params/build_prod_params.py | 22 +- dev/refresh_images.sh | 2 - docs/docker-stack/build-arg-ref.rst | 5 - docs/docker-stack/changelog.rst | 3 +- images/breeze/output-build-prod-image.svg | 352 +++++++++--------- images/breeze/output-commands-hash.txt | 2 +- scripts/ci/libraries/_initialization.sh | 7 - 12 files changed, 208 insertions(+), 250 deletions(-) diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 11413ac95be5c..e8a678e85f358 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -319,12 +319,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ ${{ hashFiles('.pre-commit-config.yaml') }}" restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} - if: needs.build-info.outputs.default-branch == 'main' - name: "Regenerate dependencies in case they was modified manually so that we can build an image" run: > breeze static-checks --type update-providers-dependencies --all-files --show-diff-on-failure --color always || true - if: needs.build-info.outputs.default-branch == 'main' - name: > Pull CI image for PROD build: ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} @@ -341,6 +339,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --package-list-file ./scripts/ci/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 + if: needs.build-info.outputs.default-branch == 'main' - name: "Prepare airflow package" run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 - name: "Move dist packages to docker-context files" @@ -354,8 +353,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --tag-as-latest --push-image --install-packages-from-context - --disable-airflow-repo-cache - --airflow-is-in-context env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83830c2154d5f..e3715ceede8e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -434,7 +434,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" breeze prepare-provider-packages --package-list-file ./scripts/ci/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 - if: needs.build-info.outputs.in-workflow-build == 'true' + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' - name: "Prepare airflow package" run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 if: needs.build-info.outputs.in-workflow-build == 'true' @@ -450,8 +452,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --run-in-parallel --push-image --install-packages-from-context - --disable-airflow-repo-cache - --airflow-is-in-context env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} @@ -1720,6 +1720,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* + - name: "Prepare airflow package for PROD build" + run: breeze prepare-airflow-package --package-format wheel + env: + VERSION_SUFFIX_FOR_PYPI: "dev0" - name: "Prepare providers packages for PROD build" run: > breeze prepare-provider-packages @@ -1727,10 +1731,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --package-format wheel env: VERSION_SUFFIX_FOR_PYPI: "dev0" - - name: "Prepare airflow package for PROD build" - run: breeze prepare-airflow-package --package-format wheel - env: - VERSION_SUFFIX_FOR_PYPI: "dev0" + if: needs.build-info.outputs.default-branch == 'main' - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh if: matrix.platform == 'linux/arm64' @@ -1746,17 +1747,17 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files + if: needs.build-info.outputs.default-branch == 'main' - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" run: > breeze build-prod-image --builder airflow_cache - --airflow-is-in-context --install-packages-from-context --prepare-buildx-cache - --disable-airflow-repo-cache --platform ${{ matrix.platform }} env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + if: needs.build-info.outputs.default-branch == 'main' - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() && matrix.platform == 'linux/arm64' diff --git a/Dockerfile b/Dockerfile index fdcbb6fa11977..24669bb0ebcea 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1283,16 +1283,23 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ COPY --from=scripts common.sh install_pip_version.sh \ install_airflow_dependencies_from_branch_tip.sh /scripts/docker/ +# We can set this value to true in case we want to install .whl .tar.gz packages placed in the +# docker-context-files folder. This can be done for both - additional packages you want to install +# as well as Airflow and Provider packages (it will be automatically detected if airflow +# is installed from docker-context files rather than from PyPI +ARG INSTALL_PACKAGES_FROM_CONTEXT="false" + # In case of Production build image segment we want to pre-install main version of airflow # dependencies from GitHub so that we do not have to always reinstall it from the scratch. # The Airflow (and providers in case INSTALL_PROVIDERS_FROM_SOURCES is "false") # are uninstalled, only dependencies remain # the cache is only used when "upgrade to newer dependencies" is not set to automatically -# account for removed dependencies (we do not install them in the first place) -# Upgrade to specific PIP version +# account for removed dependencies (we do not install them in the first place) and in case +# INSTALL_PACKAGES_FROM_CONTEXT is not set (because then caching it from main makes no sense). RUN bash /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ - ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ + ${INSTALL_PACKAGES_FROM_CONTEXT} == "false" && \ + ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi @@ -1302,8 +1309,7 @@ COPY --chown=airflow:0 ${AIRFLOW_SOURCES_WWW_FROM} ${AIRFLOW_SOURCES_WWW_TO} # hadolint ignore=SC2086, SC2010 RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ # only prepare node modules and compile assets if the prod image is build from sources - # otherwise they are already compiled-in. We should do it in one step with removing artifacts \ - # as we want to keep the final image small + # otherwise they are already compiled-in the package. bash /scripts/docker/prepare_node_modules.sh; \ REMOVE_ARTIFACTS="true" BUILD_TYPE="prod" bash /scripts/docker/compile_www_assets.sh; \ # Copy generated dist folder (otherwise it will be overridden by the COPY step below) @@ -1319,14 +1325,7 @@ RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ # Add extra python dependencies ARG ADDITIONAL_PYTHON_DEPS="" -# We can set this value to true in case we want to install .whl .tar.gz packages placed in the -# docker-context-files folder. This can be done for both - additional packages you want to install -# and for airflow as well (you have to set AIRFLOW_IS_IN_CONTEXT to true in this case) -ARG INSTALL_PACKAGES_FROM_CONTEXT="false" -# By default we install latest airflow from PyPI or sources. You can set this parameter to false -# if Airflow is in the .whl or .tar.gz packages placed in `docker-context-files` folder and you want -# to skip installing Airflow/Providers from PyPI or sources. -ARG AIRFLOW_IS_IN_CONTEXT="false" + # Those are additional constraints that are needed for some extras but we do not want to # Force them on the main Airflow package. # * dill<0.3.3 required by apache-beam @@ -1334,7 +1333,6 @@ ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \ - AIRFLOW_IS_IN_CONTEXT=${AIRFLOW_IS_IN_CONTEXT} \ EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} WORKDIR ${AIRFLOW_HOME} @@ -1345,7 +1343,8 @@ COPY --from=scripts install_from_docker_context_files.sh install_airflow.sh \ # hadolint ignore=SC2086, SC2010 RUN if [[ ${INSTALL_PACKAGES_FROM_CONTEXT} == "true" ]]; then \ bash /scripts/docker/install_from_docker_context_files.sh; \ - elif [[ ${AIRFLOW_IS_IN_CONTEXT} == "false" ]]; then \ + fi; \ + if ! airflow version 2>/dev/null >/dev/null; then \ bash /scripts/docker/install_airflow.sh; \ fi; \ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ diff --git a/IMAGES.rst b/IMAGES.rst index 2f9fc7234b13e..8819851427df5 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -165,8 +165,7 @@ You can also skip installing airflow and install it from locally provided files .. code-block:: bash - breeze build-prod-image --python 3.7 --additional-extras=trino \ - --airflow-is-in-context-pypi --install-packages-from-context + breeze build-prod-image --python 3.7 --additional-extras=trino --install-packages-from-context In this case you airflow and all packages (.whl files) should be placed in ``docker-context-files`` folder. diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 2ee164645d4b1..cafca59fdeefb 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -143,7 +143,6 @@ "name": "Customization options (for specific customization needs)", "options": [ "--install-packages-from-context", - "--airflow-is-in-context", "--cleanup-context", "--disable-mysql-client-installation", "--disable-mssql-client-installation", @@ -266,14 +265,10 @@ def run_build_in_parallel( type=BetterChoice(ALLOWED_INSTALLATION_METHODS), ) @option_install_providers_from_sources -@click.option( - '--airflow-is-in-context', - help="If set Airflow is installed from docker-context-files only rather than from PyPI or sources.", - is_flag=True, -) @click.option( '--install-packages-from-context', - help='Install wheels from local docker-context-files when building image.', + help='Install wheels from local docker-context-files when building image. ' + 'Implies --disable-airflow-repo-cache.', is_flag=True, ) @click.option( diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index c37569f85f136..65e92400c4d04 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -43,7 +43,6 @@ class BuildProdParams(CommonBuildParams): airflow_constraints_mode: str = "constraints" default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_constraints_reference: str = "" - airflow_is_in_context: bool = False cleanup_context: bool = False disable_airflow_repo_cache: bool = False disable_mssql_client_installation: bool = False @@ -186,31 +185,19 @@ def extra_docker_build_flags(self) -> List[str]: @property def airflow_pre_cached_pip_packages(self) -> str: - airflow_pre_cached_pip = 'true' - if not self.airflow_is_in_context or self.disable_airflow_repo_cache: - airflow_pre_cached_pip = 'false' - return airflow_pre_cached_pip + return 'false' if self.disable_airflow_repo_cache else 'true' @property def install_mssql_client(self) -> str: - install_mssql = 'true' - if self.disable_mssql_client_installation: - install_mssql = 'false' - return install_mssql + return 'false' if self.disable_mssql_client_installation else 'true' @property def install_mysql_client(self) -> str: - install_mysql = 'true' - if self.disable_mysql_client_installation: - install_mysql = 'false' - return install_mysql + return 'false' if self.disable_mysql_client_installation else 'true' @property def install_postgres_client(self) -> str: - install_postgres = 'true' - if self.disable_postgres_client_installation: - install_postgres = 'false' - return install_postgres + return 'false' if self.disable_postgres_client_installation else 'true' @property def docker_context_files(self) -> str: @@ -233,7 +220,6 @@ def required_image_args(self) -> List[str]: "airflow_image_date_created", "airflow_image_readme_url", "airflow_image_repository", - "airflow_is_in_context", "airflow_pre_cached_pip_packages", "airflow_version", "build_id", diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index 596997d8316a8..7675db65b2206 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -47,9 +47,7 @@ mv -v ./dist/*.whl ./docker-context-files breeze build-prod-image \ --builder airflow_cache \ --run-in-parallel \ - --airflow-is-in-context \ --install-packages-from-context \ --prepare-buildx-cache \ - --disable-airflow-repo-cache \ --platform linux/amd64,linux/arm64 \ --verbose diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index fb41e551412a9..a1e4f1d9f0da5 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -252,11 +252,6 @@ You can see some examples of those in: | | | .whl and .tar.gz files placed in the | | | | ``docker-context-files``. | +------------------------------------+------------------------------------------+------------------------------------------+ -| ``AIRFLOW_IS_IN_CONTEXT`` | ``false`` | If set to true, it means that Airflow | -| | | and providers are available in context | -| | | and the image will not attempt to | -| | | install Airflow from PyPI or sources. | -+------------------------------------+------------------------------------------+------------------------------------------+ Pre-caching PIP dependencies ............................ diff --git a/docs/docker-stack/changelog.rst b/docs/docker-stack/changelog.rst index 45e57cd9923a9..c406172dd4b27 100644 --- a/docs/docker-stack/changelog.rst +++ b/docs/docker-stack/changelog.rst @@ -73,8 +73,7 @@ Airflow 2.3 * Add Multi-Platform support (AMD64/ARM64) in order to accommodate MacOS M1 users * Build parameters which control if packages and Airflow should be installed from context file were unified - * The ``INSTALL_FROM_PYPI`` arg was replaced to ``AIRFLOW_IS_IN_CONTEXT`` (with reverse meaning and - default changed to false) + * The ``INSTALL_FROM_PYPI`` arg was removed - it is automatically detected now. * The ``INSTALL_FROM_DOCKER_CONTEXT_FILES`` arg changed to ``INSTALL_PACKAGES_FROM_CONTEXT`` Airflow 2.2 diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg index 3d6122bc3f66f..dbb2b1d56ecfd 100644 --- a/images/breeze/output-build-prod-image.svg +++ b/images/breeze/output-build-prod-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - Command: build-prod-image + Command: build-prod-image - + - - -Usage: breeze build-prod-image [OPTIONS] - -Build Production image. Include building multiple images for all or selected Python versions sequentially. - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag the image after building it(TEXT)│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ -│--airflow-constraints-modeMode of constraints for PROD image building                            │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye                                    │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.│ -│--airflow-is-in-contextIf set Airflow is installed from docker-context-files only rather than    │ -│from PyPI or sources.                                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands(TEXT)│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze build-prod-image [OPTIONS] + +Build Production image. Include building multiple images for all or selected Python versions sequentially. + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag the image after building it(TEXT)│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│--airflow-constraints-modeMode of constraints for PROD image building                            │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye                                    │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--push-imagePush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands(TEXT)│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 7631c8a61f2f6..3c0626593d073 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -5,7 +5,7 @@ main:fa4319079b275ce966502346f083f2e3 build-docs:74c301f05bbd185a19fd185f1873e38a build-image:b62509a59badf3aa230e4562df751002 -build-prod-image:1902ec077a6d70336de6038d13472ef3 +build-prod-image:b9aa5dcefcd7067a6698f68c05adc2b8 cleanup:9a94bd1063296ea86e895f671db0b330 command-hash-export:83bc6a4a8c60b62da3d0f00e81d2c3ea config:92653afc11889e1b78e3a2e38f41107f diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 41466152bd212..a5292e4befaae 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -459,11 +459,6 @@ function initialization::initialize_image_build_variables() { # Determines which providers are used to generate constraints - source, pypi or no providers export AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE:="constraints-source-providers"} - # By default we install latest airflow from PyPI or sources. You can set this parameter to false - # if Airflow is in the .whl or .tar.gz packages placed in `docker-context-files` folder and you want - # to skip installing Airflow/Providers from PyPI or sources. - export AIRFLOW_IS_IN_CONTEXT="${AIRFLOW_IS_IN_CONTEXT:="false"}" - # whether installation should be performed from the local wheel packages in "docker-context-files" folder export INSTALL_PACKAGES_FROM_CONTEXT="${INSTALL_PACKAGES_FROM_CONTEXT:="false"}" @@ -715,7 +710,6 @@ Common image build variables: INSTALL_AIRFLOW_VERSION: '${INSTALL_AIRFLOW_VERSION}' INSTALL_AIRFLOW_REFERENCE: '${INSTALL_AIRFLOW_REFERENCE}' - AIRFLOW_IS_IN_CONTEXT: '${AIRFLOW_IS_IN_CONTEXT}' AIRFLOW_PRE_CACHED_PIP_PACKAGES: '${AIRFLOW_PRE_CACHED_PIP_PACKAGES}' UPGRADE_TO_NEWER_DEPENDENCIES: '${UPGRADE_TO_NEWER_DEPENDENCIES}' CHECK_IMAGE_FOR_REBUILD: '${CHECK_IMAGE_FOR_REBUILD}' @@ -858,7 +852,6 @@ function initialization::make_constants_read_only() { readonly IMAGE_TAG readonly AIRFLOW_PRE_CACHED_PIP_PACKAGES - readonly AIRFLOW_IS_IN_CONTEXT readonly INSTALL_PACKAGES_FROM_CONTEXT readonly AIRFLOW_CONSTRAINTS_REFERENCE readonly AIRFLOW_CONSTRAINTS_LOCATION From 23fd28f18be04beac8e6e076c6a9f98b8265c0b9 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Tue, 19 Jul 2022 13:04:27 +0200 Subject: [PATCH 2/2] Update Dockerfile Co-authored-by: Jed Cunningham <66968678+jedcunningham@users.noreply.github.com> --- Dockerfile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Dockerfile b/Dockerfile index 24669bb0ebcea..b4ddddef80ac1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1283,10 +1283,10 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ COPY --from=scripts common.sh install_pip_version.sh \ install_airflow_dependencies_from_branch_tip.sh /scripts/docker/ -# We can set this value to true in case we want to install .whl .tar.gz packages placed in the -# docker-context-files folder. This can be done for both - additional packages you want to install +# We can set this value to true in case we want to install .whl/.tar.gz packages placed in the +# docker-context-files folder. This can be done for both additional packages you want to install # as well as Airflow and Provider packages (it will be automatically detected if airflow -# is installed from docker-context files rather than from PyPI +# is installed from docker-context files rather than from PyPI) ARG INSTALL_PACKAGES_FROM_CONTEXT="false" # In case of Production build image segment we want to pre-install main version of airflow