diff --git a/.github/workflows/build-images.yml b/.github/workflows/build-images.yml index 11413ac95be5c..e8a678e85f358 100644 --- a/.github/workflows/build-images.yml +++ b/.github/workflows/build-images.yml @@ -319,12 +319,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" key: "pre-commit-${{steps.host-python-version.outputs.host-python-version}}-\ ${{ hashFiles('.pre-commit-config.yaml') }}" restore-keys: pre-commit-${{steps.host-python-version.outputs.host-python-version}} - if: needs.build-info.outputs.default-branch == 'main' - name: "Regenerate dependencies in case they was modified manually so that we can build an image" run: > breeze static-checks --type update-providers-dependencies --all-files --show-diff-on-failure --color always || true - if: needs.build-info.outputs.default-branch == 'main' - name: > Pull CI image for PROD build: ${{ needs.build-info.outputs.default-python-version }}:${{ env.IMAGE_TAG_FOR_THE_BUILD }} @@ -341,6 +339,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --package-list-file ./scripts/ci/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 + if: needs.build-info.outputs.default-branch == 'main' - name: "Prepare airflow package" run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 - name: "Move dist packages to docker-context files" @@ -354,8 +353,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --tag-as-latest --push-image --install-packages-from-context - --disable-airflow-repo-cache - --airflow-is-in-context env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 83830c2154d5f..e3715ceede8e2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -434,7 +434,9 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" breeze prepare-provider-packages --package-list-file ./scripts/ci/installed_providers.txt --package-format wheel --version-suffix-for-pypi dev0 - if: needs.build-info.outputs.in-workflow-build == 'true' + if: > + needs.build-info.outputs.in-workflow-build == 'true' && + needs.build-info.outputs.default-branch == 'main' - name: "Prepare airflow package" run: breeze prepare-airflow-package --package-format wheel --version-suffix-for-pypi dev0 if: needs.build-info.outputs.in-workflow-build == 'true' @@ -450,8 +452,6 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --run-in-parallel --push-image --install-packages-from-context - --disable-airflow-repo-cache - --airflow-is-in-context env: UPGRADE_TO_NEWER_DEPENDENCIES: ${{ needs.build-info.outputs.upgrade-to-newer-dependencies }} DOCKER_CACHE: ${{ needs.build-info.outputs.cache-directive }} @@ -1720,6 +1720,10 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" IMAGE_TAG: ${{ env.IMAGE_TAG_FOR_THE_BUILD }} - name: "Cleanup dist and context file" run: rm -fv ./dist/* ./docker-context-files/* + - name: "Prepare airflow package for PROD build" + run: breeze prepare-airflow-package --package-format wheel + env: + VERSION_SUFFIX_FOR_PYPI: "dev0" - name: "Prepare providers packages for PROD build" run: > breeze prepare-provider-packages @@ -1727,10 +1731,7 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" --package-format wheel env: VERSION_SUFFIX_FOR_PYPI: "dev0" - - name: "Prepare airflow package for PROD build" - run: breeze prepare-airflow-package --package-format wheel - env: - VERSION_SUFFIX_FOR_PYPI: "dev0" + if: needs.build-info.outputs.default-branch == 'main' - name: "Start ARM instance" run: ./scripts/ci/images/ci_start_arm_instance_and_connect_to_docker.sh if: matrix.platform == 'linux/arm64' @@ -1746,17 +1747,17 @@ ${{ hashFiles('.pre-commit-config.yaml') }}" PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} - name: "Move dist packages to docker-context files" run: mv -v ./dist/*.whl ./docker-context-files + if: needs.build-info.outputs.default-branch == 'main' - name: "Push PROD cache ${{ matrix.python-version }} ${{ matrix.platform }}" run: > breeze build-prod-image --builder airflow_cache - --airflow-is-in-context --install-packages-from-context --prepare-buildx-cache - --disable-airflow-repo-cache --platform ${{ matrix.platform }} env: PYTHON_MAJOR_MINOR_VERSION: ${{ matrix.python-version }} + if: needs.build-info.outputs.default-branch == 'main' - name: "Stop ARM instance" run: ./scripts/ci/images/ci_stop_arm_instance.sh if: always() && matrix.platform == 'linux/arm64' diff --git a/Dockerfile b/Dockerfile index fdcbb6fa11977..b4ddddef80ac1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1283,16 +1283,23 @@ ENV AIRFLOW_PIP_VERSION=${AIRFLOW_PIP_VERSION} \ COPY --from=scripts common.sh install_pip_version.sh \ install_airflow_dependencies_from_branch_tip.sh /scripts/docker/ +# We can set this value to true in case we want to install .whl/.tar.gz packages placed in the +# docker-context-files folder. This can be done for both additional packages you want to install +# as well as Airflow and Provider packages (it will be automatically detected if airflow +# is installed from docker-context files rather than from PyPI) +ARG INSTALL_PACKAGES_FROM_CONTEXT="false" + # In case of Production build image segment we want to pre-install main version of airflow # dependencies from GitHub so that we do not have to always reinstall it from the scratch. # The Airflow (and providers in case INSTALL_PROVIDERS_FROM_SOURCES is "false") # are uninstalled, only dependencies remain # the cache is only used when "upgrade to newer dependencies" is not set to automatically -# account for removed dependencies (we do not install them in the first place) -# Upgrade to specific PIP version +# account for removed dependencies (we do not install them in the first place) and in case +# INSTALL_PACKAGES_FROM_CONTEXT is not set (because then caching it from main makes no sense). RUN bash /scripts/docker/install_pip_version.sh; \ if [[ ${AIRFLOW_PRE_CACHED_PIP_PACKAGES} == "true" && \ - ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ + ${INSTALL_PACKAGES_FROM_CONTEXT} == "false" && \ + ${UPGRADE_TO_NEWER_DEPENDENCIES} == "false" ]]; then \ bash /scripts/docker/install_airflow_dependencies_from_branch_tip.sh; \ fi @@ -1302,8 +1309,7 @@ COPY --chown=airflow:0 ${AIRFLOW_SOURCES_WWW_FROM} ${AIRFLOW_SOURCES_WWW_TO} # hadolint ignore=SC2086, SC2010 RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ # only prepare node modules and compile assets if the prod image is build from sources - # otherwise they are already compiled-in. We should do it in one step with removing artifacts \ - # as we want to keep the final image small + # otherwise they are already compiled-in the package. bash /scripts/docker/prepare_node_modules.sh; \ REMOVE_ARTIFACTS="true" BUILD_TYPE="prod" bash /scripts/docker/compile_www_assets.sh; \ # Copy generated dist folder (otherwise it will be overridden by the COPY step below) @@ -1319,14 +1325,7 @@ RUN if [[ ${AIRFLOW_INSTALLATION_METHOD} == "." ]]; then \ # Add extra python dependencies ARG ADDITIONAL_PYTHON_DEPS="" -# We can set this value to true in case we want to install .whl .tar.gz packages placed in the -# docker-context-files folder. This can be done for both - additional packages you want to install -# and for airflow as well (you have to set AIRFLOW_IS_IN_CONTEXT to true in this case) -ARG INSTALL_PACKAGES_FROM_CONTEXT="false" -# By default we install latest airflow from PyPI or sources. You can set this parameter to false -# if Airflow is in the .whl or .tar.gz packages placed in `docker-context-files` folder and you want -# to skip installing Airflow/Providers from PyPI or sources. -ARG AIRFLOW_IS_IN_CONTEXT="false" + # Those are additional constraints that are needed for some extras but we do not want to # Force them on the main Airflow package. # * dill<0.3.3 required by apache-beam @@ -1334,7 +1333,6 @@ ARG EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS="dill<0.3.3" ENV ADDITIONAL_PYTHON_DEPS=${ADDITIONAL_PYTHON_DEPS} \ INSTALL_PACKAGES_FROM_CONTEXT=${INSTALL_PACKAGES_FROM_CONTEXT} \ - AIRFLOW_IS_IN_CONTEXT=${AIRFLOW_IS_IN_CONTEXT} \ EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS=${EAGER_UPGRADE_ADDITIONAL_REQUIREMENTS} WORKDIR ${AIRFLOW_HOME} @@ -1345,7 +1343,8 @@ COPY --from=scripts install_from_docker_context_files.sh install_airflow.sh \ # hadolint ignore=SC2086, SC2010 RUN if [[ ${INSTALL_PACKAGES_FROM_CONTEXT} == "true" ]]; then \ bash /scripts/docker/install_from_docker_context_files.sh; \ - elif [[ ${AIRFLOW_IS_IN_CONTEXT} == "false" ]]; then \ + fi; \ + if ! airflow version 2>/dev/null >/dev/null; then \ bash /scripts/docker/install_airflow.sh; \ fi; \ if [[ -n "${ADDITIONAL_PYTHON_DEPS}" ]]; then \ diff --git a/IMAGES.rst b/IMAGES.rst index 2f9fc7234b13e..8819851427df5 100644 --- a/IMAGES.rst +++ b/IMAGES.rst @@ -165,8 +165,7 @@ You can also skip installing airflow and install it from locally provided files .. code-block:: bash - breeze build-prod-image --python 3.7 --additional-extras=trino \ - --airflow-is-in-context-pypi --install-packages-from-context + breeze build-prod-image --python 3.7 --additional-extras=trino --install-packages-from-context In this case you airflow and all packages (.whl files) should be placed in ``docker-context-files`` folder. diff --git a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py index 2ee164645d4b1..cafca59fdeefb 100644 --- a/dev/breeze/src/airflow_breeze/commands/production_image_commands.py +++ b/dev/breeze/src/airflow_breeze/commands/production_image_commands.py @@ -143,7 +143,6 @@ "name": "Customization options (for specific customization needs)", "options": [ "--install-packages-from-context", - "--airflow-is-in-context", "--cleanup-context", "--disable-mysql-client-installation", "--disable-mssql-client-installation", @@ -266,14 +265,10 @@ def run_build_in_parallel( type=BetterChoice(ALLOWED_INSTALLATION_METHODS), ) @option_install_providers_from_sources -@click.option( - '--airflow-is-in-context', - help="If set Airflow is installed from docker-context-files only rather than from PyPI or sources.", - is_flag=True, -) @click.option( '--install-packages-from-context', - help='Install wheels from local docker-context-files when building image.', + help='Install wheels from local docker-context-files when building image. ' + 'Implies --disable-airflow-repo-cache.', is_flag=True, ) @click.option( diff --git a/dev/breeze/src/airflow_breeze/params/build_prod_params.py b/dev/breeze/src/airflow_breeze/params/build_prod_params.py index c37569f85f136..65e92400c4d04 100644 --- a/dev/breeze/src/airflow_breeze/params/build_prod_params.py +++ b/dev/breeze/src/airflow_breeze/params/build_prod_params.py @@ -43,7 +43,6 @@ class BuildProdParams(CommonBuildParams): airflow_constraints_mode: str = "constraints" default_constraints_branch: str = DEFAULT_AIRFLOW_CONSTRAINTS_BRANCH airflow_constraints_reference: str = "" - airflow_is_in_context: bool = False cleanup_context: bool = False disable_airflow_repo_cache: bool = False disable_mssql_client_installation: bool = False @@ -186,31 +185,19 @@ def extra_docker_build_flags(self) -> List[str]: @property def airflow_pre_cached_pip_packages(self) -> str: - airflow_pre_cached_pip = 'true' - if not self.airflow_is_in_context or self.disable_airflow_repo_cache: - airflow_pre_cached_pip = 'false' - return airflow_pre_cached_pip + return 'false' if self.disable_airflow_repo_cache else 'true' @property def install_mssql_client(self) -> str: - install_mssql = 'true' - if self.disable_mssql_client_installation: - install_mssql = 'false' - return install_mssql + return 'false' if self.disable_mssql_client_installation else 'true' @property def install_mysql_client(self) -> str: - install_mysql = 'true' - if self.disable_mysql_client_installation: - install_mysql = 'false' - return install_mysql + return 'false' if self.disable_mysql_client_installation else 'true' @property def install_postgres_client(self) -> str: - install_postgres = 'true' - if self.disable_postgres_client_installation: - install_postgres = 'false' - return install_postgres + return 'false' if self.disable_postgres_client_installation else 'true' @property def docker_context_files(self) -> str: @@ -233,7 +220,6 @@ def required_image_args(self) -> List[str]: "airflow_image_date_created", "airflow_image_readme_url", "airflow_image_repository", - "airflow_is_in_context", "airflow_pre_cached_pip_packages", "airflow_version", "build_id", diff --git a/dev/refresh_images.sh b/dev/refresh_images.sh index 596997d8316a8..7675db65b2206 100755 --- a/dev/refresh_images.sh +++ b/dev/refresh_images.sh @@ -47,9 +47,7 @@ mv -v ./dist/*.whl ./docker-context-files breeze build-prod-image \ --builder airflow_cache \ --run-in-parallel \ - --airflow-is-in-context \ --install-packages-from-context \ --prepare-buildx-cache \ - --disable-airflow-repo-cache \ --platform linux/amd64,linux/arm64 \ --verbose diff --git a/docs/docker-stack/build-arg-ref.rst b/docs/docker-stack/build-arg-ref.rst index fb41e551412a9..a1e4f1d9f0da5 100644 --- a/docs/docker-stack/build-arg-ref.rst +++ b/docs/docker-stack/build-arg-ref.rst @@ -252,11 +252,6 @@ You can see some examples of those in: | | | .whl and .tar.gz files placed in the | | | | ``docker-context-files``. | +------------------------------------+------------------------------------------+------------------------------------------+ -| ``AIRFLOW_IS_IN_CONTEXT`` | ``false`` | If set to true, it means that Airflow | -| | | and providers are available in context | -| | | and the image will not attempt to | -| | | install Airflow from PyPI or sources. | -+------------------------------------+------------------------------------------+------------------------------------------+ Pre-caching PIP dependencies ............................ diff --git a/docs/docker-stack/changelog.rst b/docs/docker-stack/changelog.rst index 45e57cd9923a9..c406172dd4b27 100644 --- a/docs/docker-stack/changelog.rst +++ b/docs/docker-stack/changelog.rst @@ -73,8 +73,7 @@ Airflow 2.3 * Add Multi-Platform support (AMD64/ARM64) in order to accommodate MacOS M1 users * Build parameters which control if packages and Airflow should be installed from context file were unified - * The ``INSTALL_FROM_PYPI`` arg was replaced to ``AIRFLOW_IS_IN_CONTEXT`` (with reverse meaning and - default changed to false) + * The ``INSTALL_FROM_PYPI`` arg was removed - it is automatically detected now. * The ``INSTALL_FROM_DOCKER_CONTEXT_FILES`` arg changed to ``INSTALL_PACKAGES_FROM_CONTEXT`` Airflow 2.2 diff --git a/images/breeze/output-build-prod-image.svg b/images/breeze/output-build-prod-image.svg index 3d6122bc3f66f..dbb2b1d56ecfd 100644 --- a/images/breeze/output-build-prod-image.svg +++ b/images/breeze/output-build-prod-image.svg @@ -1,4 +1,4 @@ - + - - + + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - + - - - - Command: build-prod-image + Command: build-prod-image - + - - -Usage: breeze build-prod-image [OPTIONS] - -Build Production image. Include building multiple images for all or selected Python versions sequentially. - -╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--python-pPython major/minor version used in Airflow image for images.│ -│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ -│[default: 3.7]                                              │ -│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ -│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ -│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ -│--image-tag-tTag the image after building it(TEXT)│ -│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ -│when you build or pull image with --image-tag.                                 │ -│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ -│[default: registry]                          │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ -│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ -│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ -│[default: 4; 1<=x<=8]                                                      │ -│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ -│[default: 3.7 3.8 3.9 3.10]                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ -│--install-providers-from-sourcesInstall providers from sources when installing.│ -│--airflow-extrasExtras to install by default.                                                   │ -│(TEXT)                                                                          │ -│[default:                                                                       │ -│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ -│--airflow-constraints-modeMode of constraints for PROD image building                            │ -│(constraints | constraints-no-providers | constraints-source-providers)│ -│[default: constraints]                                                 │ -│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ -│--python-imageIf specified this is the base python image used to build the image. Should be   │ -│something like: python:VERSION-slim-bullseye                                    │ -│(TEXT)                                                                          │ -│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ -│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ -│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ -│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ -│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ -│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ -│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ -│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ -│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ -│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ -│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ -│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ -│--install-packages-from-contextInstall wheels from local docker-context-files when building image.│ -│--airflow-is-in-contextIf set Airflow is installed from docker-context-files only rather than    │ -│from PyPI or sources.                                                     │ -│--cleanup-contextClean up docker context files before running build (cannot be used        │ -│together with --install-packages-from-context).                           │ -│--disable-mysql-client-installationDo not install MySQL client.│ -│--disable-mssql-client-installationDo not install MsSQl client.│ -│--disable-postgres-client-installationDo not install Postgres client.│ -│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ -│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ -│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ -│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ -│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ -│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ -│--push-imagePush image after building it.│ -│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ -│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ -│image).                                                                                   │ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ -╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ -│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ -│--builderBuildx builder used to perform `docker buildx build` commands(TEXT)│ -│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ -│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ -│--verbose-vPrint verbose information about performed steps.│ -│--help-hShow this message and exit.│ -╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ + + +Usage: breeze build-prod-image [OPTIONS] + +Build Production image. Include building multiple images for all or selected Python versions sequentially. + +╭─ Basic usage ────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--python-pPython major/minor version used in Airflow image for images.│ +│(>3.7< | 3.8 | 3.9 | 3.10)                                  │ +│[default: 3.7]                                              │ +│--install-airflow-version-VInstall version of Airflow from PyPI.(TEXT)│ +│--upgrade-to-newer-dependencies-uWhen set, upgrade all PIP packages to latest.│ +│--debian-versionDebian version used for the image.(bullseye | buster)[default: bullseye]│ +│--image-tag-tTag the image after building it(TEXT)│ +│--tag-as-latestTags the image as latest and update checksum of all files after pulling. Useful│ +│when you build or pull image with --image-tag.                                 │ +│--docker-cache-cCache option for image used during the build.(registry | local | disabled)│ +│[default: registry]                          │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Building images in parallel ────────────────────────────────────────────────────────────────────────────────────────╮ +│--run-in-parallelRun the operation in parallel on all or selected subset of Python versions.│ +│--parallelismMaximum number of processes to use while running the operation in parallel.(INTEGER RANGE)│ +│[default: 4; 1<=x<=8]                                                      │ +│--python-versionsSpace separated list of python versions used for build with multiple versions.(TEXT)│ +│[default: 3.7 3.8 3.9 3.10]                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options for customizing images ─────────────────────────────────────────────────────────────────────────────────────╮ +│--install-providers-from-sourcesInstall providers from sources when installing.│ +│--airflow-extrasExtras to install by default.                                                   │ +│(TEXT)                                                                          │ +│[default:                                                                       │ +│amazon,async,celery,cncf.kubernetes,dask,docker,elasticsearch,ftp,google,google…│ +│--airflow-constraints-modeMode of constraints for PROD image building                            │ +│(constraints | constraints-no-providers | constraints-source-providers)│ +│[default: constraints]                                                 │ +│--airflow-constraints-referenceConstraint reference to use when building the image.(TEXT)│ +│--python-imageIf specified this is the base python image used to build the image. Should be   │ +│something like: python:VERSION-slim-bullseye                                    │ +│(TEXT)                                                                          │ +│--additional-python-depsAdditional python dependencies to use when building the images.(TEXT)│ +│--additional-extrasAdditional extra package while installing Airflow in the image.(TEXT)│ +│--additional-runtime-apt-depsAdditional apt runtime dependencies to use when building the images.(TEXT)│ +│--additional-runtime-apt-envAdditional environment variables set when adding runtime dependencies.(TEXT)│ +│--additional-runtime-apt-commandAdditional command executed before runtime apt deps are installed.(TEXT)│ +│--additional-dev-apt-depsAdditional apt dev dependencies to use when building the images.(TEXT)│ +│--additional-dev-apt-envAdditional environment variables set when adding dev dependencies.(TEXT)│ +│--additional-dev-apt-commandAdditional command executed before dev apt deps are installed.(TEXT)│ +│--runtime-apt-depsApt runtime dependencies to use when building the images.(TEXT)│ +│--runtime-apt-commandCommand executed before runtime apt deps are installed.(TEXT)│ +│--dev-apt-depsApt dev dependencies to use when building the images.(TEXT)│ +│--dev-apt-commandCommand executed before dev apt deps are installed.(TEXT)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Customization options (for specific customization needs) ───────────────────────────────────────────────────────────╮ +│--install-packages-from-contextInstall wheels from local docker-context-files when building image.       │ +│Implies --disable-airflow-repo-cache.                                     │ +│--cleanup-contextClean up docker context files before running build (cannot be used        │ +│together with --install-packages-from-context).                           │ +│--disable-mysql-client-installationDo not install MySQL client.│ +│--disable-mssql-client-installationDo not install MsSQl client.│ +│--disable-postgres-client-installationDo not install Postgres client.│ +│--disable-airflow-repo-cacheDisable cache from Airflow repository during building.│ +│--install-airflow-referenceInstall Airflow using GitHub tag or branch.(TEXT)│ +│--installation-methodInstall Airflow from: sources or PyPI.(. | apache-airflow)│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Preparing cache and push (for maintainers and CI) ──────────────────────────────────────────────────────────────────╮ +│--github-tokenThe token used to authenticate to GitHub.(TEXT)│ +│--github-usernameThe user name used to authenticate to GitHub.(TEXT)│ +│--platformPlatform for Airflow image.(linux/amd64 | linux/arm64 | linux/amd64,linux/arm64)│ +│--push-imagePush image after building it.│ +│--empty-imagePrepare empty image tagged with the same name as the Airflow image.│ +│--prepare-buildx-cachePrepares build cache (this is done as separate per-platform steps instead of building the │ +│image).                                                                                   │ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ +╭─ Options ────────────────────────────────────────────────────────────────────────────────────────────────────────────╮ +│--github-repository-gGitHub repository used to pull, push run images.(TEXT)[default: apache/airflow]│ +│--builderBuildx builder used to perform `docker buildx build` commands(TEXT)│ +│--answer-aForce answer to questions.(y | n | q | yes | no | quit)│ +│--dry-run-DIf dry-run is set, commands are only printed, not executed.│ +│--verbose-vPrint verbose information about performed steps.│ +│--help-hShow this message and exit.│ +╰──────────────────────────────────────────────────────────────────────────────────────────────────────────────────────╯ diff --git a/images/breeze/output-commands-hash.txt b/images/breeze/output-commands-hash.txt index 7631c8a61f2f6..3c0626593d073 100644 --- a/images/breeze/output-commands-hash.txt +++ b/images/breeze/output-commands-hash.txt @@ -5,7 +5,7 @@ main:fa4319079b275ce966502346f083f2e3 build-docs:74c301f05bbd185a19fd185f1873e38a build-image:b62509a59badf3aa230e4562df751002 -build-prod-image:1902ec077a6d70336de6038d13472ef3 +build-prod-image:b9aa5dcefcd7067a6698f68c05adc2b8 cleanup:9a94bd1063296ea86e895f671db0b330 command-hash-export:83bc6a4a8c60b62da3d0f00e81d2c3ea config:92653afc11889e1b78e3a2e38f41107f diff --git a/scripts/ci/libraries/_initialization.sh b/scripts/ci/libraries/_initialization.sh index 41466152bd212..a5292e4befaae 100644 --- a/scripts/ci/libraries/_initialization.sh +++ b/scripts/ci/libraries/_initialization.sh @@ -459,11 +459,6 @@ function initialization::initialize_image_build_variables() { # Determines which providers are used to generate constraints - source, pypi or no providers export AIRFLOW_CONSTRAINTS_MODE=${AIRFLOW_CONSTRAINTS_MODE:="constraints-source-providers"} - # By default we install latest airflow from PyPI or sources. You can set this parameter to false - # if Airflow is in the .whl or .tar.gz packages placed in `docker-context-files` folder and you want - # to skip installing Airflow/Providers from PyPI or sources. - export AIRFLOW_IS_IN_CONTEXT="${AIRFLOW_IS_IN_CONTEXT:="false"}" - # whether installation should be performed from the local wheel packages in "docker-context-files" folder export INSTALL_PACKAGES_FROM_CONTEXT="${INSTALL_PACKAGES_FROM_CONTEXT:="false"}" @@ -715,7 +710,6 @@ Common image build variables: INSTALL_AIRFLOW_VERSION: '${INSTALL_AIRFLOW_VERSION}' INSTALL_AIRFLOW_REFERENCE: '${INSTALL_AIRFLOW_REFERENCE}' - AIRFLOW_IS_IN_CONTEXT: '${AIRFLOW_IS_IN_CONTEXT}' AIRFLOW_PRE_CACHED_PIP_PACKAGES: '${AIRFLOW_PRE_CACHED_PIP_PACKAGES}' UPGRADE_TO_NEWER_DEPENDENCIES: '${UPGRADE_TO_NEWER_DEPENDENCIES}' CHECK_IMAGE_FOR_REBUILD: '${CHECK_IMAGE_FOR_REBUILD}' @@ -858,7 +852,6 @@ function initialization::make_constants_read_only() { readonly IMAGE_TAG readonly AIRFLOW_PRE_CACHED_PIP_PACKAGES - readonly AIRFLOW_IS_IN_CONTEXT readonly INSTALL_PACKAGES_FROM_CONTEXT readonly AIRFLOW_CONSTRAINTS_REFERENCE readonly AIRFLOW_CONSTRAINTS_LOCATION