diff --git a/CI.rst b/CI.rst index 61b0d503cf739..6cc97cccb0bb2 100644 --- a/CI.rst +++ b/CI.rst @@ -124,7 +124,7 @@ You can use those variables when you try to reproduce the build locally. | | | | | case of Postgres or MySQL. However, | | | | | | it requires to perform manual init/reset | | | | | | if you stop the environment. | -+-----------------------------------------+-------------+--------------+-------------+-------------------------------------------------+ ++-----------------------------------------+-------------+--------------+-------------+------------------------------------------------+ | Mount variables | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ | ``MOUNT_SELECTED_LOCAL_SOURCES`` | true | false | false | Determines whether local sources are | @@ -209,7 +209,7 @@ You can use those variables when you try to reproduce the build locally. +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ | Git variables | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ -| COMMIT_SHA | | GITHUB_SHA | GITHUB_SHA | SHA of the commit of the build is run | +| ``COMMIT_SHA`` | | GITHUB_SHA | GITHUB_SHA | SHA of the commit of the build is run | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ | Verbosity variables | +-----------------------------------------+-------------+--------------+------------+-------------------------------------------------+ @@ -255,7 +255,7 @@ You can use those variables when you try to reproduce the build locally. | | | | | of the airflow repository | | | | | | ("constraints-main, "constraints-2-0") | | | | | | but when this flag is set to anything but false | -| | | | | (for example commit SHA), they are not used | +| | | | | (for example random value), they are not used | | | | | | used and "eager" upgrade strategy is used | | | | | | when installing dependencies. We set it | | | | | | to true in case of direct pushes (merges) | @@ -265,7 +265,7 @@ You can use those variables when you try to reproduce the build locally. | | | | | we automatically push latest set of | | | | | | "tested" constraints to the repository. | | | | | | | -| | | | | Setting the value to commit SHA is best way | +| | | | | Setting the value to random value is best way | | | | | | to assure that constraints are upgraded even if | | | | | | there is no change to setup.py | | | | | | | diff --git a/scripts/ci/selective_ci_checks.sh b/scripts/ci/selective_ci_checks.sh index 15020ad366538..10aa128c7614d 100755 --- a/scripts/ci/selective_ci_checks.sh +++ b/scripts/ci/selective_ci_checks.sh @@ -43,16 +43,14 @@ else fi function check_upgrade_to_newer_dependencies_needed() { - # shellcheck disable=SC2153 - if [[ "${UPGRADE_TO_NEWER_DEPENDENCIES}" != "false" || - ${GITHUB_EVENT_NAME=} == 'push' || ${GITHUB_EVENT_NAME=} == "scheduled" ]]; then - # Trigger upgrading to latest constraints when we are in push or schedule event or when it is forced - # By UPGRADE_TO_NEWER_DEPENDENCIES set to non-false. The variable is set to - # SHA of the merge commit - so that it always triggers rebuilding layer in the docker image + if [[ ${GITHUB_EVENT_NAME=} == 'push' || ${GITHUB_EVENT_NAME=} == "scheduled" ]]; then + # Trigger upgrading to latest constraints when we are in push or schedule event. We use the + # random string so that it always triggers rebuilding layer in the docker image # Each build that upgrades to latest constraints will get truly latest constraints, not those - # Cached in the image if we set it to "true". This upgrade_to_newer_dependencies variable - # can later be overridden in case we find that setup.* files changed (see below) - upgrade_to_newer_dependencies="${INCOMING_COMMIT_SHA}" + # cached in the image because docker layer will get invalidated. + # This upgrade_to_newer_dependencies variable can later be overridden + # in case we find that any of the setup.* files changed (see below) + upgrade_to_newer_dependencies="${RANDOM}" fi } @@ -353,8 +351,9 @@ function check_if_setup_files_changed() { if [[ $(count_changed_files) != "0" ]]; then # In case the setup files changed, we automatically force upgrading to newer dependencies - # no matter what was set before - upgrade_to_newer_dependencies="${INCOMING_COMMIT_SHA}" + # no matter what was set before. We set it to random number to make sure that it will be + # always invalidating the layer in Docker that triggers installing the dependencies + upgrade_to_newer_dependencies="${RANDOM}" fi start_end::group_end } diff --git a/scripts/in_container/run_generate_constraints.sh b/scripts/in_container/run_generate_constraints.sh index d71fc9d9a8bac..402ac853541a8 100755 --- a/scripts/in_container/run_generate_constraints.sh +++ b/scripts/in_container/run_generate_constraints.sh @@ -20,6 +20,12 @@ CONSTRAINTS_DIR="/files/constraints-${PYTHON_MAJOR_MINOR_VERSION}" +LATEST_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/original-${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" +CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" + +mkdir -pv "${CONSTRAINTS_DIR}" + + if [[ ${GENERATE_CONSTRAINTS_MODE} == "no-providers" ]]; then AIRFLOW_CONSTRAINTS="constraints-no-providers" NO_PROVIDERS_EXTRAS=$(python -c 'import setup; print(",".join(setup.CORE_EXTRAS_REQUIREMENTS.keys()))') @@ -31,11 +37,34 @@ if [[ ${GENERATE_CONSTRAINTS_MODE} == "no-providers" ]]; then echo "Install airflow with [${NO_PROVIDERS_EXTRAS}] extras only (uninstall all packages first)." echo install_local_airflow_with_eager_upgrade "[${NO_PROVIDERS_EXTRAS}]" + cat <"${CURRENT_CONSTRAINT_FILE}" +# +# This constraints file was automatically generated on $(date -u +'%Y-%m-%dT%H:%M:%SZ') +# via "eager-upgrade" mechanism of PIP. For the "${DEFAULT_BRANCH}" branch of Airflow. +# This variant of constraints install just the 'bare' 'apache-airflow' package build from the HEAD of +# the branch, without installing any of the providers. +# +# Those constraints represent the "newest" dependencies airflow could use, if providers did not limit +# Airflow in any way. +# +EOF elif [[ ${GENERATE_CONSTRAINTS_MODE} == "source-providers" ]]; then AIRFLOW_CONSTRAINTS="constraints-source-providers" echo echo "Providers are already installed from sources." echo + cat <"${CURRENT_CONSTRAINT_FILE}" +# +# This constraints file was automatically generated on $(date -u +'%Y-%m-%dT%H:%M:%SZ') +# via "eager-upgrade" mechanism of PIP. For the "${DEFAULT_BRANCH}" branch of Airflow. +# This variant of constraints install uses the HEAD of the branch version of both +# 'apache-airflow' package nd all available community provider packages. +# +# Those constraints represent the dependencies that are used by all pull requests when they are build in CI. +# They represent "latest" and greatest set of constraints that HEAD of the "apache-airflow" package should +# Install with "HEAD" of providers. Those are the only constraints that are used by our CI builds. +# +EOF elif [[ ${GENERATE_CONSTRAINTS_MODE} == "pypi-providers" ]]; then AIRFLOW_CONSTRAINTS="constraints" echo @@ -43,6 +72,18 @@ elif [[ ${GENERATE_CONSTRAINTS_MODE} == "pypi-providers" ]]; then echo install_all_providers_from_pypi_with_eager_upgrade else + cat <"${CURRENT_CONSTRAINT_FILE}" +# +# This constraints file was automatically generated on $(date -u +'%Y-%m-%dT%H:%M:%SZ') +# via "eager-upgrade" mechanism of PIP. For the "${DEFAULT_BRANCH}" branch of Airflow. +# This variant of constraints install uses the HEAD of the branch version for 'apache-airflow' but installs +# the providers from PIP-released packages at the moment of the constraint generation. +# +# Those constraints are actually those that that regular users use to install released version of Airflow. +# We also use those constraints after "apache-airflow" is released and the constraints are tagged with +# "constraints-X.Y.Z" tag to build the production image for that version. +# +EOF echo echo "${COLOR_RED}Error! GENERATE_CONSTRAINTS_MODE has wrong value: '${GENERATE_CONSTRAINTS_MODE}' ${COLOR_RESET}" echo @@ -51,11 +92,6 @@ fi readonly AIRFLOW_CONSTRAINTS -LATEST_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/original-${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" -CURRENT_CONSTRAINT_FILE="${CONSTRAINTS_DIR}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" - -mkdir -pv "${CONSTRAINTS_DIR}" - CONSTRAINTS_LOCATION="https://raw.githubusercontent.com/${CONSTRAINTS_GITHUB_REPOSITORY}/${DEFAULT_CONSTRAINTS_BRANCH}/${AIRFLOW_CONSTRAINTS}-${PYTHON_MAJOR_MINOR_VERSION}.txt" readonly CONSTRAINTS_LOCATION @@ -69,7 +105,7 @@ echo pip freeze | sort | \ grep -v "apache_airflow" | \ grep -v "@" | \ - grep -v "/opt/airflow" >"${CURRENT_CONSTRAINT_FILE}" + grep -v "/opt/airflow" >>"${CURRENT_CONSTRAINT_FILE}" echo echo "Constraints generated in ${CURRENT_CONSTRAINT_FILE}"