From 6a46efe115ac1275dff9b1dda4eb157ecd89d615 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 1 Feb 2020 14:07:19 +0100 Subject: [PATCH 1/3] [AIRFLOW-6701] Rat is downloaded from stable backup/mirrors Also curl options are now using long format and include --fail to protect against some temporary errors (5xx). Also RAT download uses now two possible sources of downloads and fallbacks to the second if first is not available. --- Dockerfile | 58 +++++++++++++++++++++++++++++++++++++++--------------- 1 file changed, 42 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 7195d378263c0..d965a606024ee 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ RUN apt-get update \ && rm -rf /var/lib/apt/lists/* # Install basic apt dependencies -RUN curl -L https://deb.nodesource.com/setup_10.x | bash - \ +RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \ && echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list \ && apt-get update \ @@ -161,7 +161,7 @@ RUN HADOOP_DISTRO="cdh" \ && HADOOP_DOWNLOAD_URL="${HADOOP_URL}hadoop-${HADOOP_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" \ && HADOOP_TMP_FILE="/tmp/hadoop.tar.gz" \ && mkdir -pv "${HADOOP_HOME}" \ - && curl -L "${HADOOP_DOWNLOAD_URL}" -o "${HADOOP_TMP_FILE}" \ + && curl --fail --location "${HADOOP_DOWNLOAD_URL}" --output "${HADOOP_TMP_FILE}" \ && tar xzf "${HADOOP_TMP_FILE}" --absolute-names --strip-components 1 -C "${HADOOP_HOME}" \ && rm "${HADOOP_TMP_FILE}" \ && echo "Installing Hive" \ @@ -173,7 +173,7 @@ RUN HADOOP_DISTRO="cdh" \ && mkdir -pv "/user/hive/warehouse" \ && chmod -R 777 "${HIVE_HOME}" \ && chmod -R 777 "/user/" \ - && curl -L "${HIVE_URL}" -o "${HIVE_TMP_FILE}" \ + && curl --fail --location "${HIVE_URL}" --output "${HIVE_TMP_FILE}" \ && tar xzf "${HIVE_TMP_FILE}" --strip-components 1 -C "${HIVE_HOME}" \ && rm "${HIVE_TMP_FILE}" @@ -187,12 +187,12 @@ RUN MINICLUSTER_BASE="https://github.com/bolkedebruin/minicluster/releases/downl && MINICLUSTER_URL="${MINICLUSTER_BASE}${MINICLUSTER_VER}/minicluster-${MINICLUSTER_VER}-SNAPSHOT-bin.zip" \ && MINICLUSTER_TMP_FILE="/tmp/minicluster.zip" \ && mkdir -pv "${MINICLUSTER_HOME}" \ - && curl -L "${MINICLUSTER_URL}" -o "${MINICLUSTER_TMP_FILE}" \ + && curl --fail --location "${MINICLUSTER_URL}" --output "${MINICLUSTER_TMP_FILE}" \ && unzip "${MINICLUSTER_TMP_FILE}" -d "/opt" \ && rm "${MINICLUSTER_TMP_FILE}" # Install Docker -RUN curl -L https://download.docker.com/linux/debian/gpg | apt-key add - \ +RUN curl --fail --location https://download.docker.com/linux/debian/gpg | apt-key add - \ && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" \ && apt-get update \ && apt-get -y install --no-install-recommends docker-ce \ @@ -203,28 +203,54 @@ RUN curl -L https://download.docker.com/linux/debian/gpg | apt-key add - \ ARG KUBECTL_VERSION="v1.15.3" RUN KUBECTL_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \ - && curl -L "${KUBECTL_URL}" -o "/usr/local/bin/kubectl" \ + && curl --fail --location "${KUBECTL_URL}" --output "/usr/local/bin/kubectl" \ && chmod +x /usr/local/bin/kubectl # Install Kind ARG KIND_VERSION="v0.6.1" RUN KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \ - && curl -L "${KIND_URL}" -o "/usr/local/bin/kind" \ + && curl --fail --location "${KIND_URL}" --output "/usr/local/bin/kind" \ && chmod +x /usr/local/bin/kind # Install Apache RAT ARG RAT_VERSION="0.13" - -RUN RAT_URL="https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar" \ +ARG RAT_BACKUP_SITE_1="https://www-eu.apache.org/dist/creadur" +ARG RAT_BACKUP_SITE_2="https://www-us.apache.org/dist/creadur" +# It's OK to use HTTP rather than https here as we verify it later with gpg from the +# offcial backup servers of Apache! +ARG RAT_MIRROR_1="http://mirror.serverion.com/apache/creadur" +ARG RAT_MIRROR_2="http://mirror.cc.columbia.edu/pub/software/apache/creadur" + +RUN RAT_TARGZ_FILE_NAME="apache-rat-${RAT_VERSION}-bin.tar.gz" \ + && RAT_FOLDER="apache-rat-${RAT_VERSION}" \ + && RAT_KEYS_URL_1="${RAT_BACKUP_SITE_1}/KEYS" \ + && RAT_KEYS_URL_2="${RAT_BACKUP_SITE_2}/KEYS" \ + && RAT_ASC_URL_1="${RAT_BACKUP_SITE_1}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}.asc" \ + && RAT_ASC_URL_2="${RAT_BACKUP_SITE_2}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}.asc" \ + && RAT_URL_1="${RAT_MIRROR_1}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}" \ + && RAT_URL_2="${RAT_MIRROR_2}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}" \ + && RAT_TAR_GZ="/opt/${RAT_TARGZ_FILE_NAME}" \ + && RAT_TAR_GZ_ASC="/opt/${RAT_TARGZ_FILE_NAME}.asc" \ + && RAT_KEYS="/opt/KEYS" \ + && RAT_JAR_IN_TAR="${RAT_FOLDER}/apache-rat-${RAT_VERSION}.jar" \ && RAT_JAR="/opt/apache-rat.jar" \ - && RAT_JAR_MD5="${RAT_JAR}.md5" \ - && RAT_URL_MD5="${RAT_URL}.md5" \ - && echo "Downloading RAT from ${RAT_URL} to ${RAT_JAR}" \ - && curl -L "${RAT_URL}" -o "${RAT_JAR}" \ - && curl -L "${RAT_URL_MD5}" -o "${RAT_JAR_MD5}" \ - && jar -tf "${RAT_JAR}" > /dev/null \ - && md5sum -c <<<"$(cat "${RAT_JAR_MD5}") ${RAT_JAR}" + && echo "Downloading KEYS from backup Apache servers: ${RAT_KEYS_URL_1}, ${RAT_KEYS_URL_2}" \ + && (curl --fail --location "${RAT_KEYS_URL_1}" --output "${RAT_KEYS}" || \ + curl --fail --location "${RAT_KEYS_URL_2}" --output "${RAT_KEYS}") \ + && echo "Downloading ASC from backup Apache servers: ${RAT_ASC_URL_1}, ${RAT_ASC_URL_2}" \ + && (curl --fail --location "${RAT_ASC_URL_1}" --output "${RAT_TAR_GZ_ASC}" || \ + curl --fail --location "${RAT_ASC_URL_2}" --output "${RAT_TAR_GZ_ASC}") \ + && echo "Downloading RAT from mirrors: ${RAT_URL_1}, ${RAT_URL_2} to ${RAT_JAR}" \ + && (curl --fail --location "${RAT_URL_1}" --output "${RAT_TAR_GZ}" || \ + curl --fail --location "${RAT_URL_2}" --output "${RAT_TAR_GZ}") \ + && gpg --import ${RAT_KEYS} \ + && gpg --verify "${RAT_TAR_GZ_ASC}" "${RAT_TAR_GZ}" \ + && tar --extract --gzip --file "${RAT_TAR_GZ}" -C /opt "${RAT_JAR_IN_TAR}" \ + && mv -v /opt/"${RAT_JAR_IN_TAR}" "${RAT_JAR}" \ + && rm -vrf "${RAT_TAR_GZ}" "/opt/${RAT_FOLDER}" \ + && rm -f "${RAT_KEYS}" \ + && jar -tf "${RAT_JAR}" >/dev/null # Setup PIP # By default PIP install run without cache to make image smaller From faf8dfa86132d8b046937e951ffa542da4006514 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sun, 2 Feb 2020 08:37:21 +0100 Subject: [PATCH 2/3] Revert "[AIRFLOW-6662] Switch to --init docker flag for signal propagation (#7278)" This reverts commit d1bf343ffec505270d4b2ee4b0e9fa5dbbedd891. --- Dockerfile | 2 +- scripts/ci/_utils.sh | 55 +++++++------------ scripts/ci/docker-compose/backend-mysql.yml | 1 - .../ci/docker-compose/backend-postgres.yml | 1 - .../docker-compose/integration-cassandra.yml | 1 - .../docker-compose/integration-kerberos.yml | 1 - .../ci/docker-compose/integration-mongo.yml | 1 - .../docker-compose/integration-openldap.yml | 1 - .../docker-compose/integration-rabbitmq.yml | 1 - .../ci/docker-compose/integration-redis.yml | 1 - setup.py | 1 + 11 files changed, 22 insertions(+), 44 deletions(-) diff --git a/Dockerfile b/Dockerfile index d965a606024ee..9eaedb6b5d1d7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -431,6 +431,6 @@ ENV PATH="${HOME}:${PATH}" EXPOSE 8080 -ENTRYPOINT ["/entrypoint.sh"] +ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/entrypoint.sh"] CMD ["--help"] diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index 9ebcc71cf1a66..aad7291f76212 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -640,30 +640,27 @@ function run_flake8() { if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "${FILES[@]}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -682,31 +679,29 @@ function run_bats_tests() { function run_docs() { docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - --entrypoint \ - "/opt/airflow/docs/build.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/docs/build.sh" \ | tee -a "${OUTPUT_LOG}" } function run_check_license() { docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \ | tee -a "${OUTPUT_LOG}" } @@ -714,31 +709,27 @@ function run_mypy() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "airflow" "tests" "docs" \ + "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "airflow" "tests" "docs" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \ --rm \ "${AIRFLOW_CI_IMAGE}" \ - "${FILES[@]}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -747,30 +738,27 @@ function run_pylint_main() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "${FILES[@]}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } @@ -780,30 +768,27 @@ function run_pylint_tests() { FILES=("$@") if [[ "${#FILES[@]}" == "0" ]]; then docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ "${AIRFLOW_CI_IMAGE}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ | tee -a "${OUTPUT_LOG}" else docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --init \ + --entrypoint "/usr/local/bin/dumb-init" \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ --rm \ - --entrypoint \ - "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \ "${AIRFLOW_CI_IMAGE}" \ - "${FILES[@]}" \ + "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" "${FILES[@]}" \ | tee -a "${OUTPUT_LOG}" fi } diff --git a/scripts/ci/docker-compose/backend-mysql.yml b/scripts/ci/docker-compose/backend-mysql.yml index d2bfd8dad9e49..d9773aac1a97b 100644 --- a/scripts/ci/docker-compose/backend-mysql.yml +++ b/scripts/ci/docker-compose/backend-mysql.yml @@ -25,7 +25,6 @@ services: depends_on: - mysql mysql: - init: true image: mysql:5.6 environment: - MYSQL_ALLOW_EMPTY_PASSWORD=true diff --git a/scripts/ci/docker-compose/backend-postgres.yml b/scripts/ci/docker-compose/backend-postgres.yml index 9cfdc7b54b44f..10a6fd5227bbb 100644 --- a/scripts/ci/docker-compose/backend-postgres.yml +++ b/scripts/ci/docker-compose/backend-postgres.yml @@ -25,7 +25,6 @@ services: depends_on: - postgres postgres: - init: true image: postgres:9.6 environment: - POSTGRES_USER=postgres diff --git a/scripts/ci/docker-compose/integration-cassandra.yml b/scripts/ci/docker-compose/integration-cassandra.yml index 304c36c9d51f8..12a5537156b30 100644 --- a/scripts/ci/docker-compose/integration-cassandra.yml +++ b/scripts/ci/docker-compose/integration-cassandra.yml @@ -18,7 +18,6 @@ version: "2.2" services: cassandra: - init: true image: cassandra:3.0 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/integration-kerberos.yml b/scripts/ci/docker-compose/integration-kerberos.yml index 79d841f21a2fa..6714a104f5819 100644 --- a/scripts/ci/docker-compose/integration-kerberos.yml +++ b/scripts/ci/docker-compose/integration-kerberos.yml @@ -18,7 +18,6 @@ version: "2.2" services: kerberos: - init: true image: godatadriven/krb5-kdc-server hostname: kerberos domainname: example.com diff --git a/scripts/ci/docker-compose/integration-mongo.yml b/scripts/ci/docker-compose/integration-mongo.yml index 06dfea113b18a..5d48c5022761e 100644 --- a/scripts/ci/docker-compose/integration-mongo.yml +++ b/scripts/ci/docker-compose/integration-mongo.yml @@ -18,7 +18,6 @@ version: "2.2" services: mongo: - init: true image: mongo:3 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/integration-openldap.yml b/scripts/ci/docker-compose/integration-openldap.yml index 44e2c2f2de50e..cf8cfd3019c4b 100644 --- a/scripts/ci/docker-compose/integration-openldap.yml +++ b/scripts/ci/docker-compose/integration-openldap.yml @@ -18,7 +18,6 @@ version: "2.2" services: openldap: - init: true image: osixia/openldap:1.2.0 command: --copy-service environment: diff --git a/scripts/ci/docker-compose/integration-rabbitmq.yml b/scripts/ci/docker-compose/integration-rabbitmq.yml index 1864122f8d34b..d14db8aaef984 100644 --- a/scripts/ci/docker-compose/integration-rabbitmq.yml +++ b/scripts/ci/docker-compose/integration-rabbitmq.yml @@ -18,7 +18,6 @@ version: "2.2" services: rabbitmq: - init: true image: rabbitmq:3.7 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/scripts/ci/docker-compose/integration-redis.yml b/scripts/ci/docker-compose/integration-redis.yml index 819a35416f99c..6893af7074860 100644 --- a/scripts/ci/docker-compose/integration-redis.yml +++ b/scripts/ci/docker-compose/integration-redis.yml @@ -18,7 +18,6 @@ version: "2.2" services: redis: - init: true image: redis:5.0.1 volumes: - /dev/urandom:/dev/random # Required to get non-blocking entropy source diff --git a/setup.py b/setup.py index 0bd35a73d6f84..b9616327376a1 100644 --- a/setup.py +++ b/setup.py @@ -376,6 +376,7 @@ def write_version(filename: str = os.path.join(*["airflow", "git_version"])): 'click==6.7', 'contextdecorator;python_version<"3.4"', 'coverage', + 'dumb-init>=1.2.2', 'flake8>=3.6.0', 'flake8-colors', 'flaky', From 244025272cc4428de1c48b35032822d1033be9bc Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Thu, 30 Jan 2020 21:39:45 +0100 Subject: [PATCH 3/3] [AIRFLOW-6662] return back the dumb-init - installed by apt We had stability problems with tests with --init flag so we are going back to it --- Dockerfile | 6 +++++- scripts/ci/_utils.sh | 6 ++++-- setup.py | 1 - 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9eaedb6b5d1d7..24434cad0c668 100644 --- a/Dockerfile +++ b/Dockerfile @@ -57,6 +57,7 @@ RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \ apt-utils \ build-essential \ dirmngr \ + dumb-init \ freetds-bin \ freetds-dev \ git \ @@ -353,6 +354,9 @@ RUN \ && pip uninstall --yes apache-airflow; \ fi +# Link dumb-init for backwards compatibility (so that older images also work) +RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init + # Install NPM dependencies here. The NPM dependencies don't change that often and we already have pip # installed dependencies in case of CI optimised build, so it is ok to install NPM deps here # Rather than after setup.py is added. @@ -431,6 +435,6 @@ ENV PATH="${HOME}:${PATH}" EXPOSE 8080 -ENTRYPOINT ["/usr/local/bin/dumb-init", "--", "/entrypoint.sh"] +ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint.sh"] CMD ["--help"] diff --git a/scripts/ci/_utils.sh b/scripts/ci/_utils.sh index aad7291f76212..fffb373eca448 100644 --- a/scripts/ci/_utils.sh +++ b/scripts/ci/_utils.sh @@ -841,13 +841,15 @@ function filter_out_files_from_pylint_todo_list() { function refresh_pylint_todo() { docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \ - --entrypoint /opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \ --env PYTHONDONTWRITEBYTECODE \ --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \ --env AIRFLOW_CI_SILENT \ --env HOST_USER_ID="$(id -ur)" \ --env HOST_GROUP_ID="$(id -gr)" \ - "${AIRFLOW_CI_IMAGE}" | tee -a "${OUTPUT_LOG}" + --rm \ + "${AIRFLOW_CI_IMAGE}" \ + /opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \ + | tee -a "${OUTPUT_LOG}" } function rebuild_all_images_if_needed_and_confirmed() { diff --git a/setup.py b/setup.py index b9616327376a1..0bd35a73d6f84 100644 --- a/setup.py +++ b/setup.py @@ -376,7 +376,6 @@ def write_version(filename: str = os.path.join(*["airflow", "git_version"])): 'click==6.7', 'contextdecorator;python_version<"3.4"', 'coverage', - 'dumb-init>=1.2.2', 'flake8>=3.6.0', 'flake8-colors', 'flaky',