[AIRFLOW-6662] install dumb init

Dockerfile

@@ -49,14 +49,15 @@ RUN apt-get update \
     && rm -rf /var/lib/apt/lists/*
 
 # Install basic apt dependencies
-RUN curl -L https://deb.nodesource.com/setup_10.x | bash - \
+RUN curl --fail --location https://deb.nodesource.com/setup_10.x | bash - \
     && curl https://dl.yarnpkg.com/debian/pubkey.gpg | apt-key add - > /dev/null \
     && echo "deb https://dl.yarnpkg.com/debian/ stable main" > /etc/apt/sources.list.d/yarn.list \
     && apt-get update \
     && apt-get install -y --no-install-recommends \
            apt-utils \
            build-essential \
            dirmngr \
+           dumb-init \
            freetds-bin \
            freetds-dev \
            git \
@@ -161,7 +162,7 @@ RUN HADOOP_DISTRO="cdh" \
     && HADOOP_DOWNLOAD_URL="${HADOOP_URL}hadoop-${HADOOP_VERSION}-${HADOOP_DISTRO}${HADOOP_DISTRO_VERSION}.tar.gz" \
     && HADOOP_TMP_FILE="/tmp/hadoop.tar.gz" \
     && mkdir -pv "${HADOOP_HOME}" \
-    && curl -L "${HADOOP_DOWNLOAD_URL}" -o "${HADOOP_TMP_FILE}" \
+    && curl --fail --location "${HADOOP_DOWNLOAD_URL}" --output "${HADOOP_TMP_FILE}" \
     && tar xzf "${HADOOP_TMP_FILE}" --absolute-names --strip-components 1 -C "${HADOOP_HOME}" \
     && rm "${HADOOP_TMP_FILE}" \
     && echo "Installing Hive" \
@@ -173,7 +174,7 @@ RUN HADOOP_DISTRO="cdh" \
     && mkdir -pv "/user/hive/warehouse" \
     && chmod -R 777 "${HIVE_HOME}" \
     && chmod -R 777 "/user/" \
-    && curl -L "${HIVE_URL}" -o "${HIVE_TMP_FILE}" \
+    && curl --fail --location  "${HIVE_URL}" --output "${HIVE_TMP_FILE}" \
     && tar xzf "${HIVE_TMP_FILE}" --strip-components 1 -C "${HIVE_HOME}" \
     && rm "${HIVE_TMP_FILE}"
 
@@ -187,12 +188,12 @@ RUN MINICLUSTER_BASE="https://github.com/bolkedebruin/minicluster/releases/downl
     && MINICLUSTER_URL="${MINICLUSTER_BASE}${MINICLUSTER_VER}/minicluster-${MINICLUSTER_VER}-SNAPSHOT-bin.zip" \
     && MINICLUSTER_TMP_FILE="/tmp/minicluster.zip" \
     && mkdir -pv "${MINICLUSTER_HOME}" \
-    && curl -L "${MINICLUSTER_URL}" -o "${MINICLUSTER_TMP_FILE}" \
+    && curl --fail --location "${MINICLUSTER_URL}" --output "${MINICLUSTER_TMP_FILE}" \
     && unzip "${MINICLUSTER_TMP_FILE}" -d "/opt" \
     && rm "${MINICLUSTER_TMP_FILE}"
 
 # Install Docker
-RUN curl -L https://download.docker.com/linux/debian/gpg | apt-key add - \
+RUN curl --fail --location https://download.docker.com/linux/debian/gpg | apt-key add - \
     && add-apt-repository "deb [arch=amd64] https://download.docker.com/linux/debian stretch stable" \
     && apt-get update \
     && apt-get -y install --no-install-recommends docker-ce \
@@ -203,28 +204,54 @@ RUN curl -L https://download.docker.com/linux/debian/gpg | apt-key add - \
 ARG KUBECTL_VERSION="v1.15.3"
 
 RUN KUBECTL_URL="https://storage.googleapis.com/kubernetes-release/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" \
-  && curl -L "${KUBECTL_URL}" -o "/usr/local/bin/kubectl" \
+  && curl --fail --location  "${KUBECTL_URL}" --output "/usr/local/bin/kubectl" \
   && chmod +x /usr/local/bin/kubectl
 
 # Install Kind
 ARG KIND_VERSION="v0.6.1"
 
 RUN KIND_URL="https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-linux-amd64" \
-   && curl -L "${KIND_URL}" -o "/usr/local/bin/kind" \
+   && curl --fail --location "${KIND_URL}" --output "/usr/local/bin/kind" \
    && chmod +x /usr/local/bin/kind
 
 # Install Apache RAT
 ARG RAT_VERSION="0.13"
-
-RUN RAT_URL="https://repo1.maven.org/maven2/org/apache/rat/apache-rat/${RAT_VERSION}/apache-rat-${RAT_VERSION}.jar" \
+ARG RAT_BACKUP_SITE_1="https://www-eu.apache.org/dist/creadur"
+ARG RAT_BACKUP_SITE_2="https://www-us.apache.org/dist/creadur"
+# It's OK to use HTTP rather than https here as we verify it later with gpg from the
+# offcial backup servers of Apache!
+ARG RAT_MIRROR_1="http://mirror.serverion.com/apache/creadur"
+ARG RAT_MIRROR_2="http://mirror.cc.columbia.edu/pub/software/apache/creadur"
+
+RUN RAT_TARGZ_FILE_NAME="apache-rat-${RAT_VERSION}-bin.tar.gz" \
+    && RAT_FOLDER="apache-rat-${RAT_VERSION}" \
+    && RAT_KEYS_URL_1="${RAT_BACKUP_SITE_1}/KEYS" \
+    && RAT_KEYS_URL_2="${RAT_BACKUP_SITE_2}/KEYS"  \
+    && RAT_ASC_URL_1="${RAT_BACKUP_SITE_1}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}.asc" \
+    && RAT_ASC_URL_2="${RAT_BACKUP_SITE_2}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}.asc" \
+    && RAT_URL_1="${RAT_MIRROR_1}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}" \
+    && RAT_URL_2="${RAT_MIRROR_2}/${RAT_FOLDER}/${RAT_TARGZ_FILE_NAME}"  \
+    && RAT_TAR_GZ="/opt/${RAT_TARGZ_FILE_NAME}" \
+    && RAT_TAR_GZ_ASC="/opt/${RAT_TARGZ_FILE_NAME}.asc" \
+    && RAT_KEYS="/opt/KEYS" \
+    && RAT_JAR_IN_TAR="${RAT_FOLDER}/apache-rat-${RAT_VERSION}.jar" \
     && RAT_JAR="/opt/apache-rat.jar" \
-    && RAT_JAR_MD5="${RAT_JAR}.md5" \
-    && RAT_URL_MD5="${RAT_URL}.md5" \
-    && echo "Downloading RAT from ${RAT_URL} to ${RAT_JAR}" \
-    && curl -L "${RAT_URL}" -o "${RAT_JAR}" \
-    && curl -L "${RAT_URL_MD5}" -o "${RAT_JAR_MD5}" \
-    && jar -tf "${RAT_JAR}" > /dev/null \
-    && md5sum -c <<<"$(cat "${RAT_JAR_MD5}") ${RAT_JAR}"
+    && echo "Downloading KEYS from backup Apache servers: ${RAT_KEYS_URL_1}, ${RAT_KEYS_URL_2}" \
+    && (curl --fail --location "${RAT_KEYS_URL_1}" --output "${RAT_KEYS}" || \
+        curl --fail --location "${RAT_KEYS_URL_2}" --output "${RAT_KEYS}") \
+    && echo "Downloading ASC from backup Apache servers: ${RAT_ASC_URL_1}, ${RAT_ASC_URL_2}" \
+    && (curl --fail --location "${RAT_ASC_URL_1}" --output "${RAT_TAR_GZ_ASC}" || \
+        curl --fail --location "${RAT_ASC_URL_2}" --output "${RAT_TAR_GZ_ASC}") \
+    && echo "Downloading RAT from mirrors: ${RAT_URL_1}, ${RAT_URL_2} to ${RAT_JAR}" \
+    && (curl --fail --location "${RAT_URL_1}" --output "${RAT_TAR_GZ}" || \
+        curl --fail --location "${RAT_URL_2}" --output "${RAT_TAR_GZ}") \
+    && gpg --import ${RAT_KEYS} \
+    && gpg --verify "${RAT_TAR_GZ_ASC}" "${RAT_TAR_GZ}" \
+    && tar --extract --gzip --file "${RAT_TAR_GZ}" -C /opt "${RAT_JAR_IN_TAR}" \
+    && mv -v /opt/"${RAT_JAR_IN_TAR}" "${RAT_JAR}" \
+    && rm -vrf "${RAT_TAR_GZ}" "/opt/${RAT_FOLDER}" \
+    && rm -f "${RAT_KEYS}" \
+    && jar -tf "${RAT_JAR}" >/dev/null
 
 # Setup PIP
 # By default PIP install run without cache to make image smaller
@@ -327,6 +354,9 @@ RUN \
         && pip uninstall --yes apache-airflow; \
     fi
 
+# Link dumb-init for backwards compatibility (so that older images also work)
+RUN ln -sf /usr/bin/dumb-init /usr/local/bin/dumb-init
+
 # Install NPM dependencies here. The NPM dependencies don't change that often and we already have pip
 # installed dependencies in case of CI optimised build, so it is ok to install NPM deps here
 # Rather than after setup.py is added.
@@ -405,6 +435,6 @@ ENV PATH="${HOME}:${PATH}"
 
 EXPOSE 8080
 
-ENTRYPOINT ["/entrypoint.sh"]
+ENTRYPOINT ["/usr/bin/dumb-init", "--", "/entrypoint.sh"]
 
 CMD ["--help"]

scripts/ci/_utils.sh

@@ -640,30 +640,27 @@ function run_flake8() {
 
     if [[ "${#FILES[@]}" == "0" ]]; then
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \
             --rm \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \
             | tee -a "${OUTPUT_LOG}"
     else
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_flake8.sh" \
             "${AIRFLOW_CI_IMAGE}" \
-            "${FILES[@]}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_flake8.sh" "${FILES[@]}" \
             | tee -a "${OUTPUT_LOG}"
     fi
 }
@@ -682,63 +679,57 @@ function run_bats_tests() {
 
 function run_docs() {
     docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
-            --entrypoint \
-            "/opt/airflow/docs/build.sh" \
             --rm \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/docs/build.sh" \
             | tee -a "${OUTPUT_LOG}"
 }
 
 function run_check_license() {
     docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" -t \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \
             --rm \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_check_licence.sh" \
             | tee -a "${OUTPUT_LOG}"
 }
 
 function run_mypy() {
     FILES=("$@")
     if [[ "${#FILES[@]}" == "0" ]]; then
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \
             "${AIRFLOW_CI_IMAGE}" \
-            "airflow" "tests" "docs" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "airflow" "tests" "docs" \
             | tee -a "${OUTPUT_LOG}"
     else
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init" \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_mypy.sh" \
             --rm \
             "${AIRFLOW_CI_IMAGE}" \
-            "${FILES[@]}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_mypy.sh" "${FILES[@]}" \
             | tee -a "${OUTPUT_LOG}"
     fi
 }
@@ -747,30 +738,27 @@ function run_pylint_main() {
     FILES=("$@")
     if [[ "${#FILES[@]}" == "0" ]]; then
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \
             | tee -a "${OUTPUT_LOG}"
     else
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init" \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" \
             "${AIRFLOW_CI_IMAGE}" \
-            "${FILES[@]}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_pylint_main.sh" "${FILES[@]}" \
             | tee -a "${OUTPUT_LOG}"
     fi
 }
@@ -780,30 +768,27 @@ function run_pylint_tests() {
     FILES=("$@")
     if [[ "${#FILES[@]}" == "0" ]]; then
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \
             "${AIRFLOW_CI_IMAGE}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \
             | tee -a "${OUTPUT_LOG}"
     else
         docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-            --init \
+            --entrypoint "/usr/local/bin/dumb-init"  \
             --env PYTHONDONTWRITEBYTECODE \
             --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
             --env AIRFLOW_CI_SILENT \
             --env HOST_USER_ID="$(id -ur)" \
             --env HOST_GROUP_ID="$(id -gr)" \
             --rm \
-            --entrypoint \
-            "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" \
             "${AIRFLOW_CI_IMAGE}" \
-            "${FILES[@]}" \
+            "--" "/opt/airflow/scripts/ci/in_container/run_pylint_tests.sh" "${FILES[@]}" \
             | tee -a "${OUTPUT_LOG}"
     fi
 }
@@ -856,13 +841,15 @@ function filter_out_files_from_pylint_todo_list() {
 
 function refresh_pylint_todo() {
     docker run "${AIRFLOW_CONTAINER_EXTRA_DOCKER_FLAGS[@]}" \
-        --entrypoint /opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \
         --env PYTHONDONTWRITEBYTECODE \
         --env AIRFLOW_CI_VERBOSE="${VERBOSE}" \
         --env AIRFLOW_CI_SILENT \
         --env HOST_USER_ID="$(id -ur)" \
         --env HOST_GROUP_ID="$(id -gr)" \
-        "${AIRFLOW_CI_IMAGE}" | tee -a "${OUTPUT_LOG}"
+        --rm \
+        "${AIRFLOW_CI_IMAGE}" \
+        /opt/airflow/scripts/ci/in_container/refresh_pylint_todo.sh \
+        | tee -a "${OUTPUT_LOG}"
 }
 
 function rebuild_all_images_if_needed_and_confirmed() {

scripts/ci/docker-compose/backend-mysql.yml

@@ -25,7 +25,6 @@ services:
     depends_on:
       - mysql
   mysql:
-    init: true
     image: mysql:5.6
     environment:
       - MYSQL_ALLOW_EMPTY_PASSWORD=true

scripts/ci/docker-compose/backend-postgres.yml

@@ -25,7 +25,6 @@ services:
     depends_on:
       - postgres
   postgres:
-    init: true
     image: postgres:9.6
     environment:
       - POSTGRES_USER=postgres

scripts/ci/docker-compose/integration-cassandra.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   cassandra:
-    init: true
     image: cassandra:3.0
     volumes:
       - /dev/urandom:/dev/random   # Required to get non-blocking entropy source

scripts/ci/docker-compose/integration-kerberos.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   kerberos:
-    init: true
     image: godatadriven/krb5-kdc-server
     hostname: kerberos
     domainname: example.com

scripts/ci/docker-compose/integration-mongo.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   mongo:
-    init: true
     image: mongo:3
     volumes:
       - /dev/urandom:/dev/random   # Required to get non-blocking entropy source

scripts/ci/docker-compose/integration-openldap.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   openldap:
-    init: true
     image: osixia/openldap:1.2.0
     command: --copy-service
     environment:

scripts/ci/docker-compose/integration-rabbitmq.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   rabbitmq:
-    init: true
     image: rabbitmq:3.7
     volumes:
       - /dev/urandom:/dev/random   # Required to get non-blocking entropy source

scripts/ci/docker-compose/integration-redis.yml

@@ -18,7 +18,6 @@
 version: "2.2"
 services:
   redis:
-    init: true
     image: redis:5.0.1
     volumes:
       - /dev/urandom:/dev/random   # Required to get non-blocking entropy source