From d1d4509c54da74ebae1dc15100dee2c8a44a2080 Mon Sep 17 00:00:00 2001 From: Jarek Potiuk Date: Sat, 20 May 2023 12:23:34 +0200 Subject: [PATCH] Add libgeos-dev to dev dependencies in CI image When running some Mlengine tests, pytest (when running assert rewrite) attempts to load libgeos-dev library, which breaks collection of some Google Provider tests on ARM image (it is used by Shapely library imported by recent BigQuery library) This PR Adds the library as preinstalled in the CI image. --- Dockerfile | 2 +- Dockerfile.ci | 5 ++-- scripts/docker/install_os_dependencies.sh | 2 +- scripts/in_container/verify_providers.py | 29 +++++++++-------------- 4 files changed, 15 insertions(+), 23 deletions(-) diff --git a/Dockerfile b/Dockerfile index ade09d91df389..8ddf2b5574bbd 100644 --- a/Dockerfile +++ b/Dockerfile @@ -94,7 +94,7 @@ fi function get_dev_apt_deps() { if [[ "${DEV_APT_DEPS=}" == "" ]]; then DEV_APT_DEPS="apt-transport-https apt-utils build-essential ca-certificates dirmngr \ -freetds-bin freetds-dev git gosu graphviz graphviz-dev krb5-user ldap-utils libffi-dev \ +freetds-bin freetds-dev git gosu graphviz graphviz-dev krb5-user ldap-utils libffi-dev libgeos-dev \ libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \ libssl-dev locales lsb-release openssh-client sasl2-bin \ software-properties-common sqlite3 sudo unixodbc unixodbc-dev" diff --git a/Dockerfile.ci b/Dockerfile.ci index 6169b6066f1e3..38f02c71467f9 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -54,7 +54,7 @@ fi function get_dev_apt_deps() { if [[ "${DEV_APT_DEPS=}" == "" ]]; then DEV_APT_DEPS="apt-transport-https apt-utils build-essential ca-certificates dirmngr \ -freetds-bin freetds-dev git gosu graphviz graphviz-dev krb5-user ldap-utils libffi-dev \ +freetds-bin freetds-dev git gosu graphviz graphviz-dev krb5-user ldap-utils libffi-dev libgeos-dev \ libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \ libssl-dev locales lsb-release openssh-client sasl2-bin \ software-properties-common sqlite3 sudo unixodbc unixodbc-dev" @@ -1233,12 +1233,11 @@ ENV PYTHON_BASE_IMAGE=${PYTHON_BASE_IMAGE} \ RUN echo "Base image version: ${PYTHON_BASE_IMAGE}" -ARG ADDITIONAL_DEV_APT_DEPS="git graphviz gosu libpq-dev netcat rsync" ARG DEV_APT_COMMAND="" ARG ADDITIONAL_DEV_APT_COMMAND="" ARG ADDITIONAL_DEV_ENV_VARS="" ARG ADDITIONAL_DEV_APT_DEPS="bash-completion dumb-init git graphviz gosu krb5-user \ -less libenchant-2-2 libgcc-10-dev libpq-dev net-tools netcat \ +less libenchant-2-2 libgcc-10-dev libgeos-dev libpq-dev net-tools netcat \ openssh-server postgresql-client software-properties-common rsync tmux unzip vim xxd" ARG ADDITIONAL_DEV_APT_ENV="" diff --git a/scripts/docker/install_os_dependencies.sh b/scripts/docker/install_os_dependencies.sh index 50aec97fa5895..e39b9cfee09b6 100644 --- a/scripts/docker/install_os_dependencies.sh +++ b/scripts/docker/install_os_dependencies.sh @@ -36,7 +36,7 @@ fi function get_dev_apt_deps() { if [[ "${DEV_APT_DEPS=}" == "" ]]; then DEV_APT_DEPS="apt-transport-https apt-utils build-essential ca-certificates dirmngr \ -freetds-bin freetds-dev git gosu graphviz graphviz-dev krb5-user ldap-utils libffi-dev \ +freetds-bin freetds-dev git gosu graphviz graphviz-dev krb5-user ldap-utils libffi-dev libgeos-dev \ libkrb5-dev libldap2-dev libleveldb1d libleveldb-dev libsasl2-2 libsasl2-dev libsasl2-modules \ libssl-dev locales lsb-release openssh-client sasl2-bin \ software-properties-common sqlite3 sudo unixodbc unixodbc-dev" diff --git a/scripts/in_container/verify_providers.py b/scripts/in_container/verify_providers.py index 5a5838351b4c1..4da365b9165c7 100755 --- a/scripts/in_container/verify_providers.py +++ b/scripts/in_container/verify_providers.py @@ -21,7 +21,6 @@ import logging import os import pkgutil -import platform import re import subprocess import sys @@ -280,23 +279,17 @@ def onerror(_): try: with warnings.catch_warnings(record=True) as w: warnings.filterwarnings("always", category=DeprecationWarning) - try: - _module = importlib.import_module(modinfo.name) - for attribute_name in dir(_module): - class_name = modinfo.name + "." + attribute_name - attribute = getattr(_module, attribute_name) - if isclass(attribute): - imported_classes.append(class_name) - if isclass(attribute) and ( - issubclass(attribute, logging.Handler) - or issubclass(attribute, BaseSecretsBackend) - ): - classes_with_potential_circular_import.append(class_name) - except OSError as e: - if "geos_c" in str(e) and platform.machine() in ("aarch64", "arm64"): - # we ignore the missing geos_c library on Apple Silicon - continue - raise + _module = importlib.import_module(modinfo.name) + for attribute_name in dir(_module): + class_name = modinfo.name + "." + attribute_name + attribute = getattr(_module, attribute_name) + if isclass(attribute): + imported_classes.append(class_name) + if isclass(attribute) and ( + issubclass(attribute, logging.Handler) + or issubclass(attribute, BaseSecretsBackend) + ): + classes_with_potential_circular_import.append(class_name) if w: all_warnings.extend(w) except AirflowOptionalProviderFeatureException: