From 8ba59fbeeb373ed89b4d95bf2ca6f1c03b2dcdcf Mon Sep 17 00:00:00 2001
From: Tzu-ping Chung <tp@astronomer.io>
Date: Tue, 19 Oct 2021 12:56:31 +0800
Subject: [PATCH] Remove distutils usages for Python 3.10

Version parsing capabilities when building provider packages is now
provided by packaging.version.Version instead. Non-Python software
version parsing (sqlite) is re-implemented with regex.

Usages of strtobool are replaced by the to_boolean utility function in
the airflow.utils.strings module. This function is a bit more permissive
than strtobool, but should be good enough for how Airflow uses it for.
This function is available since 2.0 and should be safe for usages in
provider packages as well. Some usages of strtobool in providers are
replaced by copying the implementation from distutils since to_boolean's
different behavior would cause backward incompatibilities.

The copy_file usage is replaced by shutils.copy2(), which is good enough
for Airflow's needs.

The distutils reference in setup.py is kept since the file is only used
at build-time against setuptools, which patches the distutils import
with a vendored copy and is not deprecated.
---
 airflow/configuration.py                     | 19 +++++++++++-----
 airflow/operators/sql.py                     | 16 ++++++++++++--
 airflow/providers/jenkins/hooks/jenkins.py   |  9 ++------
 airflow/providers/tableau/hooks/tableau.py   | 23 ++++++++++++++------
 airflow/utils/strings.py                     | 14 +++++++++---
 dev/provider_packages/remove_old_releases.py | 10 +++++----
 docs/exts/sphinx_script_update.py            |  9 ++++++--
 setup.py                                     |  5 ++++-
 8 files changed, 74 insertions(+), 31 deletions(-)

diff --git a/airflow/configuration.py b/airflow/configuration.py
index b1f0095d37efb..c57ac2f6f0a36 100644
--- a/airflow/configuration.py
+++ b/airflow/configuration.py
@@ -32,7 +32,7 @@
 # Ignored Mypy on configparser because it thinks the configparser module has no _UNSET attribute
 from configparser import _UNSET, ConfigParser, NoOptionError, NoSectionError  # type: ignore
 from json.decoder import JSONDecodeError
-from typing import Any, Dict, List, Optional, Union
+from typing import Any, Dict, List, Optional, Tuple, Union
 
 from airflow.exceptions import AirflowConfigException
 from airflow.secrets import DEFAULT_SECRETS_SEARCH_PATH, BaseSecretsBackend
@@ -47,6 +47,15 @@
     warnings.filterwarnings(action='default', category=DeprecationWarning, module='airflow')
     warnings.filterwarnings(action='default', category=PendingDeprecationWarning, module='airflow')
 
+_SQLITE3_VERSION_PATTERN = re.compile(r"(?P<version>^\d+(?:\.\d+)*)\D?.*$")
+
+
+def _parse_sqlite_version(s: str) -> Tuple[int, ...]:
+    match = _SQLITE3_VERSION_PATTERN.match(s)
+    if match is None:
+        return ()
+    return tuple(int(p) for p in match.group("version").split("."))
+
 
 def expand_env_var(env_var):
     """
@@ -267,15 +276,15 @@ def _validate_config_dependencies(self):
             raise AirflowConfigException(f"error: cannot use sqlite with the {self.get('core', 'executor')}")
         if is_sqlite:
             import sqlite3
-            from distutils.version import StrictVersion
 
             from airflow.utils.docs import get_docs_url
 
             # Some of the features in storing rendered fields require sqlite version >= 3.15.0
-            min_sqlite_version = '3.15.0'
-            if StrictVersion(sqlite3.sqlite_version) < StrictVersion(min_sqlite_version):
+            min_sqlite_version = (3, 15, 0)
+            if _parse_sqlite_version(sqlite3.sqlite_version) < min_sqlite_version:
+                min_sqlite_version_str = ".".join(str(s) for s in min_sqlite_version)
                 raise AirflowConfigException(
-                    f"error: sqlite C library version too old (< {min_sqlite_version}). "
+                    f"error: sqlite C library version too old (< {min_sqlite_version_str}). "
                     f"See {get_docs_url('howto/set-up-database.html#setting-up-a-sqlite-database')}"
                 )
 
diff --git a/airflow/operators/sql.py b/airflow/operators/sql.py
index 8347bcb41504a..ab916c00e4012 100644
--- a/airflow/operators/sql.py
+++ b/airflow/operators/sql.py
@@ -15,7 +15,6 @@
 # KIND, either express or implied.  See the License for the
 # specific language governing permissions and limitations
 # under the License.
-from distutils.util import strtobool
 from typing import Any, Dict, Iterable, List, Mapping, Optional, SupportsAbs, Union
 
 from airflow.compat.functools import cached_property
@@ -25,6 +24,19 @@
 from airflow.models import BaseOperator, SkipMixin
 
 
+def parse_boolean(val: str) -> Union[str, bool]:
+    """Try to parse a string into boolean.
+
+    Raises ValueError if the input is not a valid true- or false-like string value.
+    """
+    val = val.lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1'):
+        return True
+    if val in ('n', 'no', 'f', 'false', 'off', '0'):
+        return False
+    raise ValueError(f"{val!r} is not a boolean-like string value")
+
+
 class BaseSQLOperator(BaseOperator):
     """
     This is a base class for generic SQL Operator to get a DB Hook
@@ -538,7 +550,7 @@ def execute(self, context: Dict):
                     follow_branch = self.follow_task_ids_if_true
             elif isinstance(query_result, str):
                 # return result is not Boolean, try to convert from String to Boolean
-                if bool(strtobool(query_result)):
+                if parse_boolean(query_result):
                     follow_branch = self.follow_task_ids_if_true
             elif isinstance(query_result, int):
                 if bool(query_result):
diff --git a/airflow/providers/jenkins/hooks/jenkins.py b/airflow/providers/jenkins/hooks/jenkins.py
index 534e8390f58df..55782b7b0da43 100644
--- a/airflow/providers/jenkins/hooks/jenkins.py
+++ b/airflow/providers/jenkins/hooks/jenkins.py
@@ -17,11 +17,10 @@
 # under the License.
 #
 
-from distutils.util import strtobool
-
 import jenkins
 
 from airflow.hooks.base import BaseHook
+from airflow.utils.strings import to_boolean
 
 
 class JenkinsHook(BaseHook):
@@ -38,11 +37,7 @@ def __init__(self, conn_id: str = default_conn_name) -> None:
         self.connection = connection
         connection_prefix = 'http'
         # connection.extra contains info about using https (true) or http (false)
-        if connection.extra is None or connection.extra == '':
-            connection.extra = 'false'
-            # set a default value to connection.extra
-            # to avoid rising ValueError in strtobool
-        if strtobool(connection.extra):
+        if to_boolean(connection.extra):
             connection_prefix = 'https'
         url = f'{connection_prefix}://{connection.host}:{connection.port}'
         self.log.info('Trying to connect to %s', url)
diff --git a/airflow/providers/tableau/hooks/tableau.py b/airflow/providers/tableau/hooks/tableau.py
index 8ec18ec86ca7e..1b9e41fc50c5f 100644
--- a/airflow/providers/tableau/hooks/tableau.py
+++ b/airflow/providers/tableau/hooks/tableau.py
@@ -16,9 +16,8 @@
 # under the License.
 import time
 import warnings
-from distutils.util import strtobool
 from enum import Enum
-from typing import Any, Optional
+from typing import Any, Optional, Union
 
 from tableauserverclient import Pager, PersonalAccessTokenAuth, Server, TableauAuth
 from tableauserverclient.server import Auth
@@ -27,6 +26,19 @@
 from airflow.hooks.base import BaseHook
 
 
+def parse_boolean(val: str) -> Union[str, bool]:
+    """Try to parse a string into boolean.
+
+    The string is returned as-is if it does not look like a boolean value.
+    """
+    val = val.lower()
+    if val in ('y', 'yes', 't', 'true', 'on', '1'):
+        return True
+    if val in ('n', 'no', 'f', 'false', 'off', '0'):
+        return False
+    return val
+
+
 class TableauJobFailedException(AirflowException):
     """An exception that indicates that a Job failed to complete."""
 
@@ -71,12 +83,9 @@ def __init__(self, site_id: Optional[str] = None, tableau_conn_id: str = default
         self.conn = self.get_connection(self.tableau_conn_id)
         self.site_id = site_id or self.conn.extra_dejson.get('site_id', '')
         self.server = Server(self.conn.host)
-        verify = self.conn.extra_dejson.get('verify', True)
+        verify: Any = self.conn.extra_dejson.get('verify', True)
         if isinstance(verify, str):
-            try:
-                verify = bool(strtobool(verify))
-            except ValueError:
-                pass
+            verify = parse_boolean(verify)
         self.server.add_http_options(
             options_dict={'verify': verify, 'cert': self.conn.extra_dejson.get('cert', None)}
         )
diff --git a/airflow/utils/strings.py b/airflow/utils/strings.py
index c1823ff919a5f..72c7fa687baf0 100644
--- a/airflow/utils/strings.py
+++ b/airflow/utils/strings.py
@@ -18,6 +18,7 @@
 
 import string
 from random import choice
+from typing import Optional
 
 
 def get_random_string(length=8, choices=string.ascii_letters + string.digits):
@@ -25,6 +26,13 @@ def get_random_string(length=8, choices=string.ascii_letters + string.digits):
     return ''.join(choice(choices) for _ in range(length))
 
 
-def to_boolean(astring):
-    """Convert a string to a boolean"""
-    return False if astring is None else astring.lower() in ['true', 't', 'y', 'yes', '1']
+TRUE_LIKE_VALUES = {"on", "t", "true", "y", "yes", "1"}
+
+
+def to_boolean(astring: Optional[str]) -> bool:
+    """Convert a string to a boolean."""
+    if astring is None:
+        return False
+    if astring.lower() in TRUE_LIKE_VALUES:
+        return True
+    return False
diff --git a/dev/provider_packages/remove_old_releases.py b/dev/provider_packages/remove_old_releases.py
index 2c8c62d6feb7a..63fa73d311e78 100644
--- a/dev/provider_packages/remove_old_releases.py
+++ b/dev/provider_packages/remove_old_releases.py
@@ -24,19 +24,21 @@
 """
 import argparse
 import glob
+import operator
 import os
 import subprocess
 from collections import defaultdict
-from distutils.version import LooseVersion
 from typing import Dict, List, NamedTuple
 
+from packaging.version import Version
+
 
 class VersionedFile(NamedTuple):
     base: str
     version: str
     suffix: str
     type: str
-    comparable_version: LooseVersion
+    comparable_version: Version
 
 
 def split_version_and_suffix(file_name: str, suffix: str) -> VersionedFile:
@@ -47,7 +49,7 @@ def split_version_and_suffix(file_name: str, suffix: str) -> VersionedFile:
         version=version,
         suffix=suffix,
         type=no_version_file + "-" + suffix,
-        comparable_version=LooseVersion(version),
+        comparable_version=Version(version),
     )
 
 
@@ -60,7 +62,7 @@ def process_all_files(directory: str, suffix: str, execute: bool):
         package_types_dicts[versioned_file.type].append(versioned_file)
 
     for package_types in package_types_dicts.values():
-        package_types.sort(key=lambda x: x.comparable_version)
+        package_types.sort(key=operator.attrgetter("comparable_version"))
 
     for package_types in package_types_dicts.values():
         if len(package_types) == 1:
diff --git a/docs/exts/sphinx_script_update.py b/docs/exts/sphinx_script_update.py
index 2fa3b527ae5bb..1926727d4b100 100644
--- a/docs/exts/sphinx_script_update.py
+++ b/docs/exts/sphinx_script_update.py
@@ -17,9 +17,9 @@
 import hashlib
 import json
 import os
+import shutil
 import sys
 import tempfile
-from distutils.file_util import copy_file
 from functools import lru_cache
 from typing import Dict
 
@@ -30,6 +30,11 @@
 log = logging.getLogger(__name__)
 
 
+def _copy_file(src: str, dst: str) -> None:
+    log.info("Copying %s -> %s", src, dst)
+    shutil.copy2(src, dst, follow_symlinks=False)
+
+
 def _gethash(string: str):
     hash_object = hashlib.sha256(string.encode())
     return hash_object.hexdigest()
@@ -107,7 +112,7 @@ def build_finished(app, exception):
     output_filename = "script.js"
 
     cache_filepath = fetch_and_cache(script_url, output_filename)
-    copy_file(cache_filepath, os.path.join(app.builder.outdir, '_static', "redoc.js"))
+    _copy_file(cache_filepath, os.path.join(app.builder.outdir, '_static', "redoc.js"))
 
 
 def setup(app):
diff --git a/setup.py b/setup.py
index 49ddbd5dbbdae..69092c8a7aad3 100644
--- a/setup.py
+++ b/setup.py
@@ -23,7 +23,6 @@
 import sys
 import unittest
 from copy import deepcopy
-from distutils import log
 from os.path import dirname, relpath
 from textwrap import wrap
 from typing import Dict, List
@@ -32,6 +31,10 @@
 from setuptools.command.develop import develop as develop_orig
 from setuptools.command.install import install as install_orig
 
+# Setuptools patches this import to point to a vendored copy instead of the
+# stdlib, which is deprecated in Python 3.10 and will be removed in 3.12.
+from distutils import log  # isort: skip
+
 # Controls whether providers are installed from packages or directly from sources
 # It is turned on by default in case of development environments such as Breeze
 # And it is particularly useful when you add a new provider and there is no
