From 40546123c39c758ddaf69bb086ee5a403cc4ca23 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 18 Nov 2022 09:07:41 -0600 Subject: [PATCH 1/2] apply markers from wheelhouse.txt during installation --- lib/charms/layer/basic.py | 61 ++++++++++++++++++++++++++++++++------- 1 file changed, 51 insertions(+), 10 deletions(-) diff --git a/lib/charms/layer/basic.py b/lib/charms/layer/basic.py index 3e741fa..bc8dae1 100644 --- a/lib/charms/layer/basic.py +++ b/lib/charms/layer/basic.py @@ -2,10 +2,13 @@ import sys import re import shutil +from contextlib import contextmanager +from collections import defaultdict from distutils.version import LooseVersion -from pkg_resources import Requirement +from pkg_resources import Requirement, parse_requirements from glob import glob from subprocess import check_call, check_output, CalledProcessError +from tempfile import NamedTemporaryFile from time import sleep from charms import layer @@ -217,15 +220,29 @@ def bootstrap_charm_deps(): # This ensures that pip 20.3.4+ will install the packages from the # wheelhouse without (erroneously) flagging an error. pkgs = _add_back_versions(_pkgs_set, _versions) - reinstall_flag = '--force-reinstall' - # if not cfg.get('use_venv', True) and pre_eoan: - if not cfg.get('use_venv', True): - reinstall_flag = '--ignore-installed' - if not pkgs: - continue - check_call([pip, 'install', '-U', reinstall_flag, '--no-index', - '--no-cache-dir', '-f', 'wheelhouse'] + list(pkgs), - env=_get_subprocess_env()) + with _marked_requirements(pkgs) as marked_file: + reinstall_flag = '--force-reinstall' + # if not cfg.get('use_venv', True) and pre_eoan: + if not cfg.get('use_venv', True): + reinstall_flag = '--ignore-installed' + if not pkgs: + continue + check_call( + [ + pip, + 'install', + '-U', + reinstall_flag, + '--no-index', + '--no-cache-dir', + '-f', + 'wheelhouse', + '-r', + marked_file + ], + env=_get_subprocess_env() + ) + # re-enable installation from pypi os.remove('/root/.pydistutils.cfg') @@ -358,6 +375,30 @@ def _maybe_add_version(pkg): return [_maybe_add_version(pkg) for pkg in pkgs] +@contextmanager +def _marked_requirements(pkgs): + """Apply and custom markers from wheelhouse.txt to list of packages.""" + wheelhouse_reqs = defaultdict(list) + with open("wheelhouse.txt") as wheelhouse: + for req in parse_requirements(wheelhouse.read()): + wheelhouse_reqs[req.project_name].append(req) + + reqs = [ + str(mark) + "\n" + for pkg in pkgs + for mark in wheelhouse_reqs.get( + # use each matching marker by project_name + Requirement(pkg).project_name, + # or default to using the incoming pkg + [pkg] + ) + ] + with NamedTemporaryFile(mode='w') as requirements: + requirements.writelines(reqs) + requirements.flush() + yield requirements.name + + def _update_if_newer(pip, pkgs): installed = _load_installed_versions(pip) wheelhouse = _load_wheelhouse_versions() From 744b0017fe5253e5abc4bfdeef6a871d1eddc905 Mon Sep 17 00:00:00 2001 From: Adam Dyess Date: Fri, 18 Nov 2022 09:15:11 -0600 Subject: [PATCH 2/2] fix misspelling --- lib/charms/layer/basic.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/charms/layer/basic.py b/lib/charms/layer/basic.py index bc8dae1..ace5395 100644 --- a/lib/charms/layer/basic.py +++ b/lib/charms/layer/basic.py @@ -377,7 +377,7 @@ def _maybe_add_version(pkg): @contextmanager def _marked_requirements(pkgs): - """Apply and custom markers from wheelhouse.txt to list of packages.""" + """Apply any custom markers from wheelhouse.txt to list of packages.""" wheelhouse_reqs = defaultdict(list) with open("wheelhouse.txt") as wheelhouse: for req in parse_requirements(wheelhouse.read()):