From c06a94022a59e1e1cd255f5cfd7f79e3a5f0493e Mon Sep 17 00:00:00 2001 From: Ian Hurst Date: Fri, 18 Oct 2024 18:41:48 +0000 Subject: [PATCH] Automatically import binary extensions whose suffixes include the python version --- .github/workflows/tests.yml | 2 +- README.md | 14 --- rpm/__init__.py | 164 ++++++++++++++++++++++++++++++------ 3 files changed, 138 insertions(+), 42 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index ef05eab..044b421 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -16,7 +16,7 @@ jobs: - base_image: centos:stream9 install_deps_cmd: dnf -y install git-core python3-pip python3-rpm - base_image: registry.access.redhat.com/ubi9:latest - install_deps_cmd: dnf -y install git-core python3-pip python3-rpm && python3 -m pip install -U requests + install_deps_cmd: dnf -y install git-core python{3,3.11}-pip python3-rpm && python3 -m pip install -U requests && python3.11 -m pip install -U requests - base_image: ubi8:latest install_deps_cmd: dnf -y install git-core python3-pip python3-rpm - base_image: opensuse/leap:latest diff --git a/README.md b/README.md index fe76399..4296288 100644 --- a/README.md +++ b/README.md @@ -44,17 +44,3 @@ Successfully installed rpm-0.1.0 (env) $ python -c "import rpm; print(rpm.__version__)" 4.18.0 ``` - -## Using RPM bindings with a different Python version than the system Python - -On many systems, the shim module will be able to find the system-installed RPM bindings, even if you use a different version of Python (e.g. Fedora 38 ships with Python 3.11 by default, but the shim will also work in a Python 3.10 virtualenv). - -On some distributions (especially Debian/Ubuntu ones), it will not work and raise a `ModuleNotFoundError: No module named 'rpm._rpm'`. This is because those distributions encode the Python version in the name of the `_rpm.so` file: `_rpm.cpython-38-x86_64-linux-gnu.so`. - -You can make the shim module work on such systems by creating a symlink to the generic `_rpm.so` name: - -```bash -for file in /usr/lib/python3/dist-packages/rpm/_rpm*.cpython-*.so; do - sudo ln -s ${file} $(echo ${file} | sed 's/\.cpython[^.]*//'); -done -``` diff --git a/rpm/__init__.py b/rpm/__init__.py index 1742526..64f346f 100644 --- a/rpm/__init__.py +++ b/rpm/__init__.py @@ -6,13 +6,16 @@ """ import importlib +import importlib.util import json import logging import platform +import pprint import subprocess import sys +import textwrap from pathlib import Path -from typing import List +from typing import Dict, List PROJECT_NAME = "rpm-shim" MODULE_NAME = "rpm" @@ -24,7 +27,7 @@ class ShimAlreadyInitializingError(Exception): pass -def get_system_sitepackages() -> List[str]: +def get_system_sitepackages_and_suffixes() -> List[Dict[str, List[str]]]: """ Gets a list of sitepackages directories of system Python interpreter(s). @@ -32,13 +35,24 @@ def get_system_sitepackages() -> List[str]: List of paths. """ - def get_sitepackages(interpreter): - command = [ - interpreter, - "-c", - "import json, site; print(json.dumps(site.getsitepackages()))", - ] - output = subprocess.check_output(command) + def get_sitepackages_and_suffixes(interpreter): + script = textwrap.dedent( + """ + import importlib + import importlib.machinery + import json + import site + print( + json.dumps( + { + "sitepackages": site.getsitepackages(), + "suffixes": importlib.machinery.EXTENSION_SUFFIXES, + } + ) + ) + """ + ) + output = subprocess.check_output([interpreter], input=script.encode()) return json.loads(output.decode()) majorver, minorver, _ = platform.python_version_tuple() @@ -52,25 +66,28 @@ def get_sitepackages(interpreter): for interpreter in interpreters: if not Path(interpreter).is_file(): continue - sitepackages = get_sitepackages(interpreter) - formatted_list = "\n".join(sitepackages) - logger.debug(f"Collected sitepackages for {interpreter}:\n{formatted_list}") - result.extend(sitepackages) + sitepackages_and_suffixes = get_sitepackages_and_suffixes(interpreter) + logger.debug( + f"Collected sitepackages and extension suffixes for {interpreter}:\n" + + pprint.pformat(sitepackages_and_suffixes) + ) + result.append(sitepackages_and_suffixes) return result -def try_path(path: str) -> bool: +def try_path(path: str, suffixes: List[str]) -> bool: """ Tries to load system RPM module from the specified path. Returns: True if successful, False otherwise. """ - if not (Path(path) / MODULE_NAME).is_dir(): + module_path = Path(path) / MODULE_NAME + if not module_path.is_dir(): return False sys.path.insert(0, path) try: - importlib.reload(sys.modules[__name__]) + reload_module(module_path, suffixes) # sanity check confdir = sys.modules[__name__].expandMacro("%getconfdir") return Path(confdir).is_dir() @@ -79,21 +96,114 @@ def try_path(path: str) -> bool: return False -def initialize() -> None: +def reload_module(path: Path, suffixes: List[str]) -> None: """ - Initializes the shim. Tries to load system RPM module and replace itself with it. + Reloads the `rpm` module. In case some of the required binary submodules fail to import, + tries to import them directly by path using valid extension suffixes. + + Args: + path: Absolute path to the `rpm` module. + suffixes: List of extension suffixes valid for the Python interpreter associated + with the path. """ - for path in get_system_sitepackages(): - logger.debug(f"Trying {path}") + attempted_modules = [] + while True: try: - if try_path(path): - logger.debug("Import successfull") - return - except ShimAlreadyInitializingError: - continue - except Exception as e: - logger.debug(f"Exception: {type(e)}: {e}") + importlib.reload(sys.modules[__name__]) + except ModuleNotFoundError as e: + if e.name is None: + raise + if e.name in attempted_modules: + logger.debug(f"Already tried {e.name} in {path}, giving up") + raise + attempted_modules.append(e.name) + logger.debug( + f"Module {e.name} not found in {path}, " + "looking for alternative filenames trying valid extension suffixes" + ) + try_import_binary_extension(path, e.name, suffixes) + else: + logger.debug(f"Reloaded {__name__}") + return + + +def try_import_binary_extension(path: Path, module: str, suffixes: List[str]) -> bool: + """ + Tries to find and import a binary extension in the specified path based on name + and valid extension suffixes. + + Args: + path: Path to the module, e.g. /usr/lib64/python3.9/site-packages/rpm/ + module: Name of the module to import, e.g. `rpm._rpm` + suffixes: List of extension suffixes to check for + (see `importlib.machinery.EXTENSION_SUFFIXES`). + + Returns: + True if the module was loaded, False otherwise. + """ + # get the submodule module name, e.g. just '_rpm' from 'rpm._rpm' + submodule = module.rpartition(".")[-1] + for suffix in suffixes: + # a file named {submodule}{suffix} may exist in {path}, e.g.: + # + # /usr/lib64/python3.9/site-packages/rpm/_rpm.cpython-39-x86_64-linux-gnu.so + # + # if so we'll try loading it as rpm._rpm + so = path / f"{submodule}{suffix}" + if not so.is_file(): + logger.debug(f"{so} isn't a file, ignoring") continue + if load_module_by_path(module, so): + return True + else: + logger.debug( + f"No combination of {submodule} and valid suffixes found in {path}, giving up" + ) + return False + + +def load_module_by_path(module_name: str, path: Path) -> bool: + """ + Imports a Python module by path. + + Args: + module_name: Name of the module to import. + path: Absolute path to the module to import. + + Returns: + True if the import succeeded, otherwise False. + """ + logger.debug(f"Trying to load {module_name} from {path}") + spec = importlib.util.spec_from_file_location(module_name, path) + if spec is None: + logger.debug(f"No spec for {module_name} in {path}") + return False + if spec.loader is None: + logger.debug(f"No loader in spec for {module_name}") + return False + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + sys.modules[module_name] = module + logger.debug(f"Loaded {module_name} from {path}") + return True + + +def initialize() -> None: + """ + Initializes the shim. Tries to load system RPM module and replace itself with it. + """ + for entry in get_system_sitepackages_and_suffixes(): + for path in entry["sitepackages"]: + logger.debug(f"Trying {path}") + try: + if try_path(path, entry["suffixes"]): + logger.debug("Import successfull") + return + except ShimAlreadyInitializingError: + continue + except Exception as e: + logger.debug(f"Exception: {type(e)}: {e}") + continue else: raise ImportError( "Failed to import system RPM module. "