Find and import binary rpm extensions by path when import fails due t…

…o unexpected file suffixes.

.github/workflows/tests.yml

@@ -16,9 +16,11 @@ jobs:
           - base_image: centos:stream9
             install_deps_cmd: dnf -y install git-core python3-pip python3-rpm
           - base_image: registry.access.redhat.com/ubi9:latest
-            install_deps_cmd: dnf -y install git-core python3-pip python3-rpm && python3 -m pip install -U requests
+            install_deps_cmd: dnf -y install git-core python{3,3.11}-pip python3-rpm && python3 -m pip install -U requests && python3.11 -m pip install -U requests
           - base_image: ubi8:latest
             install_deps_cmd: dnf -y install git-core python3-pip python3-rpm
+          - base_image: ubi8:latest
+            install_deps_cmd: dnf -y install git-core python3-pip python3-rpm && rm -f /usr/lib64/python3.6/site-packages/rpm/_{rpm,rpmb,rpms}.so
           - base_image: opensuse/leap:latest
             install_deps_cmd: zypper -n install git-core python3-pip python3-rpm
           - base_image: ubuntu:latest

README.md

@@ -44,17 +44,3 @@ Successfully installed rpm-0.1.0
 (env) $ python -c "import rpm; print(rpm.__version__)"
 4.18.0
 ```
-
-## Using RPM bindings with a different Python version than the system Python
-
-On many systems, the shim module will be able to find the system-installed RPM bindings, even if you use a different version of Python (e.g. Fedora 38 ships with Python 3.11 by default, but the shim will also work in a Python 3.10 virtualenv).
-
-On some distributions (especially Debian/Ubuntu ones), it will not work and raise a `ModuleNotFoundError: No module named 'rpm._rpm'`. This is because those distributions encode the Python version in the name of the `_rpm.so` file: `_rpm.cpython-38-x86_64-linux-gnu.so`.
-
-You can make the shim module work on such systems by creating a symlink to the generic `_rpm.so` name:
-
-```bash
-for file in /usr/lib/python3/dist-packages/rpm/_rpm*.cpython-*.so; do
-  sudo ln -s ${file} $(echo ${file} | sed 's/\.cpython[^.]*//');
-done
-```

rpm/__init__.py

@@ -6,13 +6,16 @@
 """
 
 import importlib
+import importlib.util
 import json
 import logging
 import platform
+import pprint
 import subprocess
 import sys
+import textwrap
 from pathlib import Path
-from typing import List
+from typing import Dict, List
 
 PROJECT_NAME = "rpm-shim"
 MODULE_NAME = "rpm"
@@ -24,21 +27,32 @@ class ShimAlreadyInitializingError(Exception):
     pass
 
 
-def get_system_sitepackages() -> List[str]:
+def get_system_sitepackages_and_suffixes() -> List[Dict[str, List[str]]]:
     """
     Gets a list of sitepackages directories of system Python interpreter(s).
 
     Returns:
         List of paths.
     """
 
-    def get_sitepackages(interpreter):
-        command = [
-            interpreter,
-            "-c",
-            "import json, site; print(json.dumps(site.getsitepackages()))",
-        ]
-        output = subprocess.check_output(command)
+    def get_sitepackages_and_suffixes(interpreter):
+        script = textwrap.dedent(
+            """
+            import importlib
+            import importlib.machinery
+            import json
+            import site
+            print(
+                json.dumps(
+                    {
+                        "sitepackages": site.getsitepackages(),
+                        "suffixes": importlib.machinery.EXTENSION_SUFFIXES,
+                    }
+                )
+            )
+            """
+        )
+        output = subprocess.check_output([interpreter], input=script.encode())
         return json.loads(output.decode())
 
     majorver, minorver, _ = platform.python_version_tuple()
@@ -52,25 +66,28 @@ def get_sitepackages(interpreter):
     for interpreter in interpreters:
         if not Path(interpreter).is_file():
             continue
-        sitepackages = get_sitepackages(interpreter)
-        formatted_list = "\n".join(sitepackages)
-        logger.debug(f"Collected sitepackages for {interpreter}:\n{formatted_list}")
-        result.extend(sitepackages)
+        sitepackages_and_suffixes = get_sitepackages_and_suffixes(interpreter)
+        logger.debug(
+            f"Collected sitepackages and extension suffixes for {interpreter}:\n"
+            + pprint.pformat(sitepackages_and_suffixes)
+        )
+        result.append(sitepackages_and_suffixes)
     return result
 
 
-def try_path(path: str) -> bool:
+def try_path(path: str, suffixes: List[str]) -> bool:
     """
     Tries to load system RPM module from the specified path.
 
     Returns:
         True if successful, False otherwise.
     """
-    if not (Path(path) / MODULE_NAME).is_dir():
+    module_path = Path(path) / MODULE_NAME
+    if not module_path.is_dir():
         return False
     sys.path.insert(0, path)
     try:
-        importlib.reload(sys.modules[__name__])
+        reload_module(module_path, suffixes)
         # sanity check
         confdir = sys.modules[__name__].expandMacro("%getconfdir")
         return Path(confdir).is_dir()
@@ -79,21 +96,114 @@ def try_path(path: str) -> bool:
     return False
 
 
-def initialize() -> None:
+def reload_module(path: Path, suffixes: List[str]) -> None:
     """
-    Initializes the shim. Tries to load system RPM module and replace itself with it.
+    Reloads the `rpm` module. In case some of the required binary submodules fail to import,
+    tries to import them directly by path using valid extension suffixes.
+
+    Args:
+        path: Absolute path to the `rpm` module.
+        suffixes: List of extension suffixes valid for the Python interpreter associated
+                  with the path.
     """
-    for path in get_system_sitepackages():
-        logger.debug(f"Trying {path}")
+    attempted_modules = []
+    while True:
         try:
-            if try_path(path):
-                logger.debug("Import successfull")
-                return
-        except ShimAlreadyInitializingError:
-            continue
-        except Exception as e:
-            logger.debug(f"Exception: {type(e)}: {e}")
+            importlib.reload(sys.modules[__name__])
+        except ModuleNotFoundError as e:
+            if e.name is None:
+                raise
+            if e.name in attempted_modules:
+                logger.debug(f"Already tried {e.name} in {path}, giving up")
+                raise
+            attempted_modules.append(e.name)
+            logger.debug(
+                f"Module {e.name} not found in {path}, "
+                "looking for alternative filenames trying valid extension suffixes"
+            )
+            try_import_binary_extension(path, e.name, suffixes)
+        else:
+            logger.debug(f"Reloaded {__name__}")
+            return
+
+
+def try_import_binary_extension(path: Path, module: str, suffixes: List[str]) -> bool:
+    """
+    Tries to find and import a binary extension in the specified path based on name
+    and valid extension suffixes.
+
+    Args:
+        path: Path to the module, e.g. /usr/lib64/python3.9/site-packages/rpm/
+        module: Name of the module to import, e.g. `rpm._rpm`
+        suffixes: List of extension suffixes to check for
+                  (see `importlib.machinery.EXTENSION_SUFFIXES`).
+
+    Returns:
+        True if the module was loaded, False otherwise.
+    """
+    # get the submodule module name, e.g. just '_rpm' from 'rpm._rpm'
+    submodule = module.rpartition(".")[-1]
+    for suffix in suffixes:
+        # a file named {submodule}{suffix} may exist in {path}, e.g.:
+        #
+        # /usr/lib64/python3.9/site-packages/rpm/_rpm.cpython-39-x86_64-linux-gnu.so
+        #
+        # if so we'll try loading it as rpm._rpm
+        so = path / f"{submodule}{suffix}"
+        if not so.is_file():
+            logger.debug(f"{so} isn't a file, ignoring")
             continue
+        if load_module_by_path(module, so):
+            return True
+    else:
+        logger.debug(
+            f"No combination of {submodule} and valid suffixes found in {path}, giving up"
+        )
+        return False
+
+
+def load_module_by_path(module_name: str, path: Path) -> bool:
+    """
+    Imports a Python module by path.
+
+    Args:
+        module_name: Name of the module to import.
+        path: Absolute path to the module to import.
+
+    Returns:
+        True if the import succeeded, otherwise False.
+    """
+    logger.debug(f"Trying to load {module_name} from {path}")
+    spec = importlib.util.spec_from_file_location(module_name, path)
+    if spec is None:
+        logger.debug(f"No spec for {module_name} in {path}")
+        return False
+    if spec.loader is None:
+        logger.debug(f"No loader in spec for {module_name}")
+        return False
+    module = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(module)
+    sys.modules[module_name] = module
+    logger.debug(f"Loaded {module_name} from {path}")
+    return True
+
+
+def initialize() -> None:
+    """
+    Initializes the shim. Tries to load system RPM module and replace itself with it.
+    """
+    for entry in get_system_sitepackages_and_suffixes():
+        for path in entry["sitepackages"]:
+            logger.debug(f"Trying {path}")
+            try:
+                if try_path(path, entry["suffixes"]):
+                    logger.debug("Import successfull")
+                    return
+            except ShimAlreadyInitializingError:
+                continue
+            except Exception as e:
+                logger.debug(f"Exception: {type(e)}: {e}")
+                continue
     else:
         raise ImportError(
             "Failed to import system RPM module. "