From 55f5b3919a3c03307a7b381b02374af4b53835c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Thu, 15 Aug 2024 11:24:34 -0700 Subject: [PATCH] Check for pip configuration when an externally managed environment is detected. (#979) * Raise an error when an externally managed environment is detected. rosdep is designed to treat pip like an alternative system-level package manager. Deviating from this approach is not easily achievable without a significant rethinking of how pip packages are managed. In the meantime, we can at least instruct users how to restore the prior functionality. Rather than inject the environment variable / config on behalf of the user, this change instructs them to make the necessary config changes themself, keeping them informed of the change their making to the system's new default. * Fix typos and punctuation. Co-authored-by: Christophe Bedard * Use tuple rather than list literal. Co-authored-by: Christophe Bedard * Consolidate version check. * Pass necessary environment variable via sudo. * Update test to expect sudo --preserve-env for pip. * flake8 cleanup * Run pip tests with PIP_BREAK_SYSTEM_PACKAGES=1. * Add documentation for pip configuration. * Add doc link to error output. * Use inline monospace font to refer to rosdep the cli tool. * Briefly note that sudo configuration could prevent this from working. * Recommend a specific config file to use and format user config as a warning. * Fix errors in config checker. The fallback configuration was over-indented and would never be checked. * Change formatting of rosdep. Use monospace formatting when referring to the `rosdep` command / executable name and simply 'rosdep' when referring to the project. The preferred capitalization of rosdep is rosdep not Rosdep or ROSdep (and certainly not ROSDep). * Complete a sentence I stopped writing. * Add period to end of sentence. Co-authored-by: Christophe Bedard * Add period to end of sentence. Co-authored-by: Christophe Bedard * Invert conditional for an earlier return. Co-authored-by: Christophe Bedard * Edit text for clarity and typos. Co-authored-by: Christophe Bedard * Reflow conditional for easier reading. * Fix control flow after inverting the conditional. This is a fixup after two earlier changes inverted the conditional and reformatted an internal check. * Add test to confirm that get_install_command handles externally managed environments. * Use ConfigParser.getboolean to check config value. Using the dict access method will raise a KeyError when the config file is present but this section or value is missing. --------- Co-authored-by: Christophe Bedard --- doc/contents.rst | 1 + doc/pip_and_pep_668.rst | 74 ++++++++++++++++++++++++++++++++++++ src/rosdep2/platforms/pip.py | 59 ++++++++++++++++++++++++++++ test/test_rosdep_pip.py | 19 ++++++++- 4 files changed, 152 insertions(+), 1 deletion(-) create mode 100644 doc/pip_and_pep_668.rst diff --git a/doc/contents.rst b/doc/contents.rst index f017c60ec..f5bed8c0d 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -11,3 +11,4 @@ Contents sources_list developers_guide rosdep2_api + pip_and_pep_668 diff --git a/doc/pip_and_pep_668.rst b/doc/pip_and_pep_668.rst new file mode 100644 index 000000000..38ca6db10 --- /dev/null +++ b/doc/pip_and_pep_668.rst @@ -0,0 +1,74 @@ +Pip installation after PEP 668 +============================== + +`PEP-668`_ introduced `externally managed environments `_ to Python packaging. + +rosdep is designed to use pip as an alternative system package manager, rosdep installation of pip packages requires installing packages globally as root. +Starting with Python 3.11, `PEP-668`_ compliance requires you to allow pip to install alongside externally managed packages using the ``break-system-packages`` option. + +There are multiple ways to configure pip so that rosdep will succeed. + + +Configure using environment variable +------------------------------------ + +This is the way that we recommend configuring pip for rosdep usage. +We recommend configuring pip using the system environment. +Setting environment variables in your login profile, ``PIP_BREAK_SYSTEM_PACKAGES`` in your environment. +The value of the environment variable can be any of ``1``, ``yes``, or ``true``. +The string values are not case sensitive. + +rosdep is designed to use ``sudo`` in order to gain root privileges for installation when not run as root. +If your system's sudo configuration prohibits the passing of environment variables use the :ref:`pip.conf ` method below. + + +.. _configure-using-pip.conf: + +Configure using pip.conf +------------------------ + +`Pip configuration files `_ can be used to set the desired behavior. +Pip checks for global configuration files in ``XDG_CONFIG_DIRS``, as well as ``/etc/pip.conf``. +For details on ``XDG_CONFIG_DIRS`` refer to the `XDG base directories specification `_. +If you're unsure which configuration file is in use by your system, ``/etc/pip.conf`` seems like the most generic. + +.. code-block:: ini + + [install] + break-system-packages = true + + +.. warning:: Creating a pip.conf in your user account's ``XDG_CONFIG_HOME`` (e.g. ``~/.config/pip/pip.conf``) does not appear to be sufficent when installing packages globally. + + +Configuring for CI setup +------------------------ + +Either environment variables or configuration files can be used with your CI system. +Which one you choose will depend on how your CI environment is configured. +Perhaps the most straightforward will be to set the environent variable in the shell or script execution context before invoking ``rosdep``. + +.. code-block:: bash + + sudo rosdep init + rosdep update + PIP_BREAK_SYSTEM_PACKAGES=1 rosdep install -r rolling --from-paths src/ + +If ``rosdep`` is invoked by internal processes in your CI and you need to set the configuration without having direct control over how ``rosdep install`` is run, setting the environment variable globally would also work. + +.. code-block:: bash + + export PIP_BREAK_SYSTEM_PACKAGES=1 + ./path/to/ci-script.sh + + +If you cannot set environment variables but you can create configuration files, you can set ``/etc/pip.conf`` with the necessary configuration. + +.. code-block:: bash + + printf "[install]\nbreak-system-packages = true\n" | sudo tee -a /etc/pip.conf + +.. _PEP-668: https://peps.python.org/pep-0668/ +.. _pip-configuration: https://pip.pypa.io/en/stable/topics/configuration/ +.. _externally-managed-environments: https://packaging.python.org/en/latest/specifications/externally-managed-environments/ +.. _xdg-base-dirs: https://specifications.freedesktop.org/basedir-spec/latest/ diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 5c2db4152..7f183b892 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -31,6 +31,9 @@ import subprocess import sys +from configparser import ConfigParser +from pathlib import Path + try: import importlib.metadata as importlib_metadata except ImportError: @@ -43,6 +46,17 @@ # pip package manager key PIP_INSTALLER = 'pip' +EXTERNALLY_MANAGED_EXPLAINER = """ +rosdep installation of pip packages requires installing packages globally as root. +When using Python >= 3.11, PEP 668 compliance requires you to allow pip to install alongside +externally managed packages using the 'break-system-packages' option. +The recommeded way to set this option when using rosdep is to set the environment variable +PIP_BREAK_SYSTEM_PACKAGES=1 +in your environment. + +For more information refer to http://docs.ros.org/en/independent/api/rosdep/html/pip_and_pep_668.html +""" + def register_installers(context): context.set_installer(PIP_INSTALLER, PipInstaller()) @@ -70,6 +84,45 @@ def get_pip_command(): return None +def externally_managed_installable(): + """ + PEP 668 enacted in Python 3.11 blocks pip from working in "externally + managed" environments such as operating systems with included package + managers. If we're on Python 3.11 or greater, we need to check that pip + is configured to allow installing system-wide packages with the + flagrantly named "break system packages" config option or environment + variable. + """ + + # This doesn't affect Python versions before 3.11 + if sys.version_info < (3, 11): + return True + + if ( + 'PIP_BREAK_SYSTEM_PACKAGES' in os.environ and + os.environ['PIP_BREAK_SYSTEM_PACKAGES'].lower() in ('yes', '1', 'true') + ): + return True + + # Check the same configuration directories as pip does per + # https://pip.pypa.io/en/stable/topics/configuration/ + pip_config = ConfigParser() + if 'XDG_CONFIG_DIRS' in os.environ: + for xdg_dir in os.environ['XDG_CONFIG_DIRS'].split(':'): + pip_config_file = Path(xdg_dir) / 'pip' / 'pip.conf' + pip_config.read(pip_config_file) + if pip_config.getboolean('install', 'break-system-packages', fallback=False): + return True + + fallback_config = Path('/etc/pip.conf') + pip_config.read(fallback_config) + if pip_config.getboolean('install', 'break-system-packages', fallback=False): + return True + # On Python 3.11 and later, when no explicit configuration is present, + # global pip installation will not work. + return False + + def is_cmd_available(cmd): try: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -131,6 +184,10 @@ class PipInstaller(PackageManagerInstaller): def __init__(self): super(PipInstaller, self).__init__(pip_detect, supports_depends=True) + # Pass necessary environment for pip functionality via sudo + if self.as_root and self.sudo_command != '': + self.sudo_command += ' --preserve-env=PIP_BREAK_SYSTEM_PACKAGES' + def get_version_strings(self): pip_version = importlib_metadata.version('pip') # keeping the name "setuptools" for backward compatibility @@ -145,6 +202,8 @@ def get_install_command(self, resolved, interactive=True, reinstall=False, quiet pip_cmd = get_pip_command() if not pip_cmd: raise InstallFailed((PIP_INSTALLER, 'pip is not installed')) + if not externally_managed_installable(): + raise InstallFailed((PIP_INSTALLER, EXTERNALLY_MANAGED_EXPLAINER)) packages = self.get_packages_to_install(resolved, reinstall=reinstall) if not packages: return [] diff --git a/test/test_rosdep_pip.py b/test/test_rosdep_pip.py index 4fd359d0d..ab9a879dd 100644 --- a/test/test_rosdep_pip.py +++ b/test/test_rosdep_pip.py @@ -67,6 +67,23 @@ def test_PipInstaller_get_depends(): assert ['foo'] == installer.get_depends(dict(depends=['foo'])) +@patch('rosdep2.platforms.pip.externally_managed_installable') +def test_PipInstaller_handles_externally_managed_environment(externally_managed_installable): + from rosdep2 import InstallFailed + from rosdep2.platforms.pip import EXTERNALLY_MANAGED_EXPLAINER, PipInstaller + + externally_managed_installable.return_value = False + installer = PipInstaller() + try: + installer.get_install_command(['whatever']) + assert False, 'should have raised' + except InstallFailed as e: + assert e.failures == [('pip', EXTERNALLY_MANAGED_EXPLAINER)] + externally_managed_installable.return_value = True + assert installer.get_install_command(['whatever'], interactive=False) + + +@patch.dict(os.environ, {'PIP_BREAK_SYSTEM_PACKAGES': '1'}) def test_PipInstaller(): from rosdep2 import InstallFailed from rosdep2.platforms.pip import PipInstaller @@ -104,7 +121,7 @@ def test(expected_prefix, mock_method, mock_get_pip_command): try: if hasattr(os, 'geteuid'): with patch('rosdep2.installers.os.geteuid', return_value=1): - test(['sudo', '-H']) + test(['sudo', '-H', '--preserve-env=PIP_BREAK_SYSTEM_PACKAGES']) with patch('rosdep2.installers.os.geteuid', return_value=0): test([]) else: