From 696be7b474aa944ce2f8e0b0b9349a3e3fccd620 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Wed, 7 Aug 2024 23:46:56 -0700 Subject: [PATCH 01/24] Raise an error when an externally managed environment is detected. rosdep is designed to treat pip like an alternative system-level package manager. Deviating from this approach is not easily achievable without a significant rethinking of how pip packages are managed. In the meantime, we can at least instruct users how to restore the prior functionality. Rather than inject the environment variable / config on behalf of the user, this change instructs them to make the necessary config changes themself, keeping them informed of the change their making to the system's new default. --- src/rosdep2/platforms/pip.py | 43 ++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 5c2db4152..7c77b5e0b 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -31,6 +31,9 @@ import subprocess import sys +from configparser import ConfigParser +from pathlib import Path + try: import importlib.metadata as importlib_metadata except ImportError: @@ -43,6 +46,15 @@ # pip package manager key PIP_INSTALLER = 'pip' +EXTERNALLY_MANAGED_EXPLAINER = """ +rosdep installation of pip packages requires installing packages packages globally as root +When using Python >= 3.11, PEP 668 compliance requires you to allow pip to install alongside +externally managed packages using the 'break-system-packages' option. +The recommeded way to set this option when using rosdep is to set the environment variable +PIP_BREAK_SYSTEM_PACKAGES=1 +in your environment. +""" + def register_installers(context): context.set_installer(PIP_INSTALLER, PipInstaller()) @@ -70,6 +82,35 @@ def get_pip_command(): return None +def externally_managed_installable(): + """ + PEP 668 enacted in Python 3.11 blocks pip from working in "externally + managed" envrionments such operating systems with included package + managers. If we're on Python 3.11 or greater, we need to check that pip + is configured to allow installing system-wide packages with the + flagrantly named "break system packages" config option or environment + variable. + """ + if sys.version_info[0] == 3 and sys.version_info[1] >= 11: + if "PIP_BREAK_SYSTEM_PACKAGES" in os.environ and os.environ[ + "PIP_BREAK_SYSTEM_PACKAGES" + ].lower() in ["yes", "1", "true"]: + return True + if 'XDG_CONFIG_DIRS' in os.environ: + global_config = ConfigParser() + for dir in os.environ['XDG_CONFIG_DIRS'].split(":"): + global_config_file = Path(dir) / "pip" / "pip.conf" + global_config.read(global_config_file) + if global_config['install']['break-system-packages']: + return True + fallback_config = Path('/etc/pip.conf') + global_config.read(fallback_config) + if global_config['install']['break-system-packages']: + return True + return False + return True + + def is_cmd_available(cmd): try: proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) @@ -145,6 +186,8 @@ def get_install_command(self, resolved, interactive=True, reinstall=False, quiet pip_cmd = get_pip_command() if not pip_cmd: raise InstallFailed((PIP_INSTALLER, 'pip is not installed')) + if not externally_managed_installable(): + raise InstallFailed((PIP_INSTALLER, EXTERNALLY_MANAGED_EXPLAINER)) packages = self.get_packages_to_install(resolved, reinstall=reinstall) if not packages: return [] From c121579c3b8396a4fa30c44bcc03f7fbfed21538 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Thu, 8 Aug 2024 11:28:07 -0700 Subject: [PATCH 02/24] Fix typos and punctuation. Co-authored-by: Christophe Bedard --- src/rosdep2/platforms/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 7c77b5e0b..fc3a4d9e8 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -47,7 +47,7 @@ PIP_INSTALLER = 'pip' EXTERNALLY_MANAGED_EXPLAINER = """ -rosdep installation of pip packages requires installing packages packages globally as root +rosdep installation of pip packages requires installing packages globally as root. When using Python >= 3.11, PEP 668 compliance requires you to allow pip to install alongside externally managed packages using the 'break-system-packages' option. The recommeded way to set this option when using rosdep is to set the environment variable From e9ad440ef94d0f0c88cfe7d36ad5aa36828f9a03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Thu, 8 Aug 2024 11:28:27 -0700 Subject: [PATCH 03/24] Use tuple rather than list literal. Co-authored-by: Christophe Bedard --- src/rosdep2/platforms/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index fc3a4d9e8..c42e1e351 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -94,7 +94,7 @@ def externally_managed_installable(): if sys.version_info[0] == 3 and sys.version_info[1] >= 11: if "PIP_BREAK_SYSTEM_PACKAGES" in os.environ and os.environ[ "PIP_BREAK_SYSTEM_PACKAGES" - ].lower() in ["yes", "1", "true"]: + ].lower() in ("yes", "1", "true"): return True if 'XDG_CONFIG_DIRS' in os.environ: global_config = ConfigParser() From 8ce4a754194586b1980e1a181cb39031f9f192c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 08:01:41 -0700 Subject: [PATCH 04/24] Consolidate version check. --- src/rosdep2/platforms/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index c42e1e351..2055ff68c 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -91,7 +91,7 @@ def externally_managed_installable(): flagrantly named "break system packages" config option or environment variable. """ - if sys.version_info[0] == 3 and sys.version_info[1] >= 11: + if sys.version_info >= (3, 11): if "PIP_BREAK_SYSTEM_PACKAGES" in os.environ and os.environ[ "PIP_BREAK_SYSTEM_PACKAGES" ].lower() in ("yes", "1", "true"): From 2c76686afa24ff09f52642a4b72116c2051ae5e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 08:02:50 -0700 Subject: [PATCH 05/24] Pass necessary environment variable via sudo. --- src/rosdep2/platforms/pip.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 2055ff68c..933fd3180 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -172,6 +172,10 @@ class PipInstaller(PackageManagerInstaller): def __init__(self): super(PipInstaller, self).__init__(pip_detect, supports_depends=True) + # Pass necessary environment for pip functionality via sudo + if self.as_root and self.sudo_command != '': + self.sudo_command += ' --preserve-env=PIP_BREAK_SYSTEM_PACKAGES' + def get_version_strings(self): pip_version = importlib_metadata.version('pip') # keeping the name "setuptools" for backward compatibility From 29ec17688013eb95a262508ec5cdafdb1f64a417 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 08:22:46 -0700 Subject: [PATCH 06/24] Update test to expect sudo --preserve-env for pip. --- test/test_rosdep_pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_rosdep_pip.py b/test/test_rosdep_pip.py index 4fd359d0d..8886ef9c0 100644 --- a/test/test_rosdep_pip.py +++ b/test/test_rosdep_pip.py @@ -104,7 +104,7 @@ def test(expected_prefix, mock_method, mock_get_pip_command): try: if hasattr(os, 'geteuid'): with patch('rosdep2.installers.os.geteuid', return_value=1): - test(['sudo', '-H']) + test(['sudo', '-H', '--preserve-env=PIP_BREAK_SYSTEM_PACKAGES']) with patch('rosdep2.installers.os.geteuid', return_value=0): test([]) else: From 48b9da28b2cd0a4d24d5618725591319d6898801 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 08:25:18 -0700 Subject: [PATCH 07/24] flake8 cleanup --- src/rosdep2/platforms/pip.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 933fd3180..8e6c42673 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -92,14 +92,14 @@ def externally_managed_installable(): variable. """ if sys.version_info >= (3, 11): - if "PIP_BREAK_SYSTEM_PACKAGES" in os.environ and os.environ[ - "PIP_BREAK_SYSTEM_PACKAGES" - ].lower() in ("yes", "1", "true"): + if 'PIP_BREAK_SYSTEM_PACKAGES' in os.environ and os.environ[ + 'PIP_BREAK_SYSTEM_PACKAGES' + ].lower() in ('yes', '1', 'true'): return True if 'XDG_CONFIG_DIRS' in os.environ: global_config = ConfigParser() - for dir in os.environ['XDG_CONFIG_DIRS'].split(":"): - global_config_file = Path(dir) / "pip" / "pip.conf" + for xdg_dir in os.environ['XDG_CONFIG_DIRS'].split(':'): + global_config_file = Path(xdg_dir) / 'pip' / 'pip.conf' global_config.read(global_config_file) if global_config['install']['break-system-packages']: return True From a5e10e71497db1f604e30bc1ceb83be311aa48aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 09:04:05 -0700 Subject: [PATCH 08/24] Run pip tests with PIP_BREAK_SYSTEM_PACKAGES=1. --- test/test_rosdep_pip.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/test_rosdep_pip.py b/test/test_rosdep_pip.py index 8886ef9c0..d203865ab 100644 --- a/test/test_rosdep_pip.py +++ b/test/test_rosdep_pip.py @@ -67,6 +67,7 @@ def test_PipInstaller_get_depends(): assert ['foo'] == installer.get_depends(dict(depends=['foo'])) +@patch.dict(os.environ, {'PIP_BREAK_SYSTEM_PACKAGES': '1'}) def test_PipInstaller(): from rosdep2 import InstallFailed from rosdep2.platforms.pip import PipInstaller From f042758dcb8aeb1ab77d93875eac25cb85df8c33 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 13:02:45 -0700 Subject: [PATCH 09/24] Add documentation for pip configuration. --- doc/contents.rst | 1 + doc/pip_and_pep_668.rst | 67 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 doc/pip_and_pep_668.rst diff --git a/doc/contents.rst b/doc/contents.rst index f017c60ec..f5bed8c0d 100644 --- a/doc/contents.rst +++ b/doc/contents.rst @@ -11,3 +11,4 @@ Contents sources_list developers_guide rosdep2_api + pip_and_pep_668 diff --git a/doc/pip_and_pep_668.rst b/doc/pip_and_pep_668.rst new file mode 100644 index 000000000..4a6bc85a9 --- /dev/null +++ b/doc/pip_and_pep_668.rst @@ -0,0 +1,67 @@ +Pip installation after PEP 668 +============================== + +`PEP-668`_ introduced `externally managed environments `_ to Python packaging. + +Rosdep is designed to use pip as an alternative system package manager, rosdep installation of pip packages requires installing packages globally as root. +Starting with Python 3.11, `PEP-668`_ compliance requires you to allow pip to install alongside externally managed packages using the ``break-system-packages`` option. + +There are multiple ways to configure pip so that rosdep will succeed. + + +Configure using environment variable +------------------------------------ + +This is the way that we recommend configuring pip for rosdep usage. +We recommend configuring pip using the system environment. +Setting environment variables in your login profile, ``PIP_BREAK_SYSTEM_PACKAGES`` in your environment. +The value of the environment variable can be any of ``1``, ``yes``, or ``true``. +The string values are not case sensitive. + + +Configure using pip.conf +------------------------ + +`Pip configuration files `_ can be used to set the desired behavior +Pip checks for global configuration files in ``XDG_CONFIG_DIRS``, as well as ``/etc/pip.conf`` +For details on ``XDG_CONFIG_DIRS`` refer to the `XDG base directories specification `_. + +Creating a pip.conf in your user account's ``XDG_CONFIG_HOME`` (e.g. ``~/.config/pip/pip.conf``) does not appear to be sufficent when installing packages globally. + +.. code-block:: ini + + [install] + break-system-packages = true + + +Configuring for CI setup +------------------------ + +Either environment variables or configuration files can be used with your CI system. +Which one you choose will depend on how your CI environment is configured. +Perhaps the most straightforward will be to set the environent variable in the shell or script execution context before invoking rosdep. + +.. code-block:: bash + + sudo rosdep init + rosdep update + PIP_BREAK_SYSTEM_PACKAGES=1 rosdep install -r rolling --from-paths src/ + +If rosdep is invoked by internal processes in your CI and you need to set the configuration without having direct control over how ``rosdep install`` is run, setting the environment variable globally would also work. + +.. code-block:: bash + + export PIP_BREAK_SYSTEM_PACKAGES=1 + ./path/to/ci-script.sh + + +If you cannot set environment variables but you can create configuration files, you can set ``/etc/pip.conf`` with the necessary configuration. + +.. code-block:: bash + + printf "[install]\nbreak-system-packages = true\n" | sudo tee -a /etc/pip.conf + +.. _PEP-668: https://peps.python.org/pep-0668/ +.. _pip-configuration: https://pip.pypa.io/en/stable/topics/configuration/ +.. _externally-managed-environments: https://packaging.python.org/en/latest/specifications/externally-managed-environments/ +.. _xdg-base-dirs: https://specifications.freedesktop.org/basedir-spec/latest/ From a2adfe8660cffa5087cdcbcfa3933de153fab36f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 13:23:17 -0700 Subject: [PATCH 10/24] Add doc link to error output. --- src/rosdep2/platforms/pip.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 8e6c42673..0a683e5f9 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -53,6 +53,8 @@ The recommeded way to set this option when using rosdep is to set the environment variable PIP_BREAK_SYSTEM_PACKAGES=1 in your environment. + +For more information refer to http://docs.ros.org/en/independent/api/rosdep/html/pip_and_pep_668.html """ From 5b2cc64eb713ae0a7f6445946fa7dcfe6b0fd5ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 13:42:07 -0700 Subject: [PATCH 11/24] Use inline monospace font to refer to rosdep the cli tool. --- doc/pip_and_pep_668.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pip_and_pep_668.rst b/doc/pip_and_pep_668.rst index 4a6bc85a9..fd2849373 100644 --- a/doc/pip_and_pep_668.rst +++ b/doc/pip_and_pep_668.rst @@ -3,7 +3,7 @@ Pip installation after PEP 668 `PEP-668`_ introduced `externally managed environments `_ to Python packaging. -Rosdep is designed to use pip as an alternative system package manager, rosdep installation of pip packages requires installing packages globally as root. +``rosdep`` is designed to use pip as an alternative system package manager, rosdep installation of pip packages requires installing packages globally as root. Starting with Python 3.11, `PEP-668`_ compliance requires you to allow pip to install alongside externally managed packages using the ``break-system-packages`` option. There are multiple ways to configure pip so that rosdep will succeed. From 73ed28a9fe5b31ff82e6a91c853632fe41b2cb40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 13:43:02 -0700 Subject: [PATCH 12/24] Briefly note that sudo configuration could prevent this from working. --- doc/pip_and_pep_668.rst | 3 +++ 1 file changed, 3 insertions(+) diff --git a/doc/pip_and_pep_668.rst b/doc/pip_and_pep_668.rst index fd2849373..6d7ea5a29 100644 --- a/doc/pip_and_pep_668.rst +++ b/doc/pip_and_pep_668.rst @@ -18,6 +18,9 @@ Setting environment variables in your login profile, ``PIP_BREAK_SYSTEM_PACKAGES The value of the environment variable can be any of ``1``, ``yes``, or ``true``. The string values are not case sensitive. +``rosdep`` is designed to use ``sudo`` in order to gain root privileges for installation when not run as root. +If your system's sudo configuration prohibits the passing of environment variables + Configure using pip.conf ------------------------ From f1dffc6c8969f2c1eca9b03360146bfa8bc79cfa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 13:43:39 -0700 Subject: [PATCH 13/24] Recommend a specific config file to use and format user config as a warning. --- doc/pip_and_pep_668.rst | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/doc/pip_and_pep_668.rst b/doc/pip_and_pep_668.rst index 6d7ea5a29..fd7dfed32 100644 --- a/doc/pip_and_pep_668.rst +++ b/doc/pip_and_pep_668.rst @@ -28,8 +28,7 @@ Configure using pip.conf `Pip configuration files `_ can be used to set the desired behavior Pip checks for global configuration files in ``XDG_CONFIG_DIRS``, as well as ``/etc/pip.conf`` For details on ``XDG_CONFIG_DIRS`` refer to the `XDG base directories specification `_. - -Creating a pip.conf in your user account's ``XDG_CONFIG_HOME`` (e.g. ``~/.config/pip/pip.conf``) does not appear to be sufficent when installing packages globally. +If you're unsure which configuration file is in use by your system, ``/etc/pip.conf`` seems like the most generic. .. code-block:: ini @@ -37,6 +36,9 @@ Creating a pip.conf in your user account's ``XDG_CONFIG_HOME`` (e.g. ``~/.config break-system-packages = true +.. warning:: Creating a pip.conf in your user account's ``XDG_CONFIG_HOME`` (e.g. ``~/.config/pip/pip.conf``) does not appear to be sufficent when installing packages globally. + + Configuring for CI setup ------------------------ From f6857691e9a23e3a3f39e55189195dc3e4bf0770 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 13:49:47 -0700 Subject: [PATCH 14/24] Fix errors in config checker. The fallback configuration was over-indented and would never be checked. --- src/rosdep2/platforms/pip.py | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 0a683e5f9..1e42f3dae 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -98,19 +98,25 @@ def externally_managed_installable(): 'PIP_BREAK_SYSTEM_PACKAGES' ].lower() in ('yes', '1', 'true'): return True + # Check the same configuration directories as pip does per + # https://pip.pypa.io/en/stable/topics/configuration/ + pip_config = ConfigParser() if 'XDG_CONFIG_DIRS' in os.environ: - global_config = ConfigParser() for xdg_dir in os.environ['XDG_CONFIG_DIRS'].split(':'): - global_config_file = Path(xdg_dir) / 'pip' / 'pip.conf' - global_config.read(global_config_file) - if global_config['install']['break-system-packages']: + pip_config_file = Path(xdg_dir) / 'pip' / 'pip.conf' + pip_config.read(pip_config_file) + if pip_config['install']['break-system-packages']: return True - fallback_config = Path('/etc/pip.conf') - global_config.read(fallback_config) - if global_config['install']['break-system-packages']: - return True + + fallback_config = Path('/etc/pip.conf') + pip_config.read(fallback_config) + if pip_config['install']['break-system-packages']: + return True + # On Python 3.11 and later, when no explicit configuration is present, + # global pip installation will not work. return False - return True + else: + return True def is_cmd_available(cmd): From 65c71529615dc34b5dbfede9db40d91b98488198 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 16:08:59 -0700 Subject: [PATCH 15/24] Change formatting of rosdep. Use monospace formatting when referring to the `rosdep` command / executable name and simply 'rosdep' when referring to the project. The preferred capitalization of rosdep is rosdep not Rosdep or ROSdep (and certainly not ROSDep). --- doc/pip_and_pep_668.rst | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc/pip_and_pep_668.rst b/doc/pip_and_pep_668.rst index fd7dfed32..e43d11340 100644 --- a/doc/pip_and_pep_668.rst +++ b/doc/pip_and_pep_668.rst @@ -3,7 +3,7 @@ Pip installation after PEP 668 `PEP-668`_ introduced `externally managed environments `_ to Python packaging. -``rosdep`` is designed to use pip as an alternative system package manager, rosdep installation of pip packages requires installing packages globally as root. +rosdep is designed to use pip as an alternative system package manager, rosdep installation of pip packages requires installing packages globally as root. Starting with Python 3.11, `PEP-668`_ compliance requires you to allow pip to install alongside externally managed packages using the ``break-system-packages`` option. There are multiple ways to configure pip so that rosdep will succeed. @@ -18,7 +18,7 @@ Setting environment variables in your login profile, ``PIP_BREAK_SYSTEM_PACKAGES The value of the environment variable can be any of ``1``, ``yes``, or ``true``. The string values are not case sensitive. -``rosdep`` is designed to use ``sudo`` in order to gain root privileges for installation when not run as root. +rosdep is designed to use ``sudo`` in order to gain root privileges for installation when not run as root. If your system's sudo configuration prohibits the passing of environment variables @@ -44,7 +44,7 @@ Configuring for CI setup Either environment variables or configuration files can be used with your CI system. Which one you choose will depend on how your CI environment is configured. -Perhaps the most straightforward will be to set the environent variable in the shell or script execution context before invoking rosdep. +Perhaps the most straightforward will be to set the environent variable in the shell or script execution context before invoking ``rosdep``. .. code-block:: bash @@ -52,7 +52,7 @@ Perhaps the most straightforward will be to set the environent variable in the s rosdep update PIP_BREAK_SYSTEM_PACKAGES=1 rosdep install -r rolling --from-paths src/ -If rosdep is invoked by internal processes in your CI and you need to set the configuration without having direct control over how ``rosdep install`` is run, setting the environment variable globally would also work. +If ``rosdep`` is invoked by internal processes in your CI and you need to set the configuration without having direct control over how ``rosdep install`` is run, setting the environment variable globally would also work. .. code-block:: bash From f8472d2413b8c8ce040435c5608951946f00dcae Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 16:14:49 -0700 Subject: [PATCH 16/24] Complete a sentence I stopped writing. --- doc/pip_and_pep_668.rst | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/doc/pip_and_pep_668.rst b/doc/pip_and_pep_668.rst index e43d11340..88ca2e2c7 100644 --- a/doc/pip_and_pep_668.rst +++ b/doc/pip_and_pep_668.rst @@ -19,9 +19,11 @@ The value of the environment variable can be any of ``1``, ``yes``, or ``true``. The string values are not case sensitive. rosdep is designed to use ``sudo`` in order to gain root privileges for installation when not run as root. -If your system's sudo configuration prohibits the passing of environment variables +If your system's sudo configuration prohibits the passing of environment variables use the :ref:`pip.conf ` method below. +.. _configure-using-pip.conf: + Configure using pip.conf ------------------------ From 70775f4553bd3fb4a9c86bf1afe197fbe32bc32d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 16:19:45 -0700 Subject: [PATCH 17/24] Add period to end of sentence. Co-authored-by: Christophe Bedard --- doc/pip_and_pep_668.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pip_and_pep_668.rst b/doc/pip_and_pep_668.rst index 88ca2e2c7..0c66b6fca 100644 --- a/doc/pip_and_pep_668.rst +++ b/doc/pip_and_pep_668.rst @@ -27,7 +27,7 @@ If your system's sudo configuration prohibits the passing of environment variabl Configure using pip.conf ------------------------ -`Pip configuration files `_ can be used to set the desired behavior +`Pip configuration files `_ can be used to set the desired behavior. Pip checks for global configuration files in ``XDG_CONFIG_DIRS``, as well as ``/etc/pip.conf`` For details on ``XDG_CONFIG_DIRS`` refer to the `XDG base directories specification `_. If you're unsure which configuration file is in use by your system, ``/etc/pip.conf`` seems like the most generic. From 25ac0e0cafc6431f3b0b1b66d04e35fe3ad50865 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 16:20:09 -0700 Subject: [PATCH 18/24] Add period to end of sentence. Co-authored-by: Christophe Bedard --- doc/pip_and_pep_668.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/pip_and_pep_668.rst b/doc/pip_and_pep_668.rst index 0c66b6fca..38ca6db10 100644 --- a/doc/pip_and_pep_668.rst +++ b/doc/pip_and_pep_668.rst @@ -28,7 +28,7 @@ Configure using pip.conf ------------------------ `Pip configuration files `_ can be used to set the desired behavior. -Pip checks for global configuration files in ``XDG_CONFIG_DIRS``, as well as ``/etc/pip.conf`` +Pip checks for global configuration files in ``XDG_CONFIG_DIRS``, as well as ``/etc/pip.conf``. For details on ``XDG_CONFIG_DIRS`` refer to the `XDG base directories specification `_. If you're unsure which configuration file is in use by your system, ``/etc/pip.conf`` seems like the most generic. From 8c57dbab0701aeadf1cf99a6824fbc708edf00f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 16:21:40 -0700 Subject: [PATCH 19/24] Invert conditional for an earlier return. Co-authored-by: Christophe Bedard --- src/rosdep2/platforms/pip.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 1e42f3dae..d5da29380 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -93,7 +93,8 @@ def externally_managed_installable(): flagrantly named "break system packages" config option or environment variable. """ - if sys.version_info >= (3, 11): + if sys.version_info < (3, 11): + return True if 'PIP_BREAK_SYSTEM_PACKAGES' in os.environ and os.environ[ 'PIP_BREAK_SYSTEM_PACKAGES' ].lower() in ('yes', '1', 'true'): From 56a745aef731872ee43517444fcadec9d9eac335 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 16:22:01 -0700 Subject: [PATCH 20/24] Edit text for clarity and typos. Co-authored-by: Christophe Bedard --- src/rosdep2/platforms/pip.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index d5da29380..6eff32a4d 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -87,7 +87,7 @@ def get_pip_command(): def externally_managed_installable(): """ PEP 668 enacted in Python 3.11 blocks pip from working in "externally - managed" envrionments such operating systems with included package + managed" environments such as operating systems with included package managers. If we're on Python 3.11 or greater, we need to check that pip is configured to allow installing system-wide packages with the flagrantly named "break system packages" config option or environment From a2e9a1ebf4eaeaf92eb8196265101a1d4a76369d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 17:07:50 -0700 Subject: [PATCH 21/24] Reflow conditional for easier reading. --- src/rosdep2/platforms/pip.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 6eff32a4d..ef3a1edfd 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -95,9 +95,10 @@ def externally_managed_installable(): """ if sys.version_info < (3, 11): return True - if 'PIP_BREAK_SYSTEM_PACKAGES' in os.environ and os.environ[ - 'PIP_BREAK_SYSTEM_PACKAGES' - ].lower() in ('yes', '1', 'true'): + if ( + 'PIP_BREAK_SYSTEM_PACKAGES' in os.environ and + os.environ['PIP_BREAK_SYSTEM_PACKAGES'].lower() in ('yes', '1', 'true') + ): return True # Check the same configuration directories as pip does per # https://pip.pypa.io/en/stable/topics/configuration/ From f14c6dee76d6a7f81d4f432420721eae0aa9692f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 17:09:23 -0700 Subject: [PATCH 22/24] Fix control flow after inverting the conditional. This is a fixup after two earlier changes inverted the conditional and reformatted an internal check. --- src/rosdep2/platforms/pip.py | 48 +++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 23 deletions(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index ef3a1edfd..48fe4f231 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -93,32 +93,34 @@ def externally_managed_installable(): flagrantly named "break system packages" config option or environment variable. """ + + # This doesn't affect Python versions before 3.11 if sys.version_info < (3, 11): return True - if ( - 'PIP_BREAK_SYSTEM_PACKAGES' in os.environ and - os.environ['PIP_BREAK_SYSTEM_PACKAGES'].lower() in ('yes', '1', 'true') - ): - return True - # Check the same configuration directories as pip does per - # https://pip.pypa.io/en/stable/topics/configuration/ - pip_config = ConfigParser() - if 'XDG_CONFIG_DIRS' in os.environ: - for xdg_dir in os.environ['XDG_CONFIG_DIRS'].split(':'): - pip_config_file = Path(xdg_dir) / 'pip' / 'pip.conf' - pip_config.read(pip_config_file) - if pip_config['install']['break-system-packages']: - return True - - fallback_config = Path('/etc/pip.conf') - pip_config.read(fallback_config) - if pip_config['install']['break-system-packages']: - return True - # On Python 3.11 and later, when no explicit configuration is present, - # global pip installation will not work. - return False - else: + + if ( + 'PIP_BREAK_SYSTEM_PACKAGES' in os.environ and + os.environ['PIP_BREAK_SYSTEM_PACKAGES'].lower() in ('yes', '1', 'true') + ): + return True + + # Check the same configuration directories as pip does per + # https://pip.pypa.io/en/stable/topics/configuration/ + pip_config = ConfigParser() + if 'XDG_CONFIG_DIRS' in os.environ: + for xdg_dir in os.environ['XDG_CONFIG_DIRS'].split(':'): + pip_config_file = Path(xdg_dir) / 'pip' / 'pip.conf' + pip_config.read(pip_config_file) + if pip_config['install']['break-system-packages']: + return True + + fallback_config = Path('/etc/pip.conf') + pip_config.read(fallback_config) + if pip_config['install']['break-system-packages']: return True + # On Python 3.11 and later, when no explicit configuration is present, + # global pip installation will not work. + return False def is_cmd_available(cmd): From 8b48f1356a499ec34eb6e59c24608e0ba7f7bf4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Fri, 9 Aug 2024 17:45:22 -0700 Subject: [PATCH 23/24] Add test to confirm that get_install_command handles externally managed environments. --- test/test_rosdep_pip.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/test/test_rosdep_pip.py b/test/test_rosdep_pip.py index d203865ab..ab9a879dd 100644 --- a/test/test_rosdep_pip.py +++ b/test/test_rosdep_pip.py @@ -67,6 +67,22 @@ def test_PipInstaller_get_depends(): assert ['foo'] == installer.get_depends(dict(depends=['foo'])) +@patch('rosdep2.platforms.pip.externally_managed_installable') +def test_PipInstaller_handles_externally_managed_environment(externally_managed_installable): + from rosdep2 import InstallFailed + from rosdep2.platforms.pip import EXTERNALLY_MANAGED_EXPLAINER, PipInstaller + + externally_managed_installable.return_value = False + installer = PipInstaller() + try: + installer.get_install_command(['whatever']) + assert False, 'should have raised' + except InstallFailed as e: + assert e.failures == [('pip', EXTERNALLY_MANAGED_EXPLAINER)] + externally_managed_installable.return_value = True + assert installer.get_install_command(['whatever'], interactive=False) + + @patch.dict(os.environ, {'PIP_BREAK_SYSTEM_PACKAGES': '1'}) def test_PipInstaller(): from rosdep2 import InstallFailed From 8e94ef1ffb01394b23cea9dcc12fbb6fc92f10d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Steven!=20Ragnar=C3=B6k?= Date: Mon, 12 Aug 2024 16:49:55 -0700 Subject: [PATCH 24/24] Use ConfigParser.getboolean to check config value. Using the dict access method will raise a KeyError when the config file is present but this section or value is missing. --- src/rosdep2/platforms/pip.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/rosdep2/platforms/pip.py b/src/rosdep2/platforms/pip.py index 48fe4f231..7f183b892 100644 --- a/src/rosdep2/platforms/pip.py +++ b/src/rosdep2/platforms/pip.py @@ -111,12 +111,12 @@ def externally_managed_installable(): for xdg_dir in os.environ['XDG_CONFIG_DIRS'].split(':'): pip_config_file = Path(xdg_dir) / 'pip' / 'pip.conf' pip_config.read(pip_config_file) - if pip_config['install']['break-system-packages']: + if pip_config.getboolean('install', 'break-system-packages', fallback=False): return True fallback_config = Path('/etc/pip.conf') pip_config.read(fallback_config) - if pip_config['install']['break-system-packages']: + if pip_config.getboolean('install', 'break-system-packages', fallback=False): return True # On Python 3.11 and later, when no explicit configuration is present, # global pip installation will not work.