WIP

tue-robotics · Feb 3, 2024 · b824ee8 · b824ee8
1 parent 9ff9f5f
commit b824ee8
Show file tree

Hide file tree

Showing 4 changed files with 165 additions and 40 deletions.
diff --git a/installer/bootstrap.bash b/installer/bootstrap.bash
@@ -58,6 +58,11 @@ function main
     TUE_ROS_DISTRO=
     TUE_ROS_VERSION=
 
+    # Python configuration
+    create_virtualenv="true"
+    virtualenv_include_system_site_packages="false"
+    TUE_PPM="pip"  # Default to pip
+
     for i in "$@"
     do
         case $i in
@@ -73,13 +78,29 @@ function main
             --create-virtualenv=* )
                 create_virtualenv="${i#*=}"
                 ;;
+            --virtualenv-include-system-site-packages=* )
+                virtualenv_include_system_site_packages="${i#*=}"
+                ;;
+            --pip )
+                TUE_PPM="pip"
+                ;;
+            --poetry )
+                TUE_PPM="poetry"
+                ;;
             * )
                 echo "[tue-env](bootstrap) Error! Unknown argument '${i}' provided to bootstrap script."
-                return 1
+                exit 1
                 ;;
         esac
     done
 
+    # Poetry should only be used in combination with a virtualenv
+    if [[ "${TUE_PPM}" == "poetry"  && "${create_virtualenv}" == "false" ]]
+    then
+        echo "[tue-env](bootstrap) Error! Poetry is not installed."
+        return 1
+    fi
+
     case ${DISTRIB_RELEASE} in
         "20.04")
             if [[ "${ros_version}" -eq 2 ]]
@@ -200,6 +221,13 @@ function main
         git clone "${env_url}" "${env_dir}"
     fi
 
+# Install Poetry when needed
+    if [[ "${TUE_PPM}" == "poetry" ]]
+    then
+        echo "[tue-env](bootstrap) Installing Poetry"
+        curl -sSL https://install.python-poetry.org | python3 -
+    fi
+
     # Source the installer commands
     # No need to follow to a file which is already checked by CI
     # shellcheck disable=SC1090
@@ -209,11 +237,15 @@ function main
     mkdir -p "${workspace_dir}"
 
     # Initialize ros environment directory incl. targets
-    tue-env init "${workspace}" "${workspace_dir}" "--create-virtualenv=${create_virtualenv}" "--targets-url=${env_targets_url}"
+    tue-env init "${workspace}" "${workspace_dir}" \
+    "--create-virtualenv=${create_virtualenv}" \
+    "--virtualenv-include-system-site-packages=${virtualenv_include_system_site_packages}" \
+    "--targets-url=${env_targets_url}"
 
     # Configure environment
     tue-env config "${workspace}" set "TUE_ROS_DISTRO" "${TUE_ROS_DISTRO}"
     tue-env config "${workspace}" set "TUE_ROS_VERSION" "${TUE_ROS_VERSION}"
+    tue-env config "${workspace}" set "TUE_PPM" "${TUE_PPM}"
 
     # Add loading of TU/e tools (tue-env, tue-get, etc) to bashrc
     # shellcheck disable=SC2088

diff --git a/installer/check-pip-pkg-installed-version.py b/installer/check-pip-pkg-installed-version.py
@@ -1,23 +1,20 @@
 #! /usr/bin/env python3
 
-import sys
+import site
+from typing import List
 from pip._internal.req.constructors import install_req_from_line
 from pip._internal.utils.virtualenv import running_under_virtualenv
 
 
-def main() -> int:
-    if len(sys.argv) < 2:
-        print("Usage: check-pip-pkg-installed-version.py requirement [requirements]")
-        return 2
-
+def main(req_strs: List[str]) -> int:
     return_code = 0
     pkg_installed = []
 
     try:
-        for arg in sys.argv[1:]:
-            req = install_req_from_line(arg)
+        for req_str in req_strs:
+            req = install_req_from_line(req_str)
 
-            req.check_if_exists(not running_under_virtualenv())
+            req.check_if_exists(not running_under_virtualenv)
 
             if req.satisfied_by:
                 pkg_installed.append(str(req.satisfied_by).replace(" ", "^"))
@@ -34,4 +31,14 @@ def main() -> int:
 
 
 if __name__ == "__main__":
-    sys.exit(main())
+    import argparse
+    import sys
+
+    parser = argparse.ArgumentParser(
+        description="Check if a set of pip package is installed, meeting a requirement string."
+    )
+    parser.add_argument("req_strs", nargs="+")
+
+    args = parser.parse_args()
+
+    sys.exit(main(**vars(args)))
diff --git a/installer/tue-install-impl.bash b/installer/tue-install-impl.bash
@@ -168,7 +168,7 @@ function _remove_old_target_dep_recursively
     old_dep_dep_file="${TUE_INSTALL_DEPENDENCIES_DIR}"/"${old_dep_target}"
     if [[ -f "${old_dep_dep_file}" ]]
     then
-        # Iterate over all depencies of old_dep_target, which is removed.
+        # Iterate over all dependencies of old_dep_target, which is removed.
         while read -r dep_of_old_dep
         do
             # Actually remove the deps
@@ -183,11 +183,11 @@ function _remove_old_target_dep_recursively
         done < "${old_dep_dep_file}"
         rm -f "${old_dep_dep_file}"
     else
-        tue-install-debug "[remove_old_dep] No depencies file exist for target: ${old_dep_target}"
+        tue-install-debug "[remove_old_dep] No dependencies file exist for target: ${old_dep_target}"
     fi
 
     tue-install-debug "[remove_old_dep] Uninstalled '${old_dep_target}' as a dependency of '${parent_target}'"
-    tue-install-info "[remove_old_dep] '${old_dep_target}' has been uninstalled, you can remove it from the workspace or deinstall it in another way"
+    tue-install-info "[remove_old_dep] '${old_dep_target}' has been uninstalled, you can remove it from the workspace or uninstall it in another way"
     return ${error_code}
 }
 
@@ -309,7 +309,7 @@ function tue-install-target
                 [ "$now" == "true" ] && now_cmd="--now"
                 # Do not use 'local cmds=' because it does not preserve command output status ($?)
                 local cmds
-                if cmds=$("$TUE_INSTALL_SCRIPTS_DIR"/parse_install_yaml.py "$install_file".yaml $now_cmd)
+                if cmds=$(/usr/bin/python "${TUE_INSTALL_SCRIPTS_DIR}"/parse_install_yaml.py "${install_file}".yaml ${now_cmd})
                 then
                     for cmd in $cmds
                     do
@@ -352,7 +352,7 @@ function tue-install-target
         old_deps_removed=$(comm -23 <(echo "${old_deps}") <(echo "${new_deps}"))
         if [[ -n ${old_deps_removed} ]]
         then
-            tue-install-debug "Following dropped depedencies need to be removed:\n${old_deps_removed}"
+            tue-install-debug "Following dropped dependencies need to be removed:\n${old_deps_removed}"
         else
             tue-install-debug "No dropped dependencies to be removed"
         fi
@@ -361,7 +361,7 @@ function tue-install-target
         do
             # Remove this target from dep-on file of dep
             # When the dep-on file is now empty, remove it
-            # Recurisvely -> Remove it from the dep-on files of its deps
+            # Recursively -> Remove it from the dep-on files of its deps
             tue-install-debug "Going to remove '${dep}' as a dependency"
             _remove_old_target_dep_recursively "${target}" "${dep}" || tue-install-error "Something went wrong while removing '${dep}' as a dependency"
         done
@@ -1093,6 +1093,49 @@ function tue-install-pip3
 
 # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # #
 
+function _handle_python_sites
+{
+    local pv
+    pv=$1
+    shift
+    # requires the parent function to have declared the following local variables:
+    # python_exec, site_arg, sudo_cmd, user_arg
+    tue-install-debug "_handle_python${pv}_sites $*"
+
+    local site
+    site=$1
+
+     case ${site} in
+        "system" )
+            python_exec=/usr/bin/python"${pv}"
+            site_arg="-s"
+            sudo_cmd="sudo -H"
+            user_arg=""
+            ;;
+        "user" )
+            python_exec=/usr/bin/python"${pv}"
+            site_arg=""
+            sudo_cmd=""
+            user_arg="--user"
+            ;;
+        "venv" )
+            python_exec=python"${pv}"
+            site_arg="-s"
+            sudo_cmd=""
+            user_arg=""
+            ;;
+        * )
+            tue-install-error "_handle_python${pv}_sites: Unknown input variable ${i}"
+            ;;
+    esac
+
+    tue-install-debug "python_site: ${python_site}"
+    tue-install-debug "python_exec: ${python_exec}"
+    tue-install-debug "site_arg: ${site_arg}"
+    tue-install-debug "sudo_cmd: ${sudo_cmd}"
+    tue-install-debug "user_arg: ${user_arg}"
+}
+
 function _tue-install-pip-now
 {
     local pv
@@ -1105,25 +1148,42 @@ function _tue-install-pip-now
         tue-install-error "Invalid tue-install-pip${pv}-now call: needs package as argument."
     fi
 
-    local user_arg
-    [[ -z "${VIRTUAL_ENV}" ]] && user_arg="--user"
+    local pips python_site
+    { [[ -n ${VIRTUAL_ENV} ]] && python_site="venv"; } || python_site="user"
+    if [[ -n $1 ]]
+    then
+        for i in "$@"
+        do
+            case $i in
+                --python-site=* )
+                    python_site="${i#*=}"
+                    ;;
+                --* )
+                    tue-install-error "tue-install-pip${pv}-now: Unknown input variable ${i}" ;;
+                * )
+                    pips+=("$i") ;;
+            esac
+        done
+    fi
+
+    local python_exec site_arg sudo_cmd user_arg
+    _handle_python_sites "${pv}" "${python_site}"
 
     # Make sure pip is up-to-date before checking version and installing
     local pip_version desired_pip_version
-    pip_version=$(python"${pv}" -m pip --version | awk '{print $2}')
+    pip_version=$(python"${pv}" ${site_arg} -m pip --version | awk '{print $2}')
     desired_pip_version="23"
     if version_gt "$desired_pip_version" "$pip_version"
     then
         tue-install-debug "pip${pv} not yet version >=$desired_pip_version, but $pip_version"
-        tue-install-pipe python"${pv}" -m pip install ${user_arg} --upgrade pip
+        tue-install-pipe python"${pv}" ${site_arg} -m pip install "${user_arg}" --upgrade pip
         hash -r
     else
         tue-install-debug "Already pip${pv}>=$desired_pip_version"
     fi
 
     local pips_to_check pips_to_check_w_options pips_to_install pips_to_install_w_options git_pips_to_install
-    # shellcheck disable=SC2048
-    for pkg in $*
+    for pkg in "${pips[@]}"
     do
         if [[ "$pkg" == "git+"* ]]
         then
@@ -1143,7 +1203,7 @@ function _tue-install-pip-now
     then
         local indexes_to_install
         # shellcheck disable=SC2086
-        _tue-install-pip-check "$pv" $pips_to_check
+        _tue-install-pip-check "${pv}" --python-site=${python_site} ${pips_to_check}
 
         read -r -a pips_to_check <<< "$pips_to_check"
 
@@ -1165,7 +1225,7 @@ function _tue-install-pip-now
             pips_to_check_options_removed="$pips_to_check_options_removed ${pkg_split[0]}"
         done
         # shellcheck disable=SC2086
-        _tue-install-pip-check "$pv" $pips_to_check_options_removed
+        _tue-install-pip-check "${pv}" --python-site=${python_site} ${pips_to_check_options_removed}
 
         read -r -a pips_to_check_w_options <<< "$pips_to_check_w_options"
 
@@ -1181,15 +1241,15 @@ function _tue-install-pip-now
     if [ -n "$pips_to_install" ]
     then
         # shellcheck disable=SC2048,SC2086
-        tue-install-pipe python"${pv}" -m pip install ${user_arg} $pips_to_install <<< yes || tue-install-error "An error occurred while installing pip${pv} packages."
+        tue-install-pipe ${sudo_cmd} ${python_exec} ${site_arg} -m pip install ${user_arg} $pips_to_install <<< yes || tue-install-error "An error occurred while installing pip${pv} packages."
     fi
 
     if [ -n "$pips_to_install_w_options" ]
     then
         for pkg in $pips_to_install_w_options
         do
             # shellcheck disable=SC2048,SC2086
-            tue-install-pipe python"${pv}" -m pip install ${user_arg} ${pkg//^/ } <<< yes || tue-install-error "An error occurred while installing pip${pv} packages with options."
+            tue-install-pipe ${sudo_cmd} ${python_exec} ${site_arg} -m pip install ${user_arg} ${pkg//^/ } <<< yes || tue-install-error "An error occurred while installing pip${pv} packages with options."
         done
     fi
 
@@ -1198,7 +1258,7 @@ function _tue-install-pip-now
         for pkg in $git_pips_to_install
         do
             # shellcheck disable=SC2048,SC2086
-            tue-install-pipe python"${pv}" -m pip install ${user_arg} ${pkg} <<< yes || tue-install-error "An error occurred while installing pip${pv} git packages."
+            tue-install-pipe ${sudo_cmd} ${python_exec} ${site_arg} -m pip install ${user_arg} ${pkg} <<< yes || tue-install-error "An error occurred while installing pip${pv} git packages."
         done
     fi
 }
@@ -1210,12 +1270,36 @@ function _tue-install-pip-check
     pv=$1
     shift
 
-    local pips_to_check installed_versions
-    pips_to_check=("$@")
+    local installed_versions python_site pips_to_check
+    if [[ -n $1 ]]
+    then
+        for i in "$@"
+        do
+            case $i in
+                --python-site=* )
+                    python_site="${i#*=}"
+                    ;;
+                --* )
+                    tue-install-error "Unknown input variable ${i}" ;;
+                * )
+                    pips_to_check+=("$i") ;;
+            esac
+        done
+    fi
+
+    if [[ -z "${VIRTUAL_ENV}" && "${site}" == "venv" ]]
+    then
+        tue-install-error "Trying to check pip packages in a virtualenv, but no virtualenv is activated"
+    fi
+
+    # Set by _handle_python_sites
+    local python_exec sudo_cmd site_arg user_arg
+    _handle_python_sites "${pv}" "${python_site}"
+
     if [ ${#pips_to_check[@]} -gt 0 ]
     then
         local error_code
-        installed_versions=$(python"${pv}" "$TUE_INSTALL_SCRIPTS_DIR"/check-pip-pkg-installed-version.py "${pips_to_check[@]}")
+        installed_versions=$(${sudo_cmd} "${python_exec}" ${site_arg} "${TUE_INSTALL_SCRIPTS_DIR}"/check-pip-pkg-installed-version.py "${pips_to_check[@]}")
         error_code=$?
         if [ "$error_code" -gt 1 ]
         then
@@ -1493,10 +1577,11 @@ function tue-install-ros
             local pkg_xml="$ros_pkg_dir"/package.xml
             if [ -f "$pkg_xml" ]
             then
-                # Catkin
+                # catkin/ament/colcon
                 tue-install-debug "Parsing $pkg_xml"
+
                 local deps
-                deps=$("$TUE_INSTALL_SCRIPTS_DIR"/parse_package_xml.py "$pkg_xml")
+                deps=$(/usr/bin/python "${TUE_INSTALL_SCRIPTS_DIR}"/parse_package_xml.py "${pkg_xml}")
                 tue-install-debug "Parsed package.xml\n$deps"
 
                 for dep in $deps
@@ -1636,7 +1721,8 @@ TUE_INSTALL_INFOS=
 # Make sure tools used by this installer are installed
 tue-install-system-now curl git jq python-is-python3 python3-pip wget
 
-tue-install-pip3-now catkin-pkg PyYAML
+# Install in user or system site-packages
+tue-install-pip3-now --python-site="user" catkin-pkg PyYAML
 
 
 # Handling of targets
@@ -1674,7 +1760,7 @@ do
         tue-install-debug "[$target] marked as installed after a successful install"
         touch "$TUE_INSTALL_INSTALLED_DIR"/"$target"
     else
-        tue-install-debug "[$target] succesfully updated"
+        tue-install-debug "[$target] successfully updated"
     fi
 done
 
@@ -1746,6 +1832,6 @@ fi
 
 TUE_INSTALL_CURRENT_TARGET="main-loop"
 
-tue-install-echo "Installer completed succesfully"
+tue-install-echo "Installer completed successfully"
 
 return 0