Skip to content

use uv as package manager for resolving python lib dependencies #209

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 4 commits into from
Jul 7, 2025
Merged

use uv as package manager for resolving python lib dependencies #209

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
e4db689
Update espidf.py
Jason2866 Jul 5, 2025
a3a15af
Update arduino.py
Jason2866 Jul 5, 2025
2112e03
Update main.py
Jason2866 Jul 5, 2025
c6a9fc7
PR #213
Jason2866 Jul 6, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
63 changes: 1 addition & 62 deletions builder/frameworks/arduino.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,6 @@
http://arduino.cc/en/Reference/HomePage
"""

import subprocess
import json
import os
import sys
import shutil
Expand All @@ -34,23 +32,12 @@
from pathlib import Path
from typing import Union, List

import semantic_version
from SCons.Script import DefaultEnvironment, SConscript
from platformio import fs
from platformio.package.version import pepver_to_semver
from platformio.package.manager.tool import ToolPackageManager

IS_WINDOWS = sys.platform.startswith("win")

python_deps = {
"wheel": ">=0.35.1",
"rich-click": ">=1.8.6",
"PyYAML": ">=6.0.2",
"intelhex": ">=2.3.0",
"rich": ">=14.0.0",
"esp-idf-size": ">=1.6.1"
}

# Constants for better performance
UNICORE_FLAGS = {
"CORE32SOLO1",
Expand Down Expand Up @@ -601,53 +588,6 @@ def has_unicore_flags():
env.Replace(BUILD_UNFLAGS=new_build_unflags)


def get_packages_to_install(deps, installed_packages):
"""Generator for packages to install"""
for package, spec in deps.items():
if package not in installed_packages:
yield package
else:
version_spec = semantic_version.Spec(spec)
if not version_spec.match(installed_packages[package]):
yield package


def install_python_deps():
def _get_installed_pip_packages():
result = {}
try:
pip_output = subprocess.check_output([
env.subst("$PYTHONEXE"),
"-m", "pip", "list", "--format=json",
"--disable-pip-version-check"
])
packages = json.loads(pip_output)
for p in packages:
result[p["name"]] = pepver_to_semver(p["version"])
except Exception:
print("Warning! Couldn't extract the list of installed Python "
"packages.")

return result

installed_packages = _get_installed_pip_packages()
packages_to_install = list(get_packages_to_install(python_deps,
installed_packages))

if packages_to_install:
packages_str = " ".join(f'"{p}{python_deps[p]}"'
for p in packages_to_install)
env.Execute(
env.VerboseAction(
f'"$PYTHONEXE" -m pip install -U -q -q -q {packages_str}',
"Installing Arduino Python dependencies",
)
)


install_python_deps()


def get_MD5_hash(phrase):
return hashlib.md5(phrase.encode('utf-8')).hexdigest()[:16]

Expand Down Expand Up @@ -955,6 +895,7 @@ def get_frameworks_in_current_env():

if ("arduino" in pioframework and "espidf" not in pioframework and
arduino_lib_compile_flag in ("Inactive", "True")):

# try to remove not needed include path if an lib_ignore entry exists
from component_manager import ComponentManager
component_manager = ComponentManager(env)
Expand All @@ -965,8 +906,6 @@ def get_frameworks_in_current_env():
env.AddPostAction("checkprogsize", silent_action)

if IS_WINDOWS:
# Smart include path optimization based on bleeding edge configurable
# threshold
env.AddBuildMiddleware(smart_include_length_shorten)

build_script_path = join(FRAMEWORK_DIR, "tools", "pioarduino-build.py")
Expand Down
106 changes: 19 additions & 87 deletions builder/frameworks/espidf.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,68 +56,6 @@
if os.path.exists(map_file):
os.remove(map_file)

def install_standard_python_deps():
def _get_installed_standard_pip_packages():
result = {}
packages = {}
pip_output = subprocess.check_output(
[
env.subst("$PYTHONEXE"),
"-m",
"pip",
"list",
"--format=json",
"--disable-pip-version-check",
]
)
try:
packages = json.loads(pip_output)
except:
print("Warning! Couldn't extract the list of installed Python packages.")
return {}
for p in packages:
result[p["name"]] = pepver_to_semver(p["version"])

return result

deps = {
"wheel": ">=0.35.1",
"rich-click": ">=1.8.6",
"PyYAML": ">=6.0.2",
"intelhex": ">=2.3.0",
"rich": ">=14.0.0",
"esp-idf-size": ">=1.6.1"
}

installed_packages = _get_installed_standard_pip_packages()
packages_to_install = []
for package, spec in deps.items():
if package not in installed_packages:
packages_to_install.append(package)
else:
version_spec = semantic_version.Spec(spec)
if not version_spec.match(installed_packages[package]):
packages_to_install.append(package)

if packages_to_install:
env.Execute(
env.VerboseAction(
(
'"$PYTHONEXE" -m pip install -U -q -q -q '
+ " ".join(
[
'"%s%s"' % (p, deps[p])
for p in packages_to_install
]
)
),
"Installing standard Python dependencies",
)
)
return

install_standard_python_deps()

# Allow changes in folders of managed components
os.environ["IDF_COMPONENT_OVERWRITE_MANAGED_COMPONENTS"] = "1"

Expand Down Expand Up @@ -173,7 +111,7 @@ def create_silent_action(action_func):
os.rename(ARDUINO_FRAMEWORK_DIR, new_path)
ARDUINO_FRAMEWORK_DIR = new_path
assert ARDUINO_FRAMEWORK_DIR and os.path.isdir(ARDUINO_FRAMEWORK_DIR)
arduino_libs_mcu = join(platform.get_package_dir("framework-arduinoespressif32-libs"),mcu)
arduino_libs_mcu = join(platform.get_package_dir("framework-arduinoespressif32-libs"), mcu)

BUILD_DIR = env.subst("$BUILD_DIR")
PROJECT_DIR = env.subst("$PROJECT_DIR")
Expand Down Expand Up @@ -1548,24 +1486,17 @@ def generate_mbedtls_bundle(sdk_config):


def install_python_deps():
def _get_installed_pip_packages(python_exe_path):
def _get_installed_uv_packages(python_exe_path):
result = {}
packages = {}
pip_output = subprocess.check_output(
[
python_exe_path,
"-m",
"pip",
"list",
"--format=json",
"--disable-pip-version-check",
]
)
try:
packages = json.loads(pip_output)
except:
print("Warning! Couldn't extract the list of installed Python packages.")
uv_output = subprocess.check_output([
"uv", "pip", "list", "--python", python_exe_path, "--format=json"
])
packages = json.loads(uv_output)
except (subprocess.CalledProcessError, json.JSONDecodeError, OSError) as e:
print(f"Warning! Couldn't extract the list of installed Python packages: {e}")
return {}

for p in packages:
result[p["name"]] = pepver_to_semver(p["version"])

Expand All @@ -1576,7 +1507,7 @@ def _get_installed_pip_packages(python_exe_path):
return

deps = {
"wheel": ">=0.35.1",
"uv": ">=0.1.0",
# https://github.com/platformio/platformio-core/issues/4614
"urllib3": "<2",
# https://github.com/platformio/platform-espressif32/issues/635
Expand All @@ -1590,7 +1521,7 @@ def _get_installed_pip_packages(python_exe_path):
deps["chardet"] = ">=3.0.2,<4"

python_exe_path = get_python_exe()
installed_packages = _get_installed_pip_packages(python_exe_path)
installed_packages = _get_installed_uv_packages(python_exe_path)
packages_to_install = []
for package, spec in deps.items():
if package not in installed_packages:
Expand All @@ -1601,21 +1532,22 @@ def _get_installed_pip_packages(python_exe_path):
packages_to_install.append(package)

if packages_to_install:
packages_str = " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install])

# Use uv to install packages in the specific Python environment
env.Execute(
env.VerboseAction(
(
'"%s" -m pip install -U -q -q -q ' % python_exe_path
+ " ".join(['"%s%s"' % (p, deps[p]) for p in packages_to_install])
),
"Installing ESP-IDF's Python dependencies",
f'uv pip install --python "{python_exe_path}" {packages_str}',
"Installing ESP-IDF's Python dependencies with uv",
)
)

if IS_WINDOWS and "windows-curses" not in installed_packages:
# Install windows-curses in the IDF Python environment
env.Execute(
env.VerboseAction(
'"%s" -m pip install -q -q -q windows-curses' % python_exe_path,
"Installing windows-curses package",
f'uv pip install --python "{python_exe_path}" windows-curses',
"Installing windows-curses package with uv",
)
)

Expand Down
Loading