From 960ff4b1bbdd75ca4a62fb6069a076df2f10364e Mon Sep 17 00:00:00 2001 From: Daniil Fajnberg <60156134+daniil-berg@users.noreply.github.com> Date: Thu, 16 Jan 2025 18:54:27 +0100 Subject: [PATCH] Fix TOML parsing in Python ecosystem (fixes #10523) (#10540) * Fix TOML parsing in Python ecosystem (fixes #10523) Use `tomli` instead of outdated `toml` package * Adds test cases --------- Co-authored-by: sachin-sandhu --- python/helpers/lib/parser.py | 9 +++-- python/helpers/requirements.txt | 4 +- .../pyproject_files_parser_spec.rb | 39 +++++++++++++++++++ .../pyproject_files/pyproject_1_0_0.toml | 22 +++++++++++ .../pyproject_1_0_0_nodeps.toml | 19 +++++++++ .../pyproject_1_0_0_optional_deps.toml | 29 ++++++++++++++ 6 files changed, 116 insertions(+), 6 deletions(-) create mode 100644 python/spec/fixtures/pyproject_files/pyproject_1_0_0.toml create mode 100644 python/spec/fixtures/pyproject_files/pyproject_1_0_0_nodeps.toml create mode 100644 python/spec/fixtures/pyproject_files/pyproject_1_0_0_optional_deps.toml diff --git a/python/helpers/lib/parser.py b/python/helpers/lib/parser.py index c2cfc8ff409..17c5e6f80b7 100644 --- a/python/helpers/lib/parser.py +++ b/python/helpers/lib/parser.py @@ -14,9 +14,9 @@ ) from packaging.requirements import InvalidRequirement, Requirement -# TODO: Replace 3p package `toml` with 3.11's new stdlib `tomllib` once we drop -# support for Python 3.10. -import toml +# TODO: Replace 3p package `tomli` with 3.11's new stdlib `tomllib` once we +# drop support for Python 3.10. +import tomli # Inspired by pips internal check: # https://github.com/pypa/pip/blob/0bb3ac87f5bb149bd75cceac000844128b574385/src/pip/_internal/req/req_file.py#L35 @@ -24,7 +24,8 @@ def parse_pep621_dependencies(pyproject_path): - project_toml = toml.load(pyproject_path) + with open(pyproject_path, "rb") as file: + project_toml = tomli.load(file) def parse_toml_section_pep621_dependencies(pyproject_path, dependencies): requirement_packages = [] diff --git a/python/helpers/requirements.txt b/python/helpers/requirements.txt index f5998ed5e96..f16e2759389 100644 --- a/python/helpers/requirements.txt +++ b/python/helpers/requirements.txt @@ -7,8 +7,8 @@ hashin==1.0.3; python_version >= '3.9' pipenv==2024.0.2 plette==2.1.0 poetry==1.8.5 -# TODO: Replace 3p package `toml` with 3.11's new stdlib `tomllib` once we drop support for Python 3.10. -toml==0.10.2 +# TODO: Replace 3p package `tomli` with 3.11's new stdlib `tomllib` once we drop support for Python 3.10. +tomli==2.0.1 # Some dependencies will only install if Cython is present Cython==3.0.10 diff --git a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb index cd042f22eed..e2737b4141d 100644 --- a/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb +++ b/python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb @@ -376,5 +376,44 @@ its(:length) { is_expected.to be > 0 } end + + describe "parse standard python files" do + subject(:dependencies) { parser.dependency_set.dependencies } + + let(:pyproject_fixture_name) { "pyproject_1_0_0.toml" } + + # fixture has 1 build system requires and plus 1 dependencies exists + + its(:length) { is_expected.to eq(1) } + + context "with a string declaration" do + subject(:dependency) { dependencies.first } + + it "has the right details" do + expect(dependency).to be_a(Dependabot::Dependency) + expect(dependency.name).to eq("pydantic") + expect(dependency.version).to eq("2.7.0") + end + end + + context "without dependencies" do + subject(:dependencies) { parser.dependency_set.dependencies } + + let(:pyproject_fixture_name) { "pyproject_1_0_0_nodeps.toml" } + + # fixture has 1 build system requires and no dependencies or + # optional dependencies exists + + its(:length) { is_expected.to eq(0) } + end + + context "with optional dependencies only" do + subject(:dependencies) { parser.dependency_set.dependencies } + + let(:pyproject_fixture_name) { "pyproject_1_0_0_optional_deps.toml" } + + its(:length) { is_expected.to be > 0 } + end + end end end diff --git a/python/spec/fixtures/pyproject_files/pyproject_1_0_0.toml b/python/spec/fixtures/pyproject_files/pyproject_1_0_0.toml new file mode 100644 index 00000000000..998a8616ec0 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/pyproject_1_0_0.toml @@ -0,0 +1,22 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "dependabot-pyproject-toml-error" +description = '''foo''' +version = "0.0.1" +requires-python = ">=3.12" +license = { text = "GNU General Public License v3 or later (GPLv3+)" } +classifiers = [ + '''Development Status :: 4 - Beta''', + "Programming Language :: Python :: 3 :: Only", +] +dependencies = [ + '''pydantic==2.7.0''', +] + +[tool.coverage.report] +exclude_also = [ + '''if __name__ == ['"]__main__['"]:''', +] diff --git a/python/spec/fixtures/pyproject_files/pyproject_1_0_0_nodeps.toml b/python/spec/fixtures/pyproject_files/pyproject_1_0_0_nodeps.toml new file mode 100644 index 00000000000..ee091069eb9 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/pyproject_1_0_0_nodeps.toml @@ -0,0 +1,19 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "dependabot-pyproject-toml-error" +description = '''foo''' +version = "0.0.1" +requires-python = ">=3.12" +license = { text = "GNU General Public License v3 or later (GPLv3+)" } +classifiers = [ + '''Development Status :: 4 - Beta''', + "Programming Language :: Python :: 3 :: Only", +] + +[tool.coverage.report] +exclude_also = [ + '''if __name__ == ['"]__main__['"]:''', +] diff --git a/python/spec/fixtures/pyproject_files/pyproject_1_0_0_optional_deps.toml b/python/spec/fixtures/pyproject_files/pyproject_1_0_0_optional_deps.toml new file mode 100644 index 00000000000..7bad5cb6734 --- /dev/null +++ b/python/spec/fixtures/pyproject_files/pyproject_1_0_0_optional_deps.toml @@ -0,0 +1,29 @@ +[build-system] +requires = ["setuptools", "setuptools-scm"] +build-backend = "setuptools.build_meta" + +[project] +name = "dependabot-pyproject-toml-error" +description = '''foo''' +version = "0.0.1" +requires-python = ">=3.12" +license = { text = "GNU General Public License v3 or later (GPLv3+)" } +classifiers = [ + '''Development Status :: 4 - Beta''', + "Programming Language :: Python :: 3 :: Only", +] +dependencies = [ + '''pydantic==2.7.0''', +] + +[tool.coverage.report] +exclude_also = [ + '''if __name__ == ['"]__main__['"]:''', +] +[project.optional-dependencies] +socks = [ 'PySocks >= 1.5.6, != 1.5.7, < 2' ] +tests = [ + 'ddt >= 1.2.2, < 2', + 'pytest < 6', + 'mock >= 1.0.1, < 4; python_version < "3.4"', +]