Fix TOML parsing in Python ecosystem (fixes #10523) (#10540)

* Fix TOML parsing in Python ecosystem (fixes #10523)

Use `tomli` instead of outdated `toml` package

* Adds test cases

---------

Co-authored-by: sachin-sandhu <[email protected]>

python/helpers/lib/parser.py

@@ -14,17 +14,18 @@
 )
 
 from packaging.requirements import InvalidRequirement, Requirement
-# TODO: Replace 3p package `toml` with 3.11's new stdlib `tomllib` once we drop
-# support for Python 3.10.
-import toml
+# TODO: Replace 3p package `tomli` with 3.11's new stdlib `tomllib` once we
+#       drop support for Python 3.10.
+import tomli
 
 # Inspired by pips internal check:
 # https://github.com/pypa/pip/blob/0bb3ac87f5bb149bd75cceac000844128b574385/src/pip/_internal/req/req_file.py#L35
 COMMENT_RE = re.compile(r'(^|\s+)#.*$')
 
 
 def parse_pep621_dependencies(pyproject_path):
-    project_toml = toml.load(pyproject_path)
+    with open(pyproject_path, "rb") as file:
+        project_toml = tomli.load(file)
 
     def parse_toml_section_pep621_dependencies(pyproject_path, dependencies):
         requirement_packages = []

python/helpers/requirements.txt

@@ -7,8 +7,8 @@ hashin==1.0.3; python_version >= '3.9'
 pipenv==2024.0.2
 plette==2.1.0
 poetry==1.8.5
-# TODO: Replace 3p package `toml` with 3.11's new stdlib `tomllib` once we drop support for Python 3.10.
-toml==0.10.2
+# TODO: Replace 3p package `tomli` with 3.11's new stdlib `tomllib` once we drop support for Python 3.10.
+tomli==2.0.1
 
 # Some dependencies will only install if Cython is present
 Cython==3.0.10

python/spec/dependabot/python/file_parser/pyproject_files_parser_spec.rb

@@ -376,5 +376,44 @@
 
       its(:length) { is_expected.to be > 0 }
     end
+
+    describe "parse standard python files" do
+      subject(:dependencies) { parser.dependency_set.dependencies }
+
+      let(:pyproject_fixture_name) { "pyproject_1_0_0.toml" }
+
+      # fixture has 1 build system requires and plus 1 dependencies exists
+
+      its(:length) { is_expected.to eq(1) }
+
+      context "with a string declaration" do
+        subject(:dependency) { dependencies.first }
+
+        it "has the right details" do
+          expect(dependency).to be_a(Dependabot::Dependency)
+          expect(dependency.name).to eq("pydantic")
+          expect(dependency.version).to eq("2.7.0")
+        end
+      end
+
+      context "without dependencies" do
+        subject(:dependencies) { parser.dependency_set.dependencies }
+
+        let(:pyproject_fixture_name) { "pyproject_1_0_0_nodeps.toml" }
+
+        # fixture has 1 build system requires and no dependencies or
+        # optional dependencies exists
+
+        its(:length) { is_expected.to eq(0) }
+      end
+
+      context "with optional dependencies only" do
+        subject(:dependencies) { parser.dependency_set.dependencies }
+
+        let(:pyproject_fixture_name) { "pyproject_1_0_0_optional_deps.toml" }
+
+        its(:length) { is_expected.to be > 0 }
+      end
+    end
   end
 end

python/spec/fixtures/pyproject_files/pyproject_1_0_0.toml

@@ -0,0 +1,22 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dependabot-pyproject-toml-error"
+description = '''foo'''
+version = "0.0.1"
+requires-python = ">=3.12"
+license = { text = "GNU General Public License v3 or later (GPLv3+)" }
+classifiers = [
+    '''Development Status :: 4 - Beta''',
+    "Programming Language :: Python :: 3 :: Only",
+]
+dependencies = [
+    '''pydantic==2.7.0''',
+]
+
+[tool.coverage.report]
+exclude_also = [
+    '''if __name__ == ['"]__main__['"]:''',
+]

python/spec/fixtures/pyproject_files/pyproject_1_0_0_nodeps.toml

@@ -0,0 +1,19 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dependabot-pyproject-toml-error"
+description = '''foo'''
+version = "0.0.1"
+requires-python = ">=3.12"
+license = { text = "GNU General Public License v3 or later (GPLv3+)" }
+classifiers = [
+    '''Development Status :: 4 - Beta''',
+    "Programming Language :: Python :: 3 :: Only",
+]
+
+[tool.coverage.report]
+exclude_also = [
+    '''if __name__ == ['"]__main__['"]:''',
+]

python/spec/fixtures/pyproject_files/pyproject_1_0_0_optional_deps.toml

@@ -0,0 +1,29 @@
+[build-system]
+requires = ["setuptools", "setuptools-scm"]
+build-backend = "setuptools.build_meta"
+
+[project]
+name = "dependabot-pyproject-toml-error"
+description = '''foo'''
+version = "0.0.1"
+requires-python = ">=3.12"
+license = { text = "GNU General Public License v3 or later (GPLv3+)" }
+classifiers = [
+    '''Development Status :: 4 - Beta''',
+    "Programming Language :: Python :: 3 :: Only",
+]
+dependencies = [
+    '''pydantic==2.7.0''',
+]
+
+[tool.coverage.report]
+exclude_also = [
+    '''if __name__ == ['"]__main__['"]:''',
+]
+[project.optional-dependencies]
+socks = [ 'PySocks >= 1.5.6, != 1.5.7, < 2' ]
+tests = [
+  'ddt >= 1.2.2, < 2',
+  'pytest < 6',
+  'mock >= 1.0.1, < 4; python_version < "3.4"',
+]