⬆️ Allow docutils 0.22

.github/workflows/tests.yml

@@ -74,7 +74,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        docutils-version: ["0.20", "0.21"]
+        docutils-version: ["0.20", "0.21", "0.22"]
 
     steps:
     - name: Checkout source

AGENTS.md

@@ -85,7 +85,8 @@ tox -e py311-sphinx8
 # Run with coverage
 tox -- --cov=myst_parser
 
-# Update regression test fixtures
+# Update regression test fixtures (this will initially produce an error code if the files change)
+# but note, these files must pass for all python/sphinx/docutils versions
 tox -- --regen-file-failure --force-regen
 ```
 
@@ -337,5 +338,6 @@ flowchart TB
 - [markdown-it-py Documentation](https://markdown-it-py.readthedocs.io/)
 - [Docutils Repository](https://github.com/live-clones/docutils)
 - [Docutils Documentation](https://docutils.sourceforge.io/)
+- [Docutils release log](https://docutils.sourceforge.io/RELEASE-NOTES.html)
 - [Sphinx Repository](https://github.com/sphinx-doc/sphinx)
 - [Sphinx Extension Development](https://www.sphinx-doc.org/en/master/extdev/index.html)

myst_parser/mocking.py

@@ -454,12 +454,18 @@ def run(self) -> list[nodes.Element]:
             literal_block.line = 1  # TODO don;t think this should be 1?
             self.add_name(literal_block)
             if "number-lines" in self.options:
-                try:
-                    startline = int(self.options["number-lines"] or 1)
-                except ValueError as err:
-                    raise DirectiveError(
-                        3, ":number-lines: with non-integer start value"
-                    ) from err
+                # note starting in docutils 0.22 this option is now an integer instead of a string, see: https://github.com/live-clones/docutils/commit/f39ac1413e56a330c8fea6e0d080fed0ff2b8483
+                if self.options["number-lines"] is None:
+                    startline = 1
+                elif isinstance(self.options["number-lines"], int):
+                    startline = self.options["number-lines"]
+                else:
+                    try:
+                        startline = int(self.options["number-lines"] or 1)
+                    except ValueError as err:
+                        raise DirectiveError(
+                            3, ":number-lines: with non-integer start value"
+                        ) from err
                 endline = startline + len(file_content.splitlines())
                 file_content = file_content.removesuffix("\n")
                 tokens = NumberLines([([], file_content)], startline, endline)

pyproject.toml

@@ -34,7 +34,7 @@ keywords = [
 ]
 requires-python = ">=3.11"
 dependencies = [
-    "docutils>=0.20,<0.22",
+    "docutils>=0.20,<0.23",
     "jinja2", # required for substitutions, but let sphinx choose version
     "markdown-it-py~=4.0",
     "mdit-py-plugins~=0.5",
@@ -146,6 +146,7 @@ disallow_any_generics = false
 [tool.pytest.ini_options]
 filterwarnings = [
     "ignore:.*The default for the setting.*:FutureWarning",
+    "ignore:.*:PendingDeprecationWarning",
 ]
 
 [tool.coverage.run]

tests/conftest.py

@@ -0,0 +1,54 @@
+"""Shared pytest configuration for all tests."""
+
+import re
+
+from docutils import __version_info__ as docutils_version_info
+
+DOCUTILS_0_22_PLUS = docutils_version_info >= (0, 22)
+
+
+def normalize_doctree_xml(text: str) -> str:
+    """Normalize docutils XML output for cross-version compatibility.
+
+    In docutils 0.22+, boolean attributes are serialized as "1"/"0"
+    instead of "True"/"False". This function normalizes to the old format
+    for consistent test fixtures.
+    """
+    if DOCUTILS_0_22_PLUS:
+        # Normalize new format (1/0) to old format (1/0)
+        # Only replace when it's clearly a boolean attribute value
+        # Pattern: attribute="1" or attribute="0"
+        attrs = [
+            "force",
+            "glob",
+            "hidden",
+            "id_link",
+            "includehidden",
+            "inline",
+            "internal",
+            "is_div",
+            "linenos",
+            "multi_line_parameter_list",
+            "multi_line_trailing_comma",
+            "no-contents-entry",
+            "no-index",
+            "no-index-entry",
+            "no-typesetting",
+            "no-wrap",
+            "nocontentsentry",
+            "noindex",
+            "noindexentry",
+            "nowrap",
+            "refexplicit",
+            "refspecific",
+            "refwarn",
+            "sorted",
+            "titlesonly",
+            "toctree",
+            "translatable",
+        ]
+        text = re.sub(rf' ({"|".join(attrs)})="1"', r' \1="True"', text)
+        text = re.sub(rf' ({"|".join(attrs)})="0"', r' \1="False"', text)
+        # numbered is changed in math_block, but not in toctree, so we have to be more precise
+        text = re.sub(r' numbered="1" xml:space', r' numbered="True" xml:space', text)
+    return text

tests/test_html/test_html_to_nodes.py

@@ -6,6 +6,7 @@
 
 from myst_parser.config.main import MdParserConfig
 from myst_parser.mdit_to_docutils.html_to_nodes import html_to_nodes
+from tests.conftest import normalize_doctree_xml
 
 FIXTURE_PATH = Path(__file__).parent
 
@@ -32,4 +33,4 @@ def _run_directive(name: str, first_line: str, content: str, position: int):
 def test_html_to_nodes(file_params, mock_renderer):
     output = nodes.container()
     output += html_to_nodes(file_params.content, line_number=0, renderer=mock_renderer)
-    file_params.assert_expected(output.pformat(), rstrip=True)
+    file_params.assert_expected(normalize_doctree_xml(output.pformat()), rstrip=True)

tests/test_renderers/fixtures/docutil_link_resolution.md

@@ -128,14 +128,15 @@
         Test
     <subtitle ids="other test-1" names="other test">
         Other
-    <system_message backrefs="test-1" level="1" line="3" source="<src>/index.md" type="INFO">
-        <paragraph>
-            Duplicate implicit target name: "test".
     <target refid="test-1">
     <paragraph>
         <reference id_link="True" refid="test-1">
             <inline classes="std std-ref">
                 Other
+
+
+<src>/index.md:3: (INFO/1) Target name overrides implicit target name "test".
+<src>/index.md:3: (INFO/1) Hyperlink target "test-1" is not referenced.
 .
 
 [id-with-spaces] 

tests/test_renderers/fixtures/myst-config.txt

@@ -263,6 +263,12 @@ a
 <document source="<string>">
     <paragraph>
         a
+    <section classes="system-messages">
+        <title>
+            Docutils System Messages
+        <system_message level="2" source="<string>" type="WARNING">
+            <paragraph>
+                The `attrs_image` extension is deprecated, please use `attrs_inline` instead. [myst.deprecated]
 
 <string>:: (WARNING/2) The `attrs_image` extension is deprecated, please use `attrs_inline` instead. [myst.deprecated]
 .

tests/test_renderers/test_error_reporting.py

@@ -20,4 +20,9 @@ def test_basic(file_params):
         parser=Parser(),
         settings_overrides={"warning_stream": report_stream},
     )
-    file_params.assert_expected(report_stream.getvalue(), rstrip=True)
+    text = report_stream.getvalue()
+    # changed in docutils 0.23
+    text = text.replace(
+        "corresponding footnote available", "corresponding footnotes available"
+    )
+    file_params.assert_expected(text, rstrip=True)

tests/test_renderers/test_fixtures_docutils.py

@@ -11,6 +11,8 @@
 from typing import Any
 
 import pytest
+from conftest import normalize_doctree_xml
+from docutils import __version_info__ as docutils_version
 from docutils.core import Publisher, publish_doctree
 from pytest_param_files import ParamTestData
 
@@ -37,7 +39,9 @@ def _apply_transforms(self):
     )
 
     # in docutils 0.18 footnote ids have changed
-    outcome = doctree.pformat().replace('"footnote-reference-1"', '"id1"')
+    outcome = normalize_doctree_xml(doctree.pformat()).replace(
+        '"footnote-reference-1"', '"id1"'
+    )
     outcome = outcome.replace(' language=""', "")
     file_params.assert_expected(outcome, rstrip_lines=True)
 
@@ -48,13 +52,18 @@ def test_link_resolution(file_params: ParamTestData):
     settings = settings_from_cmdline(file_params.description)
     report_stream = StringIO()
     settings["warning_stream"] = report_stream
+    if file_params.title == "explicit>implicit":
+        if docutils_version < (0, 22):
+            # reporting changed in docutils 0.22
+            pytest.skip("different in docutils>=0.22")
+        settings["report_level"] = 0
     doctree = publish_doctree(
         file_params.content,
         source_path="<src>/index.md",
         parser=Parser(),
         settings_overrides=settings,
     )
-    outcome = doctree.pformat()
+    outcome = normalize_doctree_xml(doctree.pformat())
     if report_stream.getvalue().strip():
         outcome += "\n\n" + report_stream.getvalue().strip()
     file_params.assert_expected(outcome, rstrip_lines=True)
@@ -75,7 +84,9 @@ def _apply_transforms(self):
         parser=Parser(),
     )
 
-    file_params.assert_expected(doctree.pformat(), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(doctree.pformat()), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "docutil_directives.md")
@@ -95,7 +106,9 @@ def _apply_transforms(self):
         parser=Parser(),
     )
 
-    file_params.assert_expected(doctree.pformat(), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(doctree.pformat()), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "docutil_syntax_extensions.txt")
@@ -109,7 +122,9 @@ def test_syntax_extensions(file_params: ParamTestData):
         parser=Parser(),
         settings_overrides=settings,
     )
-    file_params.assert_expected(doctree.pformat(), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(doctree.pformat()), rstrip_lines=True
+    )
 
 
 def settings_from_cmdline(cmdline: str | None) -> dict[str, Any]:

tests/test_renderers/test_fixtures_sphinx.py

@@ -17,6 +17,7 @@
 from sphinx_pytest.plugin import CreateDoctree
 
 from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer
+from tests.conftest import normalize_doctree_xml
 
 FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures")
 
@@ -36,7 +37,7 @@ def _apply_transforms(self):
         monkeypatch.setattr(SphinxTransformer, "apply_transforms", _apply_transforms)
 
     result = sphinx_doctree(file_params.content, "index.md")
-    pformat = result.pformat("index")
+    pformat = normalize_doctree_xml(result.pformat("index"))
     replacements = {
         # changed in docutils 0.20.1
         '<literal classes="code" language="">': '<literal classes="code">',
@@ -58,7 +59,7 @@ def test_link_resolution(file_params: ParamTestData, sphinx_doctree: CreateDoctr
     sphinx_doctree.srcdir.joinpath("test.txt").touch()
     sphinx_doctree.srcdir.joinpath("other.rst").write_text(":orphan:\n\nTest\n====")
     result = sphinx_doctree(file_params.content, "index.md")
-    outcome = result.pformat("index")
+    outcome = normalize_doctree_xml(result.pformat("index"))
     if result.warnings.strip():
         outcome += "\n\n" + result.warnings.strip()
     file_params.assert_expected(outcome, rstrip_lines=True)
@@ -80,7 +81,9 @@ def settings_from_json(string: str | None):
 def test_tables(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDoctree):
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "directive_options.md")
@@ -89,7 +92,9 @@ def test_directive_options(
 ):
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "sphinx_directives.md")
@@ -102,7 +107,9 @@ def test_sphinx_directives(
         pytest.skip(file_params.title)
 
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
-    pformat = sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index")
+    pformat = normalize_doctree_xml(
+        sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index")
+    )
     # see https://github.com/executablebooks/MyST-Parser/issues/522
     if sys.maxsize == 2147483647:
         pformat = pformat.replace('"2147483647"', '"9223372036854775807"')
@@ -115,7 +122,9 @@ def test_sphinx_roles(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDo
         pytest.skip(file_params.title)
 
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
-    pformat = sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index")
+    pformat = normalize_doctree_xml(
+        sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index")
+    )
     # sphinx 3 adds a parent key
     pformat = re.sub('cpp:parent_key="[^"]*"', 'cpp:parent_key=""', pformat)
     # sphinx >= 4.5.0 adds a trailing slash to PEP URLs,
@@ -136,7 +145,9 @@ def test_dollarmath(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDoct
         {"extensions": ["myst_parser"], "myst_enable_extensions": ["dollarmath"]}
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "amsmath.md")
@@ -148,7 +159,9 @@ def test_amsmath(
         {"extensions": ["myst_parser"], "myst_enable_extensions": ["amsmath"]}
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "containers.md")
@@ -160,7 +173,9 @@ def test_containers(
         {"extensions": ["myst_parser"], "myst_enable_extensions": ["colon_fence"]}
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "eval_rst.md")
@@ -169,7 +184,9 @@ def test_evalrst_elements(
 ):
     sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]})
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "definition_lists.md")
@@ -180,7 +197,9 @@ def test_definition_lists(
         {"extensions": ["myst_parser"], "myst_enable_extensions": ["deflist"]}
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )
 
 
 @pytest.mark.param_file(FIXTURE_PATH / "attributes.md")
@@ -192,4 +211,6 @@ def test_attributes(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDoct
         }
     )
     result = sphinx_doctree_no_tr(file_params.content, "index.md")
-    file_params.assert_expected(result.pformat("index"), rstrip_lines=True)
+    file_params.assert_expected(
+        normalize_doctree_xml(result.pformat("index")), rstrip_lines=True
+    )