diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5a54798b..f68f84dd 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -74,7 +74,7 @@ jobs: strategy: fail-fast: false matrix: - docutils-version: ["0.20", "0.21"] + docutils-version: ["0.20", "0.21", "0.22"] steps: - name: Checkout source diff --git a/AGENTS.md b/AGENTS.md index 798417b7..2851b1d5 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -85,7 +85,8 @@ tox -e py311-sphinx8 # Run with coverage tox -- --cov=myst_parser -# Update regression test fixtures +# Update regression test fixtures (this will initially produce an error code if the files change) +# but note, these files must pass for all python/sphinx/docutils versions tox -- --regen-file-failure --force-regen ``` @@ -337,5 +338,6 @@ flowchart TB - [markdown-it-py Documentation](https://markdown-it-py.readthedocs.io/) - [Docutils Repository](https://github.com/live-clones/docutils) - [Docutils Documentation](https://docutils.sourceforge.io/) +- [Docutils release log](https://docutils.sourceforge.io/RELEASE-NOTES.html) - [Sphinx Repository](https://github.com/sphinx-doc/sphinx) - [Sphinx Extension Development](https://www.sphinx-doc.org/en/master/extdev/index.html) diff --git a/myst_parser/mocking.py b/myst_parser/mocking.py index 8a800bcc..f809c6d7 100644 --- a/myst_parser/mocking.py +++ b/myst_parser/mocking.py @@ -454,12 +454,18 @@ def run(self) -> list[nodes.Element]: literal_block.line = 1 # TODO don;t think this should be 1? self.add_name(literal_block) if "number-lines" in self.options: - try: - startline = int(self.options["number-lines"] or 1) - except ValueError as err: - raise DirectiveError( - 3, ":number-lines: with non-integer start value" - ) from err + # note starting in docutils 0.22 this option is now an integer instead of a string, see: https://github.com/live-clones/docutils/commit/f39ac1413e56a330c8fea6e0d080fed0ff2b8483 + if self.options["number-lines"] is None: + startline = 1 + elif isinstance(self.options["number-lines"], int): + startline = self.options["number-lines"] + else: + try: + startline = int(self.options["number-lines"] or 1) + except ValueError as err: + raise DirectiveError( + 3, ":number-lines: with non-integer start value" + ) from err endline = startline + len(file_content.splitlines()) file_content = file_content.removesuffix("\n") tokens = NumberLines([([], file_content)], startline, endline) diff --git a/pyproject.toml b/pyproject.toml index 068a474b..1f9440f9 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,7 +34,7 @@ keywords = [ ] requires-python = ">=3.11" dependencies = [ - "docutils>=0.20,<0.22", + "docutils>=0.20,<0.23", "jinja2", # required for substitutions, but let sphinx choose version "markdown-it-py~=4.0", "mdit-py-plugins~=0.5", @@ -146,6 +146,7 @@ disallow_any_generics = false [tool.pytest.ini_options] filterwarnings = [ "ignore:.*The default for the setting.*:FutureWarning", + "ignore:.*:PendingDeprecationWarning", ] [tool.coverage.run] diff --git a/tests/conftest.py b/tests/conftest.py new file mode 100644 index 00000000..7952e9f2 --- /dev/null +++ b/tests/conftest.py @@ -0,0 +1,54 @@ +"""Shared pytest configuration for all tests.""" + +import re + +from docutils import __version_info__ as docutils_version_info + +DOCUTILS_0_22_PLUS = docutils_version_info >= (0, 22) + + +def normalize_doctree_xml(text: str) -> str: + """Normalize docutils XML output for cross-version compatibility. + + In docutils 0.22+, boolean attributes are serialized as "1"/"0" + instead of "True"/"False". This function normalizes to the old format + for consistent test fixtures. + """ + if DOCUTILS_0_22_PLUS: + # Normalize new format (1/0) to old format (1/0) + # Only replace when it's clearly a boolean attribute value + # Pattern: attribute="1" or attribute="0" + attrs = [ + "force", + "glob", + "hidden", + "id_link", + "includehidden", + "inline", + "internal", + "is_div", + "linenos", + "multi_line_parameter_list", + "multi_line_trailing_comma", + "no-contents-entry", + "no-index", + "no-index-entry", + "no-typesetting", + "no-wrap", + "nocontentsentry", + "noindex", + "noindexentry", + "nowrap", + "refexplicit", + "refspecific", + "refwarn", + "sorted", + "titlesonly", + "toctree", + "translatable", + ] + text = re.sub(rf' ({"|".join(attrs)})="1"', r' \1="True"', text) + text = re.sub(rf' ({"|".join(attrs)})="0"', r' \1="False"', text) + # numbered is changed in math_block, but not in toctree, so we have to be more precise + text = re.sub(r' numbered="1" xml:space', r' numbered="True" xml:space', text) + return text diff --git a/tests/test_html/test_html_to_nodes.py b/tests/test_html/test_html_to_nodes.py index 207a6274..08423a79 100644 --- a/tests/test_html/test_html_to_nodes.py +++ b/tests/test_html/test_html_to_nodes.py @@ -6,6 +6,7 @@ from myst_parser.config.main import MdParserConfig from myst_parser.mdit_to_docutils.html_to_nodes import html_to_nodes +from tests.conftest import normalize_doctree_xml FIXTURE_PATH = Path(__file__).parent @@ -32,4 +33,4 @@ def _run_directive(name: str, first_line: str, content: str, position: int): def test_html_to_nodes(file_params, mock_renderer): output = nodes.container() output += html_to_nodes(file_params.content, line_number=0, renderer=mock_renderer) - file_params.assert_expected(output.pformat(), rstrip=True) + file_params.assert_expected(normalize_doctree_xml(output.pformat()), rstrip=True) diff --git a/tests/test_renderers/fixtures/docutil_link_resolution.md b/tests/test_renderers/fixtures/docutil_link_resolution.md index e686e17c..3942a915 100644 --- a/tests/test_renderers/fixtures/docutil_link_resolution.md +++ b/tests/test_renderers/fixtures/docutil_link_resolution.md @@ -128,14 +128,15 @@ Test Other - - - Duplicate implicit target name: "test". Other + + +/index.md:3: (INFO/1) Target name overrides implicit target name "test". +/index.md:3: (INFO/1) Hyperlink target "test-1" is not referenced. . [id-with-spaces] diff --git a/tests/test_renderers/fixtures/myst-config.txt b/tests/test_renderers/fixtures/myst-config.txt index 765b0108..1f009a66 100644 --- a/tests/test_renderers/fixtures/myst-config.txt +++ b/tests/test_renderers/fixtures/myst-config.txt @@ -263,6 +263,12 @@ a a +
+ + Docutils System Messages + <system_message level="2" source="<string>" type="WARNING"> + <paragraph> + The `attrs_image` extension is deprecated, please use `attrs_inline` instead. [myst.deprecated] <string>:: (WARNING/2) The `attrs_image` extension is deprecated, please use `attrs_inline` instead. [myst.deprecated] . diff --git a/tests/test_renderers/test_error_reporting.py b/tests/test_renderers/test_error_reporting.py index 196da30d..17d68bcf 100644 --- a/tests/test_renderers/test_error_reporting.py +++ b/tests/test_renderers/test_error_reporting.py @@ -20,4 +20,9 @@ def test_basic(file_params): parser=Parser(), settings_overrides={"warning_stream": report_stream}, ) - file_params.assert_expected(report_stream.getvalue(), rstrip=True) + text = report_stream.getvalue() + # changed in docutils 0.23 + text = text.replace( + "corresponding footnote available", "corresponding footnotes available" + ) + file_params.assert_expected(text, rstrip=True) diff --git a/tests/test_renderers/test_fixtures_docutils.py b/tests/test_renderers/test_fixtures_docutils.py index 86c7d095..c46e51c7 100644 --- a/tests/test_renderers/test_fixtures_docutils.py +++ b/tests/test_renderers/test_fixtures_docutils.py @@ -11,6 +11,8 @@ from typing import Any import pytest +from conftest import normalize_doctree_xml +from docutils import __version_info__ as docutils_version from docutils.core import Publisher, publish_doctree from pytest_param_files import ParamTestData @@ -37,7 +39,9 @@ def _apply_transforms(self): ) # in docutils 0.18 footnote ids have changed - outcome = doctree.pformat().replace('"footnote-reference-1"', '"id1"') + outcome = normalize_doctree_xml(doctree.pformat()).replace( + '"footnote-reference-1"', '"id1"' + ) outcome = outcome.replace(' language=""', "") file_params.assert_expected(outcome, rstrip_lines=True) @@ -48,13 +52,18 @@ def test_link_resolution(file_params: ParamTestData): settings = settings_from_cmdline(file_params.description) report_stream = StringIO() settings["warning_stream"] = report_stream + if file_params.title == "explicit>implicit": + if docutils_version < (0, 22): + # reporting changed in docutils 0.22 + pytest.skip("different in docutils>=0.22") + settings["report_level"] = 0 doctree = publish_doctree( file_params.content, source_path="<src>/index.md", parser=Parser(), settings_overrides=settings, ) - outcome = doctree.pformat() + outcome = normalize_doctree_xml(doctree.pformat()) if report_stream.getvalue().strip(): outcome += "\n\n" + report_stream.getvalue().strip() file_params.assert_expected(outcome, rstrip_lines=True) @@ -75,7 +84,9 @@ def _apply_transforms(self): parser=Parser(), ) - file_params.assert_expected(doctree.pformat(), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(doctree.pformat()), rstrip_lines=True + ) @pytest.mark.param_file(FIXTURE_PATH / "docutil_directives.md") @@ -95,7 +106,9 @@ def _apply_transforms(self): parser=Parser(), ) - file_params.assert_expected(doctree.pformat(), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(doctree.pformat()), rstrip_lines=True + ) @pytest.mark.param_file(FIXTURE_PATH / "docutil_syntax_extensions.txt") @@ -109,7 +122,9 @@ def test_syntax_extensions(file_params: ParamTestData): parser=Parser(), settings_overrides=settings, ) - file_params.assert_expected(doctree.pformat(), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(doctree.pformat()), rstrip_lines=True + ) def settings_from_cmdline(cmdline: str | None) -> dict[str, Any]: diff --git a/tests/test_renderers/test_fixtures_sphinx.py b/tests/test_renderers/test_fixtures_sphinx.py index 02157dbe..c061febc 100644 --- a/tests/test_renderers/test_fixtures_sphinx.py +++ b/tests/test_renderers/test_fixtures_sphinx.py @@ -17,6 +17,7 @@ from sphinx_pytest.plugin import CreateDoctree from myst_parser.mdit_to_docutils.sphinx_ import SphinxRenderer +from tests.conftest import normalize_doctree_xml FIXTURE_PATH = Path(__file__).parent.joinpath("fixtures") @@ -36,7 +37,7 @@ def _apply_transforms(self): monkeypatch.setattr(SphinxTransformer, "apply_transforms", _apply_transforms) result = sphinx_doctree(file_params.content, "index.md") - pformat = result.pformat("index") + pformat = normalize_doctree_xml(result.pformat("index")) replacements = { # changed in docutils 0.20.1 '<literal classes="code" language="">': '<literal classes="code">', @@ -58,7 +59,7 @@ def test_link_resolution(file_params: ParamTestData, sphinx_doctree: CreateDoctr sphinx_doctree.srcdir.joinpath("test.txt").touch() sphinx_doctree.srcdir.joinpath("other.rst").write_text(":orphan:\n\nTest\n====") result = sphinx_doctree(file_params.content, "index.md") - outcome = result.pformat("index") + outcome = normalize_doctree_xml(result.pformat("index")) if result.warnings.strip(): outcome += "\n\n" + result.warnings.strip() file_params.assert_expected(outcome, rstrip_lines=True) @@ -80,7 +81,9 @@ def settings_from_json(string: str | None): def test_tables(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDoctree): sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) result = sphinx_doctree_no_tr(file_params.content, "index.md") - file_params.assert_expected(result.pformat("index"), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(result.pformat("index")), rstrip_lines=True + ) @pytest.mark.param_file(FIXTURE_PATH / "directive_options.md") @@ -89,7 +92,9 @@ def test_directive_options( ): sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) result = sphinx_doctree_no_tr(file_params.content, "index.md") - file_params.assert_expected(result.pformat("index"), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(result.pformat("index")), rstrip_lines=True + ) @pytest.mark.param_file(FIXTURE_PATH / "sphinx_directives.md") @@ -102,7 +107,9 @@ def test_sphinx_directives( pytest.skip(file_params.title) sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) - pformat = sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index") + pformat = normalize_doctree_xml( + sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index") + ) # see https://github.com/executablebooks/MyST-Parser/issues/522 if sys.maxsize == 2147483647: pformat = pformat.replace('"2147483647"', '"9223372036854775807"') @@ -115,7 +122,9 @@ def test_sphinx_roles(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDo pytest.skip(file_params.title) sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) - pformat = sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index") + pformat = normalize_doctree_xml( + sphinx_doctree_no_tr(file_params.content, "index.md").pformat("index") + ) # sphinx 3 adds a parent key pformat = re.sub('cpp:parent_key="[^"]*"', 'cpp:parent_key=""', pformat) # sphinx >= 4.5.0 adds a trailing slash to PEP URLs, @@ -136,7 +145,9 @@ def test_dollarmath(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDoct {"extensions": ["myst_parser"], "myst_enable_extensions": ["dollarmath"]} ) result = sphinx_doctree_no_tr(file_params.content, "index.md") - file_params.assert_expected(result.pformat("index"), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(result.pformat("index")), rstrip_lines=True + ) @pytest.mark.param_file(FIXTURE_PATH / "amsmath.md") @@ -148,7 +159,9 @@ def test_amsmath( {"extensions": ["myst_parser"], "myst_enable_extensions": ["amsmath"]} ) result = sphinx_doctree_no_tr(file_params.content, "index.md") - file_params.assert_expected(result.pformat("index"), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(result.pformat("index")), rstrip_lines=True + ) @pytest.mark.param_file(FIXTURE_PATH / "containers.md") @@ -160,7 +173,9 @@ def test_containers( {"extensions": ["myst_parser"], "myst_enable_extensions": ["colon_fence"]} ) result = sphinx_doctree_no_tr(file_params.content, "index.md") - file_params.assert_expected(result.pformat("index"), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(result.pformat("index")), rstrip_lines=True + ) @pytest.mark.param_file(FIXTURE_PATH / "eval_rst.md") @@ -169,7 +184,9 @@ def test_evalrst_elements( ): sphinx_doctree_no_tr.set_conf({"extensions": ["myst_parser"]}) result = sphinx_doctree_no_tr(file_params.content, "index.md") - file_params.assert_expected(result.pformat("index"), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(result.pformat("index")), rstrip_lines=True + ) @pytest.mark.param_file(FIXTURE_PATH / "definition_lists.md") @@ -180,7 +197,9 @@ def test_definition_lists( {"extensions": ["myst_parser"], "myst_enable_extensions": ["deflist"]} ) result = sphinx_doctree_no_tr(file_params.content, "index.md") - file_params.assert_expected(result.pformat("index"), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(result.pformat("index")), rstrip_lines=True + ) @pytest.mark.param_file(FIXTURE_PATH / "attributes.md") @@ -192,4 +211,6 @@ def test_attributes(file_params: ParamTestData, sphinx_doctree_no_tr: CreateDoct } ) result = sphinx_doctree_no_tr(file_params.content, "index.md") - file_params.assert_expected(result.pformat("index"), rstrip_lines=True) + file_params.assert_expected( + normalize_doctree_xml(result.pformat("index")), rstrip_lines=True + ) diff --git a/tests/test_renderers/test_include_directive.py b/tests/test_renderers/test_include_directive.py index 51d421e6..e7065747 100644 --- a/tests/test_renderers/test_include_directive.py +++ b/tests/test_renderers/test_include_directive.py @@ -3,6 +3,7 @@ from pathlib import Path import pytest +from conftest import normalize_doctree_xml from docutils.core import publish_doctree from myst_parser.parsers.docutils_ import Parser @@ -24,7 +25,11 @@ def test_render(file_params, tmp_path, monkeypatch): ) doctree["source"] = "tmpdir/test.md" - output = doctree.pformat().replace(str(tmp_path) + os.sep, "tmpdir/").rstrip() + output = ( + normalize_doctree_xml(doctree.pformat()) + .replace(str(tmp_path) + os.sep, "tmpdir/") + .rstrip() + ) file_params.assert_expected(output, rstrip=True) diff --git a/tests/test_renderers/test_myst_config.py b/tests/test_renderers/test_myst_config.py index 8ee5f8db..9a276cce 100644 --- a/tests/test_renderers/test_myst_config.py +++ b/tests/test_renderers/test_myst_config.py @@ -5,6 +5,8 @@ from pathlib import Path import pytest +from conftest import normalize_doctree_xml +from docutils import __version_info__ as docutils_version from docutils.core import Publisher, publish_string from pytest_param_files import ParamTestData @@ -17,6 +19,9 @@ @pytest.mark.param_file(FIXTURE_PATH / "myst-config.txt") def test_cmdline(file_params: ParamTestData): """The description is parsed as a docutils commandline""" + if file_params.title == "attrs_image" and docutils_version < (0, 22): + # loose system messages are also output to ast in 0.22 https://github.com/live-clones/docutils/commit/dc4e16315b4fbe391417a6f7aad215b9389a9c74 + pytest.skip("different in docutils>=0.22") pub = Publisher(parser=Parser()) try: pub.process_command_line(shlex.split(file_params.description or "")) @@ -36,6 +41,7 @@ def test_cmdline(file_params: ParamTestData): writer_name="pseudoxml", settings_overrides=settings, ) + output = normalize_doctree_xml(output) warnings = report_stream.getvalue() if warnings: output += "\n" + warnings diff --git a/tests/test_renderers/test_myst_refs.py b/tests/test_renderers/test_myst_refs.py index 8e853619..fcdfd619 100644 --- a/tests/test_renderers/test_myst_refs.py +++ b/tests/test_renderers/test_myst_refs.py @@ -1,6 +1,7 @@ import sys import pytest +from conftest import normalize_doctree_xml from sphinx.util.console import strip_colors from sphinx_pytest.plugin import CreateDoctree @@ -55,7 +56,7 @@ def test_parse( doctree["source"] = "root/index.md" doctree.attributes.pop("translation_progress", None) - outcome = doctree.pformat() + outcome = normalize_doctree_xml(doctree.pformat()) if result.warnings.strip(): outcome += "\n\n" + strip_colors(result.warnings.strip()) file_regression.check(outcome, basename=test_name, extension=".xml") diff --git a/tests/test_sphinx/conftest.py b/tests/test_sphinx/conftest.py index 83d2c01f..ddace0d6 100644 --- a/tests/test_sphinx/conftest.py +++ b/tests/test_sphinx/conftest.py @@ -41,6 +41,7 @@ def test_basic(app, status, warning, get_sphinx_app_output): from docutils import nodes from myst_parser._compat import findall +from tests.conftest import normalize_doctree_xml SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "sourcedirs")) @@ -125,7 +126,7 @@ def read( node.attributes.pop("translated", None) if regress: - text = doctree.pformat() # type: str + text = normalize_doctree_xml(doctree.pformat()) # type: str for find, rep in (replace or {}).items(): text = text.replace(find, rep) if rstrip_lines: diff --git a/tests/test_sphinx/sourcedirs/includes/index.md b/tests/test_sphinx/sourcedirs/includes/index.md index 9957b6d9..3f3c4cf0 100644 --- a/tests/test_sphinx/sourcedirs/includes/index.md +++ b/tests/test_sphinx/sourcedirs/includes/index.md @@ -14,7 +14,7 @@ ```{include} include_code.py :code: python -:number-lines: 0 +:number-lines: 1 ``` ```{include} include_literal.txt diff --git a/tests/test_sphinx/test_sphinx_builds.py b/tests/test_sphinx/test_sphinx_builds.py index 6ea05b40..0276b247 100644 --- a/tests/test_sphinx/test_sphinx_builds.py +++ b/tests/test_sphinx/test_sphinx_builds.py @@ -13,6 +13,8 @@ import pytest from sphinx.util.console import strip_colors +from tests.conftest import normalize_doctree_xml + SOURCE_DIR = os.path.abspath(os.path.join(os.path.dirname(__file__), "sourcedirs")) @@ -441,7 +443,9 @@ def test_substitutions( try: get_sphinx_app_doctree(app, docname="index", regress=True) file_regression.check( - get_sphinx_app_doctree(app, docname="other").pformat(), + normalize_doctree_xml( + get_sphinx_app_doctree(app, docname="other").pformat() + ), extension=".other.xml", ) finally: diff --git a/tests/test_sphinx/test_sphinx_builds/test_includes.html b/tests/test_sphinx/test_sphinx_builds/test_includes.html index 1a2faee1..251d0e72 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_includes.html +++ b/tests/test_sphinx/test_sphinx_builds/test_includes.html @@ -115,8 +115,8 @@ <h2> </pre> </div> </div> - <pre class="code python literal-block"><small class="ln">0 </small><code data-lineno="0 "><span class="keyword">def</span><span class="whitespace"> </span><span class="name function">a_func</span><span class="punctuation">(</span><span class="name">param</span><span class="punctuation">):</span><span class="whitespace"> -</span></code><small class="ln">1 </small><code data-lineno="1 "><span class="whitespace"></span> <span class="name builtin">print</span><span class="punctuation">(</span><span class="name">param</span><span class="punctuation">)</span></code></pre> + <pre class="code python literal-block"><small class="ln">1 </small><code data-lineno="1 "><span class="keyword">def</span><span class="whitespace"> </span><span class="name function">a_func</span><span class="punctuation">(</span><span class="name">param</span><span class="punctuation">):</span><span class="whitespace"> +</span></code><small class="ln">2 </small><code data-lineno="2 "><span class="whitespace"></span> <span class="name builtin">print</span><span class="punctuation">(</span><span class="name">param</span><span class="punctuation">)</span></code></pre> <div class="highlight-default notranslate"> <div class="highlight"> <pre><span></span><span class="n">This</span> <span class="n">should</span> <span class="n">be</span> <span class="o">*</span><span class="n">literal</span><span class="o">*</span> diff --git a/tests/test_sphinx/test_sphinx_builds/test_includes.xml b/tests/test_sphinx/test_sphinx_builds/test_includes.xml index 8da26cb0..c12a9922 100644 --- a/tests/test_sphinx/test_sphinx_builds/test_includes.xml +++ b/tests/test_sphinx/test_sphinx_builds/test_includes.xml @@ -76,7 +76,7 @@ ) <literal_block classes="code python" source="include_code.py" xml:space="preserve"> <inline classes="ln"> - 0 + 1 <inline classes="keyword"> def <inline classes="whitespace"> @@ -92,7 +92,7 @@ <inline classes="whitespace"> <inline classes="ln"> - 1 + 2 <inline classes="whitespace"> <inline classes="name builtin">