From a24cf324a18fd0cd4fa3b0a602e4bacac3842504 Mon Sep 17 00:00:00 2001 From: TheTripleV Date: Fri, 13 Sep 2024 15:12:15 -0400 Subject: [PATCH 1/2] fix code block args --- source/_extensions/redown.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/source/_extensions/redown.py b/source/_extensions/redown.py index de40c94a51..e872ebd0af 100644 --- a/source/_extensions/redown.py +++ b/source/_extensions/redown.py @@ -60,17 +60,26 @@ def redown(text: str) -> str: def replace(match: re.Match) -> str: start = match.group("start") btindent = match.group("btindent") - lang = match.group("lang") + lang: str = match.group("lang") cindent = match.group("cindent") code = match.group("code") end = match.group("end") ret = "" ret += start - ret += btindent + f".. code-block:: {lang}\n\n" + ret += btindent + f".. code-block:: {lang}\n" cindent = 3 * " " + in_args = True + for line in code.splitlines(keepends=True): + if in_args and re.search(r"^\s*:.+:", line) and not re.match(r"re?st", lang.lower()): + ret += cindent + line + continue + elif in_args: + in_args = False + ret += "\n" + if line.strip() == "": ret += "\n" else: From 8ad2f972b357d5ddfdf4268a9ff5e2393144d382 Mon Sep 17 00:00:00 2001 From: TheTripleV Date: Fri, 13 Sep 2024 20:39:39 -0400 Subject: [PATCH 2/2] chatgpt rewrote the whole thing into single pass --- scripts/doc8_redown.py | 5 +- source/_extensions/redown.py | 213 ++++++++++++++++------------------- 2 files changed, 100 insertions(+), 118 deletions(-) diff --git a/scripts/doc8_redown.py b/scripts/doc8_redown.py index 15fcce7d6d..b639f42a5e 100644 --- a/scripts/doc8_redown.py +++ b/scripts/doc8_redown.py @@ -11,8 +11,9 @@ def new_setattr(self, name, value): if name == "_raw_content" and value is not None: - value = redown(value.decode("utf-8").replace("\r", "")).encode("utf-8") - # Path(self.filename).with_suffix(".rd").write_text(value.decode("utf-8")) + value = redown(value.decode("utf-8").replace("\r", ""))[0].encode("utf-8") + Path(self.filename).with_suffix(".rd").write_text(value.decode("utf-8")) + old_setattr(self, name, value) diff --git a/source/_extensions/redown.py b/source/_extensions/redown.py index e872ebd0af..80b363c6e6 100644 --- a/source/_extensions/redown.py +++ b/source/_extensions/redown.py @@ -10,12 +10,10 @@ """ import re - from pathlib import Path from sphinx.application import Sphinx from dataclasses import dataclass - LINK_CORE = r""" \[ (?P[^\[\]]*?) \] # link brackets + text w/o brackets - allows spaces in text \( @@ -49,127 +47,110 @@ re.VERBOSE, # whitespace and comments are ignored ) - -def redown(text: str) -> str: +def redown(text: str): """21""" - - # replace md code blocks with reST code blocks - "redown, redown, redown, redown" - find = r"(?P^|\n)(?P *?)(?P```+)(?P\S+) *?(?:\n\s*?)+(?P *?)(?P.*?)(?P=ticks) *?(?P\n|$|\Z)" - - def replace(match: re.Match) -> str: - start = match.group("start") - btindent = match.group("btindent") - lang: str = match.group("lang") - cindent = match.group("cindent") - code = match.group("code") - end = match.group("end") - - ret = "" - ret += start - ret += btindent + f".. code-block:: {lang}\n" - cindent = 3 * " " - - in_args = True - - for line in code.splitlines(keepends=True): - if in_args and re.search(r"^\s*:.+:", line) and not re.match(r"re?st", lang.lower()): - ret += cindent + line - continue - elif in_args: - in_args = False - ret += "\n" - - if line.strip() == "": - ret += "\n" + """Transforms markdown-like syntax into reStructuredText and maps input to output line numbers.""" + input_lines = text.splitlines(keepends=True) + output_lines = [] + line_mapping = [] # List of input line numbers for each output line + i = 0 # Input line index + while i < len(input_lines): + line = input_lines[i] + # Check for code block start + code_block_match = re.match(r'^( *)(`{3,})(\S+)?\s*$', line) + if code_block_match: + # Start of code block + btindent = code_block_match.group(1) or '' + lang = code_block_match.group(3) or '' + is_rst: bool = re.search(r"re?st", lang.lower()) is not None + + params_lines = [] + params_input_line_numbers = [] + code_content_lines = [] + code_content_input_line_numbers = [] + i += 1 + while i < len(input_lines) and not re.match(r'^( *)`{3,}\s*$', input_lines[i]): + code_line = input_lines[i] + if not is_rst and not code_content_lines and re.match(r'^\s*:.*:', code_line): + # Parameter line + params_lines.append(code_line) + params_input_line_numbers.append(i) + else: + # Code content line + code_content_lines.append(code_line) + code_content_input_line_numbers.append(i) + i += 1 + if i < len(input_lines): + # Closing ``` + i += 1 + # Now process the code block + code_block_directive = f'{btindent}.. code-block::{" " + lang if lang else ""}\n' + output_lines.append(code_block_directive) + line_mapping.append(i - len(code_content_lines) - len(params_lines) - 1) # Map to the opening ``` + # Add parameter lines immediately after the directive (bug 3) + option_indent = btindent + ' ' + for idx, param_line in enumerate(params_lines): + output_lines.append(option_indent + param_line) + line_mapping.append(params_input_line_numbers[idx]) + # Add a blank line before the code content + output_lines.append('\n') + if params_input_line_numbers: + line_mapping.append(params_input_line_numbers[-1]) # Map to last param line else: - ret += cindent + line - - return ret - - code = lambda: re.sub(find, replace, text, flags=re.DOTALL) - text = code() - - # find rst code block ranges - "redown, redown, redown, redown" - - @dataclass - class Chunk: - is_code: bool - text: str = "" - - chunks: list[Chunk] = [] - - for line in text.splitlines(keepends=True): - in_code_block = chunks and chunks[-1].is_code - is_code_block_directive = re.match(r"^\s*\.\. code-block::", line) - - if not in_code_block and is_code_block_directive: - chunks.append(Chunk(is_code=True)) - elif in_code_block: - indent = len(re.match(r"^(\s*)", line).group(1).rstrip("\n")) - code_block_indent = len( - re.match(r"^(\s*)", chunks[-1].text).group(1).rstrip("\n") - ) - if len(line.strip()) and indent <= code_block_indent: - if is_code_block_directive: - chunks.append(Chunk(is_code=True)) + line_mapping.append(i - len(code_content_lines) - 1) # Map to the opening ``` + # Add code content lines + for idx, code_line in enumerate(code_content_lines): + # Handle bug 2: Don't indent empty lines + if code_line.strip(): + output_lines.append(' ' + code_line) else: - chunks.append(Chunk(is_code=False)) - - if not chunks: - chunks.append(Chunk(is_code=False)) - chunks[-1].text += line # existing block - - # dont operate on code blocks - for chunk in chunks: - if chunk.is_code: - continue - text = chunk.text - - "redown, redown, redown, redown" - heading = lambda prefix, underfix: re.sub( - rf"(^|\n){prefix} +(.+?)(?:$|\n|\Z)", - lambda m: f"{m.group(1)}{m.group(2)}\n{underfix * len(m.group(2))}\n", - text, - ) - - text = heading("#", "=") - text = heading("##", "-") - text = heading("###", "^") - text = heading("####", "~") - - "redown, redown, redown, redown" - role_links = lambda: ROLE_LINK_RE.sub( - lambda m: f"{m.group('role')}`{(t:=m.group('text'))}{' ' if len(t) else ''}<{m.group('link')}>`", - text, - ) - text = role_links() - - "redown, redown, redown, redown" - links = lambda: LINK_RE.sub( - lambda m: f"`{(t:=m.group('text'))}{' ' if len(t) else ''}<{m.group('link')}>`__", - text, - ) - text = links() - - "redown, redown, redown, redown" - math = lambda: re.sub( - r"(\b|\s|^)\$([^$\n]+)\$(\b|\s|[^\w]|$)", r"\1:math:`\2`\3", text - ) - text = math() - - chunk.text = text - - text = "".join(chunk.text for chunk in chunks) - - return text - + output_lines.append('\n') + line_mapping.append(code_content_input_line_numbers[idx]) + # Add a blank line after the code content + if code_content_lines: + output_lines.append('\n') + line_mapping.append(code_content_input_line_numbers[idx]) + else: + # Process headings + heading_match = re.match(r'^(#+) (.+)$', line) + if heading_match: + level = len(heading_match.group(1)) + heading_text = heading_match.group(2).strip() + if level == 1: + underline = '=' * len(heading_text) + elif level == 2: + underline = '-' * len(heading_text) + elif level == 3: + underline = '^' * len(heading_text) + elif level == 4: + underline = '~' * len(heading_text) + else: + underline = '"' * len(heading_text) + output_lines.append(heading_text + '\n') + line_mapping.append(i) + output_lines.append(underline + '\n') + line_mapping.extend([i, i]) # Underline and extra newline map to the same input line + else: + # Process role links, regular links, and inline math + line_processed = ROLE_LINK_RE.sub( + lambda m: f"{m.group('role')}`{(t:=m.group('text'))}{' ' if len(t) else ''}<{m.group('link')}>`", + line) + line_processed = LINK_RE.sub( + lambda m: f"`{(t:=m.group('text'))}{' ' if len(t) else ''}<{m.group('link')}>`__", + line_processed) + line_processed = re.sub( + r"(\b|\s|^)\$([^$\n]+)\$(\b|\s|[^\w]|$)", r"\1:math:`\2`\3", line_processed) + output_lines.append(line_processed) + line_mapping.append(i) + i += 1 + output_text = ''.join(output_lines) + return output_text, line_mapping + "redown, redown, redown, redown" def setup(app: Sphinx): @(lambda breadcrumb: app.connect("source-read", breadcrumb)) def _(app, docname, content): - content[0] = redown(content[0]) + content[0] = redown(content[0])[0] # Path(app.srcdir, docname).with_suffix(".rd").write_text(content[0], encoding="utf8") return {