postprocess: Add TrimTransform and enable postprocessing for lua

c2rust-postprocess/postprocess/definitions/__init__.py

@@ -206,10 +206,87 @@ def walk(node: Node) -> Generator[str]:
     return get_comments_text(walk(tree.root_node))
 
 
+def _plain_comment_prefix(comment: str) -> str | None:
+    """
+    The plain-comment marker that replaces a doc-comment marker,
+    or None if `comment` is not a doc comment.
+    """
+    # Per rustc: ////... and /***...*/ and the empty /**/ are plain comments.
+    if comment.startswith("//!"):
+        return "//"
+    if comment.startswith("///") and not comment.startswith("////"):
+        return "//"
+    if comment.startswith("/*!"):
+        return "/*"
+    if (
+        comment.startswith("/**")
+        and not comment.startswith("/***")
+        and comment != "/**/"
+    ):
+        return "/*"
+    return None
+
+
+def demote_misplaced_doc_comments(code: str) -> str:
+    """
+    Rewrite doc comments as plain comments wherever rustc would reject or
+    ignore them in a function definition fragment. A /// before an expression
+    statement desugars to an attribute on the expression, which is a hard
+    error on stable (E0658, stmt_expr_attributes); elsewhere in a body it is
+    an unused_doc_comments warning. Inner doc comments (//!) are demoted
+    everywhere since the fragment is merged into the middle of a file.
+    """
+    parser = Parser(RUST_LANGUAGE)
+    code_bytes = code.encode()
+    tree = parser.parse(code_bytes)
+
+    replacements = []
+
+    def walk(node: Node) -> None:
+        if node.type in {"line_comment", "block_comment"}:
+            text = code_bytes[node.start_byte : node.end_byte].decode()
+            prefix = _plain_comment_prefix(text)
+            inner_doc = text.startswith(("//!", "/*!"))
+            misplaced = node.parent is None or node.parent.type != "source_file"
+            if prefix is not None and (inner_doc or misplaced):
+                # all doc-comment markers are 3 bytes
+                replacements.append((node.start_byte, node.end_byte, prefix + text[3:]))
+        for child in node.children:
+            walk(child)
+
+    walk(tree.root_node)
+
+    if not replacements:
+        return code
+
+    out = []
+    pos = 0
+    for start, end, new_text in replacements:
+        out.append(code_bytes[pos:start].decode())
+        out.append(new_text)
+        pos = end
+    out.append(code_bytes[pos:].decode())
+    return "".join(out)
+
+
+def rust_parse_has_errors(code: str) -> bool:
+    """True if tree-sitter finds syntax errors in the given Rust code."""
+    parser = Parser(RUST_LANGUAGE)
+    return parser.parse(code.encode()).root_node.has_error
+
+
+def is_comment_token(tok_type: Any) -> bool:
+    """True for real comments; excludes directives and include paths."""
+    return tok_type in Comment and tok_type not in {
+        Comment.Preproc,
+        Comment.PreprocFile,
+    }
+
+
 def get_comments(code: str, lexer: RegexLexer) -> list[str]:
     comments = []
     for tok_type, tok_value in lex(code, lexer):
-        if tok_type in Comment and tok_type != Comment.Preproc:
+        if is_comment_token(tok_type):
             # Keep exactly what appears, including delimiters (//, /* */)
             comments.append(tok_value)
     return comments
@@ -238,6 +315,9 @@ def get_rust_definitions(root_rust_source_file: Path) -> dict[str, str]:
 class CDefinition:
     definition: str
     preprocessed_definition: str | None
+    # 0-based line within `definition` where the decl itself begins; earlier
+    # lines hold preceding comments and other front matter.
+    decl_line: int | None = None
 
     @property
     def effective(self) -> str:
@@ -246,19 +326,6 @@ def effective(self) -> str:
             return self.preprocessed_definition
         return self.definition
 
-    @property
-    def was_changed_by_preprocessing(self) -> bool:
-        return (
-            self.preprocessed_definition is not None
-            and self.preprocessed_definition != self.definition
-        )
-
-    @property
-    def preprocessed_if_changed(self) -> str | None:
-        if self.was_changed_by_preprocessing:
-            return self.preprocessed_definition
-        return None
-
 
 def get_c_definitions(root_rust_source_file: Path) -> dict[str, CDefinition]:
     c_defs_json = root_rust_source_file.with_suffix(".c_decls.json")
@@ -270,6 +337,7 @@ def get_c_definitions(root_rust_source_file: Path) -> dict[str, CDefinition]:
             identifier: CDefinition(
                 definition=decl["definition"],
                 preprocessed_definition=decl.get("preprocessed_definition"),
+                decl_line=decl.get("decl_line"),
             )
             for identifier, decl in c_decls["definitions"].items()
         }

c2rust-postprocess/postprocess/transforms/base.py

@@ -1,47 +1,164 @@
 import logging
 import re
-from abc import ABC, abstractmethod
+from collections.abc import Callable
 from pathlib import Path
+from typing import Any
 
+from postprocess.cache import AbstractCache
 from postprocess.definitions import (
     CDefinition,
     get_c_definitions,
     get_rust_definitions,
+    update_rust_definition,
 )
 from postprocess.exclude_list import IdentifierExcludeList
+from postprocess.models import AbstractGenerativeModel, api_key_from_env
 from postprocess.utils import get_highlighted_c
 
 
 class TransformError(Exception):
     """A transform failed to process a single definition."""
 
 
-class AbstractTransform(ABC):
+class AbstractTransform:
     """
     Abstract base class for LLM-driven transforms of c2rust transpiler output.
     """
 
-    def __init__(self, system_instruction: str):
+    max_attempts = 3
+
+    def __init__(
+        self,
+        system_instruction: str,
+        cache: AbstractCache,
+        model: AbstractGenerativeModel,
+    ):
         self._system_instruction = system_instruction
+        self.cache = cache
+        self.model = model
 
     @property
     def system_instruction(self) -> str:
         return self._system_instruction
 
-    @abstractmethod
     def apply_ident(
         self,
         rust_source_file: Path,
         rust_definition: str,
         c_definition: CDefinition,
         identifier: str,
         update_rust: bool = True,
-    ) -> None:
+    ) -> str | None:
+        """
+        Apply the transform to one Rust definition and commit it through
+        merge_rust.
+        """
+        new_definition = self.try_apply_ident(
+            rust_source_file=rust_source_file,
+            rust_definition=rust_definition,
+            c_definition=c_definition,
+            identifier=identifier,
+        )
+        if new_definition is None:
+            return None
+
+        if new_definition == rust_definition:
+            logging.info(
+                f"{self.__class__.__name__}: No changes for function: {identifier}"
+            )
+            return None
+
+        if update_rust:
+            update_rust_definition(
+                root_rust_source_file=rust_source_file,
+                identifier=identifier,
+                new_definition=new_definition,
+            )
+
+        logging.info(f"{self.__class__.__name__}: Updated Rust fn {identifier}")
+        return new_definition
+
+    def try_apply_ident(
+        self,
+        rust_source_file: Path,
+        rust_definition: str,
+        c_definition: CDefinition,
+        identifier: str,
+    ) -> str | None:
+        """
+        Return a replacement Rust definition for `identifier`,
+        or None to skip it.
         """
-        Implementations should apply transform to a single Rust definition
-        with the given identifier.
+        raise NotImplementedError(
+            f"{self.__class__.__name__} must implement try_apply_ident, "
+            "or override apply_ident directly"
+        )
+
+    def generate(
+        self,
+        identifier: str,
+        messages: list[dict[str, Any]],
+        check: Callable[[str], str],
+    ) -> str | None:
+        """
+        Model call with caching, validation, and retries. `check` returns the
+        validated result or raises TransformError to trigger regeneration.
         """
-        pass
+        transform = self.__class__.__name__
+        error: TransformError | None = None
+
+        response = self.cache.lookup(
+            transform=transform,
+            identifier=identifier,
+            model=self.model.id,
+            messages=messages,
+        )
+        if response is not None:
+            try:
+                return check(response)
+            except TransformError as stale:
+                error = stale
+                logging.warning(
+                    f"{transform}: cached response for {identifier} "
+                    f"failed validation: {stale}"
+                )
+
+        if api_key_from_env(self.model.id) is None:
+            if error is not None:
+                # Can't regenerate the invalid cached response without a key.
+                raise error
+            logging.warning(
+                f"Cache miss for {identifier}; skipping since no API key was set..."
+            )
+            return None
+
+        for attempt in range(self.max_attempts):
+            try:
+                response = self.model.generate_with_tools(messages)
+                if response is None:
+                    raise TransformError(f"model returned no response for {identifier}")
+                result = check(response)
+            except TransformError as rejected:
+                error = rejected
+                logging.warning(
+                    f"{transform}: response for {identifier} rejected on attempt "
+                    f"{attempt + 1}/{self.max_attempts}: {rejected}"
+                )
+                continue
+
+            self.cache.update(
+                transform=transform,
+                identifier=identifier,
+                model=self.model.id,
+                messages=messages,
+                response=response,
+            )
+            return result
+
+        raise TransformError(
+            f"{transform} failed after {self.max_attempts} attempts "
+            f"for {identifier}: {error}"
+        ) from error
 
     def apply_dir(
         self,