[codex] Support raw image refs for multimodal rendering

docs/renderer-config.md

@@ -27,15 +27,15 @@ chat-template kwargs. Those fields are covered by parity tests against
 | --- | --- | --- | --- |
 | Qwen3 | `Qwen3RendererConfig` | `enable_thinking` | - |
 | PrimeIntellect Qwen3 | `PrimeQwen3RendererConfig` | - | - |
-| Qwen3.5 | `Qwen35RendererConfig` | `enable_thinking`, `add_vision_id` | `image_cache_max` |
-| Qwen3.6 | `Qwen36RendererConfig` | `enable_thinking`, `add_vision_id`, `preserve_thinking` | `image_cache_max` |
-| Qwen3-VL | `Qwen3VLRendererConfig` | `add_vision_id` | `image_cache_max` |
+| Qwen3.5 | `Qwen35RendererConfig` | `enable_thinking`, `add_vision_id` | - |
+| Qwen3.6 | `Qwen36RendererConfig` | `enable_thinking`, `add_vision_id`, `preserve_thinking` | - |
+| Qwen3-VL | `Qwen3VLRendererConfig` | `add_vision_id` | - |
 | GLM-5 / 5.1 | `GLM5RendererConfig` / `GLM51RendererConfig` | `enable_thinking`, `clear_thinking` | - |
 | GLM-4.5 | `GLM45RendererConfig` | `enable_thinking` | - |
 | gpt-oss | `GptOssRendererConfig` | `reasoning_effort`, `conversation_start_date` | `use_system_prompt`, `knowledge_cutoff`, `model_identity`, `auto_drop_analysis` |
 | Hy3 | `Hy3RendererConfig` | `reasoning_effort`, `preserved_thinking`, `is_training`, `raw_last_assistant`, `fallback_strategy` | - |
 | Kimi K2 | `KimiK2RendererConfig` | - | `enable_thinking` |
-| Kimi K2.5 / 2.6 | `KimiK25RendererConfig` | `thinking` | `image_cache_max` |
+| Kimi K2.5 / 2.6 | `KimiK25RendererConfig` | `thinking` | - |
 | Laguna XS.2 | `LagunaXS2RendererConfig` | `enable_thinking`, `render_assistant_messages_raw` | - |
 | Laguna XS-2.1 | `LagunaXS21RendererConfig` | `enable_thinking` | - |
 | Llama 3 | `Llama3RendererConfig` | `date_string`, `tools_in_user_message` | - |

pyproject.toml

@@ -38,6 +38,13 @@ dependencies = [
     "prime-pydantic-config>=0.3.0.dev83",
 ]
 
+[project.optional-dependencies]
+# Raw multimodal needs Pillow for dimension / PIL decode. Torch stays a
+# caller/dev concern (processed mode + processor-backed tests).
+vision = [
+    "pillow>=12.2.0",
+]
+
 [tool.hatch.version]
 source = "vcs"
 # Tags look like ``renderers-v0.1.8`` (prefix matches the publish.yml
@@ -76,13 +83,20 @@ version-file = "renderers/_version.py"
 [tool.hatch.build.targets.wheel]
 packages = ["renderers"]
 
+# Explicit so a checkout under prime-rl does not inherit prime-rl's
+# line-length=120. CI runs this repo standalone against ruff defaults (88)
+# via .github/workflows/style.yml — keep that pin and this length in sync.
+[tool.ruff]
+line-length = 88
+required-version = "==0.15.12"
+
 [dependency-groups]
 dev = [
     "pillow>=12.2.0",
     "pre-commit",
     "pytest>=7.0.0",
     "pytest-asyncio>=0.21.0",
-    "ruff",
+    "ruff==0.15.12",
     "torch>=2.11.0",
     "torchvision>=0.26.0",
     "ty>=0.0.1a29,<0.0.22",

renderers/__init__.py

@@ -58,6 +58,7 @@
     LagunaXS21RendererConfig,
     Llama3RendererConfig,
     MiniMaxM2RendererConfig,
+    MultimodalOutput,
     Nemotron3RendererConfig,
     Nemotron3UltraRendererConfig,
     PrimeQwen3RendererConfig,
@@ -156,6 +157,7 @@ def __dir__() -> list[str]:
     "Message",
     "MiniMaxM2Renderer",
     "MiniMaxM2RendererConfig",
+    "MultimodalOutput",
     "MultiModalData",
     "MultimodalRenderer",
     "Nemotron3Renderer",

renderers/base.py

@@ -49,10 +49,11 @@ class ThinkingPart(TypedDict):
 class ImagePart(TypedDict, total=False):
     """An image attached to a message.
 
-    Accepts several source shapes so callers can pass whatever they have
-    on hand — a pre-loaded PIL Image, a filesystem path, a URL, or the
-    OpenAI ``image_url`` content part verbatim. The renderer resolves
-    these to a PIL Image at render time.
+    Part shape may use ``image`` or OpenAI-style ``image_url``. For
+    ``multimodal_output="processed"``, the value must be a local source the
+    renderer can resolve without network I/O: a PIL Image, filesystem path,
+    ``file://`` URL, or ``data:image/...;base64,...`` URI. Raw mode accepts
+    only offloaded ``file://`` assets.
     """
 
     type: Literal["image", "image_url"]
@@ -205,12 +206,10 @@ class PlaceholderRange:
 class MultiModalData:
     """Multimodal sidecar produced alongside the token stream.
 
-    Renderer output is framework-agnostic: ``mm_items[modality][i]`` is a
-    plain ``dict`` mirroring the per-item output of a HuggingFace processor
-    (e.g. ``{"pixel_values": Tensor, "image_grid_thw": Tensor}`` for
-    Qwen3-VL images). Translation to engine-specific wire formats — vLLM's
-    ``MultiModalKwargsItem``, SGLang's payload, etc. — happens in the
-    inference glue layer (see ``renderers.client``).
+    ``mm_items[modality][i]`` follows the renderer's configured
+    ``multimodal_output``. The default ``"raw"`` mode emits JSON-safe image
+    descriptor envelopes for inference paths. ``"processed"`` emits
+    image-processor payloads such as ``pixel_values`` for SFT/training paths.
     """
 
     mm_hashes: dict[str, list[str]] = field(default_factory=dict)
@@ -221,6 +220,42 @@ def is_empty(self) -> bool:
         return not (self.mm_hashes or self.mm_placeholders or self.mm_items)
 
 
+def merge_multi_modal_data(
+    previous: "MultiModalData | None",
+    new_hashes: dict[str, list[str]],
+    new_placeholders: dict[str, list[PlaceholderRange]],
+    new_items: dict[str, list[dict[str, Any]]],
+) -> "MultiModalData | None":
+    """Concatenate a prior turn's sidecar with a bridge turn's new media.
+
+    Bridge callers pass the persisted previous step's sidecar as ``previous``;
+    inner lists are copied so that object is never mutated. Returns ``None``
+    when there is no media at all.
+    """
+    merged_hashes = (
+        {k: list(v) for k, v in previous.mm_hashes.items()} if previous else {}
+    )
+    merged_placeholders = (
+        {k: list(v) for k, v in previous.mm_placeholders.items()} if previous else {}
+    )
+    merged_items = (
+        {k: list(v) for k, v in previous.mm_items.items()} if previous else {}
+    )
+    for modality, hashes in new_hashes.items():
+        merged_hashes.setdefault(modality, []).extend(hashes)
+    for modality, placeholders in new_placeholders.items():
+        merged_placeholders.setdefault(modality, []).extend(placeholders)
+    for modality, items in new_items.items():
+        merged_items.setdefault(modality, []).extend(items)
+    if not (merged_hashes or merged_placeholders or merged_items):
+        return None
+    return MultiModalData(
+        mm_hashes=merged_hashes,
+        mm_placeholders=merged_placeholders,
+        mm_items=merged_items,
+    )
+
+
 @dataclass
 class RenderedTokens:
     """Result of rendering messages to tokens.
@@ -768,8 +803,8 @@ def bridge_to_next_turn(
         Text-only renderers return :class:`RenderedTokens` with
         ``multi_modal_data=None``. Multimodal renderers (see
         :class:`MultimodalRenderer`) populate ``multi_modal_data`` so
-        the caller can recover placeholder offsets + per-item processed
-        tensors for the new full prompt; they also accept a
+        the caller can recover placeholder offsets + per-item image
+        descriptors for the new full prompt; they also accept a
         ``previous_multi_modal_data`` kwarg via the
         :class:`MultimodalRenderer` Protocol override.
 
@@ -829,8 +864,8 @@ def bridge_to_next_turn(
           the combined token sequence and silently falls back to
           hash-cache lookup (or errors)
         - returns :class:`RenderedTokens` (not ``list[int]``) so the
-          caller can recover the placeholder offsets + per-item
-          processed tensors for the new full prompt
+          caller can recover the placeholder offsets + per-item image
+          descriptors for the new full prompt
         """
         ...
 
@@ -1500,7 +1535,7 @@ def _resolve_auto_config(
     model_name = getattr(tokenizer, "name_or_path", "")
     renderer_name = MODEL_RENDERER_MAP.get(model_name)
 
-    preserve_carry = {}
+    preserve_carry: dict[str, Any] = {"multimodal_output": auto.multimodal_output}
     if auto.thinking_retention is not None:
         preserve_carry["thinking_retention"] = auto.thinking_retention
 
@@ -1551,7 +1586,7 @@ def _resolve_auto_config(
         "reasoning_parser=...) to enable structured output parsing.",
         model_name or "<unnamed tokenizer>",
     )
-    return DefaultRendererConfig()
+    return DefaultRendererConfig(multimodal_output=auto.multimodal_output)
 
 
 # ---------------------------------------------------------------------------