diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index dc032741..f188918b 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -5,18 +5,20 @@ on: branches: [main] paths: - "**.py" + - "tests/golden_renderer_outputs.json" - "pyproject.toml" - ".github/workflows/test.yml" pull_request: branches: [main] paths: - "**.py" + - "tests/golden_renderer_outputs.json" - "pyproject.toml" - ".github/workflows/test.yml" jobs: - test: - name: Renderers + unit: + name: Offline unit tests (Python ${{ matrix.python-version }}) runs-on: ubuntu-latest strategy: fail-fast: false @@ -38,4 +40,64 @@ jobs: run: uv sync - name: Run tests - run: uv run pytest tests/ -v + run: uv run pytest tests/ -m "not network" -v + + model-parity: + name: Pinned model parity + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + HF_HUB_DISABLE_PROGRESS_BARS: "1" + RENDERERS_TEST_NETWORK: "1" + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Cache pinned Hugging Face assets + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: hf-renderers-${{ hashFiles('tests/model_assets.py') }} + + - name: Install dependencies + run: uv sync + + - name: Run text model parity tests + run: uv run pytest tests/ -m "network and not multimodal" -v + + multimodal-parity: + name: Pinned multimodal parity + runs-on: ubuntu-latest + timeout-minutes: 45 + env: + HF_HUB_DISABLE_PROGRESS_BARS: "1" + RENDERERS_TEST_NETWORK: "1" + steps: + - uses: actions/checkout@v4 + + - name: Set up Python 3.13 + uses: actions/setup-python@v6 + with: + python-version: "3.13" + + - name: Install uv + uses: astral-sh/setup-uv@v7 + + - name: Cache pinned Hugging Face assets + uses: actions/cache@v4 + with: + path: ~/.cache/huggingface + key: hf-renderers-${{ hashFiles('tests/model_assets.py') }} + + - name: Install dependencies + run: uv sync + + - name: Run multimodal parity tests + run: uv run pytest tests/ -m "multimodal" -v diff --git a/README.md b/README.md index 383c77a9..c1cfbdb8 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,19 @@ parsed = r.parse_response(completion_ids) For the next turn, extend the previous sampled stream instead of re-rendering history: ```python -next_prompt_ids = r.bridge_to_next_turn( +bridged = r.bridge_to_next_turn( previous_prompt_ids=prompt_ids, previous_completion_ids=completion_ids, new_messages=[{"role": "tool", "content": "..."}], ) +if bridged is None: + # The renderer could not prove that preserving the sampled prefix is safe. + # Fall back to a full render of the conversation in that case. + ... +next_prompt_ids = bridged.token_ids ``` -Hand-coded renderers ship for `qwen3`, `qwen3-vl`, `qwen3.5`, `qwen3.6`, `glm-5`, `glm-5.1`, `glm-4.5`, `minimax-m2`, `deepseek-v3`, `deepseek-r1`, `kimi-k2`, `kimi-k2.5` / `kimi-k2.6`, `nemotron-3`, `nemotron-3-ultra`, `llama-3`, `gpt-oss`, `hy3`, and `prime-qwen3`. Anything else falls back to `DefaultRenderer`, a generic `apply_chat_template` wrapper. +Hand-coded renderers ship for `qwen3`, `qwen3-vl`, `qwen3.5`, `qwen3.6`, `glm-5`, `glm-5.1`, `glm-4.5`, `minimax-m2`, `deepseek-v3`, `deepseek-r1`, `kimi-k2`, `kimi-k2.5` / `kimi-k2.6`, `laguna-xs.2`, `laguna-xs-2.1`, `nemotron-3`, `nemotron-3-ultra`, `llama-3`, `gpt-oss`, `hy3`, and `prime-qwen3`. Anything else falls back to `DefaultRenderer`, a generic `apply_chat_template` wrapper. ## API @@ -50,7 +55,7 @@ class Renderer(Protocol): def render_ids(messages, *, tools=None, add_generation_prompt=False) -> list[int]: ... def parse_response(token_ids) -> ParsedResponse: ... def get_stop_token_ids() -> list[int]: ... - def bridge_to_next_turn(prev_prompt_ids, prev_completion_ids, new_messages, *, tools=None) -> list[int] | None: ... + def bridge_to_next_turn(prev_prompt_ids, prev_completion_ids, new_messages, *, tools=None) -> RenderedTokens | None: ... ``` - `RenderedTokens` carries `token_ids` **and** `message_indices` — one entry per token attributing each to its source message (`-1` for structural scaffolding). Lets `build_training_sample` build a per-token loss mask in one render. @@ -59,7 +64,7 @@ class Renderer(Protocol): ### `bridge_to_next_turn` (the core contract) -Given `(prev_prompt_ids, prev_completion_ids)` and new environment messages, return ids for the next turn's prompt such that the result starts with `prev_prompt_ids + prev_completion_ids` byte-for-byte and continues with the new messages plus the next assistant opener. If that cannot be proven safe, return `None` and the caller falls back to a full render. +Given `(prev_prompt_ids, prev_completion_ids)` and new environment messages, return a `RenderedTokens` object for the next turn's prompt whose `token_ids` start with `prev_prompt_ids + prev_completion_ids` byte-for-byte and continue with the new messages plus the next assistant opener. If that cannot be proven safe, return `None` and the caller falls back to a full render. Attribution in a bridge result is relative to `new_messages`; the preserved prefix uses `message_indices=-1` because only its raw token IDs are available. Each hand-coded bridge: 1. Anchors at the previous turn's canonical close token. On clean stops it's already in `prev_completion_ids`. On truncation, the renderer synthesizes the close as non-loss prompt context. diff --git a/examples/README.md b/examples/README.md index 08d79a67..e7af7512 100644 --- a/examples/README.md +++ b/examples/README.md @@ -86,5 +86,9 @@ Each script runs `Qwen/Qwen3.5-4B` with `enable_thinking=True` and `False`, then ## Multimodal Note -Renderers are text-only today. For image/video demos, use the backend's message -or prompt path until renderers grow multimodal placeholder support. +Image rendering is supported for Qwen3-VL, Qwen3.5 / Qwen3.6, and Kimi K2.5 / +K2.6. Their renderers return token IDs plus a framework-agnostic +`multi_modal_data` sidecar containing placeholder ranges and processed image +features. `renderers.client` can serialize the Qwen-VL family sidecar for +vLLM's token-in endpoint; other backends need their own adapter. Video content +parts are represented by the public types but are not yet rendered. diff --git a/pyproject.toml b/pyproject.toml index dc753970..9c5e3f44 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -117,3 +117,10 @@ invalid-key = "warn" unsupported-operator = "warn" invalid-return-type = "warn" invalid-assignment = "warn" + +[tool.pytest.ini_options] +markers = [ + "network: requires pinned external model assets from Hugging Face", + "model_parity: compares a renderer with a pinned upstream tokenizer or reference encoder", + "multimodal: exercises image/video processors and multimodal sidecars", +] diff --git a/renderers/base.py b/renderers/base.py index e8c9f228..9ea02a3c 100644 --- a/renderers/base.py +++ b/renderers/base.py @@ -1190,11 +1190,22 @@ def _tokenizer_source_for(model_name_or_path: str) -> str: return TOKENIZER_SOURCE_OVERRIDES.get(model_name_or_path, model_name_or_path) -def _tokenizer_load_kwargs(model_name_or_path: str) -> dict[str, Any]: - revision = TRUSTED_REVISIONS.get(model_name_or_path) +def _tokenizer_load_kwargs( + model_name_or_path: str, *, revision: str | None = None +) -> dict[str, Any]: + trusted_revision = TRUSTED_REVISIONS.get(model_name_or_path) + if trusted_revision is not None: + if revision is not None and revision != trusted_revision: + raise ValueError( + f"{model_name_or_path!r} executes trusted remote tokenizer code " + f"only at reviewed revision {trusted_revision}; received " + f"revision={revision!r}." + ) + return {"trust_remote_code": True, "revision": trusted_revision} + kwargs: dict[str, Any] = {"trust_remote_code": False} if revision is not None: - return {"trust_remote_code": True, "revision": revision} - return {"trust_remote_code": False} + kwargs["revision"] = revision + return kwargs def _preserve_requested_tokenizer_name( @@ -1281,13 +1292,16 @@ def _load_tokenizer_via_auto(model_name_or_path: str, **kwargs) -> Any: return tok -def load_tokenizer(model_name_or_path: str): +def load_tokenizer(model_name_or_path: str, *, revision: str | None = None): """Load a tokenizer with the renderers-package security policy. Default ``trust_remote_code=False``. Models listed in ``TRUSTED_REVISIONS`` (Moonshot Kimi-K2 family) load with ``trust_remote_code=True`` AND a pinned ``revision=`` so transformers only executes the reviewed commit's tokenizer Python. + Callers may pin ``revision`` for repositories that do not execute remote + code (for example reproducible parity tests). A caller-supplied revision + may not override a reviewed ``TRUSTED_REVISIONS`` entry. ``AutoTokenizer.from_pretrained`` eagerly builds the model config to resolve the tokenizer class. If that construction raises on a @@ -1302,7 +1316,7 @@ def load_tokenizer(model_name_or_path: str): the requested Meta ID so auto-resolution still selects ``Llama3Renderer``. """ load_name_or_path = _tokenizer_source_for(model_name_or_path) - kwargs = _tokenizer_load_kwargs(load_name_or_path) + kwargs = _tokenizer_load_kwargs(load_name_or_path, revision=revision) tok = _load_tokenizer_via_auto(load_name_or_path, **kwargs) return _preserve_requested_tokenizer_name( tok, diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 00000000..ba183c5b --- /dev/null +++ b/tests/__init__.py @@ -0,0 +1 @@ +"""Test support package for renderers.""" diff --git a/tests/conftest.py b/tests/conftest.py index bd7672eb..001f6a54 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -5,11 +5,62 @@ """ import os +from pathlib import Path import pytest from renderers import create_renderer -from renderers.base import load_tokenizer from renderers.configs import config_from_name +from tests.model_assets import load_test_tokenizer + + +# Tests in these modules exercise real, pinned model assets. Keeping the tier +# declaration here avoids hundreds of repeated decorators on parameterized +# cases while still making ``pytest -m 'not network'`` a complete offline +# suite. Mixed unit/network modules use per-test decorators instead. +_MODEL_ASSET_TEST_MODULES = frozenset( + { + "test_bridge", + "test_build_helpers", + "test_deepseek_r1", + "test_disabled_thinking_stability", + "test_glm_tool_name_validation", + "test_golden_renderer_outputs", + "test_gpt_oss_harmony_parity", + "test_hy3", + "test_is_content", + "test_laguna_xs21", + "test_llama_3", + "test_message_indices", + "test_message_tool_names", + "test_multimodal", + "test_nemotron3_parity", + "test_parse_response", + "test_parse_response_robustness", + "test_parsers", + "test_preserve_thinking", + "test_prime_qwen3_parity", + "test_qwen35_size_coverage", + "test_render_ids", + "test_renderer_config_parity", + "test_roundtrip", + "test_sampled_mask", + "test_tokens_per_message", + "test_tool_arg_type_preservation", + } +) + + +def pytest_collection_modifyitems(items): + """Assign network/parity tiers without changing test parametrization.""" + for item in items: + module_name = Path(str(item.path)).stem + if module_name not in _MODEL_ASSET_TEST_MODULES: + continue + item.add_marker(pytest.mark.network) + item.add_marker(pytest.mark.model_parity) + if module_name == "test_multimodal": + item.add_marker(pytest.mark.multimodal) + # (HuggingFace model name, renderer name or "auto") # @@ -65,7 +116,7 @@ def _load(model_name: str, renderer_name: str): key = f"{model_name}:{renderer_name}" if key not in _cache: - tokenizer = load_tokenizer(model_name) + tokenizer = load_test_tokenizer(model_name) renderer = create_renderer(tokenizer, config_from_name(renderer_name)) _cache[key] = (tokenizer, renderer) return _cache[key] diff --git a/tests/generate_renderer_goldens.py b/tests/generate_renderer_goldens.py new file mode 100644 index 00000000..8d563535 --- /dev/null +++ b/tests/generate_renderer_goldens.py @@ -0,0 +1,33 @@ +"""Regenerate the checked-in renderer behavior corpus. + +Run from the repository root: + + uv run python -m tests.generate_renderer_goldens +""" + +from __future__ import annotations + +import json +from pathlib import Path + +from tests.golden_corpus import GOLDEN_CASES, build_golden_case + + +OUTPUT_PATH = Path(__file__).with_name("golden_renderer_outputs.json") + + +def main() -> None: + cases = {} + for case in GOLDEN_CASES: + print(f"rendering {case.slug} ({case.model_name})", flush=True) + cases[case.slug] = build_golden_case(case) + payload = {"schema_version": 1, "cases": cases} + OUTPUT_PATH.write_text( + json.dumps(payload, indent=2, ensure_ascii=False) + "\n", + encoding="utf-8", + ) + print(f"wrote {OUTPUT_PATH}", flush=True) + + +if __name__ == "__main__": + main() diff --git a/tests/golden_corpus.py b/tests/golden_corpus.py new file mode 100644 index 00000000..27e59750 --- /dev/null +++ b/tests/golden_corpus.py @@ -0,0 +1,270 @@ +"""Deterministic, reviewable behavior snapshots for every renderer. + +The corpus keeps rendered text in plain sight so formatting changes are easy +to review, while a token-id digest catches tokenizer-level changes that decode +to the same text. Attribution arrays are run-length encoded to keep the JSON +compact. Upstream tokenizer inputs are pinned separately in ``model_assets``. +""" + +from __future__ import annotations + +import hashlib +import json +from dataclasses import asdict, dataclass +from typing import Any, cast + +from renderers import RendererConfig, RenderedTokens, config_from_name, create_renderer +from renderers.base import ParsedResponse, trim_to_turn_close +from tests.model_assets import load_test_tokenizer, model_revision + + +@dataclass(frozen=True) +class GoldenCase: + slug: str + renderer_name: str + model_name: str + config_overrides: tuple[tuple[str, Any], ...] = () + + +# One representative for every concrete entry in RENDERER_REGISTRY. Keep the +# explicit renderer name even when auto-resolution would choose the same class: +# the corpus is a registry contract, not a model-routing test. +GOLDEN_CASES = ( + GoldenCase("default", "default", "Qwen/Qwen2.5-0.5B-Instruct"), + GoldenCase("qwen3", "qwen3", "Qwen/Qwen3-8B"), + GoldenCase("prime-qwen3", "prime-qwen3", "PrimeIntellect/Qwen3-0.6B"), + GoldenCase("qwen3-vl", "qwen3-vl", "Qwen/Qwen3-VL-4B-Instruct"), + GoldenCase("qwen3.5", "qwen3.5", "Qwen/Qwen3.5-9B"), + GoldenCase("qwen3.6", "qwen3.6", "Qwen/Qwen3.6-35B-A3B"), + GoldenCase("glm-5", "glm-5", "zai-org/GLM-5"), + GoldenCase("glm-5.1", "glm-5.1", "zai-org/GLM-5.1"), + GoldenCase("glm-4.5", "glm-4.5", "THUDM/GLM-4.5-Air"), + GoldenCase("minimax-m2", "minimax-m2", "MiniMaxAI/MiniMax-M2.5"), + GoldenCase("deepseek-v3", "deepseek-v3", "deepseek-ai/DeepSeek-V3"), + GoldenCase("deepseek-r1", "deepseek-r1", "deepseek-ai/DeepSeek-R1"), + GoldenCase("hy3", "hy3", "tencent/Hy3"), + GoldenCase("kimi-k2", "kimi-k2", "moonshotai/Kimi-K2-Instruct"), + GoldenCase("kimi-k2.5", "kimi-k2.5", "moonshotai/Kimi-K2.5"), + GoldenCase("laguna-xs.2", "laguna-xs.2", "poolside/Laguna-XS.2"), + GoldenCase("laguna-xs-2.1", "laguna-xs-2.1", "poolside/Laguna-XS-2.1"), + GoldenCase("llama-3", "llama-3", "meta-llama/Llama-3.2-1B-Instruct"), + GoldenCase( + "nemotron-3", + "nemotron-3", + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", + ), + GoldenCase( + "nemotron-3-ultra", + "nemotron-3-ultra", + "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", + ), + GoldenCase( + "gpt-oss", + "gpt-oss", + "openai/gpt-oss-20b", + (("conversation_start_date", "2025-01-15"),), + ), +) + + +SYSTEM_AND_USER = [ + {"role": "system", "content": "You are concise."}, + {"role": "user", "content": "What is 2+2?"}, +] + +ASSISTANT = { + "role": "assistant", + "reasoning_content": "Two plus two equals four.", + "content": "Four.", +} + +TOOLS = [ + { + "type": "function", + "function": { + "name": "get_weather", + "description": "Return the weather for a city.", + "parameters": { + "type": "object", + "properties": {"city": {"type": "string"}}, + "required": ["city"], + }, + }, + } +] + +# Some renderers expose a template knob that must agree with an explicit +# ``thinking_retention="all"`` bridge policy. Keep the bridge probe on valid, +# user-constructible configs instead of bypassing Pydantic validation. +_BRIDGE_CONFIG_OVERRIDES: dict[str, dict[str, Any]] = { + "qwen3.6": {"preserve_thinking": True}, + "glm-5": {"clear_thinking": False}, + "glm-5.1": {"clear_thinking": False}, + "hy3": {"preserved_thinking": True}, + "nemotron-3": {"truncate_history_thinking": False}, + "nemotron-3-ultra": {"truncate_history_thinking": False}, + "gpt-oss": {"auto_drop_analysis": False}, +} + + +def _sha256_ids(token_ids: list[int]) -> str: + encoded = json.dumps(token_ids, separators=(",", ":")).encode() + return hashlib.sha256(encoded).hexdigest() + + +def _runs(values: list[Any]) -> list[list[Any]]: + if not values: + return [] + out: list[list[Any]] = [] + value = values[0] + count = 1 + for current in values[1:]: + if current == value: + count += 1 + continue + out.append([value, count]) + value = current + count = 1 + out.append([value, count]) + return out + + +def _decode(tokenizer, token_ids: list[int]) -> str: + return tokenizer.decode( + token_ids, + skip_special_tokens=False, + clean_up_tokenization_spaces=False, + ) + + +def _token_snapshot(tokenizer, token_ids: list[int]) -> dict[str, Any]: + return { + "text": _decode(tokenizer, token_ids), + "token_count": len(token_ids), + "token_ids_sha256": _sha256_ids(token_ids), + } + + +def _render_snapshot(tokenizer, rendered: RenderedTokens) -> dict[str, Any]: + return { + **_token_snapshot(tokenizer, rendered.token_ids), + "message_indices_rle": _runs(rendered.message_indices), + "sampled_mask_rle": _runs(rendered.sampled_mask), + "is_content_rle": _runs(rendered.is_content), + "message_roles": rendered.message_roles, + "message_tool_names": rendered.message_tool_names, + } + + +def _parsed_snapshot(parsed: ParsedResponse) -> dict[str, Any]: + return { + "content": parsed.content, + "reasoning_content": parsed.reasoning_content, + "tool_calls": [asdict(tool_call) for tool_call in parsed.tool_calls], + } + + +def _renderer_for(case: GoldenCase, tokenizer, *, bridge: bool = False): + config = config_from_name(case.renderer_name) + assert config is not None + config = cast(RendererConfig, config) + config_data = config.model_dump(mode="python") + config_data.update(case.config_overrides) + if bridge and case.renderer_name != "default": + config_data["thinking_retention"] = "all" + config_data.update(_BRIDGE_CONFIG_OVERRIDES.get(case.renderer_name, {})) + config = cast(RendererConfig, type(config).model_validate(config_data)) + return create_renderer(tokenizer, config) + + +def _completion_ids( + case: GoldenCase, renderer, generation_prompt: RenderedTokens +) -> tuple[RenderedTokens, list[int]]: + completed = renderer.render(SYSTEM_AND_USER + [ASSISTANT], tools=TOOLS) + if case.renderer_name == "gpt-oss": + # Harmony's parser consumes channel blocks beginning with <|start|>. + # A historical final answer is the smallest deterministic render that + # carries that header; sampled-mask extraction deliberately omits it. + history = renderer.render(SYSTEM_AND_USER, tools=TOOLS) + completion_ids = completed.token_ids[len(history.token_ids) :] + elif completed.sampled_mask: + completion_ids = [ + token_id + for token_id, sampled in zip( + completed.token_ids, completed.sampled_mask, strict=True + ) + if sampled + ] + else: + prefix_length = len(generation_prompt.token_ids) + if completed.token_ids[:prefix_length] != generation_prompt.token_ids: + raise AssertionError( + "opaque renderer's completed turn does not extend its generation prompt" + ) + completion_ids = completed.token_ids[prefix_length:] + return completed, completion_ids + + +def _bridge_snapshot(case: GoldenCase, tokenizer) -> dict[str, Any] | None: + if case.renderer_name == "default": + return None + + renderer = _renderer_for(case, tokenizer, bridge=True) + previous_prompt = renderer.render( + SYSTEM_AND_USER, add_generation_prompt=True + ).token_ids + previous_turn = renderer.render( + SYSTEM_AND_USER + [{"role": "assistant", "content": "Prior answer."}] + ).token_ids + previous_completion = previous_turn[len(previous_prompt) :] + previous = trim_to_turn_close( + previous_prompt, + previous_completion, + set(renderer.get_stop_token_ids()), + ) + if previous is not None: + previous_completion = previous[len(previous_prompt) :] + + bridged = renderer.bridge_to_next_turn( + previous_prompt, + previous_completion, + [{"role": "user", "content": "And 3+3?"}], + ) + if bridged is None: + raise AssertionError(f"{case.slug}: hand-coded renderer declined clean bridge") + + prior_length = len(previous_prompt) + len(previous_completion) + prior = previous_prompt + previous_completion + if bridged.token_ids[:prior_length] != prior: + raise AssertionError(f"{case.slug}: bridge did not preserve its token prefix") + snapshot = _render_snapshot(tokenizer, bridged) + snapshot["config"] = renderer.config.model_dump(mode="json") + snapshot["extension"] = _token_snapshot(tokenizer, bridged.token_ids[prior_length:]) + return snapshot + + +def build_golden_case(case: GoldenCase) -> dict[str, Any]: + """Render all public behavior probes for one registered renderer.""" + tokenizer = load_test_tokenizer(case.model_name) + renderer = _renderer_for(case, tokenizer) + + generation_prompt = renderer.render( + SYSTEM_AND_USER, + tools=TOOLS, + add_generation_prompt=True, + ) + completed, completion_ids = _completion_ids(case, renderer, generation_prompt) + parsed = renderer.parse_response(completion_ids, tools=TOOLS) + + return { + "renderer": case.renderer_name, + "renderer_class": type(renderer).__name__, + "model": case.model_name, + "model_revision": model_revision(case.model_name), + "config": renderer.config.model_dump(mode="json"), + "generation_prompt": _render_snapshot(tokenizer, generation_prompt), + "completed_turn": _render_snapshot(tokenizer, completed), + "parser_input": _token_snapshot(tokenizer, completion_ids), + "parsed_completion": _parsed_snapshot(parsed), + "bridge": _bridge_snapshot(case, tokenizer), + } diff --git a/tests/golden_renderer_outputs.json b/tests/golden_renderer_outputs.json new file mode 100644 index 00000000..860f2d22 --- /dev/null +++ b/tests/golden_renderer_outputs.json @@ -0,0 +1,4242 @@ +{ + "schema_version": 1, + "cases": { + "default": { + "renderer": "default", + "renderer_class": "DefaultRenderer", + "model": "Qwen/Qwen2.5-0.5B-Instruct", + "model_revision": "7ae557604adf67be50417f59c2c2f167def9a775", + "config": { + "thinking_retention": null, + "name": "default", + "tool_parser": null, + "reasoning_parser": null + }, + "generation_prompt": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 158, + "token_ids_sha256": "080aab54d0ee370f49c843eae30a493eabc252eca32ef5ecc0b3b1ddc72ae500", + "message_indices_rle": [ + [ + 0, + 143 + ], + [ + 1, + 12 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [], + "is_content_rle": [], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\nFour.<|im_end|>\n", + "token_count": 162, + "token_ids_sha256": "7e2438ca9883ff87c221f58e1c0d88b43689ffc815ffc8cdbbcde7315dc518fb", + "message_indices_rle": [ + [ + 0, + 143 + ], + [ + 1, + 12 + ], + [ + 2, + 7 + ] + ], + "sampled_mask_rle": [], + "is_content_rle": [], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Four.<|im_end|>\n", + "token_count": 4, + "token_ids_sha256": "bedeeb025a9ae6fd87206808a83f920932b43f0a59fdf5f19aac9c0167031836" + }, + "parsed_completion": { + "content": "Four.\n", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": null + }, + "qwen3": { + "renderer": "qwen3", + "renderer_class": "Qwen3Renderer", + "model": "Qwen/Qwen3-8B", + "model_revision": "b968826d9c46dd6066d109eabc6255188de91218", + "config": { + "thinking_retention": null, + "name": "qwen3", + "enable_thinking": true + }, + "generation_prompt": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 158, + "token_ids_sha256": "080aab54d0ee370f49c843eae30a493eabc252eca32ef5ecc0b3b1ddc72ae500", + "message_indices_rle": [ + [ + 0, + 143 + ], + [ + 1, + 12 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 158 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 139 + ], + [ + true, + 7 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\nTwo plus two equals four.\n\n\nFour.<|im_end|>\n", + "token_count": 172, + "token_ids_sha256": "0a3b9ecf2923febbe4f40a14cfdf1b293e7b2e3073230ddbf5e1f00914883109", + "message_indices_rle": [ + [ + 0, + 143 + ], + [ + 1, + 12 + ], + [ + 2, + 17 + ] + ], + "sampled_mask_rle": [ + [ + false, + 158 + ], + [ + true, + 13 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 139 + ], + [ + true, + 7 + ], + [ + false, + 5 + ], + [ + true, + 13 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "\nTwo plus two equals four.\n\n\nFour.<|im_end|>", + "token_count": 13, + "token_ids_sha256": "ae59d3cb1f34a68b0109b6092c9e3b783bab325bad5fe7bd1a0aa38d78f2dba0" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "<|im_start|>system\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\n\n\n\nPrior answer.<|im_end|>\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 47, + "token_ids_sha256": "ae61a84d8b9ad2ce6d57335f11d3b25ce73936e2abd5e4fc6ed4a75b03c2747c", + "message_indices_rle": [ + [ + -1, + 33 + ], + [ + 0, + 11 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 47 + ] + ], + "is_content_rle": [ + [ + false, + 36 + ], + [ + true, + 6 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "qwen3", + "enable_thinking": true + }, + "extension": { + "text": "\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 15, + "token_ids_sha256": "cf39a84fd7c4c25c0d96678c623ef14190fed0e32995be6cfffdd29f97846b6f" + } + } + }, + "prime-qwen3": { + "renderer": "prime-qwen3", + "renderer_class": "PrimeQwen3Renderer", + "model": "PrimeIntellect/Qwen3-0.6B", + "model_revision": "15aba67c8ec68aeac96f57ba7e31d373564ee03c", + "config": { + "thinking_retention": null, + "name": "prime-qwen3" + }, + "generation_prompt": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou have access to the following functions:\n\n\n\nget_weather\nReturn the weather for a city.\n\n\ncity\nstring\n\n[\"city\"]\n\n\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 269, + "token_ids_sha256": "afa9279d4e2e2fd6567565dbea9ff22776b481affba76a5f897a477f1c4f4eeb", + "message_indices_rle": [ + [ + 0, + 254 + ], + [ + 1, + 12 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 269 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 250 + ], + [ + true, + 7 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou have access to the following functions:\n\n\n\nget_weather\nReturn the weather for a city.\n\n\ncity\nstring\n\n[\"city\"]\n\n\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\nTwo plus two equals four.\nFour.<|im_end|>\n", + "token_count": 282, + "token_ids_sha256": "6748ef4a8b646af9a799f6abe26d77a2bf0b37d4d7bffae4f4274a06378516ed", + "message_indices_rle": [ + [ + 0, + 254 + ], + [ + 1, + 12 + ], + [ + 2, + 16 + ] + ], + "sampled_mask_rle": [ + [ + false, + 269 + ], + [ + true, + 12 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 250 + ], + [ + true, + 7 + ], + [ + false, + 5 + ], + [ + true, + 12 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Two plus two equals four.\nFour.<|im_end|>", + "token_count": 12, + "token_ids_sha256": "3a82be8a37c4ce8d9789661a62aa39228fca099cff9e9d938f2d24859a6c0ddb" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "<|im_start|>system\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\nPrior answer.<|im_end|>\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 43, + "token_ids_sha256": "371d225bc816e7bf37ce3a240f424d93c8e42414c2822fb06b2765b82d72f636", + "message_indices_rle": [ + [ + -1, + 29 + ], + [ + 0, + 11 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 43 + ] + ], + "is_content_rle": [ + [ + false, + 32 + ], + [ + true, + 6 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "prime-qwen3" + }, + "extension": { + "text": "\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 15, + "token_ids_sha256": "cf39a84fd7c4c25c0d96678c623ef14190fed0e32995be6cfffdd29f97846b6f" + } + } + }, + "qwen3-vl": { + "renderer": "qwen3-vl", + "renderer_class": "Qwen3VLRenderer", + "model": "Qwen/Qwen3-VL-4B-Instruct", + "model_revision": "ebb281ec70b05090aa6165b016eac8ec08e71b17", + "config": { + "thinking_retention": null, + "name": "qwen3-vl", + "add_vision_id": false, + "image_cache_max": 256 + }, + "generation_prompt": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 158, + "token_ids_sha256": "080aab54d0ee370f49c843eae30a493eabc252eca32ef5ecc0b3b1ddc72ae500", + "message_indices_rle": [ + [ + 0, + 143 + ], + [ + 1, + 12 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 158 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 139 + ], + [ + true, + 7 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, return a json object with function name and arguments within XML tags:\n\n{\"name\": , \"arguments\": }\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\nFour.<|im_end|>\n", + "token_count": 162, + "token_ids_sha256": "7e2438ca9883ff87c221f58e1c0d88b43689ffc815ffc8cdbbcde7315dc518fb", + "message_indices_rle": [ + [ + 0, + 143 + ], + [ + 1, + 12 + ], + [ + 2, + 7 + ] + ], + "sampled_mask_rle": [ + [ + false, + 158 + ], + [ + true, + 3 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 139 + ], + [ + true, + 7 + ], + [ + false, + 5 + ], + [ + true, + 3 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Four.<|im_end|>", + "token_count": 3, + "token_ids_sha256": "8c9c840f115008c23f98f09a0ae2d0cf23c17af40eff7e0dc2001f01cb8cf14b" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": { + "text": "<|im_start|>system\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\nPrior answer.<|im_end|>\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 43, + "token_ids_sha256": "371d225bc816e7bf37ce3a240f424d93c8e42414c2822fb06b2765b82d72f636", + "message_indices_rle": [ + [ + -1, + 29 + ], + [ + 0, + 11 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 43 + ] + ], + "is_content_rle": [ + [ + false, + 32 + ], + [ + true, + 6 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "qwen3-vl", + "add_vision_id": false, + "image_cache_max": 256 + }, + "extension": { + "text": "\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n", + "token_count": 15, + "token_ids_sha256": "cf39a84fd7c4c25c0d96678c623ef14190fed0e32995be6cfffdd29f97846b6f" + } + } + }, + "qwen3.5": { + "renderer": "qwen3.5", + "renderer_class": "Qwen35Renderer", + "model": "Qwen/Qwen3.5-9B", + "model_revision": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", + "config": { + "thinking_retention": null, + "name": "qwen3.5", + "enable_thinking": true, + "add_vision_id": false, + "image_cache_max": 256 + }, + "generation_prompt": { + "text": "<|im_start|>system\n# Tools\n\nYou have access to the following functions:\n\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n\n\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 280, + "token_ids_sha256": "adbac690e6768c6f1314ddd82e2e4c53d4e1fb3d3edc56c74b0e8145434d91b8", + "message_indices_rle": [ + [ + 0, + 263 + ], + [ + 1, + 12 + ], + [ + -1, + 5 + ] + ], + "sampled_mask_rle": [ + [ + false, + 280 + ] + ], + "is_content_rle": [ + [ + false, + 257 + ], + [ + true, + 4 + ], + [ + false, + 5 + ], + [ + true, + 7 + ], + [ + false, + 7 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_start|>system\n# Tools\n\nYou have access to the following functions:\n\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n\n\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\nTwo plus two equals four.\n\n\nFour.<|im_end|>\n", + "token_count": 293, + "token_ids_sha256": "acb2513c56e51c539e4efce8245a3f761bdee03fe6b8e3909fcd42b9f6405a4d", + "message_indices_rle": [ + [ + 0, + 263 + ], + [ + 1, + 12 + ], + [ + 2, + 18 + ] + ], + "sampled_mask_rle": [ + [ + false, + 278 + ], + [ + true, + 14 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 257 + ], + [ + true, + 4 + ], + [ + false, + 5 + ], + [ + true, + 7 + ], + [ + false, + 5 + ], + [ + true, + 14 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "\nTwo plus two equals four.\n\n\nFour.<|im_end|>", + "token_count": 14, + "token_ids_sha256": "6de6a897f24e53a8431bd456c34631afbfac74dc9120d2641b0a191f0c4bc107" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "<|im_start|>system\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\n\n\nPrior answer.<|im_end|>\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 49, + "token_ids_sha256": "51a6866b82c62fcc15859f3654d7c5faeb3af6dae6a9b244ee4afbabd72c0739", + "message_indices_rle": [ + [ + -1, + 33 + ], + [ + 0, + 11 + ], + [ + -1, + 5 + ] + ], + "sampled_mask_rle": [ + [ + false, + 49 + ] + ], + "is_content_rle": [ + [ + false, + 36 + ], + [ + true, + 6 + ], + [ + false, + 7 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "qwen3.5", + "enable_thinking": true, + "add_vision_id": false, + "image_cache_max": 256 + }, + "extension": { + "text": "\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 17, + "token_ids_sha256": "6ef5d60bff9e75950e0e7ff44788b2963ac81f7a08e0380263e121dec1c0d3bb" + } + } + }, + "qwen3.6": { + "renderer": "qwen3.6", + "renderer_class": "Qwen36Renderer", + "model": "Qwen/Qwen3.6-35B-A3B", + "model_revision": "995ad96eacd98c81ed38be0c5b274b04031597b0", + "config": { + "thinking_retention": null, + "name": "qwen3.6", + "enable_thinking": true, + "add_vision_id": false, + "preserve_thinking": false, + "image_cache_max": 256 + }, + "generation_prompt": { + "text": "<|im_start|>system\n# Tools\n\nYou have access to the following functions:\n\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n\n\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 280, + "token_ids_sha256": "adbac690e6768c6f1314ddd82e2e4c53d4e1fb3d3edc56c74b0e8145434d91b8", + "message_indices_rle": [ + [ + 0, + 263 + ], + [ + 1, + 12 + ], + [ + -1, + 5 + ] + ], + "sampled_mask_rle": [ + [ + false, + 280 + ] + ], + "is_content_rle": [ + [ + false, + 257 + ], + [ + true, + 4 + ], + [ + false, + 5 + ], + [ + true, + 7 + ], + [ + false, + 7 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_start|>system\n# Tools\n\nYou have access to the following functions:\n\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n\n\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\nTwo plus two equals four.\n\n\nFour.<|im_end|>\n", + "token_count": 293, + "token_ids_sha256": "acb2513c56e51c539e4efce8245a3f761bdee03fe6b8e3909fcd42b9f6405a4d", + "message_indices_rle": [ + [ + 0, + 263 + ], + [ + 1, + 12 + ], + [ + 2, + 18 + ] + ], + "sampled_mask_rle": [ + [ + false, + 278 + ], + [ + true, + 14 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 257 + ], + [ + true, + 4 + ], + [ + false, + 5 + ], + [ + true, + 7 + ], + [ + false, + 5 + ], + [ + true, + 14 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "\nTwo plus two equals four.\n\n\nFour.<|im_end|>", + "token_count": 14, + "token_ids_sha256": "6de6a897f24e53a8431bd456c34631afbfac74dc9120d2641b0a191f0c4bc107" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "<|im_start|>system\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\n\n\nPrior answer.<|im_end|>\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 49, + "token_ids_sha256": "51a6866b82c62fcc15859f3654d7c5faeb3af6dae6a9b244ee4afbabd72c0739", + "message_indices_rle": [ + [ + -1, + 33 + ], + [ + 0, + 11 + ], + [ + -1, + 5 + ] + ], + "sampled_mask_rle": [ + [ + false, + 49 + ] + ], + "is_content_rle": [ + [ + false, + 36 + ], + [ + true, + 6 + ], + [ + false, + 7 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "qwen3.6", + "enable_thinking": true, + "add_vision_id": false, + "preserve_thinking": true, + "image_cache_max": 256 + }, + "extension": { + "text": "\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 17, + "token_ids_sha256": "6ef5d60bff9e75950e0e7ff44788b2963ac81f7a08e0380263e121dec1c0d3bb" + } + } + }, + "glm-5": { + "renderer": "glm-5", + "renderer_class": "GLM5Renderer", + "model": "zai-org/GLM-5", + "model_revision": "4e6698ba8e85059d749020e3c4d2123719f23926", + "config": { + "thinking_retention": null, + "name": "glm-5", + "enable_thinking": true, + "clear_thinking": true + }, + "generation_prompt": { + "text": "[gMASK]<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, output the function name and arguments within the following XML format:\n{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...<|system|>You are concise.<|user|>What is 2+2?<|assistant|>", + "token_count": 170, + "token_ids_sha256": "cd54a1dbbc47ee1944954bdc93e68891ac5c04856e8c272264e4d9da4ac8b0c3", + "message_indices_rle": [ + [ + -1, + 155 + ], + [ + 0, + 5 + ], + [ + 1, + 8 + ], + [ + -1, + 2 + ] + ], + "sampled_mask_rle": [ + [ + false, + 170 + ] + ], + "is_content_rle": [ + [ + false, + 156 + ], + [ + true, + 4 + ], + [ + false, + 1 + ], + [ + true, + 7 + ], + [ + false, + 2 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "[gMASK]<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, output the function name and arguments within the following XML format:\n{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...<|system|>You are concise.<|user|>What is 2+2?<|assistant|>Two plus two equals four.Four.", + "token_count": 179, + "token_ids_sha256": "76d92f011f27a028804ae908a6f3d8d0c976d00abdc818eb7196578a0169f4a6", + "message_indices_rle": [ + [ + -1, + 155 + ], + [ + 0, + 5 + ], + [ + 1, + 8 + ], + [ + 2, + 11 + ] + ], + "sampled_mask_rle": [ + [ + false, + 170 + ], + [ + true, + 9 + ] + ], + "is_content_rle": [ + [ + false, + 156 + ], + [ + true, + 4 + ], + [ + false, + 1 + ], + [ + true, + 7 + ], + [ + false, + 2 + ], + [ + true, + 9 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Two plus two equals four.Four.", + "token_count": 9, + "token_ids_sha256": "9a6a08fbb37bc5652afd010f1e4525471e4e1c839bb89c27be7e2a74c1940dc6" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "[gMASK]<|system|>You are concise.<|user|>What is 2+2?<|assistant|>Prior answer.<|endoftext|><|user|>And 3+3?<|assistant|>", + "token_count": 30, + "token_ids_sha256": "3a91feadac1aa889b8289f122d03db24d3d3a41d15f8a9147d891701c51200cf", + "message_indices_rle": [ + [ + -1, + 21 + ], + [ + 0, + 7 + ], + [ + -1, + 2 + ] + ], + "sampled_mask_rle": [ + [ + false, + 30 + ] + ], + "is_content_rle": [ + [ + false, + 22 + ], + [ + true, + 6 + ], + [ + false, + 2 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "glm-5", + "enable_thinking": true, + "clear_thinking": false + }, + "extension": { + "text": "<|endoftext|><|user|>And 3+3?<|assistant|>", + "token_count": 10, + "token_ids_sha256": "9e9a09e456ea99d7de65e6f111fd39729b7ea9b2399041a286154f6f59a12067" + } + } + }, + "glm-5.1": { + "renderer": "glm-5.1", + "renderer_class": "GLM51Renderer", + "model": "zai-org/GLM-5.1", + "model_revision": "26e1bd6e011feb778d25ae34b09b07074139d92d", + "config": { + "thinking_retention": null, + "name": "glm-5.1", + "enable_thinking": true, + "clear_thinking": true + }, + "generation_prompt": { + "text": "[gMASK]<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}\n\n\nFor each function call, output the function name and arguments within the following XML format:\n{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...<|system|>You are concise.<|user|>What is 2+2?<|assistant|>", + "token_count": 160, + "token_ids_sha256": "26b70df854681ec7abf7df7294c0d965b06167309c449e07394c21b67aef86a7", + "message_indices_rle": [ + [ + -1, + 145 + ], + [ + 0, + 5 + ], + [ + 1, + 8 + ], + [ + -1, + 2 + ] + ], + "sampled_mask_rle": [ + [ + false, + 160 + ] + ], + "is_content_rle": [ + [ + false, + 146 + ], + [ + true, + 4 + ], + [ + false, + 1 + ], + [ + true, + 7 + ], + [ + false, + 2 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "[gMASK]<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}\n\n\nFor each function call, output the function name and arguments within the following XML format:\n{function-name}{arg-key-1}{arg-value-1}{arg-key-2}{arg-value-2}...<|system|>You are concise.<|user|>What is 2+2?<|assistant|>Two plus two equals four.Four.", + "token_count": 169, + "token_ids_sha256": "c424706b3fabd58b081c00dc8793cd61a54774a934c0cd145308c600fd4b8883", + "message_indices_rle": [ + [ + -1, + 145 + ], + [ + 0, + 5 + ], + [ + 1, + 8 + ], + [ + 2, + 11 + ] + ], + "sampled_mask_rle": [ + [ + false, + 160 + ], + [ + true, + 9 + ] + ], + "is_content_rle": [ + [ + false, + 146 + ], + [ + true, + 4 + ], + [ + false, + 1 + ], + [ + true, + 7 + ], + [ + false, + 2 + ], + [ + true, + 9 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Two plus two equals four.Four.", + "token_count": 9, + "token_ids_sha256": "9a6a08fbb37bc5652afd010f1e4525471e4e1c839bb89c27be7e2a74c1940dc6" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "[gMASK]<|system|>You are concise.<|user|>What is 2+2?<|assistant|>Prior answer.<|endoftext|><|user|>And 3+3?<|assistant|>", + "token_count": 31, + "token_ids_sha256": "8d2b9fcb086c5a248ea71e0baf227da356e4a4cc246f080b8ca7dc26ed81c5e0", + "message_indices_rle": [ + [ + -1, + 22 + ], + [ + 0, + 7 + ], + [ + -1, + 2 + ] + ], + "sampled_mask_rle": [ + [ + false, + 31 + ] + ], + "is_content_rle": [ + [ + false, + 23 + ], + [ + true, + 6 + ], + [ + false, + 2 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "glm-5.1", + "enable_thinking": true, + "clear_thinking": false + }, + "extension": { + "text": "<|endoftext|><|user|>And 3+3?<|assistant|>", + "token_count": 10, + "token_ids_sha256": "9e9a09e456ea99d7de65e6f111fd39729b7ea9b2399041a286154f6f59a12067" + } + } + }, + "glm-4.5": { + "renderer": "glm-4.5", + "renderer_class": "GLM45Renderer", + "model": "THUDM/GLM-4.5-Air", + "model_revision": "a24ceef6ce4f3536971efe9b778bdaa1bab18daa", + "config": { + "thinking_retention": null, + "name": "glm-4.5", + "enable_thinking": true + }, + "generation_prompt": { + "text": "[gMASK]<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, output the function name and arguments within the following XML format:\n{function-name}\n{arg-key-1}\n{arg-value-1}\n{arg-key-2}\n{arg-value-2}\n...\n<|system|>\nYou are concise.<|user|>\nWhat is 2+2?<|assistant|>", + "token_count": 175, + "token_ids_sha256": "fa428da2ac0ee4acded50eb23683a2fc1fa342f4ff557517c60f1674b4c6972c", + "message_indices_rle": [ + [ + -1, + 159 + ], + [ + 0, + 6 + ], + [ + 1, + 9 + ], + [ + -1, + 1 + ] + ], + "sampled_mask_rle": [ + [ + false, + 175 + ] + ], + "is_content_rle": [ + [ + false, + 161 + ], + [ + true, + 4 + ], + [ + false, + 2 + ], + [ + true, + 7 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "[gMASK]<|system|>\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, output the function name and arguments within the following XML format:\n{function-name}\n{arg-key-1}\n{arg-value-1}\n{arg-key-2}\n{arg-value-2}\n...\n<|system|>\nYou are concise.<|user|>\nWhat is 2+2?<|assistant|>\nTwo plus two equals four.\nFour.", + "token_count": 187, + "token_ids_sha256": "b99d137d29fb2a312cfbad46e49322b02a283e9729f7baaa467bdec8ba8d5be7", + "message_indices_rle": [ + [ + -1, + 159 + ], + [ + 0, + 6 + ], + [ + 1, + 9 + ], + [ + 2, + 13 + ] + ], + "sampled_mask_rle": [ + [ + false, + 176 + ], + [ + true, + 11 + ] + ], + "is_content_rle": [ + [ + false, + 161 + ], + [ + true, + 4 + ], + [ + false, + 2 + ], + [ + true, + 7 + ], + [ + false, + 2 + ], + [ + true, + 11 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Two plus two equals four.\nFour.", + "token_count": 11, + "token_ids_sha256": "1a764fe4dc8e375b93a80d6a8088778782d401123423842e525961c2e26b7d64" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "[gMASK]<|system|>\nYou are concise.<|user|>\nWhat is 2+2?<|assistant|>\n\nPrior answer.<|endoftext|><|user|>\nAnd 3+3?<|assistant|>", + "token_count": 35, + "token_ids_sha256": "8ceaf09056012b41c54f7ac12ac403248de6a95c898f539d8e0c644d1d4c6298", + "message_indices_rle": [ + [ + -1, + 26 + ], + [ + 0, + 8 + ], + [ + -1, + 1 + ] + ], + "sampled_mask_rle": [ + [ + false, + 35 + ] + ], + "is_content_rle": [ + [ + false, + 28 + ], + [ + true, + 6 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "glm-4.5", + "enable_thinking": true + }, + "extension": { + "text": "<|endoftext|><|user|>\nAnd 3+3?<|assistant|>", + "token_count": 10, + "token_ids_sha256": "94403070bdd544603e0ffebd4d53a1ad125504b4c23bcaac303dbbb3fd3ee304" + } + } + }, + "minimax-m2": { + "renderer": "minimax-m2", + "renderer_class": "MiniMaxM2Renderer", + "model": "MiniMaxAI/MiniMax-M2.5", + "model_revision": "f710177d938eff80b684d42c5aa84b382612f21f", + "config": { + "thinking_retention": null, + "name": "minimax-m2", + "model_identity": "You are a helpful assistant. Your name is MiniMax-M2.5 and is built by MiniMax." + }, + "generation_prompt": { + "text": "]~!b[]~b]system\nYou are concise.\n\n# Tools\nYou may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:\n\n\n{\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}\n\n\nWhen making tool calls, use XML format to invoke tools and pass parameters:\n\n\n\nparam-value-1\nparam-value-2\n...\n\n[e~[\n]~b]user\nWhat is 2+2?[e~[\n]~b]ai\n\n", + "token_count": 196, + "token_ids_sha256": "064526018c882c61da6276d298945df7b0f5d4cf77b3e2da597a25381230332c", + "message_indices_rle": [ + [ + 0, + 179 + ], + [ + 1, + 12 + ], + [ + -1, + 5 + ] + ], + "sampled_mask_rle": [ + [ + false, + 196 + ] + ], + "is_content_rle": [ + [ + false, + 4 + ], + [ + true, + 4 + ], + [ + false, + 174 + ], + [ + true, + 7 + ], + [ + false, + 7 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "]~!b[]~b]system\nYou are concise.\n\n# Tools\nYou may call one or more tools to assist with the user query.\nHere are the tools available in JSONSchema format:\n\n\n{\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}\n\n\nWhen making tool calls, use XML format to invoke tools and pass parameters:\n\n\n\nparam-value-1\nparam-value-2\n...\n\n[e~[\n]~b]user\nWhat is 2+2?[e~[\n]~b]ai\n\nTwo plus two equals four.\n\n\nFour.[e~[\n", + "token_count": 210, + "token_ids_sha256": "a34c9c7c9b0b3db3c465ac14b95d3b774cc8961bc732287362d66055c113ed37", + "message_indices_rle": [ + [ + 0, + 179 + ], + [ + 1, + 12 + ], + [ + 2, + 19 + ] + ], + "sampled_mask_rle": [ + [ + false, + 194 + ], + [ + true, + 15 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 4 + ], + [ + true, + 4 + ], + [ + false, + 174 + ], + [ + true, + 7 + ], + [ + false, + 5 + ], + [ + true, + 15 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "\nTwo plus two equals four.\n\n\nFour.[e~[", + "token_count": 15, + "token_ids_sha256": "cacfaf93755f67ed770279017c950ab6a6bb82d15c6e6ae444712c2c97d54af9" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "]~!b[]~b]system\nYou are concise.[e~[\n]~b]user\nWhat is 2+2?[e~[\n]~b]ai\n\n.[e~[\n]~b]user\nAnd 3+3?[e~[\n]~b]ai\n\n", + "token_count": 46, + "token_ids_sha256": "3df206dd4ef94e9cffb9a11c507f92fe24a3994729831f17d2d1c2f25842ac5d", + "message_indices_rle": [ + [ + -1, + 30 + ], + [ + 0, + 11 + ], + [ + -1, + 5 + ] + ], + "sampled_mask_rle": [ + [ + false, + 46 + ] + ], + "is_content_rle": [ + [ + false, + 33 + ], + [ + true, + 6 + ], + [ + false, + 7 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "minimax-m2", + "model_identity": "You are a helpful assistant. Your name is MiniMax-M2.5 and is built by MiniMax." + }, + "extension": { + "text": "\n]~b]user\nAnd 3+3?[e~[\n]~b]ai\n\n", + "token_count": 17, + "token_ids_sha256": "81a357c832bf91a3e8929da22b490e905815c64e35165ff574520d25446ac33f" + } + } + }, + "deepseek-v3": { + "renderer": "deepseek-v3", + "renderer_class": "DeepSeekV3Renderer", + "model": "deepseek-ai/DeepSeek-V3", + "model_revision": "e815299b0bcbac849fa540c768ef21845365c9eb", + "config": { + "thinking_retention": null, + "name": "deepseek-v3" + }, + "generation_prompt": { + "text": "<｜begin of sentence｜>Youareconcise.<｜User｜>Whatis2+2?<｜Assistant｜>", + "token_count": 14, + "token_ids_sha256": "4348fa7c62ebf42108816bbb85970fce36527dba26b14919f54351f813f573a8", + "message_indices_rle": [ + [ + -1, + 1 + ], + [ + 0, + 5 + ], + [ + 1, + 7 + ], + [ + -1, + 1 + ] + ], + "sampled_mask_rle": [ + [ + false, + 14 + ] + ], + "is_content_rle": [ + [ + false, + 1 + ], + [ + true, + 5 + ], + [ + false, + 1 + ], + [ + true, + 6 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<｜begin of sentence｜>Youareconcise.<｜User｜>Whatis2+2?<｜Assistant｜>Four.<｜end of sentence｜>", + "token_count": 17, + "token_ids_sha256": "0d9b03f25f72f34dbd171601c401ebc65230d1621b412ef7f8488454140820bf", + "message_indices_rle": [ + [ + -1, + 1 + ], + [ + 0, + 5 + ], + [ + 1, + 7 + ], + [ + 2, + 4 + ] + ], + "sampled_mask_rle": [ + [ + false, + 14 + ], + [ + true, + 3 + ] + ], + "is_content_rle": [ + [ + false, + 1 + ], + [ + true, + 5 + ], + [ + false, + 1 + ], + [ + true, + 6 + ], + [ + false, + 1 + ], + [ + true, + 3 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Four.<｜end of sentence｜>", + "token_count": 3, + "token_ids_sha256": "6776aa573920f8a5b8b899e24602d2b9928ccd4cc6c85208c3ebbe40c0d17060" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": { + "text": "<｜begin of sentence｜>Youareconcise.<｜User｜>Whatis2+2?<｜Assistant｜>Prioranswer.<｜end of sentence｜><｜User｜>And3+3?<｜Assistant｜>", + "token_count": 25, + "token_ids_sha256": "676fa47450b580f66868c4d98909a31bf625d352b84ffd06517857eedd534f75", + "message_indices_rle": [ + [ + -1, + 18 + ], + [ + 0, + 6 + ], + [ + -1, + 1 + ] + ], + "sampled_mask_rle": [ + [ + false, + 25 + ] + ], + "is_content_rle": [ + [ + false, + 19 + ], + [ + true, + 5 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "deepseek-v3" + }, + "extension": { + "text": "<｜User｜>And3+3?<｜Assistant｜>", + "token_count": 7, + "token_ids_sha256": "166f3e1f2eee1c95718bb6ecd4172ae521338e099bec9ecd3109e73b96e894e0" + } + } + }, + "deepseek-r1": { + "renderer": "deepseek-r1", + "renderer_class": "DeepSeekR1Renderer", + "model": "deepseek-ai/DeepSeek-R1", + "model_revision": "56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad", + "config": { + "thinking_retention": null, + "name": "deepseek-r1" + }, + "generation_prompt": { + "text": "<｜begin of sentence｜>Youareconcise.<｜User｜>Whatis2+2?<｜Assistant｜>", + "token_count": 15, + "token_ids_sha256": "b23a074ae7df65c370ecb1f7b0207abf99af3a6329285defaf556f4554f92a5e", + "message_indices_rle": [ + [ + -1, + 1 + ], + [ + 0, + 5 + ], + [ + 1, + 7 + ], + [ + -1, + 2 + ] + ], + "sampled_mask_rle": [ + [ + false, + 15 + ] + ], + "is_content_rle": [ + [ + false, + 1 + ], + [ + true, + 5 + ], + [ + false, + 1 + ], + [ + true, + 6 + ], + [ + false, + 2 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<｜begin of sentence｜>Youareconcise.<｜User｜>Whatis2+2?<｜Assistant｜>Four.<｜end of sentence｜>", + "token_count": 17, + "token_ids_sha256": "0d9b03f25f72f34dbd171601c401ebc65230d1621b412ef7f8488454140820bf", + "message_indices_rle": [ + [ + -1, + 1 + ], + [ + 0, + 5 + ], + [ + 1, + 7 + ], + [ + 2, + 4 + ] + ], + "sampled_mask_rle": [ + [ + false, + 14 + ], + [ + true, + 3 + ] + ], + "is_content_rle": [ + [ + false, + 1 + ], + [ + true, + 5 + ], + [ + false, + 1 + ], + [ + true, + 6 + ], + [ + false, + 1 + ], + [ + true, + 3 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Four.<｜end of sentence｜>", + "token_count": 3, + "token_ids_sha256": "6776aa573920f8a5b8b899e24602d2b9928ccd4cc6c85208c3ebbe40c0d17060" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": { + "text": "<｜begin of sentence｜>Youareconcise.<｜User｜>Whatis2+2?<｜Assistant｜>answer.<｜end of sentence｜><｜User｜>And3+3?<｜Assistant｜>", + "token_count": 26, + "token_ids_sha256": "673e5a8e63f498ec99266eeba0c308d370f3b93cae3f05f6bb265e468d46931c", + "message_indices_rle": [ + [ + -1, + 18 + ], + [ + 0, + 6 + ], + [ + -1, + 2 + ] + ], + "sampled_mask_rle": [ + [ + false, + 26 + ] + ], + "is_content_rle": [ + [ + false, + 19 + ], + [ + true, + 5 + ], + [ + false, + 2 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "deepseek-r1" + }, + "extension": { + "text": "<｜User｜>And3+3?<｜Assistant｜>", + "token_count": 8, + "token_ids_sha256": "3c1247538646f180dc7b05ca2b44a7f94a74c17b0440d4a1bdb8aec3e58296e8" + } + } + }, + "hy3": { + "renderer": "hy3", + "renderer_class": "Hy3Renderer", + "model": "tencent/Hy3", + "model_revision": "a960ebc3da325ba167f069f76c41eb62c9280d22", + "config": { + "thinking_retention": null, + "name": "hy3", + "reasoning_effort": "no_think", + "preserved_thinking": null, + "is_training": false, + "raw_last_assistant": false, + "fallback_strategy": null + }, + "generation_prompt": { + "text": "<｜hy_begin_of_sentence:opensource｜>You are concise.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor function call returns, you should first print \nFor each function call, you should return object like:\n{function-name}\n{arg-key-1}\n{arg-value-1}\n{arg-key-2}\n{arg-value-2}\n...\n\nAt the end of function call returns, you should print <｜reasoning_mode:opensource｜>reasoning_effort:no_think<｜hy_User:opensource｜>What is 2+2?<｜hy_Assistant:opensource｜>", + "token_count": 203, + "token_ids_sha256": "87a7898def35d1a408436e8a96b4aa710dc6cbb5350be08219d29e379f5d10b0", + "message_indices_rle": [ + [ + -1, + 1 + ], + [ + 0, + 4 + ], + [ + -1, + 187 + ], + [ + 1, + 8 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 203 + ] + ], + "is_content_rle": [ + [ + false, + 1 + ], + [ + true, + 4 + ], + [ + false, + 188 + ], + [ + true, + 7 + ], + [ + false, + 3 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<｜hy_begin_of_sentence:opensource｜>You are concise.\n\n# Tools\n\nYou may call one or more functions to assist with the user query.\n\nYou are provided with function signatures within XML tags:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor function call returns, you should first print \nFor each function call, you should return object like:\n{function-name}\n{arg-key-1}\n{arg-value-1}\n{arg-key-2}\n{arg-value-2}\n...\n\nAt the end of function call returns, you should print <｜reasoning_mode:opensource｜>reasoning_effort:no_think<｜hy_User:opensource｜>What is 2+2?<｜hy_Assistant:opensource｜>Two plus two equals four.Four.", + "token_count": 211, + "token_ids_sha256": "01f553043c4084dc8f25ccfff3c9ae30e8ef2b0c9b787d3e67293dcd0ea5bab9", + "message_indices_rle": [ + [ + -1, + 1 + ], + [ + 0, + 4 + ], + [ + -1, + 187 + ], + [ + 1, + 8 + ], + [ + 2, + 11 + ] + ], + "sampled_mask_rle": [ + [ + false, + 209 + ], + [ + true, + 2 + ] + ], + "is_content_rle": [ + [ + false, + 1 + ], + [ + true, + 4 + ], + [ + false, + 188 + ], + [ + true, + 7 + ], + [ + false, + 9 + ], + [ + true, + 2 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Four.", + "token_count": 2, + "token_ids_sha256": "0d80ef2b0667af0bfc80d7f203a85a096c3642d78dd21830b539a0404fd22f00" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": { + "text": "<｜hy_begin_of_sentence:opensource｜>You are concise.<｜reasoning_mode:opensource｜>reasoning_effort:no_think<｜hy_User:opensource｜>What is 2+2?<｜hy_Assistant:opensource｜>Prior answer.<｜hy_eos:opensource｜><｜hy_User:opensource｜>And 3+3?<｜hy_Assistant:opensource｜>", + "token_count": 40, + "token_ids_sha256": "d3a62980d412626a15d713b5c554154a7f2dc655bc89a69f6fdcfa7f3c1a86b1", + "message_indices_rle": [ + [ + -1, + 30 + ], + [ + 0, + 7 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 40 + ] + ], + "is_content_rle": [ + [ + false, + 31 + ], + [ + true, + 6 + ], + [ + false, + 3 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "hy3", + "reasoning_effort": "no_think", + "preserved_thinking": true, + "is_training": false, + "raw_last_assistant": false, + "fallback_strategy": null + }, + "extension": { + "text": "<｜hy_eos:opensource｜><｜hy_User:opensource｜>And 3+3?<｜hy_Assistant:opensource｜>", + "token_count": 11, + "token_ids_sha256": "7a6621f04ebe88491ef220f65f80784e52ea9e07d4705d9679346c3a45f9947f" + } + } + }, + "kimi-k2": { + "renderer": "kimi-k2", + "renderer_class": "KimiK2Renderer", + "model": "moonshotai/Kimi-K2-Instruct", + "model_revision": "fd1984e2b7a3350dbf7305fe73a4ede25c14de50", + "config": { + "thinking_retention": null, + "name": "kimi-k2", + "enable_thinking": true + }, + "generation_prompt": { + "text": "<|im_system|>tool_declare<|im_middle|>[{\"function\":{\"description\":\"Return the weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"type\":\"object\"}},\"type\":\"function\"}]<|im_end|><|im_system|>system<|im_middle|>You are concise.<|im_end|><|im_user|>user<|im_middle|>What is 2+2?<|im_end|><|im_assistant|>assistant<|im_middle|>", + "token_count": 72, + "token_ids_sha256": "d723bc7a064f55fa3355339a72b4571f0888a6d26a7c76d3a4669f40b5058207", + "message_indices_rle": [ + [ + -1, + 50 + ], + [ + 0, + 8 + ], + [ + 1, + 11 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 72 + ] + ], + "is_content_rle": [ + [ + false, + 53 + ], + [ + true, + 4 + ], + [ + false, + 4 + ], + [ + true, + 7 + ], + [ + false, + 4 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_system|>tool_declare<|im_middle|>[{\"function\":{\"description\":\"Return the weather for a city.\",\"name\":\"get_weather\",\"parameters\":{\"properties\":{\"city\":{\"type\":\"string\"}},\"required\":[\"city\"],\"type\":\"object\"}},\"type\":\"function\"}]<|im_end|><|im_system|>system<|im_middle|>You are concise.<|im_end|><|im_user|>user<|im_middle|>What is 2+2?<|im_end|><|im_assistant|>assistant<|im_middle|>Four.<|im_end|>", + "token_count": 75, + "token_ids_sha256": "f3d525f2077f092358cc0716a1c4b4cfcc07c689c2202cf5ce1e67743e718612", + "message_indices_rle": [ + [ + -1, + 50 + ], + [ + 0, + 8 + ], + [ + 1, + 11 + ], + [ + 2, + 6 + ] + ], + "sampled_mask_rle": [ + [ + false, + 72 + ], + [ + true, + 3 + ] + ], + "is_content_rle": [ + [ + false, + 53 + ], + [ + true, + 4 + ], + [ + false, + 4 + ], + [ + true, + 7 + ], + [ + false, + 4 + ], + [ + true, + 3 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Four.<|im_end|>", + "token_count": 3, + "token_ids_sha256": "f00b967291aa8086eede4025d42b7ef63338b1e81a767590065333f1ca2b6f90" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": { + "text": "<|im_system|>system<|im_middle|>You are concise.<|im_end|><|im_user|>user<|im_middle|>What is 2+2?<|im_end|><|im_assistant|>assistant<|im_middle|>Prior answer.<|im_end|><|im_user|>user<|im_middle|>And 3+3?<|im_end|><|im_assistant|>assistant<|im_middle|>", + "token_count": 39, + "token_ids_sha256": "898c52d29a0d1a27d31b35c57a2f609b6fa8d69a165b96aeca605d31a0a66795", + "message_indices_rle": [ + [ + -1, + 26 + ], + [ + 0, + 10 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 39 + ] + ], + "is_content_rle": [ + [ + false, + 29 + ], + [ + true, + 6 + ], + [ + false, + 4 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "kimi-k2", + "enable_thinking": true + }, + "extension": { + "text": "<|im_user|>user<|im_middle|>And 3+3?<|im_end|><|im_assistant|>assistant<|im_middle|>", + "token_count": 13, + "token_ids_sha256": "737f09355e21e945eea13bbe80faf0f0ebec1fa7efcb03e0cdc859ecea6867db" + } + } + }, + "kimi-k2.5": { + "renderer": "kimi-k2.5", + "renderer_class": "KimiK25Renderer", + "model": "moonshotai/Kimi-K2.5", + "model_revision": "4d01dfe0332d63057c186e0b262165819efb6611", + "config": { + "thinking_retention": null, + "name": "kimi-k2.5", + "thinking": true, + "image_cache_max": 256 + }, + "generation_prompt": { + "text": "<|im_system|>tool_declare<|im_middle|># Tools\n\n## functions\nnamespace functions {\n// Return the weather for a city.\ntype get_weather = (_: {\n city: string\n}) => any;\n}\n<|im_end|><|im_system|>system<|im_middle|>You are concise.<|im_end|><|im_user|>user<|im_middle|>What is 2+2?<|im_end|><|im_assistant|>assistant<|im_middle|>", + "token_count": 64, + "token_ids_sha256": "6c57efcebad1a6790ad211f0b300bf06d33fb89004d1fd8daba7e3a8e29a532b", + "message_indices_rle": [ + [ + -1, + 41 + ], + [ + 0, + 8 + ], + [ + 1, + 11 + ], + [ + -1, + 4 + ] + ], + "sampled_mask_rle": [ + [ + false, + 64 + ] + ], + "is_content_rle": [ + [ + false, + 44 + ], + [ + true, + 4 + ], + [ + false, + 4 + ], + [ + true, + 7 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_system|>tool_declare<|im_middle|># Tools\n\n## functions\nnamespace functions {\n// Return the weather for a city.\ntype get_weather = (_: {\n city: string\n}) => any;\n}\n<|im_end|><|im_system|>system<|im_middle|>You are concise.<|im_end|><|im_user|>user<|im_middle|>What is 2+2?<|im_end|><|im_assistant|>assistant<|im_middle|>Four.<|im_end|>", + "token_count": 68, + "token_ids_sha256": "c5b43c9c5a2280ce4cc4fc785a0bffa0166973dda5efdd71599d209c8da0dd0d", + "message_indices_rle": [ + [ + -1, + 41 + ], + [ + 0, + 8 + ], + [ + 1, + 11 + ], + [ + 2, + 8 + ] + ], + "sampled_mask_rle": [ + [ + false, + 63 + ], + [ + true, + 5 + ] + ], + "is_content_rle": [ + [ + false, + 44 + ], + [ + true, + 4 + ], + [ + false, + 4 + ], + [ + true, + 7 + ], + [ + false, + 4 + ], + [ + true, + 5 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Four.<|im_end|>", + "token_count": 5, + "token_ids_sha256": "cdd71bb6844d792df08028acbfe38d937e31f3806cf363f885f212a8588ff1ad" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": { + "text": "<|im_system|>system<|im_middle|>You are concise.<|im_end|><|im_user|>user<|im_middle|>What is 2+2?<|im_end|><|im_assistant|>assistant<|im_middle|>Prior answer.<|im_end|><|im_user|>user<|im_middle|>And 3+3?<|im_end|><|im_assistant|>assistant<|im_middle|>", + "token_count": 42, + "token_ids_sha256": "9bfe9520230e132680a78bf94649d8b139d4b6c4eb13e151defb9d69149e590e", + "message_indices_rle": [ + [ + -1, + 28 + ], + [ + 0, + 10 + ], + [ + -1, + 4 + ] + ], + "sampled_mask_rle": [ + [ + false, + 42 + ] + ], + "is_content_rle": [ + [ + false, + 31 + ], + [ + true, + 6 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "kimi-k2.5", + "thinking": true, + "image_cache_max": 256 + }, + "extension": { + "text": "<|im_user|>user<|im_middle|>And 3+3?<|im_end|><|im_assistant|>assistant<|im_middle|>", + "token_count": 14, + "token_ids_sha256": "e50c295ecd43151e62240781ca6b89f2b32bf22372151a86e71c8708286dc865" + } + } + }, + "laguna-xs.2": { + "renderer": "laguna-xs.2", + "renderer_class": "LagunaXS2Renderer", + "model": "poolside/Laguna-XS.2", + "model_revision": "69e3f4046616e40fb55ac54e0e2e6accbe5cadfe", + "config": { + "thinking_retention": null, + "name": "laguna-xs.2", + "enable_thinking": false, + "render_assistant_messages_raw": false + }, + "generation_prompt": { + "text": "〈|EOS|〉\n\nYou are concise.\n\n### Tools\n\nYou may call functions to assist with the user query.\nAll available function signatures are listed below:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, return an unescaped XML-like object with function name and arguments within '' and '' tags, like here:\nfunction-name\nargument-key\nvalue-of-argument-key\n\n\n\nWhat is 2+2?\n\n\n", + "token_count": 184, + "token_ids_sha256": "41f835bc77481985b356fdda415cb4c15d282ba408641814ceddb0196b87248d", + "message_indices_rle": [ + [ + -1, + 5 + ], + [ + 0, + 4 + ], + [ + -1, + 156 + ], + [ + 1, + 16 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 184 + ] + ], + "is_content_rle": [ + [ + false, + 5 + ], + [ + true, + 4 + ], + [ + false, + 160 + ], + [ + true, + 7 + ], + [ + false, + 8 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "〈|EOS|〉\n\nYou are concise.\n\n### Tools\n\nYou may call functions to assist with the user query.\nAll available function signatures are listed below:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\n\nFor each function call, return an unescaped XML-like object with function name and arguments within '' and '' tags, like here:\nfunction-name\nargument-key\nvalue-of-argument-key\n\n\n\nWhat is 2+2?\n\n\n\nTwo plus two equals four.\n\nFour.\n\n", + "token_count": 199, + "token_ids_sha256": "ebe233315fb4946d6b23d661aa3026dca7a77735f074e041fdc985b653c01a78", + "message_indices_rle": [ + [ + -1, + 5 + ], + [ + 0, + 4 + ], + [ + -1, + 156 + ], + [ + 1, + 16 + ], + [ + 2, + 18 + ] + ], + "sampled_mask_rle": [ + [ + false, + 183 + ], + [ + true, + 15 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 5 + ], + [ + true, + 4 + ], + [ + false, + 160 + ], + [ + true, + 7 + ], + [ + false, + 7 + ], + [ + true, + 15 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "\nTwo plus two equals four.\n\nFour.\n", + "token_count": 15, + "token_ids_sha256": "6a8c980962bac4e5c4b60f094848485017c9127a20c096ba8059947bddcf755e" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "〈|EOS|〉\n\nYou are concise.\n\n\nWhat is 2+2?\n\n\n\nPrior answer.\n\nAnd 3+3?\n\n\n", + "token_count": 57, + "token_ids_sha256": "f06677006fe7c6f3577ac7f7f32b54dd4a3805ba3241edf65fb239372dfb1125", + "message_indices_rle": [ + [ + -1, + 39 + ], + [ + 0, + 15 + ], + [ + -1, + 3 + ] + ], + "sampled_mask_rle": [ + [ + false, + 57 + ] + ], + "is_content_rle": [ + [ + false, + 43 + ], + [ + true, + 6 + ], + [ + false, + 8 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "laguna-xs.2", + "enable_thinking": false, + "render_assistant_messages_raw": false + }, + "extension": { + "text": "\nAnd 3+3?\n\n\n", + "token_count": 18, + "token_ids_sha256": "3dde2bc1c5d96c09cb9c110ca55a462e7f0bad8946e47ca1be75a3db8c96d6fa" + } + } + }, + "laguna-xs-2.1": { + "renderer": "laguna-xs-2.1", + "renderer_class": "LagunaXS21Renderer", + "model": "poolside/Laguna-XS-2.1", + "model_revision": "e9df9a59996d790b94b70f3fef343fe1d9e34bdf", + "config": { + "thinking_retention": null, + "name": "laguna-xs-2.1", + "enable_thinking": false + }, + "generation_prompt": { + "text": "〈|EOS|〉You are concise.\n\n### Tools\n\nYou may call functions to assist with the user query.\nAll available function signatures are listed below:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\nWhat is 2+2?\n", + "token_count": 114, + "token_ids_sha256": "82b73c980a35330e3982069ec9bc5bf7bcef57a366c02ac3e9e33153e414a4d6", + "message_indices_rle": [ + [ + -1, + 3 + ], + [ + 0, + 4 + ], + [ + -1, + 93 + ], + [ + 1, + 12 + ], + [ + -1, + 2 + ] + ], + "sampled_mask_rle": [ + [ + false, + 114 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 95 + ], + [ + true, + 7 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "〈|EOS|〉You are concise.\n\n### Tools\n\nYou may call functions to assist with the user query.\nAll available function signatures are listed below:\n\n{\"type\": \"function\", \"function\": {\"name\": \"get_weather\", \"description\": \"Return the weather for a city.\", \"parameters\": {\"type\": \"object\", \"properties\": {\"city\": {\"type\": \"string\"}}, \"required\": [\"city\"]}}}\n\nWhat is 2+2?\nFour.\n", + "token_count": 118, + "token_ids_sha256": "217890f96a94727a65317f1cc1bbb00e1fd696ffa757cdab2c7970fdfe9fc8fa", + "message_indices_rle": [ + [ + -1, + 3 + ], + [ + 0, + 4 + ], + [ + -1, + 93 + ], + [ + 1, + 12 + ], + [ + 2, + 6 + ] + ], + "sampled_mask_rle": [ + [ + false, + 114 + ], + [ + true, + 3 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 95 + ], + [ + true, + 7 + ], + [ + false, + 5 + ], + [ + true, + 3 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Four.", + "token_count": 3, + "token_ids_sha256": "f5cfd11242c7e5df2a67a82ef629821d9ff243e2363e7a5a98fef158e13d4a0a" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": { + "text": "〈|EOS|〉You are concise.\nWhat is 2+2?\nPrior answer.\nAnd 3+3?\n", + "token_count": 43, + "token_ids_sha256": "4442f0216467b883c1747022db8260e3ac29d4fb34956385f7a7c39e887903d4", + "message_indices_rle": [ + [ + -1, + 28 + ], + [ + 0, + 13 + ], + [ + -1, + 2 + ] + ], + "sampled_mask_rle": [ + [ + false, + 43 + ] + ], + "is_content_rle": [ + [ + false, + 32 + ], + [ + true, + 6 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "laguna-xs-2.1", + "enable_thinking": false + }, + "extension": { + "text": "\nAnd 3+3?\n", + "token_count": 15, + "token_ids_sha256": "cd7a1343732f37e2dde9b6b03358f1d2d939081b07c5ba235b131884eb5fdf6a" + } + } + }, + "llama-3": { + "renderer": "llama-3", + "renderer_class": "Llama3Renderer", + "model": "meta-llama/Llama-3.2-1B-Instruct", + "model_revision": "5a8abab4a5d6f164389b1079fb721cfab8d7126c", + "config": { + "thinking_retention": null, + "name": "llama-3", + "date_string": "26 Jul 2024", + "tools_in_user_message": true + }, + "generation_prompt": { + "text": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are concise.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.Do not use variables.\n\n{\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n \"description\": \"Return the weather for a city.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\"\n ]\n }\n }\n}\n\nWhat is 2+2?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "token_count": 184, + "token_ids_sha256": "bf6e3613f5731ec844e14744b434dcd3e68e619bd26ba061e7165704ed68f6ce", + "message_indices_rle": [ + [ + -1, + 1 + ], + [ + 0, + 34 + ], + [ + 1, + 145 + ], + [ + -1, + 4 + ] + ], + "sampled_mask_rle": [ + [ + false, + 184 + ] + ], + "is_content_rle": [ + [ + false, + 30 + ], + [ + true, + 4 + ], + [ + false, + 138 + ], + [ + true, + 7 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nEnvironment: ipython\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are concise.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nGiven the following functions, please respond with a JSON for a function call with its proper arguments that best answers the given prompt.\n\nRespond in the format {\"name\": function name, \"parameters\": dictionary of argument name and its value}.Do not use variables.\n\n{\n \"type\": \"function\",\n \"function\": {\n \"name\": \"get_weather\",\n \"description\": \"Return the weather for a city.\",\n \"parameters\": {\n \"type\": \"object\",\n \"properties\": {\n \"city\": {\n \"type\": \"string\"\n }\n },\n \"required\": [\n \"city\"\n ]\n }\n }\n}\n\nWhat is 2+2?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nFour.<|eot_id|>", + "token_count": 187, + "token_ids_sha256": "a62739bb8f58556690620f7fe58e9ed4c5ad401ba1312bfea39c6cc74dedf61c", + "message_indices_rle": [ + [ + -1, + 1 + ], + [ + 0, + 34 + ], + [ + 1, + 145 + ], + [ + 2, + 7 + ] + ], + "sampled_mask_rle": [ + [ + false, + 184 + ], + [ + true, + 3 + ] + ], + "is_content_rle": [ + [ + false, + 30 + ], + [ + true, + 4 + ], + [ + false, + 138 + ], + [ + true, + 7 + ], + [ + false, + 5 + ], + [ + true, + 3 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "Four.<|eot_id|>", + "token_count": 3, + "token_ids_sha256": "7b3902220739646e69cf69a7a01cabf34b727fbf4f1b1b8a89825ca3b6044711" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": { + "text": "<|begin_of_text|><|start_header_id|>system<|end_header_id|>\n\nCutting Knowledge Date: December 2023\nToday Date: 26 Jul 2024\n\nYou are concise.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nWhat is 2+2?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\nPrior answer.<|eot_id|><|start_header_id|>user<|end_header_id|>\n\nAnd 3+3?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "token_count": 65, + "token_ids_sha256": "192ae0293aba2c5f83fb0daba913e79aa12c1e38752efe819c3d67cb882cf058", + "message_indices_rle": [ + [ + -1, + 50 + ], + [ + 0, + 11 + ], + [ + -1, + 4 + ] + ], + "sampled_mask_rle": [ + [ + false, + 65 + ] + ], + "is_content_rle": [ + [ + false, + 54 + ], + [ + true, + 6 + ], + [ + false, + 5 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "llama-3", + "date_string": "26 Jul 2024", + "tools_in_user_message": true + }, + "extension": { + "text": "<|start_header_id|>user<|end_header_id|>\n\nAnd 3+3?<|eot_id|><|start_header_id|>assistant<|end_header_id|>\n\n", + "token_count": 15, + "token_ids_sha256": "c5c629c5dec69f2eba2b66398f2483b3ffe5de5f1674c4b2b6e3857ef87e6655" + } + } + }, + "nemotron-3": { + "renderer": "nemotron-3", + "renderer_class": "Nemotron3Renderer", + "model": "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", + "model_revision": "2d59de1cbd51c0adf384eb906b766d1aee0e0517", + "config": { + "thinking_retention": null, + "name": "nemotron-3", + "enable_thinking": true, + "truncate_history_thinking": true, + "low_effort": false + }, + "generation_prompt": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou have access to the following functions:\n\n\n\nget_weather\nReturn the weather for a city.\n\n\ncity\nstring\n\n[\"city\"]\n\n\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 279, + "token_ids_sha256": "dc9ff3c24c451dfc2c85fe05c8bf1fb5f5c07fc7d242cb7b3d7638f1e095bc49", + "message_indices_rle": [ + [ + 0, + 261 + ], + [ + 1, + 12 + ], + [ + -1, + 6 + ] + ], + "sampled_mask_rle": [ + [ + false, + 279 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 257 + ], + [ + true, + 7 + ], + [ + false, + 8 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou have access to the following functions:\n\n\n\nget_weather\nReturn the weather for a city.\n\n\ncity\nstring\n\n[\"city\"]\n\n\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\nTwo plus two equals four.\n\nFour.<|im_end|>\n", + "token_count": 291, + "token_ids_sha256": "aa0e2c3ea22e3225c61b7472f49f098e29fb0d5193ee17fabb0785036a10d7f9", + "message_indices_rle": [ + [ + 0, + 261 + ], + [ + 1, + 12 + ], + [ + 2, + 18 + ] + ], + "sampled_mask_rle": [ + [ + false, + 277 + ], + [ + true, + 13 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 257 + ], + [ + true, + 7 + ], + [ + false, + 6 + ], + [ + true, + 13 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "\nTwo plus two equals four.\n\nFour.<|im_end|>", + "token_count": 13, + "token_ids_sha256": "7f34d41d7134b6706726a8332c91ec136369b3fd1a97319be00ac492361d8453" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "<|im_start|>system\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\nPrior answer.<|im_end|>\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 49, + "token_ids_sha256": "2482c5693c06c73b3171111b1e75b3956e07f3491ad66c78152fbfe62c32791c", + "message_indices_rle": [ + [ + -1, + 32 + ], + [ + 0, + 11 + ], + [ + -1, + 6 + ] + ], + "sampled_mask_rle": [ + [ + false, + 49 + ] + ], + "is_content_rle": [ + [ + false, + 35 + ], + [ + true, + 6 + ], + [ + false, + 8 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "nemotron-3", + "enable_thinking": true, + "truncate_history_thinking": false, + "low_effort": false + }, + "extension": { + "text": "\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 18, + "token_ids_sha256": "aaf62b6cf1798710dddd574ff83aa44cfbb894cb725d7a42274660b2040e409d" + } + } + }, + "nemotron-3-ultra": { + "renderer": "nemotron-3-ultra", + "renderer_class": "Nemotron3UltraRenderer", + "model": "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16", + "model_revision": "624ba927cfbef0427354998700de3d51173c8c04", + "config": { + "thinking_retention": null, + "name": "nemotron-3-ultra", + "enable_thinking": true, + "truncate_history_thinking": true, + "medium_effort": false + }, + "generation_prompt": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou have access to the following functions:\n\n\n\nget_weather\nReturn the weather for a city.\n\n\ncity\nstring\n\n[\"city\"]\n\n\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 279, + "token_ids_sha256": "dc9ff3c24c451dfc2c85fe05c8bf1fb5f5c07fc7d242cb7b3d7638f1e095bc49", + "message_indices_rle": [ + [ + 0, + 261 + ], + [ + 1, + 12 + ], + [ + -1, + 6 + ] + ], + "sampled_mask_rle": [ + [ + false, + 279 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 257 + ], + [ + true, + 7 + ], + [ + false, + 8 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|im_start|>system\nYou are concise.\n\n# Tools\n\nYou have access to the following functions:\n\n\n\nget_weather\nReturn the weather for a city.\n\n\ncity\nstring\n\n[\"city\"]\n\n\n\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n\n\n\nvalue_1\n\n\nThis is the value for the second parameter\nthat can span\nmultiple lines\n\n\n\n\n\nReminder:\n- Function calls MUST follow the specified format: an inner block must be nested within XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\nTwo plus two equals four.Four.<|im_end|>\n", + "token_count": 290, + "token_ids_sha256": "9d0876fe354b4257eb1c5648f55a126107e27ddab68093b29959d45f8eacca97", + "message_indices_rle": [ + [ + 0, + 261 + ], + [ + 1, + 12 + ], + [ + 2, + 17 + ] + ], + "sampled_mask_rle": [ + [ + false, + 277 + ], + [ + true, + 12 + ], + [ + false, + 1 + ] + ], + "is_content_rle": [ + [ + false, + 3 + ], + [ + true, + 4 + ], + [ + false, + 257 + ], + [ + true, + 7 + ], + [ + false, + 6 + ], + [ + true, + 12 + ], + [ + false, + 1 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "\nTwo plus two equals four.Four.<|im_end|>", + "token_count": 12, + "token_ids_sha256": "f1ec73673e71597f8ef09928391a9a09bdff0c894c4151348857edd2496cce3f" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": "Two plus two equals four.", + "tool_calls": [] + }, + "bridge": { + "text": "<|im_start|>system\nYou are concise.<|im_end|>\n<|im_start|>user\nWhat is 2+2?<|im_end|>\n<|im_start|>assistant\n\nPrior answer.<|im_end|>\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 49, + "token_ids_sha256": "2482c5693c06c73b3171111b1e75b3956e07f3491ad66c78152fbfe62c32791c", + "message_indices_rle": [ + [ + -1, + 32 + ], + [ + 0, + 11 + ], + [ + -1, + 6 + ] + ], + "sampled_mask_rle": [ + [ + false, + 49 + ] + ], + "is_content_rle": [ + [ + false, + 35 + ], + [ + true, + 6 + ], + [ + false, + 8 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "nemotron-3-ultra", + "enable_thinking": true, + "truncate_history_thinking": false, + "medium_effort": false + }, + "extension": { + "text": "\n<|im_start|>user\nAnd 3+3?<|im_end|>\n<|im_start|>assistant\n\n", + "token_count": 18, + "token_ids_sha256": "aaf62b6cf1798710dddd574ff83aa44cfbb894cb725d7a42274660b2040e409d" + } + } + }, + "gpt-oss": { + "renderer": "gpt-oss", + "renderer_class": "GptOssRenderer", + "model": "openai/gpt-oss-20b", + "model_revision": "6cee5e81ee83917806bbde320786a8fb61efebee", + "config": { + "thinking_retention": null, + "name": "gpt-oss", + "reasoning_effort": "medium", + "conversation_start_date": "2025-01-15", + "use_system_prompt": true, + "knowledge_cutoff": null, + "model_identity": null, + "auto_drop_analysis": true + }, + "generation_prompt": { + "text": "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\nKnowledge cutoff: 2024-06\nCurrent date: 2025-01-15\n\nReasoning: medium\n\n# Valid channels: analysis, commentary, final. Channel must be included for every message.\nCalls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Instructions\n\nYou are concise.\n\n# Tools\n\n## functions\n\nnamespace functions {\n\n// Return the weather for a city.\ntype get_weather = (_: {\ncity: string,\n}) => any;\n\n} // namespace functions<|end|><|start|>user<|message|>What is 2+2?<|end|><|start|>assistant<|channel|>analysis<|message|>", + "token_count": 137, + "token_ids_sha256": "2b3451835637d6aab0b61ba661839c2ea87edea6c24d00f5e7729907627ae14a", + "message_indices_rle": [ + [ + 0, + 121 + ], + [ + 1, + 11 + ], + [ + -1, + 5 + ] + ], + "sampled_mask_rle": [ + [ + false, + 137 + ] + ], + "is_content_rle": [ + [ + false, + 79 + ], + [ + true, + 7 + ], + [ + false, + 38 + ], + [ + true, + 7 + ], + [ + false, + 6 + ] + ], + "message_roles": [ + "system", + "user" + ], + "message_tool_names": [ + null, + null + ] + }, + "completed_turn": { + "text": "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\nKnowledge cutoff: 2024-06\nCurrent date: 2025-01-15\n\nReasoning: medium\n\n# Valid channels: analysis, commentary, final. Channel must be included for every message.\nCalls to these tools must go to the commentary channel: 'functions'.<|end|><|start|>developer<|message|># Instructions\n\nYou are concise.\n\n# Tools\n\n## functions\n\nnamespace functions {\n\n// Return the weather for a city.\ntype get_weather = (_: {\ncity: string,\n}) => any;\n\n} // namespace functions<|end|><|start|>user<|message|>What is 2+2?<|end|><|start|>assistant<|channel|>final<|message|>Four.<|return|>", + "token_count": 140, + "token_ids_sha256": "e1d61a4419e80253f3ddcae9c9299c8afde2a6a68f198bd05e35813654a44c3b", + "message_indices_rle": [ + [ + 0, + 121 + ], + [ + 1, + 11 + ], + [ + 2, + 8 + ] + ], + "sampled_mask_rle": [ + [ + false, + 137 + ], + [ + true, + 3 + ] + ], + "is_content_rle": [ + [ + false, + 79 + ], + [ + true, + 7 + ], + [ + false, + 38 + ], + [ + true, + 7 + ], + [ + false, + 6 + ], + [ + true, + 3 + ] + ], + "message_roles": [ + "system", + "user", + "assistant" + ], + "message_tool_names": [ + null, + null, + null + ] + }, + "parser_input": { + "text": "<|start|>assistant<|channel|>final<|message|>Four.<|return|>", + "token_count": 8, + "token_ids_sha256": "595367d24916debfc84b50b7741fb2c16eb2136500d3a4439c647448c0913c35" + }, + "parsed_completion": { + "content": "Four.", + "reasoning_content": null, + "tool_calls": [] + }, + "bridge": { + "text": "<|start|>system<|message|>You are ChatGPT, a large language model trained by OpenAI.\nKnowledge cutoff: 2024-06\nCurrent date: 2025-01-15\n\nReasoning: medium\n\n# Valid channels: analysis, commentary, final. Channel must be included for every message.<|end|><|start|>developer<|message|># Instructions\n\nYou are concise.<|end|><|start|>user<|message|>What is 2+2?<|end|><|start|>assistant<|channel|>analysis<|message|>Prior answer.<|return|><|start|>user<|message|>And 3+3?<|end|><|start|>assistant<|channel|>analysis<|message|>", + "token_count": 107, + "token_ids_sha256": "99ee782584e8f2bb4eb308e909d9172d76cf520351c0d57839db75887671b51f", + "message_indices_rle": [ + [ + -1, + 92 + ], + [ + 0, + 10 + ], + [ + -1, + 5 + ] + ], + "sampled_mask_rle": [ + [ + false, + 107 + ] + ], + "is_content_rle": [ + [ + false, + 95 + ], + [ + true, + 6 + ], + [ + false, + 6 + ] + ], + "message_roles": [ + "user" + ], + "message_tool_names": [ + null + ], + "config": { + "thinking_retention": "all", + "name": "gpt-oss", + "reasoning_effort": "medium", + "conversation_start_date": "2025-01-15", + "use_system_prompt": true, + "knowledge_cutoff": null, + "model_identity": null, + "auto_drop_analysis": false + }, + "extension": { + "text": "<|start|>user<|message|>And 3+3?<|end|><|start|>assistant<|channel|>analysis<|message|>", + "token_count": 15, + "token_ids_sha256": "226e613ad7f518ed4053e0c764441220d9ced9dba263db7451937f22ba2c9ebf" + } + } + } + } +} diff --git a/tests/model_assets.py b/tests/model_assets.py new file mode 100644 index 00000000..7ff218a5 --- /dev/null +++ b/tests/model_assets.py @@ -0,0 +1,83 @@ +"""Immutable Hugging Face revisions used by network-backed tests. + +Production tokenizer loading intentionally follows the requested model unless +the repository executes trusted remote code. Tests have a different contract: +parity expectations must not change because an upstream model owner moves +``main``. Every real model asset loaded by the test suite therefore goes +through :func:`load_test_tokenizer` with a full commit SHA. + +For canonical Meta Llama IDs the revision belongs to the audited ``unsloth`` +tokenizer mirror selected by ``TOKENIZER_SOURCE_OVERRIDES``. Kimi revisions +must match ``TRUSTED_REVISIONS`` because those tokenizers execute repository +code and cannot be overridden by a test pin. +""" + +from __future__ import annotations + +from typing import Any + +from renderers.base import TRUSTED_REVISIONS, load_tokenizer + + +MODEL_REVISIONS: dict[str, str] = { + "MiniMaxAI/MiniMax-M2.5": "f710177d938eff80b684d42c5aa84b382612f21f", + "PrimeIntellect/Qwen3-0.6B": "15aba67c8ec68aeac96f57ba7e31d373564ee03c", + "PrimeIntellect/Qwen3-1.7B": "61a79cd101170642a653d1d44d18f69f88f54e08", + "Qwen/Qwen2.5-0.5B-Instruct": "7ae557604adf67be50417f59c2c2f167def9a775", + "Qwen/Qwen3-0.6B": "c1899de289a04d12100db370d81485cdf75e47ca", + "Qwen/Qwen3-8B": "b968826d9c46dd6066d109eabc6255188de91218", + "Qwen/Qwen3-VL-4B-Instruct": "ebb281ec70b05090aa6165b016eac8ec08e71b17", + "Qwen/Qwen3-VL-8B-Instruct": "0c351dd01ed87e9c1b53cbc748cba10e6187ff3b", + "Qwen/Qwen3-VL-30B-A3B-Instruct": "9c4b90e1e4ba969fd3b5378b57d966d725f1b86c", + "Qwen/Qwen3.5-0.8B": "2fc06364715b967f1860aea9cf38778875588b17", + "Qwen/Qwen3.5-2B": "15852e8c16360a2fea060d615a32b45270f8a8fc", + "Qwen/Qwen3.5-4B": "851bf6e806efd8d0a36b00ddf55e13ccb7b8cd0a", + "Qwen/Qwen3.5-9B": "c202236235762e1c871ad0ccb60c8ee5ba337b9a", + "Qwen/Qwen3.5-35B-A3B": "59d61f3ce65a6d9863b86d2e96597125219dc754", + "Qwen/Qwen3.5-122B-A10B": "dc4d348443bc740c68e2d77492492c11606384d5", + "Qwen/Qwen3.5-397B-A17B": "8472618112abcbd45acbcdc58436aff4233c23f7", + "Qwen/Qwen3.6-35B-A3B": "995ad96eacd98c81ed38be0c5b274b04031597b0", + "THUDM/GLM-4.5-Air": "a24ceef6ce4f3536971efe9b778bdaa1bab18daa", + "deepseek-ai/DeepSeek-R1": "56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad", + "deepseek-ai/DeepSeek-V3": "e815299b0bcbac849fa540c768ef21845365c9eb", + "meta-llama/Llama-3.2-1B-Instruct": "5a8abab4a5d6f164389b1079fb721cfab8d7126c", + "meta-llama/Llama-3.2-3B-Instruct": "006f5dcd1393c3add266de40994ba96225e9689d", + "moonshotai/Kimi-K2-Instruct": TRUSTED_REVISIONS["moonshotai/Kimi-K2-Instruct"], + "moonshotai/Kimi-K2.5": TRUSTED_REVISIONS["moonshotai/Kimi-K2.5"], + "moonshotai/Kimi-K2.6": TRUSTED_REVISIONS["moonshotai/Kimi-K2.6"], + "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16": "2d59de1cbd51c0adf384eb906b766d1aee0e0517", + "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-BF16": "d51eab0d1f979ebc26b546e634a04f450d99158e", + "nvidia/NVIDIA-Nemotron-3-Ultra-550B-A55B-BF16": "624ba927cfbef0427354998700de3d51173c8c04", + "openai/gpt-oss-20b": "6cee5e81ee83917806bbde320786a8fb61efebee", + "poolside/Laguna-XS-2.1": "e9df9a59996d790b94b70f3fef343fe1d9e34bdf", + "poolside/Laguna-XS.2": "69e3f4046616e40fb55ac54e0e2e6accbe5cadfe", + "tencent/Hy3": "a960ebc3da325ba167f069f76c41eb62c9280d22", + "zai-org/GLM-4.7-Flash": "7dd20894a642a0aa287e9827cb1a1f7f91386b67", + "zai-org/GLM-5": "4e6698ba8e85059d749020e3c4d2123719f23926", + "zai-org/GLM-5.1": "26e1bd6e011feb778d25ae34b09b07074139d92d", +} + + +def model_revision(model_name: str) -> str: + """Return the immutable test revision for ``model_name``.""" + try: + return MODEL_REVISIONS[model_name] + except KeyError as exc: + raise KeyError( + f"No immutable test revision registered for {model_name!r}. " + "Resolve and review a full Hugging Face commit SHA before adding " + "this model to a network-backed test." + ) from exc + + +def load_test_tokenizer(model_name: str): + """Load a real tokenizer at the suite's reviewed immutable revision.""" + return load_tokenizer(model_name, revision=model_revision(model_name)) + + +def processor_load_kwargs(model_name: str) -> dict[str, Any]: + """Pinned, security-aware kwargs for ``AutoProcessor.from_pretrained``.""" + return { + "revision": model_revision(model_name), + "trust_remote_code": model_name in TRUSTED_REVISIONS, + } diff --git a/tests/test_bridge.py b/tests/test_bridge.py index fe9bd629..b4ace33d 100644 --- a/tests/test_bridge.py +++ b/tests/test_bridge.py @@ -34,7 +34,7 @@ ("moonshotai/Kimi-K2.5", "auto"), ("nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16", "auto"), ("tencent/Hy3", "auto"), - ("unsloth/Llama-3.2-1B-Instruct", "llama-3"), + ("meta-llama/Llama-3.2-1B-Instruct", "llama-3"), ("openai/gpt-oss-20b", "gpt-oss"), ] @@ -42,10 +42,10 @@ @lru_cache(maxsize=None) def _load(model_name: str, renderer_name: str): from renderers import create_renderer - from renderers.base import load_tokenizer + from tests.model_assets import load_test_tokenizer from renderers.configs import config_from_name - tok = load_tokenizer(model_name) + tok = load_test_tokenizer(model_name) return tok, create_renderer(tok, config_from_name(renderer_name)) @@ -227,10 +227,10 @@ def test_bridge_declines_across_user_query_when_template_drops_thinking(): - no prior thinking + new user query -> decline; no marker lookback. """ from renderers import create_renderer - from renderers.base import load_tokenizer + from tests.model_assets import load_test_tokenizer from renderers.configs import Qwen3RendererConfig - tok = load_tokenizer("Qwen/Qwen3-8B") + tok = load_test_tokenizer("Qwen/Qwen3-8B") im_end = tok.convert_tokens_to_ids("<|im_end|>") u1 = {"role": "user", "content": "What is 2+2?"} @@ -322,7 +322,7 @@ def test_bridge_keeps_thinking_when_history_kwarg_disables_truncation(): bridge must NOT decline across a user turn — declining would re-render and re-tokenize model-sampled thinking bytes.""" from renderers import create_renderer - from renderers.base import load_tokenizer + from tests.model_assets import load_test_tokenizer from renderers.configs import GLM5RendererConfig, Nemotron3RendererConfig asst = [ @@ -336,7 +336,7 @@ def test_bridge_keeps_thinking_when_history_kwarg_disables_truncation(): ), ] for model, cfg in cases: - r = create_renderer(load_tokenizer(model), cfg) + r = create_renderer(load_test_tokenizer(model), cfg) prev_prompt, prev_completion = _simulate_prior_turn(r, asst) bridged = r.bridge_to_next_turn( prev_prompt, prev_completion, [{"role": "user", "content": "next"}] diff --git a/tests/test_client.py b/tests/test_client.py index 3a0becd9..d054bd82 100644 --- a/tests/test_client.py +++ b/tests/test_client.py @@ -379,6 +379,9 @@ def test_generate_threads_prompt_attribution_through_prebuilt_prompt_path(): # --------------------------------------------------------------------------- +@pytest.mark.network +@pytest.mark.model_parity +@pytest.mark.multimodal @pytest.mark.parametrize( "model_id,renderer_class_path", [ @@ -403,8 +406,8 @@ def test_generate_serializes_multimodal_features_for_qwen_vl_family( from renderers.base import ( MultiModalData, PlaceholderRange, - load_tokenizer, ) + from tests.model_assets import load_test_tokenizer mod_name, cls_name = renderer_class_path.split(":") renderer_cls = getattr(importlib.import_module(mod_name), cls_name) @@ -413,7 +416,7 @@ def test_generate_serializes_multimodal_features_for_qwen_vl_family( # _build_mm_features hits the qwen branch. The tokenizer is only # touched in __init__ to grab special-token ids; render() / etc. # aren't called here because we pre-supply prompt_ids + mm_data. - tokenizer = load_tokenizer(model_id) + tokenizer = load_test_tokenizer(model_id) renderer = renderer_cls(tokenizer) # Two synthetic 1×2×2 images. Field factory expects pixel_values diff --git a/tests/test_deepseek_r1.py b/tests/test_deepseek_r1.py index bc1e8808..a98a03b9 100644 --- a/tests/test_deepseek_r1.py +++ b/tests/test_deepseek_r1.py @@ -15,18 +15,18 @@ DeepSeekV3Renderer, create_renderer, ) -from renderers.base import load_tokenizer +from tests.model_assets import load_test_tokenizer @lru_cache def _r1(): - tok = load_tokenizer("deepseek-ai/DeepSeek-R1") + tok = load_test_tokenizer("deepseek-ai/DeepSeek-R1") return tok, create_renderer(tok) @lru_cache def _v3(): - tok = load_tokenizer("deepseek-ai/DeepSeek-V3") + tok = load_test_tokenizer("deepseek-ai/DeepSeek-V3") return tok, create_renderer(tok) diff --git a/tests/test_disabled_thinking_stability.py b/tests/test_disabled_thinking_stability.py index 335b52df..c9651657 100644 --- a/tests/test_disabled_thinking_stability.py +++ b/tests/test_disabled_thinking_stability.py @@ -26,7 +26,7 @@ from renderers import create_renderer -from renderers.base import load_tokenizer +from tests.model_assets import load_test_tokenizer from renderers.configs import ( Qwen3RendererConfig, Qwen35RendererConfig, @@ -46,7 +46,7 @@ @lru_cache(maxsize=None) def _load(model_name: str): - return load_tokenizer(model_name) + return load_test_tokenizer(model_name) def pytest_generate_tests(metafunc): diff --git a/tests/test_glm_tool_name_validation.py b/tests/test_glm_tool_name_validation.py index e16f51ba..a0df4aa6 100644 --- a/tests/test_glm_tool_name_validation.py +++ b/tests/test_glm_tool_name_validation.py @@ -57,9 +57,9 @@ @lru_cache(maxsize=None) def _load(model: str, renderer_name: str): from renderers import config_from_name, create_renderer - from renderers.base import load_tokenizer + from tests.model_assets import load_test_tokenizer - tok = load_tokenizer(model) + tok = load_test_tokenizer(model) return tok, create_renderer(tok, config_from_name(renderer_name)) diff --git a/tests/test_golden_renderer_outputs.py b/tests/test_golden_renderer_outputs.py new file mode 100644 index 00000000..613ca733 --- /dev/null +++ b/tests/test_golden_renderer_outputs.py @@ -0,0 +1,23 @@ +"""Golden contract for each registered renderer's public behavior.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from tests.golden_corpus import GOLDEN_CASES, build_golden_case + + +GOLDEN_PATH = Path(__file__).with_name("golden_renderer_outputs.json") + + +@pytest.fixture(scope="module") +def expected_cases(): + return json.loads(GOLDEN_PATH.read_text(encoding="utf-8"))["cases"] + + +@pytest.mark.parametrize("case", GOLDEN_CASES, ids=lambda case: case.slug) +def test_renderer_behavior_matches_golden(case, expected_cases): + assert build_golden_case(case) == expected_cases[case.slug] diff --git a/tests/test_gpt_oss_harmony_parity.py b/tests/test_gpt_oss_harmony_parity.py index ea52af67..14729164 100644 --- a/tests/test_gpt_oss_harmony_parity.py +++ b/tests/test_gpt_oss_harmony_parity.py @@ -35,7 +35,7 @@ ) from renderers.configs import GptOssRendererConfig from renderers.gpt_oss import GptOssRenderer -from transformers import AutoTokenizer +from tests.model_assets import load_test_tokenizer GPT_OSS_MODEL = "openai/gpt-oss-20b" DATE_FOR_PARITY = datetime.now().strftime("%Y-%m-%d") @@ -43,7 +43,7 @@ @pytest.fixture(scope="module") def tokenizer(): - return AutoTokenizer.from_pretrained(GPT_OSS_MODEL) + return load_test_tokenizer(GPT_OSS_MODEL) @pytest.fixture(scope="module") diff --git a/tests/test_hy3.py b/tests/test_hy3.py index 31a2e451..74d036ae 100644 --- a/tests/test_hy3.py +++ b/tests/test_hy3.py @@ -15,7 +15,8 @@ import pytest from renderers import Hy3RendererConfig, create_renderer -from renderers.base import ToolCallParseStatus, load_tokenizer +from renderers.base import ToolCallParseStatus +from tests.model_assets import load_test_tokenizer _MODEL = "tencent/Hy3" @@ -43,7 +44,7 @@ @lru_cache(maxsize=None) def _tok(): - return load_tokenizer(_MODEL) + return load_test_tokenizer(_MODEL) def _renderer(**flags): diff --git a/tests/test_laguna_xs21.py b/tests/test_laguna_xs21.py index a91e815a..c83f3013 100644 --- a/tests/test_laguna_xs21.py +++ b/tests/test_laguna_xs21.py @@ -20,7 +20,7 @@ from functools import lru_cache from renderers import create_renderer -from renderers.base import load_tokenizer +from tests.model_assets import load_test_tokenizer from renderers.configs import LagunaXS21RendererConfig from renderers.laguna_xs2 import LagunaXS21Renderer @@ -47,7 +47,7 @@ @lru_cache(maxsize=None) def _tok(): - return load_tokenizer(_MODEL) + return load_test_tokenizer(_MODEL) def _renderer(**config_kwargs) -> LagunaXS21Renderer: diff --git a/tests/test_llama_3.py b/tests/test_llama_3.py index 8c162329..59b01863 100644 --- a/tests/test_llama_3.py +++ b/tests/test_llama_3.py @@ -16,8 +16,8 @@ MODEL_RENDERER_MAP, ParsedResponse, ToolCallParseStatus, - load_tokenizer, ) +from tests.model_assets import load_test_tokenizer # Pinned date for byte-parity tests. Matches the chat template's # strftime fallback so we don't have to override on the apply side. @@ -34,7 +34,7 @@ @pytest.fixture(scope="module", params=_MODEL_PAIRS, ids=[m for m, _ in _MODEL_PAIRS]) def llama_pair(request): canonical, mirror = request.param - tok = load_tokenizer(canonical) + tok = load_test_tokenizer(canonical) renderer = Llama3Renderer(tok, Llama3RendererConfig(date_string=_PINNED_DATE)) return canonical, mirror, tok, renderer @@ -59,7 +59,7 @@ def test_create_renderer_via_explicit_config(llama_pair): def test_create_renderer_auto_resolves_after_mirror_load(llama_pair): - """``load_tokenizer(canonical_meta_id)`` loads from the unrestricted + """``load_test_tokenizer(canonical_meta_id)`` loads from the unrestricted mirror but preserves the canonical name needed for auto-resolution.""" canonical, _, tok, _ = llama_pair assert tok.name_or_path == canonical diff --git a/tests/test_load_tokenizer.py b/tests/test_load_tokenizer.py index ea15d6a5..564ae0a1 100644 --- a/tests/test_load_tokenizer.py +++ b/tests/test_load_tokenizer.py @@ -16,6 +16,7 @@ from renderers import base from renderers.base import TOKENIZER_SOURCE_OVERRIDES, TRUSTED_REVISIONS, load_tokenizer +from tests.model_assets import load_test_tokenizer # --------------------------------------------------------------------------- @@ -61,6 +62,21 @@ def test_unlisted_model_loads_without_remote_code(mock_from_pretrained): assert kwargs == {"trust_remote_code": False} +@patch("transformers.AutoTokenizer.from_pretrained") +def test_unlisted_model_accepts_immutable_revision_without_remote_code( + mock_from_pretrained, +): + """Parity tests may pin ordinary tokenizer repositories without opting + into repository-supplied Python.""" + revision = "a" * 40 + + load_tokenizer("Qwen/Qwen3-0.6B", revision=revision) + + args, kwargs = mock_from_pretrained.call_args + assert args == ("Qwen/Qwen3-0.6B",) + assert kwargs == {"trust_remote_code": False, "revision": revision} + + @patch("transformers.AutoTokenizer.from_pretrained") def test_kimi_loads_with_pinned_revision(mock_from_pretrained): """Kimi-K2 family: trust_remote_code=True, revision pinned to the @@ -74,6 +90,14 @@ def test_kimi_loads_with_pinned_revision(mock_from_pretrained): } +@patch("transformers.AutoTokenizer.from_pretrained") +def test_kimi_rejects_caller_revision_override(mock_from_pretrained): + with pytest.raises(ValueError, match="only at reviewed revision"): + load_tokenizer("moonshotai/Kimi-K2.5", revision="a" * 40) + + mock_from_pretrained.assert_not_called() + + @patch("transformers.AutoTokenizer.from_pretrained") def test_meta_llama_loads_tokenizer_from_unsloth_mirror(mock_from_pretrained): """Canonical Meta Llama repos are gated; load their tokenizer/chat @@ -91,6 +115,20 @@ def test_meta_llama_loads_tokenizer_from_unsloth_mirror(mock_from_pretrained): assert tok.name_or_path == canonical +@patch("transformers.AutoTokenizer.from_pretrained") +def test_meta_llama_forwards_pinned_mirror_revision(mock_from_pretrained): + canonical = "meta-llama/Llama-3.2-1B-Instruct" + mirror = "unsloth/Llama-3.2-1B-Instruct" + revision = "b" * 40 + mock_from_pretrained.return_value = SimpleNamespace(name_or_path=mirror) + + load_tokenizer(canonical, revision=revision) + + args, kwargs = mock_from_pretrained.call_args + assert args == (mirror,) + assert kwargs == {"trust_remote_code": False, "revision": revision} + + @patch("transformers.AutoTokenizer.from_pretrained") def test_unknown_path_falls_through_to_no_remote_code(mock_from_pretrained): """Unknown / fine-tuned model paths — including ``moonshotai/Kimi-K2*`` @@ -145,22 +183,26 @@ def __call__(self, *args, **kwargs): # --------------------------------------------------------------------------- +@pytest.mark.network +@pytest.mark.model_parity def test_load_tokenizer_real_qwen_works_without_remote_code(): """End-to-end: an unlisted model loads successfully without trust_remote_code. Qwen tokenizers don't ship custom Python.""" - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") assert tok is not None # Smoke: the tokenizer can encode a basic string. ids = tok.encode("hello", add_special_tokens=False) assert len(ids) > 0 +@pytest.mark.network +@pytest.mark.model_parity def test_load_tokenizer_real_kimi_uses_pinned_revision(): """End-to-end: Kimi-K2.5 loads via the pinned-revision path. The parity tests already exercise this path — this test pins the contract that ``load_tokenizer`` is the only sanctioned entry point for the trusted-revision allow-list.""" - tok = load_tokenizer("moonshotai/Kimi-K2.5") + tok = load_test_tokenizer("moonshotai/Kimi-K2.5") assert tok is not None ids = tok.encode("hello", add_special_tokens=False) assert len(ids) > 0 diff --git a/tests/test_message_indices.py b/tests/test_message_indices.py index b66efb73..90d4b311 100644 --- a/tests/test_message_indices.py +++ b/tests/test_message_indices.py @@ -96,9 +96,9 @@ def test_kimi_k2_unknown_role_message_indices(): caller list of length 2. """ from renderers import create_renderer - from renderers.base import load_tokenizer + from tests.model_assets import load_test_tokenizer - tok = load_tokenizer("moonshotai/Kimi-K2-Instruct") + tok = load_test_tokenizer("moonshotai/Kimi-K2-Instruct") renderer = create_renderer(tok) msgs = [ diff --git a/tests/test_model_assets.py b/tests/test_model_assets.py new file mode 100644 index 00000000..e8dff642 --- /dev/null +++ b/tests/test_model_assets.py @@ -0,0 +1,28 @@ +"""Offline validation of immutable model revisions used by parity tests.""" + +import re + +from renderers.base import TRUSTED_REVISIONS +from tests.model_assets import MODEL_REVISIONS, model_revision + + +def test_model_revisions_are_full_shas(): + sha_re = re.compile(r"^[0-9a-f]{40}$") + for model_name, revision in MODEL_REVISIONS.items(): + assert sha_re.fullmatch(revision), ( + f"{model_name}: test revision {revision!r} is not a full commit SHA" + ) + + +def test_remote_code_revisions_match_production_security_policy(): + for model_name, revision in TRUSTED_REVISIONS.items(): + assert model_revision(model_name) == revision + + +def test_unknown_test_model_fails_loudly(): + try: + model_revision("unreviewed/example") + except KeyError as exc: + assert "No immutable test revision" in str(exc) + else: + raise AssertionError("unreviewed model unexpectedly resolved") diff --git a/tests/test_multimodal.py b/tests/test_multimodal.py index 96c65a77..717c2982 100644 --- a/tests/test_multimodal.py +++ b/tests/test_multimodal.py @@ -15,7 +15,8 @@ the combined message list. Tests skip per-pair when: -- The HF snapshot isn't cached locally (network-free CI mode). +- The HF snapshot isn't cached locally and ``RENDERERS_TEST_NETWORK`` is not + set to ``1``. - The model lists a modality the renderer doesn't yet support (``NotImplementedError`` in ``render``). - ``Pillow`` / ``torch`` are missing. @@ -33,8 +34,9 @@ Qwen3VLRenderer, create_renderer, ) -from renderers.base import MODEL_RENDERER_MAP, load_tokenizer +from renderers.base import MODEL_RENDERER_MAP from renderers.configs import _config_class_for +from tests.model_assets import load_test_tokenizer, processor_load_kwargs def _config_for_model(model_name: str, **kwargs): @@ -67,6 +69,8 @@ def _hf_snapshot_cached(model_name: str) -> bool: Mirrors the convention used elsewhere in this repo (test_qwen35_size_coverage) of relying on the user having pre-fetched relevant models. """ + if os.environ.get("RENDERERS_TEST_NETWORK") == "1": + return True cache = ( Path(os.environ.get("HF_HOME") or Path.home() / ".cache" / "huggingface") / "hub" @@ -103,27 +107,14 @@ def _modality_cases(): _loaded: dict[str, tuple] = {} -# Models whose processors need ``trust_remote_code=True`` (custom Python -# in the repo) AND a pinned revision for security. Mirrors the -# ``TRUSTED_REVISIONS`` policy in ``renderers.base`` for tokenizers. -_PROCESSOR_TRUSTED_REVISIONS: dict[str, str] = { - "moonshotai/Kimi-K2.5": "4d01dfe0332d63057c186e0b262165819efb6611", - "moonshotai/Kimi-K2.6": "2755962d07cb42aa2d988a35bcb65cd4a9c2de82", -} - - def _load_processor_and_renderer(model_name: str): if model_name not in _loaded: from transformers import AutoProcessor - tokenizer = load_tokenizer(model_name) - revision = _PROCESSOR_TRUSTED_REVISIONS.get(model_name) - if revision is not None: - processor = AutoProcessor.from_pretrained( - model_name, trust_remote_code=True, revision=revision - ) - else: - processor = AutoProcessor.from_pretrained(model_name) + tokenizer = load_test_tokenizer(model_name) + processor = AutoProcessor.from_pretrained( + model_name, **processor_load_kwargs(model_name) + ) renderer = create_renderer(tokenizer) # Inject processor so the renderer doesn't try to fetch it lazily. if hasattr(renderer, "_processor") and renderer._processor is None: @@ -688,7 +679,7 @@ def test_modality_registry_models_route_to_renderer(): for model_name in MULTIMODAL_MODELS: if not _hf_snapshot_cached(model_name): continue - tokenizer = load_tokenizer(model_name) + tokenizer = load_test_tokenizer(model_name) renderer = create_renderer(tokenizer) # We expect a hand-coded VL renderer, not the default fallback. assert not type(renderer).__name__.startswith("Default"), ( @@ -924,7 +915,7 @@ def test_qwen3_vl_renderer_exposes_image_modality(): model = "Qwen/Qwen3-VL-4B-Instruct" if not _hf_snapshot_cached(model): pytest.skip(f"{model}: HF snapshot not cached locally") - tokenizer = load_tokenizer(model) + tokenizer = load_test_tokenizer(model) renderer = create_renderer(tokenizer) assert isinstance(renderer, Qwen3VLRenderer) assert "image" in MULTIMODAL_MODELS[model] diff --git a/tests/test_nemotron3_parity.py b/tests/test_nemotron3_parity.py index 01c521b4..a3877c12 100644 --- a/tests/test_nemotron3_parity.py +++ b/tests/test_nemotron3_parity.py @@ -37,8 +37,9 @@ import pytest from renderers import create_renderer -from renderers.base import MODEL_RENDERER_MAP, load_tokenizer +from renderers.base import MODEL_RENDERER_MAP from renderers.configs import _config_class_for +from tests.model_assets import load_test_tokenizer # BF16 / FP8 share a tokenizer; only the BF16 checkpoints are cached for tests. NANO = "nvidia/NVIDIA-Nemotron-3-Nano-30B-A3B-BF16" @@ -49,7 +50,7 @@ @lru_cache def _tok(model: str): - return load_tokenizer(model) + return load_test_tokenizer(model) def _config_cls(model: str): diff --git a/tests/test_parse_response.py b/tests/test_parse_response.py index 698c4616..bc75fc49 100644 --- a/tests/test_parse_response.py +++ b/tests/test_parse_response.py @@ -7,12 +7,13 @@ from functools import lru_cache from renderers import create_renderer -from renderers.base import ToolCallParseStatus, load_tokenizer +from renderers.base import ToolCallParseStatus +from tests.model_assets import load_test_tokenizer @lru_cache def _qwen3_vl(): - tokenizer = load_tokenizer("Qwen/Qwen3-VL-4B-Instruct") + tokenizer = load_test_tokenizer("Qwen/Qwen3-VL-4B-Instruct") renderer = create_renderer(tokenizer) return tokenizer, renderer @@ -100,7 +101,7 @@ def test_qwen3_vl_malformed_tool_call_surfaces_as_invalid_json(): @lru_cache def _qwen3(): - tokenizer = load_tokenizer("Qwen/Qwen3-0.6B") + tokenizer = load_test_tokenizer("Qwen/Qwen3-0.6B") renderer = create_renderer(tokenizer) return tokenizer, renderer @@ -163,7 +164,7 @@ def test_qwen3_distinct_parallel_calls_after_think_are_preserved(): @lru_cache def _kimi_k25(): - tokenizer = load_tokenizer("moonshotai/Kimi-K2.5") + tokenizer = load_test_tokenizer("moonshotai/Kimi-K2.5") renderer = create_renderer(tokenizer) return tokenizer, renderer @@ -232,7 +233,7 @@ def test_kimi_k25_in_think_section_is_not_a_real_call(): @lru_cache def _deepseek_v3(): - tokenizer = load_tokenizer("deepseek-ai/DeepSeek-V3") + tokenizer = load_test_tokenizer("deepseek-ai/DeepSeek-V3") renderer = create_renderer(tokenizer) return tokenizer, renderer diff --git a/tests/test_parsers.py b/tests/test_parsers.py index 1204ec98..5d4f8f7c 100644 --- a/tests/test_parsers.py +++ b/tests/test_parsers.py @@ -4,7 +4,7 @@ import pytest -from renderers.base import ToolCallParseStatus, load_tokenizer +from renderers.base import ToolCallParseStatus from renderers.parsers import ( REASONING_PARSERS, TOOL_PARSERS, @@ -13,6 +13,7 @@ get_reasoning_parser, get_tool_parser, ) +from tests.model_assets import load_test_tokenizer def test_registries_nonempty(): @@ -24,7 +25,7 @@ def test_registries_nonempty(): def test_unknown_parser_errors(): - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") with pytest.raises(ValueError, match="Unknown tool_parser"): get_tool_parser("does-not-exist", tok) with pytest.raises(ValueError, match="Unknown reasoning_parser"): @@ -33,7 +34,7 @@ def test_unknown_parser_errors(): def test_qwen3_tool_parser_roundtrip(): """Tokenize a Hermes-style tool call, parse it back out.""" - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") parser = get_tool_parser("qwen3", tok) assert isinstance(parser, Qwen3ToolParser) @@ -53,7 +54,7 @@ def test_qwen3_tool_parser_roundtrip(): def test_qwen3_tool_parser_no_tool_call(): - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") parser = get_tool_parser("qwen3", tok) ids = tok.encode("just plain text response", add_special_tokens=False) content_ids, tool_calls = parser.extract(list(ids)) @@ -63,7 +64,7 @@ def test_qwen3_tool_parser_no_tool_call(): def test_qwen3_tool_parser_records_invalid_json(): """Malformed JSON in a block surfaces as INVALID_JSON, not silently dropped.""" - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") parser = get_tool_parser("qwen3", tok) completion = ( 'hi\n\n{"name": "f", "arguments": {broken json\n' @@ -77,7 +78,7 @@ def test_qwen3_tool_parser_records_invalid_json(): def test_qwen3_tool_parser_parallel_partial_success(): """Parallel calls: parser keeps the good ones AND records the broken one.""" - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") parser = get_tool_parser("qwen3", tok) completion = ( "pre\n" @@ -96,7 +97,7 @@ def test_qwen3_tool_parser_parallel_partial_success(): def test_think_reasoning_parser_extracts_block(): - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") parser = get_reasoning_parser("think", tok) assert isinstance(parser, ThinkTextReasoningParser) reasoning, content = parser.extract("let me thinkthe answer") @@ -105,7 +106,7 @@ def test_think_reasoning_parser_extracts_block(): def test_think_reasoning_parser_no_block(): - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") parser = get_reasoning_parser("think", tok) reasoning, content = parser.extract("no reasoning here") assert reasoning is None @@ -116,7 +117,7 @@ def test_default_renderer_uses_parsers(): """DefaultRenderer + parsers should extract tool calls and reasoning.""" from renderers import DefaultRendererConfig, create_renderer - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") renderer = create_renderer( tok, DefaultRendererConfig(tool_parser="qwen3", reasoning_parser="think"), @@ -137,7 +138,7 @@ def test_default_renderer_without_parsers_is_backward_compatible(): """Without parsers, DefaultRenderer still does basic extraction.""" from renderers import DefaultRendererConfig, create_renderer - tok = load_tokenizer("Qwen/Qwen3-0.6B") + tok = load_test_tokenizer("Qwen/Qwen3-0.6B") renderer = create_renderer(tok, DefaultRendererConfig()) assert renderer.supports_tools is False diff --git a/tests/test_prime_qwen3_parity.py b/tests/test_prime_qwen3_parity.py index 56cbc6ce..e6a10169 100644 --- a/tests/test_prime_qwen3_parity.py +++ b/tests/test_prime_qwen3_parity.py @@ -2,7 +2,7 @@ import pytest from renderers import create_renderer -from renderers.base import load_tokenizer +from tests.model_assets import load_test_tokenizer MODELS = [ "PrimeIntellect/Qwen3-0.6B", @@ -141,7 +141,7 @@ @lru_cache(maxsize=None) def _load(model: str): - tokenizer = load_tokenizer(model) + tokenizer = load_test_tokenizer(model) return tokenizer, create_renderer(tokenizer) diff --git a/tests/test_qwen35_size_coverage.py b/tests/test_qwen35_size_coverage.py index 366b4e73..193e52ec 100644 --- a/tests/test_qwen35_size_coverage.py +++ b/tests/test_qwen35_size_coverage.py @@ -19,7 +19,8 @@ import pytest from renderers import Qwen35Renderer, Qwen35RendererConfig, create_renderer -from renderers.base import MODEL_RENDERER_MAP, load_tokenizer +from renderers.base import MODEL_RENDERER_MAP +from tests.model_assets import load_test_tokenizer _QWEN35_IN_MAP = { @@ -76,7 +77,7 @@ def test_qwen35_enable_thinking_polarity_default(qwen35_model, expected_default) """With no explicit flag, the renderer resolves ``enable_thinking`` from the hard-coded per-model default — so big / small sizes each match their own template at the gen-prompt boundary.""" - tok = load_tokenizer(qwen35_model) + tok = load_test_tokenizer(qwen35_model) renderer = create_renderer(tok, Qwen35RendererConfig()) assert isinstance(renderer, Qwen35Renderer) assert renderer.config.enable_thinking is expected_default, ( @@ -170,7 +171,7 @@ def test_qwen35_size_parity_with_apply_chat_template( ``apply_chat_template`` output. Locks in the property that lets us share ``Qwen35Renderer`` across all seven sizes — the polarity flip on 0.8B / 2B is absorbed by the per-model default.""" - tok = load_tokenizer(qwen35_model) + tok = load_test_tokenizer(qwen35_model) renderer = create_renderer(tok, Qwen35RendererConfig()) assert isinstance(renderer, Qwen35Renderer) diff --git a/tests/test_registry_consistency.py b/tests/test_registry_consistency.py new file mode 100644 index 00000000..0dfbb60f --- /dev/null +++ b/tests/test_registry_consistency.py @@ -0,0 +1,58 @@ +"""Fast, offline checks for the renderer registration surfaces. + +Adding a renderer touches several deliberately explicit public/type surfaces. +These assertions make drift fail in a small unit test instead of at runtime for +one model family or in downstream config deserialization. +""" + +from pydantic import TypeAdapter + +import renderers +from renderers import RendererConfig +from renderers import base as base_module +from renderers import configs as configs_module +from tests.golden_corpus import GOLDEN_CASES + + +def test_runtime_registry_matches_config_registry(): + base_module._populate_registry() + + runtime_names = set(base_module.RENDERER_REGISTRY) + config_names = set(configs_module._CONFIG_BY_NAME) - {"auto"} + + assert runtime_names == config_names + + +def test_renderer_config_union_matches_config_registry(): + schema = TypeAdapter(RendererConfig).json_schema() + discriminator_names = set(schema["discriminator"]["mapping"]) + + assert discriminator_names == set(configs_module._CONFIG_BY_NAME) + + +def test_lazy_public_renderer_exports_match_runtime_classes(): + base_module._populate_registry() + + runtime_class_names = { + renderer_class.__name__ + for renderer_class in base_module.RENDERER_REGISTRY.values() + } + + assert runtime_class_names == set(renderers._LAZY_RENDERERS) + assert runtime_class_names <= set(renderers.__all__) + + +def test_model_and_multimodal_maps_only_reference_registered_renderers(): + base_module._populate_registry() + runtime_names = set(base_module.RENDERER_REGISTRY) + + assert set(base_module.MODEL_RENDERER_MAP.values()) <= runtime_names + assert set(base_module.MULTIMODAL_MODELS) <= set(base_module.MODEL_RENDERER_MAP) + + +def test_golden_corpus_covers_each_registered_renderer_once(): + base_module._populate_registry() + golden_names = [case.renderer_name for case in GOLDEN_CASES] + + assert len(golden_names) == len(set(golden_names)) + assert set(golden_names) == set(base_module.RENDERER_REGISTRY) diff --git a/tests/test_render_ids.py b/tests/test_render_ids.py index e2e4a500..7ee303fb 100644 --- a/tests/test_render_ids.py +++ b/tests/test_render_ids.py @@ -11,7 +11,7 @@ from functools import lru_cache from renderers import create_renderer -from renderers.base import load_tokenizer +from tests.model_assets import load_test_tokenizer def _expected(tokenizer, messages, **kwargs): @@ -337,7 +337,7 @@ def test_multi_step_tool_cycle(model_name, tokenizer, renderer): @lru_cache def _qwen3_vl(): - tokenizer = load_tokenizer("Qwen/Qwen3-VL-4B-Instruct") + tokenizer = load_test_tokenizer("Qwen/Qwen3-VL-4B-Instruct") renderer = create_renderer(tokenizer) return tokenizer, renderer @@ -352,7 +352,7 @@ def test_qwen3_vl_auto_renderer(): @lru_cache def _kimi_k25(): - tokenizer = load_tokenizer("moonshotai/Kimi-K2.5") + tokenizer = load_test_tokenizer("moonshotai/Kimi-K2.5") renderer = create_renderer(tokenizer) return tokenizer, renderer @@ -365,7 +365,7 @@ def test_kimi_k2_inline_think_tags_render_verbatim(): splitting out ``...`` and then discarded the extracted reasoning, producing tokens that disagreed with ``apply_chat_template``. """ - tokenizer = load_tokenizer("moonshotai/Kimi-K2-Instruct") + tokenizer = load_test_tokenizer("moonshotai/Kimi-K2-Instruct") renderer = create_renderer(tokenizer) msgs = [ {"role": "user", "content": "hi"}, diff --git a/tests/test_renderer_config_parity.py b/tests/test_renderer_config_parity.py index 21e7c2a2..8decdcf7 100644 --- a/tests/test_renderer_config_parity.py +++ b/tests/test_renderer_config_parity.py @@ -34,9 +34,9 @@ from renderers.base import ( MODEL_RENDERER_MAP, _populate_registry, - load_tokenizer, ) from renderers.configs import _config_class_for +from tests.model_assets import load_test_tokenizer # Models exercised by the parity tests. Mirrors ``conftest.RENDERER_MODELS`` @@ -354,7 +354,7 @@ def test_kwarg_values_covers_every_declared_kwarg(): @lru_cache(maxsize=None) def _tokenizer(model_name: str): - return load_tokenizer(model_name) + return load_test_tokenizer(model_name) @lru_cache(maxsize=None) diff --git a/tests/test_roundtrip.py b/tests/test_roundtrip.py index ef6e0398..949b0709 100644 --- a/tests/test_roundtrip.py +++ b/tests/test_roundtrip.py @@ -52,7 +52,7 @@ # render time, so the reasoning round-trip can't hold by design. # test_laguna_xs21.py covers the round-trip under enable_thinking=True. ("tencent/Hy3", "auto"), - ("unsloth/Llama-3.2-1B-Instruct", "llama-3"), + ("meta-llama/Llama-3.2-1B-Instruct", "llama-3"), ("openai/gpt-oss-20b", "gpt-oss"), ("Qwen/Qwen2.5-0.5B-Instruct", "default"), ] @@ -61,9 +61,9 @@ @lru_cache(maxsize=None) def _load_renderer(model_name: str, renderer_name: str): from renderers import config_from_name, create_renderer - from renderers.base import load_tokenizer + from tests.model_assets import load_test_tokenizer - tok = load_tokenizer(model_name) + tok = load_test_tokenizer(model_name) return tok, create_renderer(tok, config_from_name(renderer_name)) diff --git a/tests/test_tool_arg_type_preservation.py b/tests/test_tool_arg_type_preservation.py index 403fca0d..4984e0e4 100644 --- a/tests/test_tool_arg_type_preservation.py +++ b/tests/test_tool_arg_type_preservation.py @@ -43,9 +43,9 @@ @lru_cache(maxsize=None) def _load(model: str, renderer_name: str): from renderers import config_from_name, create_renderer - from renderers.base import load_tokenizer + from tests.model_assets import load_test_tokenizer - tok = load_tokenizer(model) + tok = load_test_tokenizer(model) return tok, create_renderer(tok, config_from_name(renderer_name))