diff --git a/.gitignore b/.gitignore index 9facfca..616b7e2 100644 --- a/.gitignore +++ b/.gitignore @@ -66,3 +66,10 @@ __pycache__/ *-config-with-keys* *-with-keys* .rocketride/ + +# E2E test outputs +demos/e2e-tests/e2e-results.txt +demos/e2e-tests/e2e.csv +!demos/e2e-tests/config-ollama.txt +demos/e2e-tests/config-groq.txt +demos/e2e-tests/config-anthropic.txt diff --git a/demos/e2e-tests/config-anthropic.txt.example b/demos/e2e-tests/config-anthropic.txt.example new file mode 100644 index 0000000..5653df2 --- /dev/null +++ b/demos/e2e-tests/config-anthropic.txt.example @@ -0,0 +1,19 @@ +# E2E validation — Anthropic (PR #71 / issue #63) +# +# Get a key at https://console.anthropic.com/settings/keys (needs billing). +# +# What this config is for: verifying the adaptive-thinking request shape. +# The review found the new code path is unreachable for every model bundled +# in models.yaml — all 7 are 4.5-or-earlier and take the extended-thinking +# branch. To exercise the adaptive branch you must set a 5-series model +# explicitly, which llm:set-model permits with a warning. +# +# Swap `model` below to claude-opus-4-5-20251101 to test the EXTENDED path, +# or to claude-sonnet-5 / claude-opus-5 to test the ADAPTIVE path. + +provider=anthropic +anthropic_api_key=sk-ant-REPLACE_WITH_YOUR_KEY +model=claude-haiku-4-5-20251001 +temperature=0.0 +max_tokens=300 +timeout_seconds=60 diff --git a/demos/e2e-tests/config-groq.txt.example b/demos/e2e-tests/config-groq.txt.example new file mode 100644 index 0000000..1309aab --- /dev/null +++ b/demos/e2e-tests/config-groq.txt.example @@ -0,0 +1,17 @@ +# E2E validation — Groq (PR #73 / issue #51) +# +# Get a free key in ~2 minutes at https://console.groq.com/keys +# Free tier, no credit card. Replace the placeholder below. +# +# What this config is for: the code review found that a plain llm:chat never +# sends reasoning_format, so Groq falls back to its documented default "raw" +# and emits thinking inside tags in the visible content. Nothing +# strips them. This config is what proves or disproves that against the real +# API — it cannot be settled without a live call. + +provider=groq +groq_api_key=gsk_REPLACE_WITH_YOUR_KEY +model=openai/gpt-oss-20b +temperature=0.0 +max_tokens=300 +timeout_seconds=60 diff --git a/demos/e2e-tests/config-ollama.txt b/demos/e2e-tests/config-ollama.txt new file mode 100644 index 0000000..58e1280 --- /dev/null +++ b/demos/e2e-tests/config-ollama.txt @@ -0,0 +1,8 @@ +# E2E validation — local Ollama. No API key required. +# Verifies the full stack end to end without touching a paid provider. +provider=ollama +model=qwen2.5vl:3b +base_url=http://localhost:11434 +temperature=0.0 +max_tokens=200 +timeout_seconds=120 diff --git a/demos/e2e-tests/e2e-tests.nlogox b/demos/e2e-tests/e2e-tests.nlogox new file mode 100644 index 0000000..33da3c0 --- /dev/null +++ b/demos/e2e-tests/e2e-tests.nlogox @@ -0,0 +1,326 @@ + + + " answer) + assert "response is a non-empty string" (is-string? answer and not empty? answer) + assert "response is not an error" (not member? "ERROR:" answer) + report-totals +end + +;; --------------------------------------------------------------------------- +;; T2 — thinking-tag leakage (the HIGH finding on PR #73) +;; +;; Groq's documented default reasoning_format is "raw", which returns the +;; model's reasoning inline in tags. The extension copies content +;; verbatim, so if the tags survive into the answer a user sees them. This +;; is the check that only a live call can settle. +;; --------------------------------------------------------------------------- + +to test-think-tag-leakage + log-line "T2 thinking-tag leakage" + let answer "" + carefully + [ set answer llm:chat "What is 2 + 2? Answer with just the number." ] + [ set answer (word "ERROR: " error-message) ] + log-line (word " -> " answer) + assert "visible answer has no tag" (not member? "" answer) + assert "visible answer has no tag" (not member? "" answer) + report-totals +end + +;; --------------------------------------------------------------------------- +;; T3 — the same check with thinking explicitly enabled +;; +;; With thinking on, reasoning_format is sent. Thinking should arrive through +;; llm:chat-with-thinking, and must NOT also be duplicated into the answer. +;; --------------------------------------------------------------------------- + +to test-thinking-path + log-line "T3 thinking enabled" + llm:set-thinking true + let result 0 + carefully + [ set result llm:chat-with-thinking "What is 7 * 6? Answer with just the number." ] + [ set result (list (word "ERROR: " error-message) "") ] + let answer item 0 result + let reasoning item 1 result + output-print (word " answer -> " answer) + output-print (word " thinking -> " reasoning) + assert "answer is non-empty" (not empty? answer) + assert "answer carries no tag" (not member? "" answer) + llm:set-thinking false + report-totals +end + +;; --------------------------------------------------------------------------- +;; T4 — llm:compile-error against the live extension (issue #52, merged) +;; +;; No API call. Confirms the merged primitive still behaves once the other +;; three branches are integrated on top of it. +;; --------------------------------------------------------------------------- + +to test-compile-error + log-line "T4 llm:compile-error" + assert "valid code reports no error" (llm:compile-error "fd 1" = "") + assert "undefined primitive is rejected" ((llm:compile-error "frobnicate") != "") + assert "turtle context accepts rt" (llm:compile-error "rt 90 fd 1" = "") + assert "banned primitive is caught" ((llm:compile-error "die" ["die"]) != "") + assert "banned name inside identifier is not flagged" + ((llm:compile-error "let diehard 1 fd diehard" ["die"]) = "") + assert "multi-line let scoping works" (llm:compile-error "let x 5\nfd x" = "") + report-totals +end + +;; --------------------------------------------------------------------------- +;; T5 — per-agent history isolation under real calls +;; --------------------------------------------------------------------------- + +to test-agent-history + log-line "T5 per-agent history" + create-turtles 3 [ setxy random-xcor random-ycor ] + ask turtles [ + carefully [ let ignored llm:chat "Say OK" ] [ ] + ] + let lengths [length llm:history] of turtles + output-print (word " history lengths -> " lengths) + assert "every agent kept its own history" (length remove-duplicates lengths <= 2) + ask turtles [ llm:clear-history ] + assert "clear-history empties each agent" + (reduce + [length llm:history] of turtles = 0) + report-totals +end + + +;; --------------------------------------------------------------------------- +;; T6 — structured output (#22) +;; +;; The point of schema-constrained replies is that fields arrive as NetLogo +;; TYPES, not text to be parsed. A provider can return well-formed JSON and +;; still fail this if the conversion drops types, so both are asserted. +;; --------------------------------------------------------------------------- + +to test-structured-output + log-line "T6 structured output" + let schema "{\"type\":\"object\",\"properties\":{\"action\":{\"type\":\"string\",\"enum\":[\"eat\",\"explore\"]},\"confidence\":{\"type\":\"number\"},\"alive\":{\"type\":\"boolean\"}},\"required\":[\"action\",\"confidence\",\"alive\"]}" + let parsed 0 + carefully + [ set parsed llm:chat-with-schema "A turtle stands on food. Decide." schema ] + [ set parsed 0 + log-line (word " chat-with-schema failed: " error-message) ] + + assert "schema reply is a list" (is-list? parsed) + if is-list? parsed [ + log-line (word " -> " parsed) + let act llm:get parsed "action" + let conf llm:get parsed "confidence" + let live llm:get parsed "alive" + assert "action is a string" (is-string? act) + assert "action honours the schema enum" (member? act ["eat" "explore"]) + assert "confidence is a NUMBER, not text" (is-number? conf) + assert "boolean field is a boolean" (is-boolean? live) + assert "missing key raises rather than defaulting" missing-key-raises? parsed + ] + + ;; The no-schema path returns raw JSON text, which must stay a string. + let raw "" + carefully + [ set raw llm:chat-json "Name two colours as a JSON object with key colours." ] + [ set raw (word "ERROR: " error-message) ] + log-line (word " chat-json -> " raw) + assert "chat-json returns a string" (is-string? raw) + assert "chat-json output looks like JSON" (member? "{" raw) + report-totals +end + +;; llm:get on an absent key must raise, so an optional field cannot be read as +;; a silent empty value. +to-report missing-key-raises? [parsed] + let raised? true + carefully + [ let ignored llm:get parsed "no-such-key-here" + set raised? false ] + [ set raised? true ] + report raised? +end + +;; Run everything in sequence. +to run-headless + carefully [ file-delete "e2e-results.txt" ] [ ] + setup + test-all +end + +to noop +end + +to test-all + clear-output + set pass-count 0 + set fail-count 0 + test-basic-chat + test-think-tag-leakage + test-thinking-path + test-compile-error + test-agent-history + test-structured-output + output-print "" + log-line (word "TOTAL passed " pass-count " failed " fail-count) +end +]]> + + + + + + + + + + pass-count + fail-count + + + .txt` beside this model. +2. Pick the provider in the chooser. +3. Press **setup**, then **run all tests**. + +`ollama` needs no key and runs locally. + +## WHAT EACH TEST COVERS + +**T1 basic chat** — a plain round trip. Proves the provider is reachable and the +request shape is accepted. + +**T2 thinking-tag leakage** — the highest-severity open finding. A plain `llm:chat` +never sends `reasoning_format`, so Groq falls back to its documented default `raw` +and returns reasoning inline in `` tags. Content is copied verbatim, so those +tags reach the user. This test fails if that happens. + +**T3 thinking enabled** — the same question with thinking on, where +`reasoning_format` IS sent. Reasoning should arrive separately, not duplicated into +the answer. + +**T4 llm:compile-error** — no API call. Confirms the merged primitive still works +with the other branches integrated on top. + +**T5 per-agent history** — each turtle keeps its own conversation, and +`llm:clear-history` empties it. + +## THINGS TO NOTICE + +A pass on Ollama does not imply a pass on Groq. T2 and T3 are provider-specific by +construction: the bug they target only appears against a provider whose default +reasoning format is `raw`. + +## SCOPE + +These tests prove the extension handles what a provider returns. They do not prove +every model in `models.yaml` is still live — providers retire models on their own +schedule, and the registry needs periodic checking against each provider's +deprecation page. +]]> + + + + + + + + + + + + + + + + + + + + + + + + + + + + run-headless + noop + + pass-count + fail-count + + + +