Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 8 additions & 7 deletions tests/e2e/test_config_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,8 +14,9 @@
may fail in test environments. We mock it to return ("cpu", ["cpu"]).

Note: The config command writes JSON to stdout via print() and Rich status
messages to stderr via Console(stderr=True). CliRunner captures both in
result.output. We extract JSON by finding the first '{' or '[' character.
messages to stderr via Console(stderr=True). We parse result.stdout (the
clean JSON channel) so that -v/--verbose log lines on stderr never corrupt
the payload. _extract_json still scans defensively for the JSON object.

Markers:
e2e: Full end-to-end test with real models
Expand Down Expand Up @@ -88,11 +89,11 @@ def _resolve_device_mock(


def _extract_json(output: str) -> dict | list:
"""Extract JSON object/array from mixed CLI output.
"""Extract JSON object/array from CLI stdout.

The config command mixes Rich status messages (stderr) with JSON
(stdout) in CliRunner output. Find the first '{' or '[' that
starts a valid JSON payload.
The config command prints JSON to stdout; this defensively scans for
the first '{' or '[' that starts a valid JSON payload in case any
non-JSON noise (e.g. a stray print) leaks onto stdout.
"""
decoder = json.JSONDecoder()
# JSON is printed as its own line; probing only line starts avoids
Expand All @@ -113,7 +114,7 @@ def _run_config(*args: str) -> dict:
runner = CliRunner()
result = runner.invoke(config, list(args), catch_exceptions=False)
assert result.exit_code == 0, f"config failed (exit {result.exit_code}):\n{result.output}"
return _extract_json(result.output)
return _extract_json(result.stdout)


def _assert_hf_config_structure(data: dict) -> None:
Expand Down
11 changes: 8 additions & 3 deletions tests/e2e/test_inspect_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -60,10 +60,15 @@ def _run_json(*args: str) -> dict:
"""Invoke inspect with *args + '-f json' and return parsed JSON.

Asserts exit_code == 0 and that the output is valid JSON.

Parses ``result.stdout`` (not ``result.output``): under Click 8.4
``result.output`` interleaves stderr into stdout, so ``-v`` log lines
emitted on stderr would corrupt the JSON. stdout carries only the
structured payload, matching how real JSON consumers read the command.
"""
result = _run(*args, "-f", "json")
assert result.exit_code == 0, f"inspect exited {result.exit_code}:\n{result.output}"
return json.loads(result.output)
return json.loads(result.stdout)


def _run_network(model: str, task: str | None = None) -> dict:
Expand All @@ -77,7 +82,7 @@ def _run_network(model: str, task: str | None = None) -> dict:
args.extend(["-t", task])
result = CliRunner().invoke(inspect, args, obj={}, catch_exceptions=False)
assert result.exit_code == 0, f"inspect failed (exit {result.exit_code}):\n{result.output}"
return json.loads(result.output)
return json.loads(result.stdout)


def _assert_common_structure(data: dict, model_id: str, expected_task: str) -> None:
Expand Down Expand Up @@ -229,7 +234,7 @@ def test_next_sentence_prediction(self):
obj={},
)
if result.exit_code == 0:
data = json.loads(result.output)
data = json.loads(result.stdout)
assert "model_id" in data
else:
assert "Traceback (most recent call last)" not in result.output
Expand Down
18 changes: 9 additions & 9 deletions tests/e2e/test_run_e2e.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,7 @@ def test_file_json_format(self, runner: CliRunner, image_model: str, test_image:
catch_exceptions=False,
)
assert result.exit_code == 0
data = json.loads(result.output)
data = json.loads(result.stdout)
assert data["task"] == "image-classification"
assert isinstance(data["predictions"], list)
assert len(data["predictions"]) > 0
Expand Down Expand Up @@ -227,7 +227,7 @@ def test_named_input(self, runner: CliRunner, image_model: str, test_image: str)
catch_exceptions=False,
)
assert result.exit_code == 0
data = json.loads(result.output)
data = json.loads(result.stdout)
assert len(data["predictions"]) > 0


Expand Down Expand Up @@ -274,7 +274,7 @@ def test_named_input(self, runner: CliRunner, text_model: str) -> None:
catch_exceptions=False,
)
assert result.exit_code == 0
data = json.loads(result.output)
data = json.loads(result.stdout)
assert "predictions" in data

def test_pipeline_param(self, runner: CliRunner, text_model: str) -> None:
Expand All @@ -295,7 +295,7 @@ def test_pipeline_param(self, runner: CliRunner, text_model: str) -> None:
catch_exceptions=False,
)
assert result.exit_code == 0
data = json.loads(result.output)
data = json.loads(result.stdout)
assert "predictions" in data


Expand Down Expand Up @@ -327,7 +327,7 @@ def test_json_format_keys(self, runner: CliRunner, image_model: str, test_image:
catch_exceptions=False,
)
assert result.exit_code == 0
data = json.loads(result.output)
data = json.loads(result.stdout)
assert {"task", "predictions", "latency_ms", "device"}.issubset(data.keys())

def test_output_to_file(
Expand Down Expand Up @@ -363,7 +363,7 @@ def test_schema_json(self, runner: CliRunner, image_model: str) -> None:
catch_exceptions=False,
)
assert result.exit_code == 0
data = json.loads(result.output)
data = json.loads(result.stdout)
assert "task" in data
assert isinstance(data["inputs"], list)

Expand Down Expand Up @@ -404,7 +404,7 @@ def test_schema(self, runner: CliRunner, pair: dict[str, str]) -> None:
catch_exceptions=False,
)
assert result.exit_code == 0, f"--schema failed (exit {result.exit_code}):\n{result.output}"
data = json.loads(result.output)
data = json.loads(result.stdout)
assert "task" in data
assert isinstance(data["inputs"], list)
if data["inputs"]:
Expand Down Expand Up @@ -446,7 +446,7 @@ def test_run(
assert schema_result.exit_code == 0, (
f"--schema failed (exit {schema_result.exit_code}):\n{schema_result.output}"
)
schema = json.loads(schema_result.output)
schema = json.loads(schema_result.stdout)

# Step 2: Build inference args
input_args = _build_inference_args(schema["inputs"], task, test_image)
Expand All @@ -465,7 +465,7 @@ def test_run(
assert result.exit_code == 0, (
f"Inference failed (exit {result.exit_code}):\n{result.output}"
)
data = _extract_json(result.output)
data = _extract_json(result.stdout)
assert "task" in data
assert "latency_ms" in data
assert data["latency_ms"] > 0
Loading