fix(llm): bound extract retry cascade to one timeout budget per chunk

graphify/llm.py

@@ -2265,6 +2265,7 @@ def _extract_with_adaptive_retry(
     _depth: int = 0,
     *,
     deep_mode: bool = False,
+    _deadline: float | None = None,
 ) -> dict:
     """Extract a chunk; if the response is truncated (`finish_reason="length"`),
     the API rejects the prompt as too large for the model's context window, or
@@ -2307,13 +2308,46 @@ def _extract_with_adaptive_retry(
     a splittable document: the slice is bisected and retried (#1369). A whole
     non-splittable file (e.g. one huge code file) can't be made smaller than
     itself, so we return what we got and warn.
+
+    Timeouts additionally carry a subtree wall-clock budget (``_deadline``): the
+    top-level call anchors it to one `GRAPHIFY_API_TIMEOUT` allowance, and every
+    split inherits the same absolute deadline rather than each getting a fresh
+    full timeout. Before #3142, a chunk that timed out at every depth re-paid
+    the full timeout on each of up to ``2**max_depth`` attempts — up to 2.5h for
+    the default 600s timeout at max_depth=3. A timeout checks the budget
+    reactively, after the attempt, and gives up rather than splitting further
+    once it is spent.
+
+    A split whose *own* attempt has not yet started also checks the budget
+    proactively, before making that attempt: if an earlier sibling elsewhere
+    in the same subtree already exhausted the budget with a real timeout of
+    its own, this split is skipped outright rather than paying for a fresh
+    full-length attempt that the shared budget can no longer afford.
     """
+    if _deadline is None:
+        _deadline = time.monotonic() + _resolve_api_timeout()
+    elif _depth > 0 and time.monotonic() >= _deadline:
+        # An earlier split in this subtree already used up the shared budget
+        # with a real timeout of its own (the reactive check below, on a
+        # prior call in this recursion). Skip this attempt entirely rather
+        # than paying for one more full-length call the budget can no longer
+        # afford — without this, a sibling reached after the budget is spent
+        # would still start (and pay for) its own fresh timeout, since the
+        # reactive check only fires after that sibling's own attempt fails.
+        print(
+            f"[graphify] chunk of {len(chunk)} at depth {_depth}: the subtree's "
+            f"{_resolve_api_timeout():g}s timeout budget is spent — skipping "
+            f"this split rather than starting a fresh attempt",
+            file=sys.stderr,
+        )
+        return {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, "model": model, "finish_reason": "stop"}
+
     def _merge_two(left_units, right_units) -> dict:
         left = _extract_with_adaptive_retry(
-            left_units, backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode
+            left_units, backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode, _deadline=_deadline
         )
         right = _extract_with_adaptive_retry(
-            right_units, backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode
+            right_units, backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode, _deadline=_deadline
         )
         return {
             "nodes": left.get("nodes", []) + right.get("nodes", []),
@@ -2363,6 +2397,19 @@ def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None":
         if not (_looks_like_context_exceeded(exc) or is_timeout):
             raise
         reason = "timed out" if is_timeout else "exceeded context"
+        if is_timeout and time.monotonic() >= _deadline:
+            # The subtree's shared timeout budget is spent — every prior split
+            # in this cascade already re-paid the full per-attempt timeout, so
+            # granting yet another one here is how a single slow chunk used to
+            # burn up to 2**max_depth timeouts (#3142). Give up on whatever is
+            # left rather than committing to another full-length attempt.
+            print(
+                f"[graphify] chunk of {len(chunk)} timed out at depth {_depth} and "
+                f"the subtree's {_resolve_api_timeout():g}s timeout budget is spent "
+                f"— giving up on this chunk instead of splitting further",
+                file=sys.stderr,
+            )
+            return {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 0, "output_tokens": 0, "model": model, "finish_reason": "stop"}
         if len(chunk) <= 1:
             halves = _split_lone_slice()
             if halves is not None:
@@ -2394,10 +2441,10 @@ def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None":
         )
         mid = len(chunk) // 2
         left = _extract_with_adaptive_retry(
-            chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode
+            chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode, _deadline=_deadline
         )
         right = _extract_with_adaptive_retry(
-            chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode
+            chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode, _deadline=_deadline
         )
         return {
             "nodes": left.get("nodes", []) + right.get("nodes", []),
@@ -2482,10 +2529,10 @@ def _split_lone_slice() -> "tuple[FileSlice, FileSlice] | None":
     )
     mid = len(chunk) // 2
     left = _extract_with_adaptive_retry(
-        chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode
+        chunk[:mid], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode, _deadline=_deadline
     )
     right = _extract_with_adaptive_retry(
-        chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode
+        chunk[mid:], backend, api_key, model, root, max_depth, _depth + 1, deep_mode=deep_mode, _deadline=_deadline
     )
 
     return {

tests/test_llm_backends.py

@@ -445,6 +445,91 @@ def fake_extract(chunk, *_, **__):
     assert "exceeded context" not in err
 
 
+def test_adaptive_retry_stops_when_timeout_budget_exhausted(tmp_path, capsys):
+    """Regression test for #3142: before this fix, every split re-paid the
+    full per-attempt timeout, so a chunk that keeps timing out could burn
+    2**max_depth timeouts (up to 2.5h for the 600s default at max_depth=3).
+    Once the shared subtree deadline has passed, a further timeout must give
+    up immediately instead of committing another chunk to a fresh timeout."""
+    import subprocess
+
+    files = [tmp_path / f"f{i}.md" for i in range(4)]
+    for f in files:
+        f.write_text("hello")
+
+    calls = {"n": 0}
+
+    def always_timeout(chunk, *_, **__):
+        calls["n"] += 1
+        raise subprocess.TimeoutExpired(["claude", "-p"], 600)
+
+    # First monotonic() call anchors the deadline at t=0 (+600s default
+    # budget). The second call, made right after the first timeout while
+    # deciding whether to split, reports t=700 -- past the budget -- so the
+    # cascade must give up rather than commit to another 600s attempt.
+    clock = iter([0.0, 700.0])
+    with patch("graphify.llm.extract_files_direct", side_effect=always_timeout), \
+         patch("graphify.llm.time.monotonic", side_effect=lambda: next(clock)):
+        result = llm._extract_with_adaptive_retry(
+            files, backend="claude-cli", api_key=None, model=None, root=tmp_path, max_depth=3
+        )
+
+    assert result["nodes"] == []
+    assert result["finish_reason"] == "stop"
+    assert calls["n"] == 1  # only the original attempt -- no bisection paid for
+    err = capsys.readouterr().err
+    assert "subtree" in err and "budget is spent" in err
+
+
+def test_adaptive_retry_skips_sibling_attempt_after_budget_exhausted_mid_tree(tmp_path, capsys):
+    """Regression test: the reactive check above only catches a timeout
+    *after* the attempt that caused it. Once some split elsewhere in the
+    subtree has already spent the shared budget with a real timeout of its
+    own, a sibling split reached afterwards must not still pay for a brand
+    new full-length attempt before discovering the same thing reactively --
+    it should never start that attempt at all.
+
+    The depth-0 chunk gets an instant (non-timeout) truncated response, so it
+    splits immediately with the budget untouched. The left half then times
+    out for real, spending the whole shared budget. The right half must be
+    skipped proactively, without ever calling extract_files_direct."""
+    import subprocess
+
+    files = [tmp_path / f"f{i}.md" for i in range(4)]
+    for f in files:
+        f.write_text("hello")
+
+    calls = []
+
+    def truncated_then_timeout(chunk, *_, **__):
+        calls.append(len(chunk))
+        if len(chunk) == 4:
+            return {"nodes": [], "edges": [], "hyperedges": [], "input_tokens": 1,
+                    "output_tokens": 1, "model": "m", "finish_reason": "length"}
+        raise subprocess.TimeoutExpired(["claude", "-p"], 600)
+
+    # t=0: depth-0 anchors the deadline (+600s). Its own attempt returns
+    # "length" instantly, so no further monotonic() call happens at depth 0.
+    # t=5: the left child's proactive check, well inside the budget -- it
+    # proceeds to attempt, and that attempt times out for real.
+    # t=605: the left child's reactive check, past the deadline -- it gives
+    # up and returns to depth 0.
+    # t=605: the right child's proactive check, also past the deadline -- it
+    # must skip its attempt entirely rather than starting a fresh one.
+    clock = iter([0.0, 5.0, 605.0, 605.0])
+    with patch("graphify.llm.extract_files_direct", side_effect=truncated_then_timeout), \
+         patch("graphify.llm.time.monotonic", side_effect=lambda: next(clock)):
+        result = llm._extract_with_adaptive_retry(
+            files, backend="claude-cli", api_key=None, model=None, root=tmp_path, max_depth=3
+        )
+
+    assert result["nodes"] == []
+    # depth-0 (len 4) + left's real attempt (len 2) -- right never attempted.
+    assert calls == [4, 2]
+    err = capsys.readouterr().err
+    assert "subtree" in err and "budget is spent" in err
+
+
 def test_adaptive_retry_timeout_caps_at_max_depth(tmp_path, capsys):
     import subprocess