fix(extract): retain provenance for legacy and data nodes

graphify/build.py

@@ -872,6 +872,34 @@ def build_from_json(extraction: dict, *, directed: bool = False, root: str | Pat
         if isinstance(edge, dict):
             _fold_edge_aliases(edge)
 
+    # Older graph.json files can contain semantic concept nodes without the
+    # schema-required source_file even though their incident edges retain the
+    # originating file.  Preserve those nodes during an incremental rebuild,
+    # but restore auditable provenance deterministically from the edge evidence
+    # before validation.  Do not invent a path when no incident edge proves one.
+    incident_sources: dict[object, set[str]] = {}
+    for edge in extraction.get("edges", []):
+        if not isinstance(edge, dict):
+            continue
+        source_file = edge.get("source_file")
+        if not isinstance(source_file, str) or not source_file.strip():
+            continue
+        for endpoint in (edge.get("source"), edge.get("target")):
+            try:
+                incident_sources.setdefault(endpoint, set()).add(source_file)
+            except TypeError:
+                # The validator reports malformed, non-hashable endpoints.
+                continue
+    for node in extraction.get("nodes", []):
+        if not isinstance(node, dict) or "source_file" in node:
+            continue
+        try:
+            candidates = incident_sources.get(node.get("id"), set())
+        except TypeError:
+            continue
+        if candidates:
+            node["source_file"] = min(candidates, key=lambda value: (value.casefold(), value))
+
     errors = validate_extraction(extraction)
     # Dangling edges (stdlib/external imports) are expected - only warn about real schema errors.
     real_errors = [e for e in errors if "does not match any node id" not in e]

graphify/extractors/json_config.py

@@ -58,11 +58,26 @@ def extract_json(path: Path) -> dict:
     (dependencies / extends / $ref / $schema / compilerOptions)."""
     _JSON_MAX_BYTES = 1_048_576  # 1 MiB — skip large fixture dumps / GeoJSON blobs
 
+    # Even deliberately-unexpanded data JSON must remain visible in the graph.
+    # Matrices, fixtures and datasets are real repository artefacts; returning no
+    # nodes made freshness/affected checks blind to thousands of files.  Emit the
+    # file identity independently of content parsing, while retaining the cap
+    # that prevents data keys from exploding into low-signal graph nodes.
+    str_path = str(path)
+    file_nid = _make_id(str_path)
+    file_node = {
+        "id": file_nid,
+        "label": path.name,
+        "file_type": "code",
+        "source_file": str_path,
+        "source_location": "L1",
+    }
+
     try:
         import tree_sitter_json as tsjson
         from tree_sitter import Language, Parser
     except ImportError:
-        return {"nodes": [], "edges": [], "error": "tree-sitter-json not installed"}
+        return {"nodes": [file_node], "edges": [], "error": "tree-sitter-json not installed"}
 
     try:
         # Bounded read instead of stat()+read() to eliminate TOCTOU (J-1):
@@ -71,19 +86,22 @@ def extract_json(path: Path) -> dict:
         with path.open("rb") as _f:
             source = _f.read(_JSON_MAX_BYTES + 1)
         if len(source) > _JSON_MAX_BYTES:
-            return {"nodes": [], "edges": [], "error": "json file too large to index"}
+            return {
+                "nodes": [file_node],
+                "edges": [],
+                "skipped": "data json content exceeds structural indexing cap",
+            }
         language = Language(tsjson.language())
         parser = Parser(language)
         tree = parser.parse(source)
         root = tree.root_node
     except Exception as e:
-        return {"nodes": [], "edges": [], "error": str(e)}
+        return {"nodes": [file_node], "edges": [], "error": str(e)}
 
     stem = _file_stem(path)
-    str_path = str(path)
-    nodes: list[dict] = []
+    nodes: list[dict] = [file_node]
     edges: list[dict] = []
-    seen_ids: set[str] = set()
+    seen_ids: set[str] = {file_nid}
 
     # Keys whose string values become imports (package.json dep blocks)
     _DEP_KEYS = frozenset({
@@ -108,9 +126,6 @@ def add_edge(src: str, tgt: str, relation: str, line: int,
             edge["context"] = context
         edges.append(edge)
 
-    file_nid = _make_id(str(path))
-    add_node(file_nid, path.name, 1)
-
     def _key_text(pair_node) -> str | None:
         """Extract the string content of a pair's key."""
         key_node = pair_node.child_by_field_name("key")
@@ -205,12 +220,12 @@ def walk_object(obj_node, parent_nid: str, parent_key: str | None,
     if doc.type == "object":
         # Only AST-extract recognized config/manifest JSON. Data JSON (fixtures,
         # datasets, GeoJSON, API dumps) is skipped so it doesn't explode into
-        # orphan key-nodes (#1224); it's left to the LLM semantic pass.
+        # orphan key-nodes (#1224). The file node above preserves its existence.
         if not _is_config_json(path, doc, source):
-            return {"nodes": [], "edges": [], "skipped": "data json (not a config/manifest)"}
+            return {"nodes": nodes, "edges": [], "skipped": "data json (not a config/manifest)"}
         walk_object(doc, file_nid, None, 0, [0])
     else:
         # Top-level array or scalar => data JSON, never a config/manifest.
-        return {"nodes": [], "edges": [], "skipped": "data json (non-object root)"}
+        return {"nodes": nodes, "edges": [], "skipped": "data json (non-object root)"}
 
     return {"nodes": nodes, "edges": edges}

tests/test_build.py

@@ -385,6 +385,44 @@ def test_edge_missing_source_file_backfilled_from_node():
     assert sf == "docs/a.md"  # backfilled from the source node
 
 
+def test_legacy_node_missing_source_file_backfilled_from_incident_edge(capsys):
+    """A preserved semantic node inherits real, deterministic edge provenance."""
+    extraction = {
+        "nodes": [
+            {"id": "concept_auth", "label": "Authentication", "file_type": "concept"},
+            {"id": "n1", "label": "A", "file_type": "document", "source_file": "docs/z.md"},
+            {"id": "n2", "label": "B", "file_type": "document", "source_file": "docs/a.md"},
+        ],
+        "edges": [
+            {"source": "n1", "target": "concept_auth", "relation": "references",
+             "confidence": "EXTRACTED", "source_file": "docs/z.md"},
+            {"source": "n2", "target": "concept_auth", "relation": "references",
+             "confidence": "EXTRACTED", "source_file": "docs/a.md"},
+        ],
+        "input_tokens": 0,
+        "output_tokens": 0,
+    }
+
+    G = build_from_json(extraction)
+
+    assert G.nodes["concept_auth"]["source_file"] == "docs/a.md"
+    assert "missing required field 'source_file'" not in capsys.readouterr().err
+
+
+def test_missing_node_source_file_without_edge_evidence_still_warns(capsys):
+    """A migration must not fabricate provenance when the graph proves none."""
+    extraction = {
+        "nodes": [{"id": "concept_auth", "label": "Authentication", "file_type": "concept"}],
+        "edges": [],
+        "input_tokens": 0,
+        "output_tokens": 0,
+    }
+
+    build_from_json(extraction)
+
+    assert "missing required field 'source_file'" in capsys.readouterr().err
+
+
 def test_build_merges_multiple_extractions():
     ext1 = {"nodes": [{"id": "n1", "label": "A", "file_type": "code", "source_file": "a.py"}],
             "edges": [], "input_tokens": 0, "output_tokens": 0}

tests/test_extract.py

@@ -3343,8 +3343,9 @@ def test_extract_json_large_file_skipped(tmp_path):
     # Write a JSON file just over 1 MiB
     big.write_bytes(b'{"x": "' + b"a" * (1_048_576) + b'"}')
     result = extract_json(big)
-    assert "error" in result
-    assert result["nodes"] == []
+    assert "skipped" in result
+    assert len(result["nodes"]) == 1
+    assert result["nodes"][0]["source_file"] == str(big)
 
 
 def test_extract_json_handles_invalid_json(tmp_path):
@@ -3367,15 +3368,16 @@ def test_extract_json_no_self_loops():
 # ---------------------------------------------------------------------------
 
 def test_extract_json_data_file_skipped(tmp_path):
-    """A data-shaped .json (eval fixture / dataset) must NOT emit per-key nodes."""
+    """Data JSON emits its file identity but no per-key nodes."""
     data = tmp_path / "cases.json"
     data.write_text(json.dumps({
         "generation": {"target": "gpt-4", "cases_file": "c.json", "num_cases": 12},
         "prompt_inputs_spec": {"a": 1, "b": 2},
         "suite": [{"name": "x"}, {"name": "y"}],
     }))
     result = extract_json(data)
-    assert result["nodes"] == []
+    assert len(result["nodes"]) == 1
+    assert result["nodes"][0]["source_file"] == str(data)
     assert result["edges"] == []
     assert "skipped" in result
 
@@ -3385,7 +3387,8 @@ def test_extract_json_top_level_array_skipped(tmp_path):
     data = tmp_path / "records.json"
     data.write_text(json.dumps([{"id": 1}, {"id": 2}]))
     result = extract_json(data)
-    assert result["nodes"] == []
+    assert len(result["nodes"]) == 1
+    assert result["nodes"][0]["source_file"] == str(data)
     assert result["edges"] == []