diff --git a/graphify/watch.py b/graphify/watch.py index 2b47c11ee..cd4714042 100644 --- a/graphify/watch.py +++ b/graphify/watch.py @@ -2,6 +2,7 @@ from __future__ import annotations import contextlib import json +import logging import os import posixpath import re @@ -12,6 +13,8 @@ # Single source of truth in graphify.paths (#1423); re-exported as _GRAPHIFY_OUT. from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT, is_absolute_any_platform + +logger = logging.getLogger(__name__) _PENDING_FILENAME = ".pending_changes" _PENDING_DRAIN_MAX_PASSES = 20 @@ -506,6 +509,180 @@ def _is_remote_source(source_file: str) -> bool: return bool(_REMOTE_SOURCE_RE.match(source_file)) +def _reconcile_markdown_links( + result: dict, + preserved_edges: list[dict], + preserved_nodes: list[dict], + code_files: list[Path], + extract_targets: list[Path], + full_rebuild: bool, + project_root: Path, + source_paths, +) -> list[dict]: + """Reconcile Markdown links without losing semantic-backed targets. + + Semantic-backed documents are deliberately skipped by the AST quick scan + (#1915/#1954), so their canonical file node can be absent. Re-point an + authored link only when both files have unique representatives. If either + side is ambiguous, retain the existing AST edge instead of guessing or + deleting it. A link removed from its owning Markdown source is pruned. + """ + from graphify.build import _is_ast_tier + from graphify.extract import _file_node_id, _safe_extract_with_xaml_root + from graphify.extractors.markdown import extract_markdown + + all_nodes = result.get("nodes", []) + preserved_nodes + nodes_by_source: dict[str, list[dict]] = {} + for node in all_nodes: + if source_file := node.get("source_file"): + normalized = source_paths.normalize(source_file) + nodes_by_source.setdefault(normalized, []).append(node) + + representatives: dict[str, str | None] = {} + for source_file, nodes in nodes_by_source.items(): + canonical_id = _file_node_id(Path(source_file)) + canonical = [node for node in nodes if node["id"] == canonical_id] + pages = [node for node in nodes if node.get("node_kind") == "page"] + semantic_documents = [ + node + for node in nodes + if node.get("file_type") == "document" and node.get("_origin") != "ast" + ] + if len(canonical) == 1: + representatives[source_file] = canonical[0]["id"] + elif len(pages) == 1: + representatives[source_file] = pages[0]["id"] + elif len(semantic_documents) == 1: + representatives[source_file] = semantic_documents[0]["id"] + else: + representatives[source_file] = None + + markdown_files = code_files if full_rebuild else extract_targets + markdown_files = [path for path in markdown_files if path.suffix.lower() == ".md"] + parsed_sources: set[str] = set() + authored_links: set[tuple[str, str]] = set() + authored_raw_pairs: set[frozenset[str]] = set() + resolved_raw_pairs: dict[frozenset[str], frozenset[str]] = {} + desired_edges: list[tuple[str, str, dict, str, str]] = [] + node_sources = { + node["id"]: source_paths.normalize(node["source_file"]) + for node in all_nodes + if node.get("source_file") + } + + for markdown_file in markdown_files: + try: + relative_source = markdown_file.relative_to(project_root) + except ValueError: + relative_source = markdown_file + source_file = source_paths.normalize(str(relative_source)) + parsed_sources.add(source_file) + source_rep = representatives.get(source_file) + + extraction = _safe_extract_with_xaml_root( + extract_markdown, markdown_file, project_root + ) + for edge in extraction.get("edges", []): + if edge.get("relation") != "references" or not edge.get("target_file"): + continue + target_path = Path(edge["target_file"]) + try: + target_path = target_path.relative_to(project_root) + except ValueError: + pass + target_file = source_paths.normalize(str(target_path)) + authored_links.add((source_file, target_file)) + raw_pair = frozenset( + (_file_node_id(Path(source_file)), _file_node_id(Path(target_file))) + ) + authored_raw_pairs.add(raw_pair) + + target_rep = representatives.get(target_file) + if not source_rep or not target_rep: + logger.debug( + "Preserving ambiguous Markdown link %s -> %s", + source_file, + target_file, + ) + continue + if source_rep != target_rep: + resolved_raw_pairs[raw_pair] = frozenset((source_rep, target_rep)) + desired_edges.append( + (source_rep, target_rep, edge, source_file, target_file) + ) + + desired_file_pairs = { + frozenset((source_file, target_file)) + for _, _, _, source_file, target_file in desired_edges + } + desired_reps_by_file_pair = { + frozenset((source_file, target_file)): frozenset((source_rep, target_rep)) + for source_rep, target_rep, _, source_file, target_file in desired_edges + } + + def _keep_edge(edge: dict) -> bool: + if not (_is_ast_tier(edge) and edge.get("relation") == "references"): + return True + owner = source_paths.normalize(edge.get("source_file")) + if owner not in parsed_sources: + return True + + raw_pair = frozenset((edge.get("source"), edge.get("target"))) + if raw_pair in authored_raw_pairs: + desired_pair = resolved_raw_pairs.get(raw_pair) + return desired_pair is None or raw_pair == desired_pair + + endpoint_files = { + node_sources.get(edge.get("source")), + node_sources.get(edge.get("target")), + } + endpoint_files.discard(None) + targets = endpoint_files - {owner} + if len(targets) == 1: + target_file = targets.pop() + elif edge.get("target_file"): + target_path = Path(edge["target_file"]) + try: + target_path = target_path.relative_to(project_root) + except ValueError: + pass + target_file = source_paths.normalize(str(target_path)) + else: + return True + + file_pair = frozenset((owner, target_file)) + if (owner, target_file) not in authored_links: + return False + if file_pair not in desired_file_pairs: + return True + return raw_pair == desired_reps_by_file_pair[file_pair] + + result["edges"] = [edge for edge in result.get("edges", []) if _keep_edge(edge)] + preserved_edges = [edge for edge in preserved_edges if _keep_edge(edge)] + existing_pairs = { + frozenset((edge.get("source"), edge.get("target"))) + for edge in result["edges"] + preserved_edges + } + for source_rep, target_rep, edge, source_file, _ in desired_edges: + pair = frozenset((source_rep, target_rep)) + # graphify uses a simple graph. Never overwrite another relation that + # already occupies the representative pair. + if pair in existing_pairs: + continue + reconciled = edge.copy() + reconciled.pop("target_file", None) + reconciled.update( + source=source_rep, + target=target_rep, + source_file=source_file, + _origin="ast", + ) + result["edges"].append(reconciled) + existing_pairs.add(pair) + + return preserved_edges + + def _reconcile_existing_graph( existing_graph: Path, result: dict, @@ -747,6 +924,17 @@ def _ignored_now(identity: str) -> bool: ) ] + preserved_edges = _reconcile_markdown_links( + result, + preserved_edges, + preserved_nodes, + code_files, + extract_targets, + full_rebuild, + project_root, + source_paths, + ) + new_hyperedge_ids = { edge.get("id") for edge in result.get("hyperedges", []) if edge.get("id") } diff --git a/tests/test_watch.py b/tests/test_watch.py index 075cc3f4a..adc8aa834 100644 --- a/tests/test_watch.py +++ b/tests/test_watch.py @@ -3798,3 +3798,232 @@ def test_read_only_events_with_real_watchdog_classes(): assert not _is_read_only_event(we.FileModifiedEvent("/tmp/x.py")) assert not _is_read_only_event(we.FileCreatedEvent("/tmp/x.py")) assert not _is_read_only_event(we.FileClosedEvent("/tmp/x.py")) + + +def _markdown_reconcile_fixture(tmp_path, files, nodes, links): + corpus = tmp_path / "corpus" + corpus.mkdir() + for relative_path, content in files.items(): + path = corpus / relative_path + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(content, encoding="utf-8") + output = corpus / "graphify-out" + output.mkdir() + graph_path = output / "graph.json" + graph_path.write_text( + json.dumps( + { + "nodes": nodes, + "links": links, + "hyperedges": [], + "directed": False, + } + ), + encoding="utf-8", + ) + return corpus, graph_path + + +def _semantic_doc(node_id, source_file): + return { + "id": node_id, + "label": node_id, + "file_type": "document", + "source_file": source_file, + "_origin": "semantic", + } + + +def _ast_reference(source, target, source_file, **extra): + return { + "source": source, + "target": target, + "relation": "references", + "source_file": source_file, + "source_location": "L1", + "_origin": "ast", + **extra, + } + + +def test_markdown_reconcile_links_new_source_to_semantic_target(tmp_path): + """#1915/#1954: a fresh source reaches a semantic-only target.""" + corpus, graph_path = _markdown_reconcile_fixture( + tmp_path, + {"a.md": "[link](b.md)\n", "b.md": "target\n"}, + [_semantic_doc("b_sem", "b.md")], + [], + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + references = [edge for edge in links if edge.get("relation") == "references"] + assert len(references) == 1 + assert {references[0]["source"], references[0]["target"]} == {"a", "b_sem"} + + +def test_markdown_reconcile_preserves_ambiguous_target(tmp_path): + """#1915/#1954: an authored link must survive target ambiguity.""" + original = _ast_reference("a_sem", "b_one", "a.md", sentinel="keep") + corpus, graph_path = _markdown_reconcile_fixture( + tmp_path, + {"a.md": "[link](b.md)\n", "b.md": "target\n"}, + [ + _semantic_doc("a_sem", "a.md"), + _semantic_doc("b_one", "b.md"), + _semantic_doc("b_two", "b.md"), + ], + [original], + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + references = [edge for edge in links if edge.get("relation") == "references"] + assert len(references) == 1 + assert references[0]["sentinel"] == "keep" + assert {references[0]["source"], references[0]["target"]} == {"a_sem", "b_one"} + + +def test_markdown_reconcile_preserves_ambiguous_source(tmp_path): + """#1915/#1954: an authored link must survive source ambiguity.""" + corpus, graph_path = _markdown_reconcile_fixture( + tmp_path, + {"a.md": "[link](b.md)\n", "b.md": "target\n"}, + [ + _semantic_doc("a_one", "a.md"), + _semantic_doc("a_two", "a.md"), + _semantic_doc("b_sem", "b.md"), + ], + [_ast_reference("a_one", "b_sem", "a.md", sentinel="keep")], + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + references = [edge for edge in links if edge.get("relation") == "references"] + assert len(references) == 1 + assert references[0]["sentinel"] == "keep" + + +def test_markdown_reconcile_prunes_removed_authored_link(tmp_path): + """#1915/#1954: removing a Markdown link removes its owned AST edge.""" + corpus, graph_path = _markdown_reconcile_fixture( + tmp_path, + {"a.md": "no link\n", "b.md": "target\n"}, + [_semantic_doc("a_sem", "a.md"), _semantic_doc("b_sem", "b.md")], + [_ast_reference("a_sem", "b_sem", "a.md")], + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + assert not any(edge.get("relation") == "references" for edge in links) + + +def test_markdown_reconcile_keeps_reverse_stored_edge_unchanged(tmp_path): + """#1915/#1954: undirected endpoint order must not churn edge data.""" + corpus, graph_path = _markdown_reconcile_fixture( + tmp_path, + {"a.md": "[link](b.md)\n", "b.md": "target\n"}, + [_semantic_doc("a_sem", "a.md"), _semantic_doc("b_sem", "b.md")], + [_ast_reference("b_sem", "a_sem", "a.md", sentinel="keep")], + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + references = [edge for edge in links if edge.get("relation") == "references"] + assert len(references) == 1 + assert references[0]["source"] == "b_sem" + assert references[0]["target"] == "a_sem" + assert references[0]["sentinel"] == "keep" + + +def test_markdown_reconcile_prefers_later_canonical_page(tmp_path): + """#1915/#1954: a later canonical page replaces a semantic fallback.""" + corpus, graph_path = _markdown_reconcile_fixture( + tmp_path, + {"a.md": "[link](b.md)\n", "b.md": "target\n"}, + [ + _semantic_doc("a_sem", "a.md"), + _semantic_doc("b_sem", "b.md"), + { + "id": "b", + "label": "b.md", + "node_kind": "page", + "file_type": "document", + "source_file": "b.md", + "source_location": "L1", + "_origin": "ast", + }, + ], + [_ast_reference("a_sem", "b_sem", "a.md")], + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + references = [edge for edge in links if edge.get("relation") == "references"] + assert len(references) == 1 + assert {references[0]["source"], references[0]["target"]} == {"a_sem", "b"} + + +def test_markdown_reconcile_keeps_existing_semantic_relation(tmp_path): + """#1915/#1954: a simple graph must not overwrite a semantic relation.""" + semantic_edge = { + "source": "a_sem", + "target": "b_sem", + "relation": "derived_from", + "source_file": "a.md", + "_origin": "semantic", + } + corpus, graph_path = _markdown_reconcile_fixture( + tmp_path, + {"a.md": "[link](b.md)\n", "b.md": "target\n"}, + [_semantic_doc("a_sem", "a.md"), _semantic_doc("b_sem", "b.md")], + [semantic_edge], + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + assert links == [semantic_edge] + + +def test_markdown_reconcile_uses_legacy_page_fallback(tmp_path): + """#1915/#1954: a unique legacy page is a safe target fallback.""" + legacy_page = { + "id": "legacy_b", + "label": "b.md", + "node_kind": "page", + "file_type": "document", + "source_file": "b.md", + "_origin": "semantic", + } + corpus, graph_path = _markdown_reconcile_fixture( + tmp_path, + {"a.md": "[link](b.md)\n", "b.md": "target\n"}, + [_semantic_doc("a_sem", "a.md"), legacy_page], + [_ast_reference("a_sem", "legacy_b", "a.md", sentinel="keep")], + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + references = [edge for edge in links if edge.get("relation") == "references"] + assert len(references) == 1 + assert {references[0]["source"], references[0]["target"]} == {"a_sem", "legacy_b"} + assert references[0]["sentinel"] == "keep" + + +def test_markdown_reconcile_distinguishes_same_basename_paths(tmp_path): + """#1915/#1954: full relative paths disambiguate equal basenames.""" + corpus, graph_path = _markdown_reconcile_fixture( + tmp_path, + {"one/a.md": "[link](../two/a.md)\n", "two/a.md": "target\n"}, + [ + _semantic_doc("one_a", "one/a.md"), + _semantic_doc("two_a", "two/a.md"), + ], + [_ast_reference("one_a", "two_a", "one/a.md", sentinel="keep")], + ) + + assert _rebuild_code(corpus, no_cluster=True, acquire_lock=False) is True + links = json.loads(graph_path.read_text(encoding="utf-8"))["links"] + references = [edge for edge in links if edge.get("relation") == "references"] + assert len(references) == 1 + assert {references[0]["source"], references[0]["target"]} == {"one_a", "two_a"}