diff --git a/graphify/cache.py b/graphify/cache.py index 622ba6cff..80271831c 100644 --- a/graphify/cache.py +++ b/graphify/cache.py @@ -17,6 +17,7 @@ # absolute path ("/shared/graphify-out"). Single source of truth in graphify.paths # (#1423); re-exported here as _GRAPHIFY_OUT for the existing call sites. from graphify.paths import GRAPHIFY_OUT as _GRAPHIFY_OUT +from graphify.paths import resolve_cached # AST cache entries are the output of graphify's own extractor code, so they # are only valid for the version that wrote them: keying purely on file @@ -321,7 +322,7 @@ def _stat_key_to_absolute(key: str, anchor: Path) -> str: def _stat_index_file(root: Path) -> Path: _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else resolve_cached(root) / _out return base / "cache" / "stat-index.json" @@ -453,7 +454,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No # graphify-out/cache/stat-index.json inside the analyzed source tree even when # the AST cache itself is redirected to CWD (#1774 completion). _ensure_stat_index(root, cache_root=cache_root) - resolved = p.resolve() + resolved = resolve_cached(p) abs_key = str(resolved) # The salt is the path component that enters the digest (relative to root, or # the absolute-path fallback). The stat-index memo MUST be keyed by it too: @@ -462,7 +463,7 @@ def file_hash(path: Path, root: Path = Path("."), cache_root: "Path | None" = No # path served whichever was computed first — making file_hash order-dependent # and poisoning the persisted stat-index across runs (#1989). Store one digest # per salt so alternating roots don't force re-reads. - resolved_root = root.resolve() + resolved_root = resolve_cached(root) try: resolved_rel = resolved.relative_to(resolved_root) except ValueError: @@ -547,7 +548,7 @@ def cached_word_count(path: Path, root: Path, compute, cache_root: "Path | None" p = _normalize_path(Path(path)) root = _normalize_path(Path(root)) _ensure_stat_index(root, cache_root=cache_root) - abs_key = str(p.resolve()) + abs_key = str(resolve_cached(p)) st: "os.stat_result | None" = None try: st = p.stat() @@ -606,6 +607,34 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None: # source_file the same way nodes/edges/hyperedges do, so it needs the same # portable-path treatment for cache entries to round-trip correctly across # machines/checkout directories. + def _relativized(source: str) -> "str | None": + """The stored form of one ``source_file`` value, or None to leave it be.""" + sp = Path(source) + if not sp.is_absolute(): + # os.path.abspath is lexical (no symlink resolution), matching + # the symbolic relativization below. + cwd_form = Path(os.path.abspath(sp)) + try: + if cwd_form == root_resolved / sp or not cwd_form.exists(): + return None # already root-relative, or a ghost path + except OSError: + return None + sp = cwd_form + try: + rel = os.path.relpath(sp, root_resolved) + except (ValueError, OSError): + return None # out-of-root (e.g. Windows cross-drive) + if rel == ".." or rel.startswith(".." + os.sep) or rel.startswith("../"): + return None # escaped root — keep absolute + return rel.replace(os.sep, "/") + + # A payload holds one file's symbols, so nearly every item repeats the same + # source_file — this loop asked the identical question once per node: 61,964 + # relpath calls and 61,964 Path constructions across 438 payloads in a + # profiled run (#3008). ``root_resolved`` is fixed for the call and the rest + # is a pure function of the string, so one answer per distinct value serves + # the whole payload. + relativized: "dict[str, str | None]" = {} for bucket in ("nodes", "edges", "hyperedges", "raw_calls"): for item in payload.get(bucket, []): if not isinstance(item, dict): @@ -613,24 +642,12 @@ def _relativize_source_files_in(payload: dict, root: Path) -> None: source = item.get("source_file") if not source: continue - sp = Path(source) - if not sp.is_absolute(): - # os.path.abspath is lexical (no symlink resolution), matching - # the symbolic relativization below. - cwd_form = Path(os.path.abspath(sp)) - try: - if cwd_form == root_resolved / sp or not cwd_form.exists(): - continue # already root-relative, or a ghost path - except OSError: - continue - sp = cwd_form try: - rel = os.path.relpath(sp, root_resolved) - except (ValueError, OSError): - continue # out-of-root (e.g. Windows cross-drive) - if rel == ".." or rel.startswith(".." + os.sep) or rel.startswith("../"): - continue # escaped root — keep absolute - item["source_file"] = rel.replace(os.sep, "/") + rel = relativized[source] + except KeyError: + rel = relativized[source] = _relativized(source) + if rel is not None: + item["source_file"] = rel def _normalize_source_file_value(src: "str | Path", root_resolved: Path) -> str: @@ -805,6 +822,29 @@ def _rewrite_id_keyed_table_keys(payload: object, fn) -> None: } +def _json_deepcopy(obj): + """Deep-copy a JSON-shaped payload: dicts, lists, immutable scalars. + + ``copy.deepcopy`` on a cache payload was 868,965 calls / 0.75s of a profiled + run (#3008), almost all of it memo bookkeeping and dispatch rather than + copying — and none of that is needed here. The payload is about to be + ``json.dumps``-ed, so it can only hold JSON types, and every JSON scalar is + immutable. + + Containers of other types are returned by reference on purpose: the two + passes that mutate a copy — :func:`_relativize_source_files_in` and + :func:`_rewrite_strings` — descend into dicts and lists only, so nothing + reachable through a tuple or a set can be written to. Unlike ``deepcopy`` + this does not preserve shared identity between two references to the same + sub-object, which is unobservable in JSON output. + """ + if isinstance(obj, dict): + return {k: _json_deepcopy(v) for k, v in obj.items()} + if isinstance(obj, list): + return [_json_deepcopy(v) for v in obj] + return obj + + def _rewrite_strings(obj: object, fn) -> None: """Apply ``fn`` to every string VALUE reachable in ``obj``, in place. @@ -944,7 +984,7 @@ def cache_dir(root: Path = Path("."), kind: str = "ast", vintage live. """ _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else resolve_cached(root) / _out d = base / "cache" / kind if kind == "ast": d = d / f"v{_EXTRACTOR_VERSION}-s{_AST_CACHE_SCHEMA}" @@ -1105,8 +1145,7 @@ def save_cached(path: Path, result: dict, root: Path = Path("."), kind: str = "a # below would then mutate the caller's dict for real. on_disk = result if isinstance(result, dict): - import copy as _copy - on_disk = _copy.deepcopy(result) + on_disk = _json_deepcopy(result) _relativize_source_files_in(on_disk, root) # Then replace the absolute root inside the ids and remaining paths, so # the entry replays portably under any root (#2257). Strictly after the @@ -1213,7 +1252,7 @@ def prune_semantic_cache(root: Path, live_hashes: set[str]) -> int: one doc on a future run, never incorrect output. """ _out = Path(_GRAPHIFY_OUT) - base = _out if _out.is_absolute() else Path(root).resolve() / _out + base = _out if _out.is_absolute() else resolve_cached(root) / _out pruned = 0 for kind in ("semantic", "semantic-deep"): semantic_dir = base / "cache" / kind diff --git a/graphify/detect.py b/graphify/detect.py index 3668eb6fc..892f83af5 100644 --- a/graphify/detect.py +++ b/graphify/detect.py @@ -11,6 +11,7 @@ import unicodedata from concurrent.futures import ThreadPoolExecutor from enum import Enum +from functools import lru_cache from pathlib import Path from typing import Callable @@ -19,7 +20,7 @@ convert_google_workspace_file, google_workspace_enabled, ) -from graphify.paths import GRAPHIFY_OUT, out_path +from graphify.paths import GRAPHIFY_OUT, clear_resolve_cache, out_path, resolve_cached class FileType(str, Enum): @@ -1248,37 +1249,6 @@ def _load_graphifyignore(root: Path, *, gitignore: bool = True) -> list[tuple[Pa return patterns -# Parsed-pattern cache: raw pattern string -> (negated, directory_only, -# path_relative, stripped_pattern). Ignore patterns are re-evaluated for every -# walked entry; parsing the same strings per entry per scan was pure waste. -# Plain dict (no LRU): the universe of keys is the distinct pattern lines in -# the corpus's ignore files, which is small and bounded per scan. A long-lived -# `graphify watch` process spanning many repos could still accumulate keys over -# time, so cap it and clear wholesale on overflow (parsing is cheap, so a rare -# full re-fill costs nothing that matters). -_PARSED_PATTERN_CACHE: dict[str, tuple[bool, bool, bool, str]] = {} -_PARSED_PATTERN_CACHE_MAX = 100_000 - - -def _parse_ignore_pattern(pattern: str) -> tuple[bool, bool, bool, str]: - """Split one gitignore-style pattern into its matching flags, cached. - - Returns (negated, directory_only, path_relative, stripped). ``stripped`` - is empty for patterns that match nothing (bare "!", bare "/"). - """ - got = _PARSED_PATTERN_CACHE.get(pattern) - if got is None: - negated = pattern.startswith("!") - raw = pattern[1:] if negated else pattern - directory_only = raw.endswith("/") - path_relative = "/" in raw.rstrip("/") - got = (negated, directory_only, path_relative, raw.strip("/")) - if len(_PARSED_PATTERN_CACHE) >= _PARSED_PATTERN_CACHE_MAX: - _PARSED_PATTERN_CACHE.clear() - _PARSED_PATTERN_CACHE[pattern] = got - return got - - # PurePath.relative_to compares casefolded parts on Windows only; mirror that # exactly, but skip the normcase pass entirely where it is a no-op (POSIX). _CASEFOLD_PATHS = os.path.normcase("Aa") != "Aa" @@ -1295,6 +1265,13 @@ def _lexical_relative( of constructing a Path object (and, on 3.12+, walking ``parents`` quadratically) per pattern per scanned entry. That construction was the dominant cost of large scans (see CHANGELOG: 76k-file vault, 50+ min). + + No longer on the scan path: ``_eval`` below derives the same relative form + by slicing the target's normcased string against a precomputed anchor + prefix, which needs no ``parts`` tuple at all (#3008). Kept because it is + the reference this equivalence is pinned against + (``test_lexical_relative_matches_pathlib_relative_to``), and because it is + the only implementation that handles a part-less anchor. """ anchor_parts = anchor.parts if not anchor_parts: @@ -1321,66 +1298,234 @@ def _lexical_relative( return _nfc("/".join(tail)) -def _match_globstar_parts( +@lru_cache(maxsize=1 << 16) +def _match_anchored_ignore_pattern(path: str, pattern: str) -> bool: + """Match an anchored gitignore pattern without letting ``*`` cross ``/``. + + Memoized on the two strings (#3008). This used to build a fresh + ``lru_cache``-wrapped recursive closure on every call, so a 46,050-path scan of + a large repo paid 2.1M cache constructions whose contents were thrown away + immediately — the memo only ever helped within a single pattern/path pair. The + function is pure in its arguments, so the cache never goes stale and needs no + per-scan clear; the bound exists only so a long-lived ``watch`` process cannot + grow it without limit. + """ + return _anchored_ignore_match( + tuple(path.split("/")), tuple(pattern.split("/")) + ) + + +def _anchored_ignore_match( path_parts: tuple[str, ...], pattern_parts: tuple[str, ...], - path_idx: int, - pattern_idx: int, - memo: dict[tuple[int, int], bool], ) -> bool: - """Recursive ``**``-aware segment match, memoized via an explicit dict. - - Lifted out of ``_match_anchored_ignore_pattern`` (was a per-call - ``@lru_cache`` closure): the decorated inner closure referenced itself, so - every call leaked a reference cycle for the GC to reclaim on this hot path. - A plain dict passed in avoids both the cycle and the per-call cache setup. + """Segment-wise match for :func:`_match_anchored_ignore_pattern`. + + The pattern is a chain of segment tests where ``**`` may absorb any number of + segments, which is exactly a small non-deterministic automaton over pattern + positions, so it is walked as one: ``frontier`` holds every pattern index the + path could be sitting at, and each path segment advances the whole set at once. + + This replaced a memoized recursion over ``(path_parts, pattern_parts, + path_idx, pattern_idx)`` (#3008). The memo was worthless in practice — the key + carries the path, so consecutive paths never share an entry — and a profile of + a 46,050-path scan showed 22.5M executions of the recursive body, each one + hashing two tuples to miss its own cache. The automaton is O(segments x + pattern parts) with no hashing and no recursion, and decides the same language: + a non-final ``**`` either steps past itself or absorbs a segment (the two + recursive branches, now the epsilon closure plus the self-loop below), a final + ``**`` demands at least one segment remain, and any other part must match its + segment positionally. """ - key = (path_idx, pattern_idx) - cached = memo.get(key) - if cached is not None: - return cached + nparts = len(pattern_parts) + last = nparts - 1 + frontier = _globstar_closure({0}, pattern_parts, last) + for segment in path_parts: + advanced: set[int] = set() + for idx in frontier: + if idx > last: + continue # pattern exhausted but the path is not: a dead state + part = pattern_parts[idx] + if part == "**": + # A trailing ``**`` accepts as soon as one segment is left to eat, + # and this loop body only runs when one is. + if idx == last: + return True + advanced.add(idx) # absorb this segment, stay put + elif fnmatch.fnmatchcase(segment, part): + advanced.add(idx + 1) + if not advanced: + return False + frontier = _globstar_closure(advanced, pattern_parts, last) + return nparts in frontier - if pattern_idx == len(pattern_parts): - result = path_idx == len(path_parts) - else: - part = pattern_parts[pattern_idx] - if part == "**": - if pattern_idx == len(pattern_parts) - 1: - result = path_idx < len(path_parts) - else: - result = _match_globstar_parts( - path_parts, pattern_parts, path_idx, pattern_idx + 1, memo - ) or ( - path_idx < len(path_parts) - and _match_globstar_parts( - path_parts, pattern_parts, path_idx + 1, pattern_idx, memo - ) - ) - else: - result = ( - path_idx < len(path_parts) - and fnmatch.fnmatchcase(path_parts[path_idx], part) - and _match_globstar_parts( - path_parts, pattern_parts, path_idx + 1, pattern_idx + 1, memo - ) + +def _globstar_closure( + states: set[int], pattern_parts: tuple[str, ...], last: int +) -> set[int]: + """Add the pattern indices reachable from *states* without consuming a segment. + + Only a non-final ``**`` is skippable: ``a/**/b`` must accept ``a/b``. A final + ``**`` is not, because it requires a segment of its own. + """ + pending = [idx for idx in states if idx <= last and pattern_parts[idx] == "**"] + if not pending: + return states + reached = set(states) + while pending: + idx = pending.pop() + if idx < last and idx + 1 not in reached: + reached.add(idx + 1) + if idx + 1 <= last and pattern_parts[idx + 1] == "**": + pending.append(idx + 1) + return reached + + +_GLOB_METACHARS = ("*", "?", "[") + + +def _matches_ignore_pattern( + rel: str, + parts: tuple[str, ...], + parts_normcased: frozenset, + p: str, + p_normcased: str, + is_literal: bool, + path_relative: bool, + pattern_parts: tuple[str, ...] | None, + has_globstar: bool, + name_nfc: str, +) -> bool: + """Does the relative path *rel* match the bare gitignore pattern *p*? + + ``parts``/``parts_normcased``/``name_nfc`` describe the target and are derived + once per distinct relative path by the caller; the rest describe the pattern + and are derived once per scan by :func:`_compile_ignore_pattern`. Both used to + be recomputed on every one of the 3.8M calls a 46,050-path scan makes (#3008). + + Two fast paths carry almost all real-world patterns, and both are exact + rewrites of the general form below for every input this module produces — + with one documented exception, noted in the first bullet: + + - A **literal** non-anchored pattern (no ``*``, ``?`` or ``[``, and no ``/`` + by construction — a pattern containing ``/`` is anchored instead) can only + match a single path component. ``fnmatch`` reduces to equality for such a + pattern, ``name_nfc`` is ``parts[-1]`` for every ``(rel, name_nfc)`` pair + :func:`_eval` builds, and every multi-part join the general loop tries + contains a separator the pattern cannot hold. So the whole cascade + collapses to one set membership test. + + The exception is ``rel == "."``, where ``parts == (".",)`` and + ``name_nfc`` is the *directory's own* name rather than ``parts[-1]``: this + branch drops the general form's ``fnmatch(name_nfc, p)`` term, so + ``rel='.' pattern='build' name='build'`` matches under the general form and + not here. ``_eval`` produces ``rel == "."`` only when the target IS the + scan root and the pattern's anchor is a strict ancestor, and nothing asks + that question: the parent-exclusion walk never ``_eval``s the root itself, + ``collect_files`` and ``extract.py`` only ask about files, and + ``watch.py`` returns early for a directory event. It is recorded rather + than fixed because changing the fast path costs every real scan and buys + an answer no caller reads. + - An **anchored** pattern with no ``**`` matches strictly positionally, so the + automaton in :func:`_anchored_ignore_match` reduces to a length check plus a + per-segment ``fnmatchcase``. + """ + if path_relative: + if pattern_parts is None: # defensive: caller always supplies these + return _match_anchored_ignore_pattern(rel, p) + if not has_globstar: + return len(parts) == len(pattern_parts) and all( + fnmatch.fnmatchcase(a, b) for a, b in zip(parts, pattern_parts) ) - memo[key] = result - return result + return _anchored_ignore_match(parts, pattern_parts) + if is_literal: + return p_normcased in parts_normcased + + if fnmatch.fnmatch(rel, p): + return True + if fnmatch.fnmatch(name_nfc, p): + return True + for i, part in enumerate(parts): + if fnmatch.fnmatch(part, p): + return True + if fnmatch.fnmatch("/".join(parts[: i + 1]), p): + return True + return False -def _match_anchored_ignore_pattern(path: str, pattern: str) -> bool: - """Match an anchored gitignore pattern without letting ``*`` cross ``/``.""" - path_parts = tuple(path.split("/")) - pattern_parts = tuple(pattern.split("/")) - # Fast path: with no ``**`` the match is a straight segment-wise fnmatch of - # equal-length paths, so skip the recursive matcher and its memo entirely. - if "**" not in pattern_parts: - if len(path_parts) != len(pattern_parts): - return False - return all( - fnmatch.fnmatchcase(pp, qp) for pp, qp in zip(path_parts, pattern_parts) - ) - return _match_globstar_parts(path_parts, pattern_parts, 0, 0, {}) + +def _compile_ignore_pattern(anchor: Path, pattern: str) -> tuple | None: + """Pre-split one ``(anchor, pattern)`` pair into the form ``_eval`` consumes. + + Returns ``None`` for a pattern that is empty once its slashes are stripped — + the case the old inline loop skipped with ``continue``. + + The tuple carries the anchor as normcased strings so membership can be tested + by string prefix instead of ``Path.relative_to``. ``os.path.normcase`` is + identity on POSIX and, on Windows, lowercases and maps ``/`` to ``\\`` — both + length-preserving, so the slice offsets computed here stay valid against the + un-normcased path (and the case-insensitive comparison matches what + ``PureWindowsPath.relative_to`` already did). + """ + negated = pattern.startswith("!") + raw = pattern[1:] if negated else pattern + directory_only = raw.endswith("/") + path_relative = "/" in raw.rstrip("/") + p = raw.strip("/") + if not p: + return None + anchor_key = os.path.normcase(str(anchor)) + prefix = anchor_key if anchor_key.endswith(os.sep) else anchor_key + os.sep + pattern_parts = tuple(p.split("/")) if path_relative else None + return ( + anchor_key, + prefix, + len(prefix), + len(anchor.parts), + p, + os.path.normcase(p), + not any(c in p for c in _GLOB_METACHARS), + negated, + directory_only, + path_relative, + pattern_parts, + pattern_parts is not None and "**" in pattern_parts, + ) + + +# Compiled pattern lists, keyed by the identity of the caller's pattern list +# (#3008). The pattern list is fixed for a scan apart from nested ignore files, +# which are only ever *appended* as the walk descends, so a compiled list can be +# extended in place rather than rebuilt. Each entry holds a strong reference to +# the source list, which is what makes `id()` safe as a key: while an entry is +# cached its list cannot be collected, so its id cannot be reused. Entries are +# dropped wholesale on overflow — the cost of a miss is one recompile. +_IGNORE_COMPILED: "dict[int, list]" = {} +_IGNORE_COMPILED_MAX = 16 + + +def _compiled_ignore_patterns(patterns: list[tuple[Path, str]]) -> list[tuple]: + """Return the compiled form of *patterns*, compiling only what is new.""" + key = id(patterns) + entry = _IGNORE_COMPILED.get(key) + if entry is not None and entry[0] is patterns and entry[1] <= len(patterns): + if entry[1] < len(patterns): + for anchor, pattern in patterns[entry[1]:]: + compiled_one = _compile_ignore_pattern(anchor, pattern) + if compiled_one is not None: + entry[2].append(compiled_one) + entry[1] = len(patterns) + return entry[2] + + compiled: list[tuple] = [] + for anchor, pattern in patterns: + compiled_one = _compile_ignore_pattern(anchor, pattern) + if compiled_one is not None: + compiled.append(compiled_one) + if len(_IGNORE_COMPILED) >= _IGNORE_COMPILED_MAX: + _IGNORE_COMPILED.clear() + _IGNORE_COMPILED[key] = [patterns, len(patterns), compiled] + return compiled def _is_ignored( @@ -1402,10 +1547,28 @@ def _is_ignored( _cache: optional dict shared across calls within the same scan. Ancestor directory results are memoised so files under the same subtree don't re-evaluate the same patterns repeatedly. + + Performance (#3008): this was the single most expensive frame in the whole + pipeline — 437.4 s of a 469 s instrumented ``detect()`` profile on a + 46,050-path repo, 93% of the stage. The work was not the matching but the + ``Path`` arithmetic around it: ``target.relative_to(anchor)`` ran once per + pattern per path (5.1M calls, each one walking ``anchor.parents`` and + constructing ``PurePath`` objects to compare), and the per-pattern flags were + re-derived from the pattern string every time. Now the pattern list is + compiled once (:func:`_compiled_ignore_patterns`), anchor membership is a + string prefix test, and everything that depends only on the target — its + root-relative form, its NFC basename, its ``is_dir()`` — is computed at most + once per target instead of once per pattern. Decisions are unchanged; the + ``_cache`` contract is unchanged too, including that ``_eval`` writes each + target exactly once (#1235). """ if not patterns: return False + compiled = _compiled_ignore_patterns(patterns) + root_key = os.path.normcase(str(root)) + root_prefix = root_key if root_key.endswith(os.sep) else root_key + os.sep + root_prefix_len = len(root_prefix) root_nparts = len(root.parts) def _eval(target: Path) -> bool: @@ -1422,87 +1585,96 @@ def _eval(target: Path) -> bool: if _cache is not None and target in _cache: return _cache[target] - target_parts = target.parts - target_name_nfc: str | None = None + target_str = str(target) + target_key = os.path.normcase(target_str) + name_nfc = _nfc(target.name) + + # Root-relative form, needed by every non-anchored pattern whose anchor + # sits above the scan root. Computed once here rather than per pattern. + if target_key == root_key: + rel_root: str | None = "." + elif target_key.startswith(root_prefix): + rel_root = _nfc(target_str[root_prefix_len:].replace(os.sep, "/")) + else: + rel_root = None + + # One anchor typically owns many patterns (every rule in one .gitignore), + # so the anchor-relative form is derived per anchor, not per pattern. + rel_by_anchor: dict[str, str | None] = {} + # Split/normcase of a relative path, keyed by that path. A target has at + # most a handful of distinct relative forms (one per anchor, plus the + # root-relative one) but is tested against every pattern, so this is + # derived once per form rather than once per pattern. + parts_by_rel: dict[str, tuple[tuple[str, ...], frozenset]] = {} target_is_dir: bool | None = None - rel_root_known = False - rel_root: str | None = None - # rel string (or None = outside anchor) and part-count per distinct - # anchor; patterns overwhelmingly share a handful of anchors. - rel_by_anchor: dict[Path, tuple[str | None, int]] = {} - # split segments + accumulated "/" prefixes per distinct rel string. - segs_by_rel: dict[str, tuple[list[str], list[str]]] = {} - - def _segments(rel: str) -> tuple[list[str], list[str]]: - got = segs_by_rel.get(rel) - if got is None: - parts = rel.split("/") - prefixes: list[str] = [] - acc = "" - for part in parts: - acc = part if not acc else acc + "/" + part - prefixes.append(acc) - got = (parts, prefixes) - segs_by_rel[rel] = got - return got - - def _matches(rel: str, p: str, path_relative: bool) -> bool: - nonlocal target_name_nfc - if path_relative: - return _match_anchored_ignore_pattern(rel, p) - if fnmatch.fnmatch(rel, p): - return True - if target_name_nfc is None: - target_name_nfc = _nfc(target.name) - if fnmatch.fnmatch(target_name_nfc, p): - return True - parts, prefixes = _segments(rel) - for part, prefix in zip(parts, prefixes): - if fnmatch.fnmatch(part, p): - return True - if fnmatch.fnmatch(prefix, p): - return True - return False result = False - for anchor, pattern in patterns: - negated, directory_only, path_relative, p = _parse_ignore_pattern(pattern) - if not p: - continue - + for ( + anchor_key, + anchor_prefix, + anchor_prefix_len, + anchor_nparts, + p, + p_normcased, + is_literal, + negated, + directory_only, + path_relative, + pattern_parts, + has_globstar, + ) in compiled: # gitignore semantics: patterns from A/.gitignore apply ONLY to paths # under A. Matching non-anchored patterns against root-relative paths # let e.g. .hypothesis/.gitignore's bare "*" ignore the ENTIRE repo # (detect() returned 0 files). The anchor dir itself is exempt — an # ignore file governs its directory's contents, not the directory. - cached_rel = rel_by_anchor.get(anchor) - if cached_rel is None: - cached_rel = ( - _lexical_relative(target, target_parts, anchor), - len(anchor.parts), + if anchor_key in rel_by_anchor: + rel_anchor = rel_by_anchor[anchor_key] + else: + if target_key == anchor_key: + rel_anchor = "." + elif target_key.startswith(anchor_prefix): + rel_anchor = _nfc( + target_str[anchor_prefix_len:].replace(os.sep, "/") + ) + else: + rel_anchor = None # outside this anchor: cannot match + rel_by_anchor[anchor_key] = rel_anchor + if rel_anchor is None or rel_anchor == ".": + continue + + rel = rel_anchor + if not path_relative and root_nparts > anchor_nparts and rel_root is not None: + rel = rel_root + cached_parts = parts_by_rel.get(rel) + if cached_parts is None: + split = tuple(rel.split("/")) + cached_parts = ( + split, + frozenset(os.path.normcase(x) for x in split), ) - rel_by_anchor[anchor] = cached_rel - rel_anchor, anchor_nparts = cached_rel - if rel_anchor is None: - continue # target outside this pattern's anchor: cannot match - matched = False - if rel_anchor != ".": - rel = rel_anchor - if not path_relative and root_nparts > anchor_nparts: - if not rel_root_known: - rel_root_known = True - rel_root = _lexical_relative(target, target_parts, root) - if rel_root is not None: - rel = rel_root - matched = _matches(rel, p, path_relative=path_relative) - if matched and directory_only: - if target_is_dir is None: - target_is_dir = target.is_dir() - if not target_is_dir: - matched = False - - if matched: - result = not negated # last match wins; ! flips to un-ignore + parts_by_rel[rel] = cached_parts + parts, parts_normcased = cached_parts + if not _matches_ignore_pattern( + rel, + parts, + parts_normcased, + p, + p_normcased, + is_literal, + path_relative, + pattern_parts, + has_globstar, + name_nfc, + ): + continue + if directory_only: + if target_is_dir is None: + target_is_dir = target.is_dir() + if not target_is_dir: + continue + + result = not negated # last match wins; ! flips to un-ignore if _cache is not None: _cache[target] = result return result @@ -1656,13 +1828,19 @@ def _auto_follow_symlinks(root: Path) -> bool: def _resolves_under_root(path: Path, root: Path) -> bool: """True when ``path`` resolves to a target inside ``root``.""" try: - path.resolve().relative_to(root.resolve()) + resolve_cached(path).relative_to(resolve_cached(root)) except (OSError, RuntimeError, ValueError): return False return True def detect(root: Path, *, follow_symlinks: bool | None = None, google_workspace: bool | None = None, extra_excludes: list[str] | None = None, cache_root: Path | None = None, gitignore: bool = True) -> dict: + # _resolves_under_root memoizes through resolve_cached, so a scan must start + # from a clean cache the same way extract() does (#3008). Without this, a + # `graphify watch` process would carry the previous cycle's symlink targets + # into this scan: extract() clears at its own start, which is too late for + # the detect that runs before it. + clear_resolve_cache() root = root.resolve() configured_out_dir = root / GRAPHIFY_OUT configured_out_names = {configured_out_dir.name} diff --git a/graphify/extract.py b/graphify/extract.py index 758f02f12..5c6a2030c 100644 --- a/graphify/extract.py +++ b/graphify/extract.py @@ -10,6 +10,7 @@ import textwrap from collections import Counter from dataclasses import dataclass, field +from functools import lru_cache from pathlib import Path, PurePath from typing import Any, Callable @@ -59,7 +60,7 @@ from graphify.extractors.verilog import extract_verilog # noqa: F401 from graphify.extractors.zig import extract_zig # noqa: F401 from graphify.security import sanitize_metadata -from graphify.paths import disambiguate_ambiguous_candidates +from graphify.paths import clear_resolve_cache, disambiguate_ambiguous_candidates, resolve_cached from graphify.extractors.models import LanguageConfig, _JS_CACHE_BYPASS_SUFFIXES, _NamespaceExportFact, _StarExportFact, _SymbolAliasFact, _SymbolDeclarationFact, _SymbolExportFact, _SymbolImportFact, _SymbolResolutionFacts, _SymbolUseFact, _WORKSPACE_PACKAGE_CACHE # noqa: E402,F401 @@ -69,15 +70,21 @@ _EXPORT_CONDITION_PRIORITY, _JS_INDEX_FILES, _JS_PRIMITIVE_TYPES, + _JS_CONFIG_DIR_CACHE, + _JS_MODULE_PATH_CACHE, _JS_RESOLVE_EXTS, + _SOURCE_KEY_CACHE, _TSCONFIG_ALIAS_CACHE, _TSCONFIG_BASEURL_CACHE, _VUE_SCRIPT_LANG_RE, _VUE_SCRIPT_RE, _WORKSPACE_MANIFEST_NAMES, + _WORKSPACE_ROOT_CACHE, _apply_symbol_resolution_facts, _augment_symbol_resolution_edges, _collect_js_symbol_resolution_facts, + _collect_python_file_facts, + _collect_python_reference_facts, _collect_python_symbol_resolution_facts, _contained_in_package, _decldef_class_stem, @@ -115,6 +122,7 @@ _python_call_identifier, _python_import_from_module, _python_imported_names, + _python_local_name_map, _python_top_level_function_bodies, _read_tsconfig_aliases, _resolve_c_include_path, @@ -141,6 +149,7 @@ _walk_js_tree, _walk_python_tree, _workspace_globs, + clear_python_tree_cache, ) from graphify.symbol_resolution import resolve_bash_source_edges # noqa: E402 @@ -207,7 +216,7 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: (ambiguous -> leave dangling, as before). Files whose package root IS the scan root are skipped (ids already coincide).""" try: - root = Path(root).resolve() + root = resolve_cached(root) except OSError: root = Path(root) node_ids = {n.get("id") for n in all_nodes if isinstance(n, dict)} @@ -216,13 +225,13 @@ def _repoint_python_package_imports(paths, all_nodes, all_edges, root) -> None: if p.suffix.lower() not in (".py", ".pyi"): continue try: - rel = Path(p).resolve().relative_to(root) + rel = resolve_cached(p).relative_to(root) except (ValueError, OSError): continue parts = rel.parts if len(parts) < 2: continue # top-level file: scan-root-relative id already matches - d = Path(p).resolve().parent + d = resolve_cached(p).parent levels = 0 # Bounded by the number of dirs between the file and the scan root, so a # pathological `/__init__.py` chain can't loop forever. @@ -1416,7 +1425,7 @@ def _rescue_js_dynamic_imports(path: Path, result: dict) -> None: tf = e.get("target_file") if tf: try: - deferred_files.add(str(Path(tf).resolve())) + deferred_files.add(str(resolve_cached(tf))) except OSError: deferred_files.add(str(tf)) # `(? None: continue if resolved_file is not None: try: - if str(resolved_file.resolve()) in deferred_files: + if str(resolve_cached(resolved_file)) in deferred_files: continue except OSError: pass @@ -1453,7 +1462,7 @@ def _rescue_js_dynamic_imports(path: Path, result: dict) -> None: # in-function dynamic import through here too, which would turn an edge case # into the common one — a hub module deferred from eight functions of the same # file would carry eight identical arrows. - emit_key = str(resolved_file.resolve()) if resolved_file is not None else raw + emit_key = str(resolve_cached(resolved_file)) if resolved_file is not None else raw if emit_key in rescued_targets: continue rescued_targets.add(emit_key) @@ -2276,11 +2285,31 @@ def _canonicalize_csharp_namespace_nodes(all_nodes: list[dict], all_edges: list[ }) +@lru_cache(maxsize=1 << 16) +def _lower_suffix(source_file: str) -> str: + """A path's lowercased extension, without building a `Path` (#3008). + + `Path(...).suffix` parses the whole path into components — 1.7M of those + accesses in a profiled 32k-file run, 5.7s, almost all of them re-asking about + a path already seen. `splitext` answers from the string alone. Two + differences from `Path.suffix`: a trailing-dot name, where it reports `"."` + and pathlib reports `""` (normalized away below), and a trailing separator, + where `_lower_suffix("foo.go/")` is `""` while `Path("foo.go/").suffix` is + `".go"` — pathlib drops the empty final component and `splitext` does not. + The second is left as-is: both callers are fed node `source_file` values, + which never carry a trailing slash. Purely a function of the + string, so unlike the filesystem caches this one needs no per-run clear; the + bound is there only so a long-lived `watch` process cannot grow without limit. + """ + suffix = os.path.splitext(source_file)[1].lower() + return "" if suffix == "." else suffix + + def _lang_is_case_insensitive(source_file: object) -> bool: """True when the file's language resolves identifiers case-insensitively (#1581).""" if not source_file: return False - return Path(str(source_file)).suffix.lower() in _CASE_INSENSITIVE_EXTS + return _lower_suffix(str(source_file)) in _CASE_INSENSITIVE_EXTS # Language interop families for cross-file call resolution. A call in one language @@ -2326,7 +2355,7 @@ def _lang_family(source_file: object) -> str | None: """Interop family of the file's language, or None when unknown/not code.""" if not source_file: return None - return _LANG_FAMILY_BY_EXT.get(Path(str(source_file)).suffix.lower()) + return _LANG_FAMILY_BY_EXT.get(_lower_suffix(str(source_file))) # A language's own built-in throwable hierarchy, keyed by the interop family of @@ -2745,11 +2774,11 @@ def _merge_csharp_partial_class_nodes( for p in paths: if p.suffix.lower() in proj_exts: try: - project_dirs.add(p.resolve().parent) + project_dirs.add(resolve_cached(p).parent) except OSError: pass try: - stop = root.resolve() + stop = resolve_cached(root) except OSError: stop = root dir_assembly: dict[Path, str] = {} @@ -2789,7 +2818,7 @@ def _assembly_of_node(nid: str) -> str: if path is None: return "" try: - d = path.resolve().parent + d = resolve_cached(path).parent except OSError: return "" return _assembly_of_dir(d) @@ -3092,13 +3121,25 @@ def _key(label: str) -> str: if alias: import_alias_by_filenode.setdefault(e.get("source"), {})[e.get("target")] = _key(alias) + # Keyed only on nid, and this function never adds to node_by_id nor rewrites + # an existing node's source_file/label — so one answer per nid holds for the + # whole pass. The inner comprehension below re-asks about the same handful of + # module nodes per call site (63,220 calls in a profiled run, #3008). + _module_stem_keys: dict[str, str] = {} + def _module_stem_key(nid: str) -> str: + hit = _module_stem_keys.get(nid) + if hit is not None: + return hit n = node_by_id.get(nid) if not n: - return "" - sf = n.get("source_file") or "" - stem = Path(sf).stem if sf else "" - return _key(stem or n.get("label", "")) + key = "" + else: + sf = n.get("source_file") or "" + stem = Path(sf).stem if sf else "" + key = _key(stem or n.get("label", "")) + _module_stem_keys[nid] = key + return key existing_pairs = {(e.get("source"), e.get("target")) for e in all_edges} @@ -4539,7 +4580,7 @@ def extract_slnx(path: Path) -> dict: def _resolve(proj_path: str) -> str: proj_path = proj_path.replace("\\", "/") try: - return str((path.parent / proj_path).resolve()) + return str(resolve_cached(path.parent / proj_path)) except Exception: return proj_path @@ -4668,7 +4709,7 @@ def find_all(tag: str): continue ref_path_norm = ref_path.replace("\\", "/") try: - abs_ref = str((path.parent / ref_path_norm).resolve()) + abs_ref = str(resolve_cached(path.parent / ref_path_norm)) except Exception: abs_ref = ref_path_norm proj_nid = _make_id(abs_ref) @@ -4956,7 +4997,7 @@ def _xaml_project_root(path: Path) -> Path: return root boundary = _XAML_ACTIVE_EXTRACT_ROOT.resolve() try: - root.resolve().relative_to(boundary) + resolve_cached(root).relative_to(boundary) return root except ValueError: return boundary @@ -4965,7 +5006,7 @@ def _xaml_project_root(path: Path) -> Path: def _xaml_csharp_class_nodes(path: Path) -> dict[str, list[dict]]: from graphify.detect import _is_ignored, _is_noise_dir, _load_graphifyignore root = _xaml_project_root(path) - cache_key = str(root.resolve()) if _XAML_ACTIVE_EXTRACT_ROOT is not None else None + cache_key = str(resolve_cached(root)) if _XAML_ACTIVE_EXTRACT_ROOT is not None else None if cache_key and cache_key in _XAML_CSHARP_CLASS_CACHE: return _XAML_CSHARP_CLASS_CACHE[cache_key] classes: dict[str, list[dict]] = {} @@ -5684,6 +5725,75 @@ def _extract_single_file(args: tuple) -> tuple[int, dict]: return idx, result +def _pool_context(): + """Multiprocessing context for the extraction pool, or None for the default. + + Prefers ``forkserver`` with this module preloaded. Under ``spawn`` — the + default on macOS and Windows — every worker is a fresh interpreter that + re-imports ``graphify.extract``, and that duplicated startup is paid once + per worker rather than once per run: measured worker CPU on a 501-file + corpus grew from 3.95s at 6 workers to 4.65s at 16, about 0.07s per extra + worker, and the pool phase got *slower* past 8 workers (1.06s at 8, 1.57s + at 16) because the added startup outran the added parallelism. A forkserver + imports this module once and forks each worker from that warm image, which + on the same corpus cut the pool phase to 0.60s at 12 workers and flattened + the worker-count curve so more workers help again; the cost grows with core + count, so the win is larger on a big machine, not smaller (#3008). + + Returns None when forkserver is unavailable (Windows, which only has spawn) + or unusable, leaving the executor's default context and the existing + BrokenProcessPool fallback in place. + """ + import multiprocessing + + if "forkserver" not in multiprocessing.get_all_start_methods(): + return None + try: + ctx = multiprocessing.get_context("forkserver") + # Import cost moves into the forkserver process, once. Workers inherit + # the loaded module by fork, so this must name the module the worker + # entrypoint lives in. + ctx.set_forkserver_preload(["graphify.extract"]) + return ctx + except (ValueError, OSError, ImportError): + return None + + +def _pool_worker_count(max_workers: "int | None", work_len: int) -> int: + """Worker count for a per-file pool over ``work_len`` items. + + Shared by every pool in this module so the env override, the CPU scaling and + the platform clamp are derived one way only. + + ``max_workers`` None means auto: honour the GRAPHIFY_MAX_WORKERS env + override, otherwise scale to the full CPU. The historical `, 8)` cap was a + safety bound for laptops in 2023 — on a 32-thread workstation it costs a 4x + slowdown (issue #792). Capping at ``work_len`` keeps small jobs from + spawning useless idle workers. + + Windows ProcessPoolExecutor hard-caps at 61 workers (CPython limitation tied + to WaitForMultipleObjects). Clamping here keeps every path — auto-compute, + GRAPHIFY_MAX_WORKERS, and --max-workers — valid on >61-core boxes (issue + #1298). Never returns 0, so an empty work list is still a legal count. + """ + if max_workers is None: + env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip() + env_cap = None + if env_raw: + try: + v = int(env_raw) + if v > 0: + env_cap = v + except ValueError: + pass + cpu_cap = env_cap if env_cap is not None else (os.cpu_count() or 4) + max_workers = min(cpu_cap, work_len) + + if sys.platform == "win32": + max_workers = min(max_workers, 61) + return max(max_workers, 1) + + def _extract_parallel( uncached_work: list[tuple[int, Path]], per_file: list[dict | None], @@ -5701,31 +5811,7 @@ def _extract_parallel( """ import concurrent.futures - if max_workers is None: - # Honour GRAPHIFY_MAX_WORKERS env override; otherwise scale to the - # full CPU. The historical `, 8)` cap was a safety bound for laptops - # in 2023 — on a 32-thread workstation it costs a 4x slowdown - # (issue #792). Capping at len(uncached_work) keeps small jobs - # from spawning useless idle workers. - env_raw = os.environ.get("GRAPHIFY_MAX_WORKERS", "").strip() - env_cap = None - if env_raw: - try: - v = int(env_raw) - if v > 0: - env_cap = v - except ValueError: - pass - cpu_cap = env_cap if env_cap is not None else (os.cpu_count() or 4) - max_workers = min(cpu_cap, len(uncached_work)) - - # Windows ProcessPoolExecutor hard-caps at 61 workers (CPython limitation - # tied to WaitForMultipleObjects). Clamp here so every path — auto-compute, - # GRAPHIFY_MAX_WORKERS, and --max-workers — stays valid on >61-core boxes - # (issue #1298). Guard against 0 from an empty work list. - if sys.platform == "win32": - max_workers = min(max_workers, 61) - max_workers = max(max_workers, 1) + max_workers = _pool_worker_count(max_workers, len(uncached_work)) # A one-worker pool buys no parallelism: it still pays process spawn plus an # IPC round trip per file, and it is the one residual case where the parent's @@ -5746,7 +5832,9 @@ def _extract_parallel( failed: list[int] = [] # positions into uncached_work whose future failed _PROGRESS_INTERVAL = 100 try: - with concurrent.futures.ProcessPoolExecutor(max_workers=max_workers) as pool: + with concurrent.futures.ProcessPoolExecutor( + max_workers=max_workers, mp_context=_pool_context() + ) as pool: futures = { pool.submit(_extract_single_file, item): pos for pos, item in enumerate(work_items) @@ -5853,6 +5941,108 @@ def _extract_sequential( print(f" AST extraction: {_done}/{_done} uncached files (100%)", flush=True) +# Python resolution passes below this many files stay in the parent. Both are a +# parse plus a tree walk per file — real work, but a pool has to be paid for. On +# a platform without forkserver (Windows only has spawn) every worker re-imports +# graphify.extract, ~0.07s each, so a small corpus would lose more to startup +# than the walks cost in the first place. Deliberately higher than +# _PARALLEL_THRESHOLD, which gates far heavier per-file extraction work. +_PY_PASS_PARALLEL_THRESHOLD = 60 + + +def _python_facts_worker(args: tuple) -> "tuple[int, dict | None]": + """Pool entrypoint for per-file Python symbol-resolution facts. + + Must be at module level (not a closure) so it can be pickled. + + Args: + args: ``(index, path_str, root_str)``; ``root`` anchors module resolution. + + Returns: + ``(index, payload)`` so results can be placed back in order. + """ + idx, path_str, root_str = args + return idx, _collect_python_file_facts(Path(path_str), Path(root_str)) + + +def _python_refs_worker(args: tuple) -> "tuple[int, dict | None]": + """Pool entrypoint for per-file Python cross-file-import reference facts. + + Args: + args: ``(index, path_str, name_to_nid)``. The name map is built in the + parent because node ids only exist there. + + Returns: + ``(index, payload)`` so results can be placed back in order. + """ + idx, path_str, name_to_nid = args + return idx, _collect_python_reference_facts(Path(path_str), name_to_nid) + + +def _map_python_pass( + worker, + work_items: list[tuple], + slots: int, + parallel: bool, + max_workers: "int | None", + label: str, +) -> "list[dict | None] | None": + """Run a per-file Python resolution pass in a worker pool. + + Both Python resolution hotspots are a parse plus a tree walk per file, over + files the extraction workers already parsed once, and only the cross-file + merge genuinely needs the global node table. Moving the per-file halves into + a pool leaves the parent doing just the merge: measured on a 501-file corpus + the two passes were 1.47s of a 2.01s serial tail (#3008). The payloads are + plain str/int data, and IPC is cheap enough not to matter — 4.56MB of + per-file results pickled in 0.017s on the same corpus. + + ``work_items`` are ``(slot_index, ...)`` tuples; each result lands at its own + slot, so the returned list is aligned with the caller's path list regardless + of completion order. Slots with no work item stay None. + + Returns None when the pass should run in the parent instead: pooling + disabled, too few files to pay for a pool, a single worker, or a pool that + broke — the same fallback contract as :func:`_extract_parallel`. + """ + if not parallel or len(work_items) < _PY_PASS_PARALLEL_THRESHOLD: + return None + workers = _pool_worker_count(max_workers, len(work_items)) + if workers == 1: + return None + # These passes only pay for themselves on a forkserver. Under spawn every + # worker of every pool re-imports graphify.extract, and these two pools are + # in addition to the extraction pool: on the 501-file corpus, moving the + # passes into spawn pools made the whole run *slower*, 3.59s -> 4.23s median, + # while the same change on a forkserver took it to 1.70s. A platform with no + # forkserver (Windows) keeps them in the parent (#3008). + ctx = _pool_context() + if ctx is None: + return None + + import concurrent.futures + + out: "list[dict | None]" = [None] * slots + try: + with concurrent.futures.ProcessPoolExecutor( + max_workers=workers, mp_context=ctx + ) as pool: + futures = [pool.submit(worker, item) for item in work_items] + for future in concurrent.futures.as_completed(futures): + idx, payload = future.result() + out[idx] = payload + except Exception as exc: + # Any failure discards the whole pass rather than half of it: the caller + # redoes it in-process, where the payloads are byte-identical, so a + # broken pool costs time and nothing else. + import logging + logging.getLogger(__name__).warning( + "parallel %s failed (%s); running in-process instead", label, exc + ) + return None + return out + + _PARALLEL_THRESHOLD = 20 @@ -5922,6 +6112,22 @@ def extract( _TSCONFIG_BASEURL_CACHE.clear() _XAML_CSHARP_CLASS_CACHE.clear() _MD_LINK_INDEX_CACHE.clear() + # Same contract for the two path caches added in #3008: which config a + # directory sees, and a source path's root-relative key, both answer + # questions about the filesystem with no mtime component. They exist because + # cross-file resolution asked them millions of times per run — `_source_key` + # alone was 36% of a profiled 32k-file run — so they must be cleared here, + # never made permanent, or a config added between two watch rebuilds (or a + # retargeted symlink) would never be observed again. + _JS_CONFIG_DIR_CACHE.clear() + _SOURCE_KEY_CACHE.clear() + _WORKSPACE_ROOT_CACHE.clear() + _JS_MODULE_PATH_CACHE.clear() + clear_resolve_cache() + # Same per-run contract: the Python parse cache treats each file's contents + # as fixed for one run, so a watch process must not carry a previous cycle's + # trees into this one. + clear_python_tree_cache() # Infer a common root for cache keys (use first diverging segment, not sum of all matches) try: @@ -6178,7 +6384,21 @@ def _describe_syntax_error(rel: str, line: "int | None", kept: int) -> str: # marker set in the per-file extractor. Populated just before the pass that uses it. callable_nids: set[str] = set() - _augment_symbol_resolution_edges(paths, all_nodes, all_edges, root) + # The per-file half of this pass — one parse plus two tree walks per Python + # file — needs no cross-file state, so hand it to a pool and let the parent + # do only the merge (#3008). + _py_fact_paths = [p for p in paths if p.suffix == ".py"] + _py_fact_payloads = _map_python_pass( + _python_facts_worker, + [(i, str(p), str(root)) for i, p in enumerate(_py_fact_paths)], + len(_py_fact_paths), + parallel, + max_workers, + "Python fact collection", + ) + _augment_symbol_resolution_edges( + paths, all_nodes, all_edges, root, py_facts_payloads=_py_fact_payloads + ) # Merge a header-declared class (and its methods) with its sibling-impl # definition into ONE node (C/C++/ObjC #1547/#1556). Runs BEFORE the id-remap @@ -6250,7 +6470,7 @@ def _portable_out_of_root_sf(p: Path) -> str: _remap_seen: set[Path] = set() for _p in paths: try: - _remap_seen.add(_p.resolve()) + _remap_seen.add(resolve_cached(_p)) except (OSError, RuntimeError): pass for _e in all_edges: @@ -6259,7 +6479,7 @@ def _portable_out_of_root_sf(p: Path) -> str: continue _raw_tp = Path(_tf) try: - _tp = _raw_tp.resolve() + _tp = resolve_cached(_raw_tp) except (OSError, RuntimeError): continue if _tp in _remap_seen: @@ -6330,7 +6550,7 @@ def _portable_out_of_root_sf(p: Path) -> str: rel = path.relative_to(root) except ValueError: try: - rel = path.resolve().relative_to(root) + rel = resolve_cached(path).relative_to(root) except ValueError: continue new_id = _file_node_id(rel) @@ -6339,14 +6559,14 @@ def _portable_out_of_root_sf(p: Path) -> str: # Also register the absolute-resolved form of the file-level id so # alias/workspace import targets (resolved via .resolve()) remap to # canonical instead of orphaning (#1529). - old_id_abs = _make_id(str(path.resolve())) + old_id_abs = _make_id(str(resolve_cached(path))) if old_id_abs != new_id: id_remap[old_id_abs] = new_id old_prefs: list[tuple[str, str]] = [] old_pref = _file_node_id(path) if old_pref != new_id: old_prefs.append((old_pref, new_id)) - old_pref_abs = _file_node_id(path.resolve()) + old_pref_abs = _file_node_id(resolve_cached(path)) if old_pref_abs != new_id and old_pref_abs != old_pref: old_prefs.append((old_pref_abs, new_id)) # Bash entrypoint node ids append "__entry" to the file-level id @@ -6365,10 +6585,10 @@ def _portable_out_of_root_sf(p: Path) -> str: if _entry_old != _entry_new: id_remap.setdefault(_entry_old, _entry_new) if old_prefs: - prefix_remap[path.resolve()] = old_prefs + prefix_remap[resolve_cached(path)] = old_prefs # Absolute form first: it is the longest, so prefix decomposition can # try forms in order without a shorter form shadowing it. - stem_forms[path.resolve()] = ( + stem_forms[resolve_cached(path)] = ( new_id, [old_pref_abs, old_pref, new_id] ) if id_remap: @@ -6420,7 +6640,7 @@ def _portable_out_of_root_sf(p: Path) -> str: if n.get("type") == "package": continue try: - entry = prefix_remap.get(Path(sf).resolve()) + entry = prefix_remap.get(resolve_cached(sf)) except Exception: continue if entry is None: @@ -6526,7 +6746,7 @@ def _edge_key(edge: dict) -> str: def _decompose(target: str, tf: str) -> "tuple[str, str] | None": try: - forms = stem_forms.get(Path(tf).resolve()) + forms = stem_forms.get(resolve_cached(tf)) except (OSError, RuntimeError): return None if not forms: @@ -6654,12 +6874,37 @@ def _learn(e: dict) -> None: py_paths = [p for p in paths if p.suffix == ".py"] if py_paths: py_results = [r for r, p in zip(per_file, paths) if p.suffix == ".py"] + # Local symbol maps are built here because node ids only exist in the + # parent, and only after the id-remap above; the walk that consumes them + # is per-file, so it goes to a pool (#3008). A file with no local symbols + # is dropped from the work list, matching the in-process skip. + _ref_name_maps = [ + _python_local_name_map(r, str(p)) for r, p in zip(py_results, py_paths) + ] + _ref_payloads = _map_python_pass( + _python_refs_worker, + [ + (i, str(p), nm) + for i, (p, nm) in enumerate(zip(py_paths, _ref_name_maps)) + if nm + ], + len(py_paths), + parallel, + max_workers, + "Python reference collection", + ) try: - cross_file_edges = _resolve_cross_file_imports(py_results, py_paths) + cross_file_edges = _resolve_cross_file_imports( + py_results, py_paths, ref_payloads=_ref_payloads + ) all_edges.extend(cross_file_edges) except Exception as exc: import logging logging.getLogger(__name__).warning("Cross-file import resolution failed, skipping: %s", exc) + # Last consumer of the shared Python trees — release them here rather + # than at the end of the run, so the remaining passes do not hold a + # parse tree per Python file for no reason. + clear_python_tree_cache() # Cross-file Java import resolution java_paths = [p for p in paths if p.suffix == ".java"] @@ -6867,6 +7112,11 @@ def _looks_like_bash(result: object) -> bool: # of these files with no import evidence is gated below (#1659). _JS_TS_CALL_SUFFIXES = (".ts", ".tsx", ".mts", ".cts", ".js", ".jsx", ".mjs", ".cjs") _go_module_cache: dict[Path, str | None] = {} + # The import-path filter below runs once per candidate per Go raw_call, and + # the candidate set for a common method name is large. Without this memo the + # lookup was 48% of extract()'s wall clock on a 2,500-file Go corpus, nearly + # all of it re-deriving the same answer for the same file. + _go_import_path_cache: dict[str, str | None] = {} for rc in all_raw_calls: callee = rc.get("callee", "") if not callee: @@ -6932,7 +7182,8 @@ def _looks_like_bash(result: object) -> bool: candidates = [ candidate for candidate in candidates if _go_import_path_for_file( - nid_to_source_file.get(candidate, ""), root, _go_module_cache + nid_to_source_file.get(candidate, ""), root, + _go_module_cache, _go_import_path_cache, ) == import_path ] if not candidates: @@ -7133,7 +7384,7 @@ def _sf_entry(sf: str, sf_path: Path) -> tuple[str, str, tuple[str, ...]]: canonical_id = _file_node_id(rel) new_sf = rel.as_posix() try: - sf_resolved = sf_path.resolve() + sf_resolved = resolve_cached(sf_path) except (OSError, RuntimeError): sf_resolved = sf_path # Learn the STEM (extension-dropped) forms too: symbol producers mint diff --git a/graphify/extractors/csharp.py b/graphify/extractors/csharp.py index f823d64c3..ad47946f5 100644 --- a/graphify/extractors/csharp.py +++ b/graphify/extractors/csharp.py @@ -373,19 +373,26 @@ def _label_for_type_ref_target(target_node: dict, source_file: str) -> str | Non return alias return stem or None + # First sourceless node per label, in `all_nodes` order — the answer + # `_dangling_stub_id` used to re-derive by scanning every node on every call + # (#3008). Nothing in this pass rewrites a node's `label` or `source_file`, so + # one pass up front is equivalent; the only new members are the stubs minted + # below, which register themselves. `setdefault` keeps the earliest match, so + # a later stub never shadows a placeholder the scan would have reached first. + placeholder_by_label: dict[object, str] = {} + for node in all_nodes: + nid = node.get("id") + if isinstance(nid, str) and _is_placeholder(node): + placeholder_by_label.setdefault(node.get("label"), nid) + def _dangling_stub_id(label: str, current_target: object) -> str: current = node_by_id.get(current_target) if _is_placeholder(current) and current.get("label") == label: return str(current_target) - for node in all_nodes: - nid = node.get("id") - if ( - isinstance(nid, str) - and node.get("label") == label - and _is_placeholder(node) - ): - return nid + hit = placeholder_by_label.get(label) + if hit is not None: + return hit stem = _make_id(label) stub_id = stem @@ -404,6 +411,7 @@ def _dangling_stub_id(label: str, current_target: object) -> str: } all_nodes.append(node) node_by_id[stub_id] = node + placeholder_by_label.setdefault(label, stub_id) return stub_id REPOINT_RELATIONS = {"implements", "inherits", "references"} diff --git a/graphify/extractors/resolution.py b/graphify/extractors/resolution.py index 4982c7d05..d6b1a430e 100644 --- a/graphify/extractors/resolution.py +++ b/graphify/extractors/resolution.py @@ -10,6 +10,7 @@ _make_id, _read_text, ) +from graphify.paths import resolve_cached import hashlib import json import os @@ -22,8 +23,32 @@ # compilerOptions.baseUrl per config path, as an absolute dir (#2153). _TSCONFIG_BASEURL_CACHE: "dict[str, Path | None]" = {} +# Nearest tsconfig/jsconfig per starting directory, including the negative answer +# (#3008). The two loaders below cache config *contents* but each called +# _find_js_config unconditionally first, so the upward directory walk — two +# `exists()` probes per ancestor — ran on every import in the corpus. On a +# 32k-file corpus that was 2.37M of the run's 2.55M `exists()` calls. +_JS_CONFIG_DIR_CACHE: "dict[str, tuple[Path, Path] | None]" = {} + +# `_source_key` results per (source_file, root). The hottest function in the +# whole pipeline: 1.7M calls / 36% of a profiled 32k-file run, each doing an +# unmemoized `resolve()` over a handful of distinct paths (#3008). +_SOURCE_KEY_CACHE: "dict[tuple[str, str], str]" = {} + _WORKSPACE_MANIFEST_NAMES = ("pnpm-workspace.yaml", "package.json") +# Nearest workspace root per starting directory, negative answer included (#3008). +# `_load_workspace_packages` already caches the package map it builds, but it called +# `_find_workspace_root` first on every one of its 13,892 calls in a profiled +# 32k-file run — and that walk reads and json-parses the `package.json` of every +# ancestor that has one, looking for a `workspaces` key. Same shape as +# `_JS_CONFIG_DIR_CACHE`, and cleared per run for the same reason. +_WORKSPACE_ROOT_CACHE: "dict[str, Path | None]" = {} + +# Resolved module specifier per `(specifier, starting directory)`, negative answer +# included (#3008). See :func:`_resolve_js_module_path`; cleared per run. +_JS_MODULE_PATH_CACHE: "dict[tuple[str, str], Path | None]" = {} + _JS_RESOLVE_EXTS = (".ts", ".tsx", ".mts", ".cts", ".svelte", ".js", ".jsx", ".mjs", ".cjs") _JS_INDEX_FILES = ("index.ts", "index.tsx", "index.svelte", "index.js", "index.jsx", "index.mjs") @@ -132,7 +157,7 @@ def _read_tsconfig_aliases(tsconfig: Path, base_dir: Path, seen: set) -> dict[st # Skip scoped npm package configs (e.g. @tsconfig/svelte) — not on disk. if not ext or ext.startswith("@"): continue - extended_path = (base_dir / ext).resolve() + extended_path = resolve_cached(base_dir / ext) if not extended_path.suffix: extended_path = extended_path.with_suffix(".json") if extended_path.exists(): @@ -194,14 +219,26 @@ def _find_js_config(start_dir: Path) -> "tuple[Path, Path] | None": configures resolution in jsconfig.json got no aliases at all (#2153). tsconfig.json wins when both sit in one directory, matching tsc and editors, which consult jsconfig.json only when there is no tsconfig.json. + + Memoized per starting directory, negative answers included, and cleared per + run alongside the two content caches below (#2917, #3008) — so a config + added or removed between two extract() calls in one watch process is seen. """ - current = start_dir.resolve() + current = resolve_cached(start_dir) + key = str(current) + if key in _JS_CONFIG_DIR_CACHE: + return _JS_CONFIG_DIR_CACHE[key] + found: "tuple[Path, Path] | None" = None for candidate in [current, *current.parents]: for name in ("tsconfig.json", "jsconfig.json"): config = candidate / name if config.exists(): - return config, candidate - return None + found = config, candidate + break + if found is not None: + break + _JS_CONFIG_DIR_CACHE[key] = found + return found def _load_tsconfig_aliases(start_dir: Path) -> dict[str, list[str]]: """Walk up from start_dir to find tsconfig/jsconfig.json and return compilerOptions.paths aliases. @@ -325,10 +362,15 @@ def _resolve_tsconfig_alias(raw: str, aliases: dict[str, list[str]], return first def _find_workspace_root(start_dir: Path) -> Path | None: - current = start_dir.resolve() + current = resolve_cached(start_dir) + key = str(current) + if key in _WORKSPACE_ROOT_CACHE: + return _WORKSPACE_ROOT_CACHE[key] + found: "Path | None" = None for candidate in [current, *current.parents]: if (candidate / "pnpm-workspace.yaml").exists(): - return candidate + found = candidate + break package_json = candidate / "package.json" if package_json.is_file(): try: @@ -336,8 +378,10 @@ def _find_workspace_root(start_dir: Path) -> Path | None: except Exception: continue if "workspaces" in data: - return candidate - return None + found = candidate + break + _WORKSPACE_ROOT_CACHE[key] = found + return found def _pnpm_workspace_globs(workspace_file: Path) -> list[str]: globs: list[str] = [] @@ -434,7 +478,7 @@ def _contained_in_package(resolved: Path, package_dir: Path) -> bool: (e.g. "./evil": "../../../etc/passwd"). Only accept paths that stay within package_dir after resolution.""" try: - return resolved.resolve().is_relative_to(package_dir.resolve()) + return resolve_cached(resolved).is_relative_to(resolve_cached(package_dir)) except ValueError: return False @@ -517,6 +561,23 @@ def _resolve_js_module_path(raw: str | Path, start_dir: Path | None = None) -> P return _resolve_js_import_path(raw) if start_dir is None: return _resolve_js_import_path(Path(raw)) + + # A specifier resolves the same way for every import that writes it from the + # same directory, and the corpus writes the same handful over and over — + # `./utils`, a package name, an alias — so this was 76,317 calls at ~174us + # each, each one probing the filesystem for candidate extensions and index + # files (#3008). Keyed by directory rather than by file, so the hit rate is + # per-directory. Filesystem-dependent, hence the per-run clear. + cache_key = (raw, str(start_dir)) + if cache_key in _JS_MODULE_PATH_CACHE: + return _JS_MODULE_PATH_CACHE[cache_key] + resolved = _resolve_js_module_path_uncached(raw, start_dir) + _JS_MODULE_PATH_CACHE[cache_key] = resolved + return resolved + + +def _resolve_js_module_path_uncached(raw: str, start_dir: Path) -> Path | None: + """The filesystem probing behind :func:`_resolve_js_module_path`.""" if raw.startswith("."): return _resolve_js_import_path(start_dir / raw) @@ -563,7 +624,7 @@ def _resolve_c_include_path(raw: str, str_path: str) -> "Path | None": """ if not raw: return None - candidate = (Path(str_path).parent / raw).resolve() + candidate = resolve_cached(Path(str_path).parent / raw) if candidate.is_file(): return candidate return None @@ -641,13 +702,26 @@ def _blank(s: str) -> str: return "".join(out), lang def _source_key(source_file: str, root: Path) -> str: + """A source path as a root-relative key, memoized for the run (#3008). + + Pure in ``(source_file, root)``, but called once per node per collision + group — 1.7M times on a 32k-file corpus, over only ~46k distinct paths — and + each call paid a full symlink walk. Cleared per run with the other + path caches (#2917). + """ if not source_file: return "" + cache_key = (source_file, str(root)) + hit = _SOURCE_KEY_CACHE.get(cache_key) + if hit is not None: + return hit source_path = Path(source_file) try: - return str(source_path.resolve().relative_to(root)) + value = str(resolve_cached(source_path).relative_to(root)) except Exception: - return str(source_path) + value = str(source_path) + _SOURCE_KEY_CACHE[cache_key] = value + return value def _node_disambiguation_source_key(node: dict, root: Path) -> str: source_file = str(node.get("source_file", "")) @@ -815,7 +889,7 @@ def _js_source_path(source_file: str, root: Path) -> Path | None: if not path.is_absolute(): path = root / path try: - return path.resolve() + return resolve_cached(path) except Exception: return path @@ -839,8 +913,8 @@ def _apply_symbol_resolution_facts( ): return - path_by_resolved = {path.resolve(): path for path in paths} - source_file_id = {path.resolve(): _make_id(str(path)) for path in paths} + path_by_resolved = {resolve_cached(path): path for path in paths} + source_file_id = {resolve_cached(path): _make_id(str(path)) for path in paths} symbol_nodes: dict[tuple[Path, str], str] = {} for node in nodes: source_path = _js_source_path(str(node.get("source_file", "")), root) @@ -851,7 +925,7 @@ def _apply_symbol_resolution_facts( symbol_nodes[(source_path, label)] = str(node["id"]) def ensure_symbol_node(path: Path, name: str, line: int) -> str: - resolved_path = path.resolve() + resolved_path = resolve_cached(path) existing = symbol_nodes.get((resolved_path, name)) if existing is not None: return existing @@ -911,15 +985,15 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s local_aliases_by_file: dict[Path, dict[str, tuple[Path, str]]] = {} for import_fact in facts.imports: - file_path = import_fact.file_path.resolve() + file_path = resolve_cached(import_fact.file_path) local_aliases_by_file.setdefault(file_path, {})[import_fact.local_name] = ( - import_fact.target_path.resolve(), + resolve_cached(import_fact.target_path), import_fact.imported_name, ) pending_aliases_by_file: dict[Path, list[_SymbolAliasFact]] = {} for alias_fact in facts.aliases: - pending_aliases_by_file.setdefault(alias_fact.file_path.resolve(), []).append(alias_fact) + pending_aliases_by_file.setdefault(resolve_cached(alias_fact.file_path), []).append(alias_fact) for file_path, aliases in pending_aliases_by_file.items(): local_aliases = local_aliases_by_file.setdefault(file_path, {}) @@ -938,8 +1012,8 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s star_exports_by_file: dict[Path, list[Path]] = {} for star_fact in facts.star_exports: - source_path = star_fact.file_path.resolve() - target_path = star_fact.target_path.resolve() + source_path = resolve_cached(star_fact.file_path) + target_path = resolve_cached(star_fact.target_path) star_exports_by_file.setdefault(source_path, []).append(target_path) source_id = source_file_id.get(source_path) if source_id is not None: @@ -955,8 +1029,8 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) for namespace_fact in facts.namespace_exports: - source_path = namespace_fact.file_path.resolve() - target_path = namespace_fact.target_path.resolve() + source_path = resolve_cached(namespace_fact.file_path) + target_path = resolve_cached(namespace_fact.target_path) namespace_id = ensure_symbol_node( namespace_fact.file_path, namespace_fact.exported_name, @@ -987,10 +1061,10 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) for export_fact in facts.exports: - file_path = export_fact.file_path.resolve() + file_path = resolve_cached(export_fact.file_path) origin: tuple[Path, str] | None = None if export_fact.target_path is not None and export_fact.target_name is not None: - origin = (export_fact.target_path.resolve(), export_fact.target_name) + origin = (resolve_cached(export_fact.target_path), export_fact.target_name) elif export_fact.local_name is not None: origin = local_aliases_by_file.get(file_path, {}).get(export_fact.local_name) if origin is None and (file_path, export_fact.local_name) in symbol_nodes: @@ -1013,7 +1087,7 @@ def add_edge(source: str, target: str, relation: str, context: str, line: int, s ) def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tuple[Path, str]] | None = None) -> tuple[Path, str]: - target_path = target_path.resolve() + target_path = resolve_cached(target_path) key = (target_path, imported_name) if seen is None: seen = set() @@ -1033,7 +1107,7 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup return key for import_fact in facts.imports: - source_id = source_file_id.get(import_fact.file_path.resolve()) + source_id = source_file_id.get(resolve_cached(import_fact.file_path)) if source_id is None: continue origin_path, origin_symbol = resolve_exported_origin( @@ -1077,7 +1151,7 @@ def resolve_exported_origin(target_path: Path, imported_name: str, seen: set[tup # canonicalizes — or drop it when no file node id is available. owned = {str(n.get("id")) for n in nodes} for use_fact in facts.uses: - file_path = use_fact.file_path.resolve() + file_path = resolve_cached(use_fact.file_path) target_id = None unresolved_origin = local_aliases_by_file.get(file_path, {}).get(use_fact.local_name) if unresolved_origin is not None: @@ -1483,20 +1557,126 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut trees: dict[Path, tuple[bytes, object]] = {} + def _collect_export_facts(node, source: bytes, path: Path) -> None: + """The export-statement half of one file's facts, for one syntax node. + + Lifted out of its own pass over every tree so it can ride along with the + declaration/import walk (#3008): the walk was 27.3M node visits in a + profiled 32k-file run, and this pass re-visited every one of them to look + at export statements alone. It appends only to `exports`, + `namespace_exports` and `star_exports`, none of which the merged-in walk + touches, so every list keeps the order it had. Each `continue` in the + original loop body became a `return` — nothing followed it. + """ + raw_module = _js_module_specifier(node, source) + export_clause = _js_export_clause(node) + # `export type { X } from ...` / `export type * from ...`: the + # statement-level `type` keyword is a bare anonymous child; the + # default binding NAMED type sits inside the clause instead (#3123). + stmt_type_only = any( + child.type == "type" and not child.is_named + for child in node.children + ) + if raw_module is not None: + target_path = _resolve_js_module_path(raw_module, path.parent) + if target_path is None: + return + target_path = resolve_cached(target_path) + namespace_name = _js_namespace_export_name(node, source) + if namespace_name is not None: + facts.namespace_exports.append( + _NamespaceExportFact( + path, + namespace_name, + target_path, + node.start_point[0] + 1, + type_only=stmt_type_only, + ) + ) + elif _js_export_statement_is_star(node): + facts.star_exports.append( + _StarExportFact(path, target_path, node.start_point[0] + 1, + type_only=stmt_type_only) + ) + if export_clause is not None: + for original_name, exported_name in _js_named_specifiers( + export_clause, source, "export_specifier" + ): + facts.exports.append( + _SymbolExportFact( + path, + exported_name, + node.start_point[0] + 1, + target_path=target_path, + target_name=original_name, + type_only=stmt_type_only, + ) + ) + return + + if export_clause is not None: + for local_name, exported_name in _js_named_specifiers( + export_clause, source, "export_specifier" + ): + facts.exports.append( + _SymbolExportFact( + path, + exported_name, + node.start_point[0] + 1, + local_name=local_name, + ) + ) + return + + for exported_name in _js_exported_declaration_names(node, source): + facts.exports.append( + _SymbolExportFact( + path, + exported_name, + node.start_point[0] + 1, + local_name=exported_name, + ) + ) + + # `export default class Foo {}` / `export default foo` exposes the + # symbol under the name "default"; record that so a default import + # (imported_name="default") resolves to it. `export { X as default }` + # is already handled via the export_clause path above. + default_name = _js_default_export_name(node, source) + if default_name is not None: + facts.exports.append( + _SymbolExportFact( + path, + "default", + node.start_point[0] + 1, + local_name=default_name, + ) + ) + for path in js_paths: - resolved_path = path.resolve() + resolved_path = resolve_cached(path) parsed = _parse_js_tree(path) if parsed is None: continue source, root_node = parsed trees[resolved_path] = parsed + # Declarations, imports, lexical aliases and exports share one DFS: the walk + # was 37.2M node visits in a profiled 32k-file run and these passes ran + # back to back over the same root (#3008). Each list is appended to + # independently and DFS order is unchanged, so per-list order is identical. for node in _walk_js_tree(root_node): + for alias, target in _js_lexical_aliases(node, source): + facts.aliases.append( + _SymbolAliasFact(path, alias, target, node.start_point[0] + 1) + ) + if node.type == "export_statement": for name in _js_exported_declaration_names(node, source): facts.declarations.append( _SymbolDeclarationFact(path, name, node.start_point[0] + 1) ) + _collect_export_facts(node, source, path) if node.type != "import_statement": continue @@ -1506,7 +1686,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut target_path = _resolve_js_module_path(raw_module, path.parent) if target_path is None: continue - target_path = target_path.resolve() + target_path = resolve_cached(target_path) for imported_name, local_name in _js_named_specifiers(node, source, "import_specifier"): facts.imports.append( _SymbolImportFact( @@ -1529,110 +1709,8 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) ) - for node in _walk_js_tree(root_node): - for alias, target in _js_lexical_aliases(node, source): - facts.aliases.append( - _SymbolAliasFact(path, alias, target, node.start_point[0] + 1) - ) - for path in js_paths: - resolved_path = path.resolve() - parsed = trees.get(resolved_path) - if parsed is None: - continue - source, root_node = parsed - - for node in _walk_js_tree(root_node): - if node.type != "export_statement": - continue - - raw_module = _js_module_specifier(node, source) - export_clause = _js_export_clause(node) - # `export type { X } from ...` / `export type * from ...`: the - # statement-level `type` keyword is a bare anonymous child; the - # default binding NAMED type sits inside the clause instead (#3123). - stmt_type_only = any( - child.type == "type" and not child.is_named - for child in node.children - ) - if raw_module is not None: - target_path = _resolve_js_module_path(raw_module, path.parent) - if target_path is None: - continue - target_path = target_path.resolve() - namespace_name = _js_namespace_export_name(node, source) - if namespace_name is not None: - facts.namespace_exports.append( - _NamespaceExportFact( - path, - namespace_name, - target_path, - node.start_point[0] + 1, - type_only=stmt_type_only, - ) - ) - elif _js_export_statement_is_star(node): - facts.star_exports.append( - _StarExportFact(path, target_path, node.start_point[0] + 1, - type_only=stmt_type_only) - ) - if export_clause is not None: - for original_name, exported_name in _js_named_specifiers( - export_clause, source, "export_specifier" - ): - facts.exports.append( - _SymbolExportFact( - path, - exported_name, - node.start_point[0] + 1, - target_path=target_path, - target_name=original_name, - type_only=stmt_type_only, - ) - ) - continue - - if export_clause is not None: - for local_name, exported_name in _js_named_specifiers( - export_clause, source, "export_specifier" - ): - facts.exports.append( - _SymbolExportFact( - path, - exported_name, - node.start_point[0] + 1, - local_name=local_name, - ) - ) - continue - - for exported_name in _js_exported_declaration_names(node, source): - facts.exports.append( - _SymbolExportFact( - path, - exported_name, - node.start_point[0] + 1, - local_name=exported_name, - ) - ) - - # `export default class Foo {}` / `export default foo` exposes the - # symbol under the name "default"; record that so a default import - # (imported_name="default") resolves to it. `export { X as default }` - # is already handled via the export_clause path above. - default_name = _js_default_export_name(node, source) - if default_name is not None: - facts.exports.append( - _SymbolExportFact( - path, - "default", - node.start_point[0] + 1, - local_name=default_name, - ) - ) - - for path in js_paths: - resolved_path = path.resolve() + resolved_path = resolve_cached(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1654,7 +1732,7 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut ) for path in js_paths: - resolved_path = path.resolve() + resolved_path = resolve_cached(path) parsed = trees.get(resolved_path) if parsed is None: continue @@ -1676,20 +1754,104 @@ def _collect_js_symbol_resolution_facts(paths: list[Path], facts: _SymbolResolut class_nid = _make_id(stem, class_name) _ts_walk_class_members(node, source, path, class_nid, facts) +# One Parser per process, and the parsed tree per file for the length of one +# run. Three separate phases parsed the same Python file — the worker's +# _extract_python_rationale, then _collect_python_symbol_resolution_facts and +# _resolve_cross_file_imports in the serial parent — 1,042 parses for a 350-file +# corpus, 1.32s of the 1.79s a profiled run spent inside tree_sitter.parse. The +# two parent-side phases share this cache; the worker's parse happens in another +# process and cannot (#3008). +# +# Peak memory is effectively unchanged: _collect_python_symbol_resolution_facts +# already held every Python tree at once in a local dict, and it runs after the +# node and edge lists exist, so trees and graph already coexist at the peak. +# extract() drops the cache as soon as the last consumer is done. +_PY_PARSER: "Any | None" = None +_PY_QUERIES: "dict[str, Any]" = {} +_PY_TREE_CACHE: "dict[Path, tuple[bytes, Any] | None]" = {} + +def clear_python_tree_cache() -> None: + """Drop the per-run Python parse cache. + + Must run at the start of every extract(), the same contract as + clear_resolve_cache(): the cache treats a file's contents as fixed for one + run, so a long-lived `graphify watch` process would otherwise resolve the + next cycle's symbols against the previous cycle's source text. + """ + _PY_TREE_CACHE.clear() + +def _python_language(): + import tree_sitter_python as tspython + from tree_sitter import Language + return Language(tspython.language()) + +def _python_query(pattern: str): + """Compiled tree-sitter query for ``pattern``, one per process.""" + query = _PY_QUERIES.get(pattern) + if query is None: + from tree_sitter import Query + query = Query(_python_language(), pattern) + _PY_QUERIES[pattern] = query + return query + +def _query_python_nodes(pattern: str, node, capture: str = "m") -> "list | None": + """Nodes under ``node`` matching ``pattern``, in pre-order. + + Stands in for a full Python-side walk plus a node-type test. Doing the + filtering in tree-sitter's C layer measured 3.6x faster for + `import_from_statement` and 3.1x for `call` over a 350-file corpus, where one + full walk visits 1.44M nodes to reach a few thousand of interest (#3008). + + Captures arrive in match-completion order, NOT pre-order — a nested match is + reported after every shallower one, so `a(b(c()))` beside `d()` comes back as + a, d, b, c. Sorting by start byte ascending then end byte descending + reproduces pre-order exactly, because a parent always starts no later than + its child and ends no earlier. That order is observable: callers append to + fact lists that flow into emitted edge order. + + Returns None when the cursor hit its match limit, so a caller can fall back + to walking rather than act on a silently truncated capture set. + """ + from tree_sitter import QueryCursor + cursor = QueryCursor(_python_query(pattern)) + captures = cursor.captures(node) + if cursor.did_exceed_match_limit: + return None + found = captures.get(capture) + if not found: + return [] + found.sort(key=lambda n: (n.start_byte, -n.end_byte)) + return found + def _parse_python_tree(path: Path): + key = resolve_cached(path) + if key in _PY_TREE_CACHE: + return _PY_TREE_CACHE[key] + global _PY_PARSER try: - import tree_sitter_python as tspython - from tree_sitter import Language, Parser + if _PY_PARSER is None: + from tree_sitter import Parser + _PY_PARSER = Parser(_python_language()) source = path.read_bytes() - parser = Parser(Language(tspython.language())) - return source, parser.parse(source).root_node + parsed = (source, _PY_PARSER.parse(source).root_node) except Exception: - return None + parsed = None + _PY_TREE_CACHE[key] = parsed + return parsed def _walk_python_tree(node): - yield node - for child in node.children: - yield from _walk_python_tree(child) + # Iterative DFS avoids Python's O(depth) generator-chain overhead, the same + # way _walk_js_tree does. Recursive yield-from relays every yielded value up + # through one generator frame per level, and a profiled run put 1.45M nodes + # at average depth ~17 — 24.7M frame resumptions for 1.45M nodes. + # reversed() is load-bearing: the stack pops LIFO, so children must be + # pushed right-to-left to come out left-to-right. Sibling order is + # observable — callers append to fact lists that flow into edge order. + stack = [node] + while stack: + n = stack.pop() + yield n + stack.extend(reversed(n.children)) def _python_import_from_module(node, source: bytes) -> tuple[int, str] | None: level = 0 @@ -1809,96 +1971,290 @@ def _python_call_identifier(node, source: bytes) -> str | None: return _read_text(function_node, source) return None +def _collect_python_file_facts(path: Path, root: Path) -> "dict | None": + """Symbol-resolution facts for ONE Python file, as plain picklable data. + + Split out of :func:`_collect_python_symbol_resolution_facts` so the parse and + the two tree walks can run in a worker pool: every fact here comes from this + file plus the filesystem, never from the cross-file node table, so a worker + can produce it and the parent only stitches the results back together in + input order. The parent's half of this pass was 0.85s of a 2.01s serial tail + on a 501-file corpus, ~80% of it in here (#3008). + + Values are str/int rather than Path/dataclass to keep the pickled payload + small; :func:`_apply_python_file_facts` rebuilds the fact objects. + + Returns None when the file does not parse, which the caller skips. + """ + parsed = _parse_python_tree(path) + if parsed is None: + return None + source, root_node = parsed + + imports: list[tuple[str, str, str, int]] = [] + module_imports: list[tuple[str, int, str]] = [] + exports: list[tuple[str, int, str, str]] = [] + uses: list[tuple[str, str, int]] = [] + is_init = path.name == "__init__.py" + + # An import_from_statement is a handful of nodes in a file of thousands, + # so let tree-sitter find them instead of walking every node here. + import_nodes = _query_python_nodes("(import_from_statement) @m", root_node) + if import_nodes is None: # match limit hit — fall back to the full walk + import_nodes = [n for n in _walk_python_tree(root_node) + if n.type == "import_from_statement"] + for node in import_nodes: + module = _python_import_from_module(node, source) + if module is None: + continue + level, module_name = module + target_path = _resolve_python_module_path(module_name, path, root, level) + if target_path is None: + continue + # #1146: `from pkg import submod` — if the target is a package + # (__init__.py) and an imported name matches a submodule file on + # disk, emit a file-level import edge to that submodule rather + # than only to the package. + pkg_dir = target_path.parent if target_path.name == "__init__.py" else None + target_str = str(target_path) + for imported_name, local_name in _python_imported_names(node, source): + line = node.start_point[0] + 1 + if pkg_dir is not None: + sub_py = pkg_dir / f"{imported_name}.py" + sub_pkg = pkg_dir / imported_name / "__init__.py" + submodule = sub_py if sub_py.is_file() else (sub_pkg if sub_pkg.is_file() else None) + if submodule is not None: + module_imports.append((str(submodule), line, local_name)) + continue + imports.append((local_name, target_str, imported_name, line)) + if is_init: + exports.append((local_name, line, target_str, imported_name)) + + for source_id, body in _python_top_level_function_bodies(path, root_node, source): + # Only `call` nodes matter here — 48k of the 1.44M nodes a full walk + # of this corpus visits. _python_call_identifier still returns None + # for a call whose function field is not a bare identifier, so the + # None check below stays. + call_nodes = _query_python_nodes("(call) @m", body) + if call_nodes is None: # match limit hit — fall back to the full walk + call_nodes = [n for n in _walk_python_tree(body) if n.type == "call"] + for node in call_nodes: + imported_name = _python_call_identifier(node, source) + if imported_name is None: + continue + uses.append((source_id, imported_name, node.start_point[0] + 1)) + + return { + "imports": imports, + "module_imports": module_imports, + "exports": exports, + "uses": uses, + } + +def _apply_python_file_facts( + py_paths: list[Path], + payloads: "list[dict | None]", + facts: _SymbolResolutionFacts, +) -> None: + """Stitch per-file Python facts into ``facts``, in input order. + + ``payloads`` must be aligned with ``py_paths`` — position i holds the facts + for py_paths[i], or None for a file that did not parse. The four fact lists + are disjoint, so appending all four per file reproduces exactly what the + sequential collector produced (each list in py_paths order); downstream edge + emission follows those lists, so the alignment is load-bearing. + """ + for path, payload in zip(py_paths, payloads): + if payload is None: + continue + for local_name, target_str, imported_name, line in payload["imports"]: + facts.imports.append( + _SymbolImportFact(path, local_name, Path(target_str), imported_name, line) + ) + for submodule_str, line, local_name in payload["module_imports"]: + facts.module_imports.append((path, Path(submodule_str), line, local_name)) + for local_name, line, target_str, target_name in payload["exports"]: + facts.exports.append( + _SymbolExportFact( + path, + local_name, + line, + target_path=Path(target_str), + target_name=target_name, + ) + ) + for source_id, imported_name, line in payload["uses"]: + facts.uses.append( + _SymbolUseFact(path, source_id, imported_name, "calls", "call", line) + ) + def _collect_python_symbol_resolution_facts( paths: list[Path], root: Path, facts: _SymbolResolutionFacts, + payloads: "list[dict | None] | None" = None, ) -> None: + """Collect Python symbol-resolution facts for every ``.py`` path. + + ``payloads`` lets a caller supply per-file facts computed elsewhere — the + extraction pool, see ``_map_python_pass`` in extract.py — one entry per + ``.py`` path in ``paths`` order. Passing None collects them in-process. + """ py_paths = [path for path in paths if path.suffix == ".py"] if not py_paths: return - - trees: dict[Path, tuple[bytes, object]] = {} - for path in py_paths: - parsed = _parse_python_tree(path) - if parsed is None: - continue - source, root_node = parsed - trees[path.resolve()] = parsed - - for node in _walk_python_tree(root_node): - if node.type != "import_from_statement": - continue - module = _python_import_from_module(node, source) - if module is None: - continue - level, module_name = module - target_path = _resolve_python_module_path(module_name, path, root, level) - if target_path is None: - continue - # #1146: `from pkg import submod` — if the target is a package - # (__init__.py) and an imported name matches a submodule file on - # disk, emit a file-level import edge to that submodule rather - # than only to the package. - pkg_dir = target_path.parent if target_path.name == "__init__.py" else None - for imported_name, local_name in _python_imported_names(node, source): - line = node.start_point[0] + 1 - if pkg_dir is not None: - sub_py = pkg_dir / f"{imported_name}.py" - sub_pkg = pkg_dir / imported_name / "__init__.py" - submodule = sub_py if sub_py.is_file() else (sub_pkg if sub_pkg.is_file() else None) - if submodule is not None: - facts.module_imports.append((path, submodule, line, local_name)) - continue - facts.imports.append( - _SymbolImportFact(path, local_name, target_path, imported_name, line) - ) - if path.name == "__init__.py": - facts.exports.append( - _SymbolExportFact( - path, - local_name, - line, - target_path=target_path, - target_name=imported_name, - ) - ) - - for path in py_paths: - parsed = trees.get(path.resolve()) - if parsed is None: - continue - source, root_node = parsed - for source_id, body in _python_top_level_function_bodies(path, root_node, source): - for node in _walk_python_tree(body): - imported_name = _python_call_identifier(node, source) - if imported_name is None: - continue - facts.uses.append( - _SymbolUseFact( - path, - source_id, - imported_name, - "calls", - "call", - node.start_point[0] + 1, - ) - ) + if payloads is None: + payloads = [_collect_python_file_facts(path, root) for path in py_paths] + _apply_python_file_facts(py_paths, payloads, facts) def _augment_symbol_resolution_edges( paths: list[Path], nodes: list[dict], edges: list[dict], root: Path, + py_facts_payloads: "list[dict | None] | None" = None, ) -> None: facts = _SymbolResolutionFacts() _collect_js_symbol_resolution_facts(paths, facts) - _collect_python_symbol_resolution_facts(paths, root, facts) + _collect_python_symbol_resolution_facts(paths, root, facts, payloads=py_facts_payloads) _apply_symbol_resolution_facts(paths, nodes, edges, root, facts) +def _python_local_name_map(file_result: dict, str_path: str) -> dict[str, str]: + """Local symbol name -> node id for one file's own nodes. + + Function labels end in "()"; the file node ends in ".py"; rationale nodes + never import (#563). First writer wins on a name collision (inherently + ambiguous within a file). Split out of :func:`_resolve_cross_file_imports` so + the map can be built in the parent — node ids only exist there — and handed + to a worker that walks the file (#3008). + """ + name_to_nid: dict[str, str] = {} + for n in file_result.get("nodes", []): + if n.get("source_file") != str_path or n.get("file_type") == "rationale": + continue + label = n.get("label", "") + if not label or label.endswith(".py"): + continue + sym_name = label[:-2] if label.endswith("()") else label + if sym_name and sym_name not in name_to_nid: + name_to_nid[sym_name] = n["id"] + return name_to_nid + +def _collect_python_reference_facts( + path: Path, name_to_nid: dict[str, str] +) -> "dict | None": + """Per-file half of cross-file import resolution, as plain picklable data. + + Walks the file once and returns (a) every ``from X import ...`` statement in + walk order, reduced to what the parent needs for the global stem lookup, and + (b) the first line at which each local symbol references each name. Neither + depends on the cross-file node table, so this half runs in a worker pool + while the parent keeps the global index and the edge emission. The walk was + ~87% of a 0.62s serial pass on a 501-file corpus (#3008). + + Each ``modules`` entry is ``(rel_fq, bares, names)``: + + * ``rel_fq`` is the directory-qualified stem of a relative import's target, + resolvable from this file's directory alone, or None. + * ``bares`` is every ``dotted_name`` child in child order, stopping at a + ``relative_import``. The parent maps them through its bare-stem index and + takes the first that resolves — the sequential code tried each in turn + while its target was still unresolved, and the imported names sit in that + same child list, so they were candidates too. + * ``names`` is ``(imported_name, local_name)`` per imported name, in order; + ``local_name`` honours ``import X as Y`` so a reference to the alias in the + body still attributes correctly. + + Returns None when the file does not parse. + """ + parsed = _parse_python_tree(path) + if parsed is None: + return None + source, tree_root = parsed + + modules: list[tuple[str | None, list[str], list[tuple[str, str]]]] = [] + # referenced name -> {source symbol nid: first reference line} + ref_sources: dict[str, dict[str, int]] = {} + parent_dir = path.parent + + def _text(n) -> str: + return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") + + def record_import(node) -> None: + # Find the module name - handles both absolute and relative imports. + # Relative: `from .models import X` → relative_import → dotted_name + # Absolute: `from models import X` → module_name field + rel_fq: str | None = None + bares: list[str] = [] + for child in node.children: + if child.type == "relative_import": + for sub in child.children: + if sub.type == "dotted_name": + bare = _text(sub).split(".")[-1] + rel_fq = _file_stem(parent_dir / f"{bare}.py") + break + break + if child.type == "dotted_name": + bares.append(_text(child).split(".")[-1]) + + # Imported names come AFTER the 'import' keyword token. For + # `import X as Y` the target is found via X but the body uses Y. + names: list[tuple[str, str]] = [] + past_import_kw = False + for child in node.children: + if child.type == "import": + past_import_kw = True + continue + if not past_import_kw: + continue + imported_name: str | None = None + local_name: str | None = None + if child.type == "dotted_name": + imported_name = local_name = _text(child) + elif child.type == "aliased_import": + name_node = child.child_by_field_name("name") + alias_node = child.child_by_field_name("alias") + if name_node is not None: + imported_name = _text(name_node) + local_name = _text(alias_node) if alias_node is not None else imported_name + if not imported_name or not local_name: + continue + names.append((imported_name, local_name)) + modules.append((rel_fq, bares, names)) + + def visit(node, current_nid: str | None) -> None: + # node.type is an attribute read that builds a str each time, and + # this walk covers 1.4M nodes per run — read it once (#3008). + ntype = node.type + # Identifiers inside an import statement are the import itself, not a + # real use — record the import here and don't descend into it. + if ntype == "import_from_statement": + record_import(node) + return + # Attribute references to the top-level symbol that contains them: a + # class is a unit (a reference inside one of its methods counts for + # the class, matching the documented DigestAuth->Response edge), and + # a module-level function is its own source. Only set at module scope + # (current_nid is None) so nested defs never override the container. + if current_nid is None and ntype in ("class_definition", "function_definition"): + name_node = node.child_by_field_name("name") + if name_node is not None: + mapped = name_to_nid.get(_text(name_node)) + if mapped is not None: + current_nid = mapped + if ntype == "identifier" and current_nid is not None: + slot = ref_sources.setdefault(_text(node), {}) + slot.setdefault(current_nid, node.start_point[0] + 1) + for child in node.children: + visit(child, current_nid) + + visit(tree_root, None) + return {"modules": modules, "refs": ref_sources} + def _resolve_cross_file_imports( per_file: list[dict], paths: list[Path], + ref_payloads: "list[dict | None] | None" = None, ) -> list[dict]: """ Two-pass import resolution: turn file-level imports into class-level edges. @@ -1915,14 +2271,11 @@ def _resolve_cross_file_imports( BasicAuth --uses--> Request [INFERRED] """ try: - import tree_sitter_python as tspython - from tree_sitter import Language, Parser + import tree_sitter_python # noqa: F401 + import tree_sitter # noqa: F401 except ImportError: return [] - language = Language(tspython.language()) - parser = Parser(language) - # Pass 1: _file_stem(path) → {ClassName: node_id} # Keyed by directory-qualified stem (e.g. "auth_models") to avoid collisions # when multiple files share the same filename in different directories. @@ -1959,119 +2312,52 @@ def _resolve_cross_file_imports( # actually references the imported name — not to every class that merely # shares the file (#2652). The edge is anchored at the real reference, not # the import line, so `source_location` points at genuine corroboration. + # The per-file walk lives in _collect_python_reference_facts; ``ref_payloads`` + # supplies its results when a caller ran it in a worker pool, one entry per + # path in ``paths`` order (None for a file with no local symbols, no parse, + # or no payload). Passing None walks in-process. new_edges: list[dict] = [] - for file_result, path in zip(per_file, paths): + for i, (file_result, path) in enumerate(zip(per_file, paths)): str_path = str(path) - # Map each local symbol (class or function) to its node id, keyed by the - # bare symbol name. Function labels end in "()"; the file node ends in - # ".py"; rationale nodes never import (#563). First writer wins on a - # name collision (inherently ambiguous within a file). - name_to_nid: dict[str, str] = {} - for n in file_result.get("nodes", []): - if n.get("source_file") != str_path or n.get("file_type") == "rationale": - continue - label = n.get("label", "") - if not label or label.endswith(".py"): + if ref_payloads is not None: + payload = ref_payloads[i] + else: + name_to_nid = _python_local_name_map(file_result, str_path) + if not name_to_nid: continue - sym_name = label[:-2] if label.endswith("()") else label - if sym_name and sym_name not in name_to_nid: - name_to_nid[sym_name] = n["id"] - if not name_to_nid: - continue - - # Parse imports from this file - try: - source = path.read_bytes() - tree = parser.parse(source) - except Exception: + # _collect_python_file_facts already parsed this file earlier in the + # same run when both passes run in one process, so this is a cache hit. + payload = _collect_python_reference_facts(path, name_to_nid) + if payload is None: continue - # local_name -> target node id (local_name honours `import X as Y`, so a - # reference to the alias in the body still attributes correctly). + # local_name -> target node id. Insertion order is import-statement walk + # order, which is the order the edges below are emitted in. import_targets: dict[str, str] = {} - # referenced name -> {source symbol nid: first reference line} - ref_sources: dict[str, dict[str, int]] = {} - - def _text(n) -> str: - return source[n.start_byte:n.end_byte].decode("utf-8", errors="replace") - - def resolve_import(node) -> None: - # Find the module name - handles both absolute and relative imports. - # Relative: `from .models import X` → relative_import → dotted_name - # Absolute: `from models import X` → module_name field + for rel_fq, bares, names in payload["modules"]: # target_fq is the directory-qualified stem used as the key in # stem_to_entities. Relative imports are resolved exactly via the # importing file's directory; absolute imports fall back to the # bare-stem secondary index (first-writer-wins when names collide). - target_fq: str | None = None - for child in node.children: - if child.type == "relative_import": - for sub in child.children: - if sub.type == "dotted_name": - bare = _text(sub).split(".")[-1] - candidate = path.parent / f"{bare}.py" - target_fq = _file_stem(candidate) - break - break - if child.type == "dotted_name" and target_fq is None: - bare = _text(child).split(".")[-1] + target_fq = rel_fq + if target_fq is None: + for bare in bares: target_fq = bare_to_qualified.get(bare) - - if not target_fq or target_fq not in stem_to_entities: - return - - # Imported names come AFTER the 'import' keyword token. For - # `import X as Y` the target is found via X but the body uses Y. - past_import_kw = False - for child in node.children: - if child.type == "import": - past_import_kw = True - continue - if not past_import_kw: - continue - imported_name: str | None = None - local_name: str | None = None - if child.type == "dotted_name": - imported_name = local_name = _text(child) - elif child.type == "aliased_import": - name_node = child.child_by_field_name("name") - alias_node = child.child_by_field_name("alias") - if name_node is not None: - imported_name = _text(name_node) - local_name = _text(alias_node) if alias_node is not None else imported_name - if not imported_name or not local_name: - continue - tgt_nid = stem_to_entities[target_fq].get(imported_name) + if target_fq is not None: + break + if not target_fq: + continue + entities = stem_to_entities.get(target_fq) + if entities is None: + continue + for imported_name, local_name in names: + tgt_nid = entities.get(imported_name) if tgt_nid: import_targets[local_name] = tgt_nid - def visit(node, current_nid: str | None) -> None: - # Identifiers inside an import statement are the import itself, not a - # real use — resolve the import here and don't descend into it. - if node.type == "import_from_statement": - resolve_import(node) - return - # Attribute references to the top-level symbol that contains them: a - # class is a unit (a reference inside one of its methods counts for - # the class, matching the documented DigestAuth->Response edge), and - # a module-level function is its own source. Only set at module scope - # (current_nid is None) so nested defs never override the container. - if current_nid is None and node.type in ("class_definition", "function_definition"): - name_node = node.child_by_field_name("name") - if name_node is not None: - mapped = name_to_nid.get(_text(name_node)) - if mapped is not None: - current_nid = mapped - if node.type == "identifier" and current_nid is not None: - slot = ref_sources.setdefault(_text(node), {}) - slot.setdefault(current_nid, node.start_point[0] + 1) - for child in node.children: - visit(child, current_nid) - - visit(tree.root_node, None) - + ref_sources = payload["refs"] for name, tgt_nid in import_targets.items(): for src_nid, line in ref_sources.get(name, {}).items(): if src_nid == tgt_nid: @@ -2408,14 +2694,44 @@ def _go_import_path_for_file( source_file: str | Path, root: Path, module_cache: dict[Path, str | None] | None = None, + result_cache: dict[str, str | None] | None = None, +) -> str | None: + """Return the canonical Go import path for a source file inside a module. + + ``module_cache`` memoizes the directory-to-module-path lookup, so a go.mod is + read and parsed once per module rather than once per file. ``result_cache`` + memoizes the whole answer per ``source_file``, which is a different and much + larger saving: the callers ask about the same file once per candidate per raw + call, and everything before the go.mod lookup — building a ``Path``, + resolving it, and materializing ``directory.parents`` — runs on every ask. + + On a 2,500-file Go corpus this function was called 125k times and accounted + for 13.4 s of a 27.7 s ``extract()``, essentially all of it ``pathlib`` + object churn rather than filesystem work. Both caches are caller-owned and + live for one ``extract()``, so a file whose module changes between runs is + not stale. + """ + key = str(source_file) + if result_cache is not None and key in result_cache: + return result_cache[key] + + answer = _go_import_path_uncached(source_file, root, module_cache) + if result_cache is not None: + result_cache[key] = answer + return answer + + +def _go_import_path_uncached( + source_file: str | Path, + root: Path, + module_cache: dict[Path, str | None] | None = None, ) -> str | None: - """Return the canonical Go import path for a source file inside a module.""" cache = module_cache if module_cache is not None else {} path = Path(source_file) if not path.is_absolute(): path = root / path try: - directory = path.resolve().parent + directory = resolve_cached(path).parent except OSError: directory = path.absolute().parent @@ -2480,6 +2796,8 @@ def _resolve_go_type_references( contained = {edge.get("target") for edge in definition_edges if edge.get("relation") == "contains"} module_cache: dict[Path, str | None] = {} + # Many nodes share one file, so the per-file answer is asked for repeatedly. + import_path_cache: dict[str, str | None] = {} fqn_to_ids: dict[str, list[str]] = {} for node in definition_nodes: source_file = str(node.get("source_file") or "") @@ -2489,7 +2807,9 @@ def _resolve_go_type_references( or not _is_type_like_definition(node)): continue actual_path = actual_path_by_file.get(source_file, Path(source_file)) - package_path = _go_import_path_for_file(actual_path, root, module_cache) + package_path = _go_import_path_for_file( + actual_path, root, module_cache, import_path_cache + ) if package_path: fqn_to_ids.setdefault(f"{package_path}.{label}", []).append(nid) diff --git a/graphify/ids.py b/graphify/ids.py index 0143ee2d1..9f49e76dd 100644 --- a/graphify/ids.py +++ b/graphify/ids.py @@ -43,13 +43,21 @@ import re import unicodedata +from functools import lru_cache __all__ = ["normalize_id", "make_id"] +@lru_cache(maxsize=1 << 17) def normalize_id(s: str) -> str: r"""Normalize a single ID string to its canonical form. + Memoized (#3008): the recipe below is a pure function of the string — a + casefold/NFKC fixpoint loop plus two regex passes, ~7.6us a call — and a + corpus asks about the same names repeatedly, 448,282 calls on a 32k-file run. + The bound is a ceiling for a long-lived `watch` process, not a correctness + concern; nothing here consults the filesystem, so there is no staleness. + Guarantees, all enforced by tests: - Idempotent: ``normalize_id(normalize_id(s)) == normalize_id(s)``. diff --git a/graphify/paths.py b/graphify/paths.py index ba15b32cc..698962aca 100644 --- a/graphify/paths.py +++ b/graphify/paths.py @@ -16,6 +16,7 @@ from __future__ import annotations +import functools import json import os import re @@ -25,6 +26,47 @@ GRAPHIFY_OUT = os.environ.get("GRAPHIFY_OUT", "graphify-out") +# Memoized `Path.resolve()` results, keyed by the input path string (a relative +# input is keyed under its cwd-joined form, so a chdir cannot serve a wrong +# answer). See :func:`resolve_cached`. +_RESOLVE_CACHE: "dict[str, Path]" = {} + + +def resolve_cached(path: "str | Path") -> Path: + """``Path(path).resolve()``, memoized for the life of one extraction run. + + ``Path.resolve()`` is not a string operation: it walks the path component by + component through ``lstat`` to follow symlinks, so one call on a path 15 + segments deep costs ~15 syscalls. Cross-file resolution asks the same + question about the same few thousand paths millions of times — a profile of + a 32k-file corpus showed 3.9M ``realpath`` calls issuing 59M ``lstat`` + syscalls, with ``_source_key`` alone accounting for 1.7M of them (#3008). + + The filesystem is treated as frozen for the duration of a run, which is the + same assumption the tsconfig/workspace caches already make. Like those, this + cache has no mtime or content component, so ``extract()`` clears it per run + (#2917) — a symlink retargeted between two ``extract()`` calls in one + ``graphify watch`` process is observed on the next run, not within a run. + + The returned ``Path`` is shared between callers. ``Path`` is immutable, so + that is safe, but callers must not rely on object identity meaning anything. + """ + key = str(path) + if not os.path.isabs(key): + # A relative input resolves against the cwd, and watch.py can chdir to + # recover a lost repo root — so the cwd has to be part of the identity. + key = os.path.join(os.getcwd(), key) + hit = _RESOLVE_CACHE.get(key) + if hit is None: + hit = path.resolve() if isinstance(path, Path) else Path(path).resolve() + _RESOLVE_CACHE[key] = hit + return hit + + +def clear_resolve_cache() -> None: + """Drop the memoized ``resolve()`` results. Called once per ``extract()`` run.""" + _RESOLVE_CACHE.clear() + def _atomic_replace(path: "str | Path", write_fn) -> None: """Atomically replace ``path`` with content written by ``write_fn(f)``. @@ -130,9 +172,19 @@ def write_json_atomic(path: "str | Path", obj, *, indent: "int | None" = None, e ) +@functools.lru_cache(maxsize=65536) def _is_test_path(path: str) -> bool: """Classify a source path as a test path (case-insensitive, segment-aware). + Cached because it is a pure function of the string — no filesystem access — + and `disambiguate_ambiguous_candidates` asks it once per candidate per + ambiguous call site. On a 2,500-file Go corpus that was 173k calls over a few + thousand distinct paths, ~2.2 s of a run, nearly all of it re-deriving the + same answer via a fresh `PurePosixPath` and up to nine regex matches. + + The bound matters: `watch` keeps one process alive across many extractions, + so an unbounded cache would grow with every path ever classified. + Shared by extract.py and symbol_resolution.py so cross-file call resolution treats test mocks/stubs identically. A path is a test path when: * any whole path segment equals a known test dir name @@ -166,6 +218,25 @@ def _is_test_path(path: str) -> bool: return False +@functools.lru_cache(maxsize=16384) +def _posix_parent_parts(norm: str) -> tuple[str, ...]: + """``PurePosixPath(norm).parent.parts``, memoized on the normalized string. + + ``_path_proximity_winner`` is called once per ambiguous bare-name call and + walks every candidate, so a corpus with a few thousand distinct source files + built tens of thousands of ``PurePosixPath`` objects to read one attribute — + pathlib's parser accounted for ~15% of extraction CPU in a profiled run + (#3008). The parse is a pure function of the string, so one construction per + distinct path serves every call. + + Deliberately still pathlib rather than ``str.rsplit``: ``.parent`` collapses + duplicate slashes, drops ``.`` components, yields ``"."`` for a bare + filename and ``"/"`` for a root child. Those cases feed the god-node guard, + where a wrong parent silently changes which candidate wins. + """ + return PurePosixPath(norm).parent.parts + + def _path_proximity_winner(call_site_file: str, candidate_files: dict[str, str]) -> str | None: """Pick the candidate whose source file is closest to the call site. @@ -183,19 +254,24 @@ def _path_proximity_winner(call_site_file: str, candidate_files: dict[str, str]) if not call_site_file: return None call_norm = str(call_site_file).replace("\\", "/") - call_dir = PurePosixPath(call_norm).parent + # All three tiers below need each candidate's separator-normalized path, and + # two of them need its parent. Normalize once per candidate instead of once + # per candidate per tier; dict order is preserved, so tier 3's scoring order + # is unchanged. + norm_items = [(cid, str(f).replace("\\", "/")) for cid, f in candidate_files.items()] # Tier 1: exact same file. - same_file = [cid for cid, f in candidate_files.items() - if str(f).replace("\\", "/") == call_norm] + same_file = [cid for cid, norm in norm_items if norm == call_norm] if len(same_file) == 1: return same_file[0] if len(same_file) > 1: return None # genuinely ambiguous within one file; bail - # Tier 2: same directory. - same_dir = [cid for cid, f in candidate_files.items() - if PurePosixPath(str(f).replace("\\", "/")).parent == call_dir] + # Tier 2: same directory. Two PurePosixPaths are equal exactly when their + # parts match, so comparing the memoized tuples is the same test. + call_parts = _posix_parent_parts(call_norm) + same_dir = [cid for cid, norm in norm_items + if _posix_parent_parts(norm) == call_parts] if len(same_dir) == 1: return same_dir[0] if len(same_dir) > 1: @@ -203,10 +279,9 @@ def _path_proximity_winner(call_site_file: str, candidate_files: dict[str, str]) # Tier 3: longest common path-prefix, computed over path segments. The # winner must be a strict unique maximum, else we bail (guard holds). - call_parts = call_dir.parts - def _common_prefix_len(f: str) -> int: - parts = PurePosixPath(str(f).replace("\\", "/")).parent.parts + def _common_prefix_len(norm: str) -> int: + parts = _posix_parent_parts(norm) n = 0 for a, b in zip(call_parts, parts): if a != b: @@ -215,7 +290,7 @@ def _common_prefix_len(f: str) -> int: return n scored = sorted( - ((cid, _common_prefix_len(f)) for cid, f in candidate_files.items()), + ((cid, _common_prefix_len(norm)) for cid, norm in norm_items), key=lambda kv: kv[1], reverse=True, )