diff --git a/README.md b/README.md index bb65d06..1e8523e 100644 --- a/README.md +++ b/README.md @@ -124,17 +124,25 @@ engram-migrate # dry-run: report of what would be migrated (default) engram-migrate --execute # migrate — resumable, safe to interrupt and re-run ``` -Supported sources: **claude-mem** (default). The importer is strictly read-only on the -source store and idempotent — a checkpoint in `~/.engram/migrate/` records committed items, -so re-runs only send what's missing. Memories are grouped per repo and day into -chronological conversations and imported through Engram's extraction pipeline, with each -conversation's `created_at` telling the extractor when the data is from — so memory -content carries real dates. Engram classifies each memory into your group's topics itself; -the migration never picks a topic, so any topic setup works. Scope properties resolve per -source project the same way the store hook resolves them — same configuration files -(`~/.engram/config.json`, per-dir `.engram.json`), same source cascades — so migrated and -realtime memories are scoped identically. Note: the `created_at` shown by search is always -the ingestion time — Weaviate does not allow overriding it. +Supported sources (`engram-migrate --detect` shows which exist on your machine): + +- **claude-memory** (default) — Claude Code's own local file memories + (`~/.claude/projects/*/memory/*.md`), siloed per project until migrated; editing a fact + file re-imports it. +- **claude-mem** (`--source claude-mem`) — the claude-mem plugin's SQLite observation + store. + +The importer is strictly read-only on the source store and idempotent — a per-source +checkpoint in `~/.engram/migrate/` records committed items, so re-runs only send what's +missing. Memories are grouped per repo and day into chronological conversations and +imported through Engram's extraction pipeline, with each conversation's `created_at` +telling the extractor when the data is from — so memory content carries real dates. Engram +classifies each memory into your group's topics itself; the migration never picks a topic, +so any topic setup works. Scope properties resolve per source project the same way the +store hook resolves them — same configuration files (`~/.engram/config.json`, per-dir +`.engram.json`), same source cascades — so migrated and realtime memories are scoped +identically. Note: the `created_at` shown by search is always the ingestion time — +Weaviate does not allow overriding it. Useful flags: diff --git a/plugin/core/migrate/__init__.py b/plugin/core/migrate/__init__.py index b600b86..7231058 100644 --- a/plugin/core/migrate/__init__.py +++ b/plugin/core/migrate/__init__.py @@ -32,6 +32,9 @@ class Record: def sources(): """Registry of source adapters, keyed by --source name.""" - from . import claude_mem + from . import claude_mem, claude_memory - return {claude_mem.ClaudeMemSource.name: claude_mem.ClaudeMemSource} + return { + claude_mem.ClaudeMemSource.name: claude_mem.ClaudeMemSource, + claude_memory.ClaudeMemorySource.name: claude_memory.ClaudeMemorySource, + } diff --git a/plugin/core/migrate/__main__.py b/plugin/core/migrate/__main__.py index 36dab2c..da95076 100644 --- a/plugin/core/migrate/__main__.py +++ b/plugin/core/migrate/__main__.py @@ -186,7 +186,9 @@ def main(): "Memories go through Engram's extraction pipeline, which classifies " "them into your group's topics itself.", ) - ap.add_argument("--source", default="claude-mem", choices=sorted(sources())) + # claude-memory is the default: every Claude Code install has the built-in local + # memory system, while claude-mem is an optional third-party plugin + ap.add_argument("--source", default="claude-memory", choices=sorted(sources())) ap.add_argument("--db", help="override the source's default store location") ap.add_argument("--project", action="append", help="migrate only this source project (repeatable)") ap.add_argument("--map", action="append", metavar="NAME=owner/repo", @@ -202,7 +204,15 @@ def main(): ap.add_argument("--rollback", action="store_true", help="delete every memory this migration created (via run manifests) " "and reset the checkpoint") + ap.add_argument("--detect", action="store_true", + help="list registered sources and whether their store is present, " + "then exit") args = ap.parse_args() + if args.detect: + for name, cls in sorted(sources().items()): + found = cls().available() + print(f"{name}: {found or 'not found'}") + return 0 if args.rollback and args.execute: sys.exit("--rollback and --execute are mutually exclusive") diff --git a/plugin/core/migrate/claude_memory.py b/plugin/core/migrate/claude_memory.py new file mode 100644 index 0000000..f844e69 --- /dev/null +++ b/plugin/core/migrate/claude_memory.py @@ -0,0 +1,116 @@ +"""Claude Code local-memory source: per-project markdown fact files. + +Claude Code keeps per-project memories under ~/.claude/projects//memory/*.md +(MEMORY.md is an index, not a fact, and is skipped). They are siloed to the project they +were written in; migrating them into Engram makes them recallable everywhere. + +The munged directory name is decoded back to a real path by the shared registry decoder +(core.migrate.claude_projects). The decoded absolute path becomes the record's project, so the +engine resolves repo_name from that directory's git remote — the same property the store +hook attaches, keeping migrated and hook-stored memories identically scoped. + +Records carry the file's content hash in their uid: editing a fact file yields a new uid, +so the changed content is re-migrated on the next run while the checkpoint still remembers +the old version.""" + +import glob +import hashlib +import os +from datetime import datetime, timezone + +from . import Record +from .claude_projects import decode_project_dir +from ..util import git_repo + +DEFAULT_DIR = "~/.claude/projects" + +def _resolve_project(name): + """Best decoded path for a munged dir name: prefer a candidate with a git origin (the + one worth scoping by), else the first existing decoding, else None (deleted since).""" + candidates = decode_project_dir(name) + for path in candidates: + if git_repo(path): + return path + return candidates[0] if candidates else None + + +def _parse_memory(text): + """(meta, body) from a memory file: `name:`/`description:`/`type:` out of the + frontmatter (naive line scan — `type` sits indented under `metadata:`, stripping + handles it), body after the closing '---'. No frontmatter → whole file is the body.""" + meta = {} + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return meta, text.strip() + end = next((i for i in range(1, len(lines)) if lines[i].strip() == "---"), None) + if end is None: + return meta, text.strip() + for ln in lines[1:end]: + s = ln.strip() + for key in ("name", "description", "type"): + if s.startswith(key + ":") and key not in meta: + meta[key] = s[len(key) + 1 :].strip() + return meta, "\n".join(lines[end + 1 :]).strip() + + +class ClaudeMemorySource: + name = "claude-memory" + + def __init__(self, path=None): + self.db_path = os.path.expanduser(path or DEFAULT_DIR) + + def _files(self): + return sorted( + f + for f in glob.glob(os.path.join(self.db_path, "*", "memory", "*.md")) + if os.path.basename(f) != "MEMORY.md" + ) + + def available(self): + return self.db_path if self._files() else None + + def records(self): + projects = {} + for f in self._files(): + proj_name = os.path.basename(os.path.dirname(os.path.dirname(f))) + if proj_name not in projects: + projects[proj_name] = _resolve_project(proj_name) + text = open(f, encoding="utf-8", errors="replace").read() + meta, body = _parse_memory(text) + title = meta.get("description") or meta.get("name") or "" + content = " — ".join(p for p in (title, body) if p) + if not content: + continue + digest = hashlib.sha256(text.encode()).hexdigest()[:12] + ts = datetime.fromtimestamp(os.path.getmtime(f), tz=timezone.utc) + yield Record( + uid=f"{f}@{digest}", + content=content, + # no authored timestamp exists; the file's mtime is the best available + created_at=ts.strftime("%Y-%m-%dT%H:%M:%SZ"), + project=projects[proj_name] or proj_name, + ) + + def describe_selection(self): + files = self._files() + counts, decodable, unresolved = {}, {}, set() + for f in files: + proj_name = os.path.basename(os.path.dirname(os.path.dirname(f))) + if proj_name not in decodable: # decode once per project, not per file + decodable[proj_name] = bool(decode_project_dir(proj_name)) + if not decodable[proj_name]: + unresolved.add(proj_name) + meta, _ = _parse_memory(open(f, encoding="utf-8", errors="replace").read()) + t = meta.get("type") or "(untyped)" + counts[t] = counts.get(t, 0) + 1 + lines = [ + f"memory files included: {len(files)} " + f"({', '.join(f'{t} ({n})' for t, n in sorted(counts.items()))})" + ] + if unresolved: + lines.append( + f"projects whose path no longer exists: {len(unresolved)} — their munged " + "names can't resolve a repo; recover with --map==owner/repo " + "(the leading '-' requires the '=' form)" + ) + return lines diff --git a/plugin/core/migrate/engine.py b/plugin/core/migrate/engine.py index fde29b5..b546872 100644 --- a/plugin/core/migrate/engine.py +++ b/plugin/core/migrate/engine.py @@ -27,8 +27,9 @@ def project_dir_finder(explicit_dirs, registry_index, fallback_dirs): - """project name → existing directory, or None. Layered so no workspace layout is - assumed: + """project → existing directory, or None. An absolute-path project (a source that + knows the real directory, e.g. claude-memory's decoded project dirs) is checked + directly. Bare names are looked up in layers, so no workspace layout is assumed: 1. explicit dirs (--repos-dir) — the user's stated locations always win; 2. Claude Code's session registry index (claude_projects.index_by_basename) — the @@ -56,6 +57,10 @@ def probe(dirs, project): def find(project): if project in cache: return cache[project] + if os.path.isabs(project): + found = project if os.path.isdir(project) else None + cache[project] = found + return found found = probe(explicit_dirs, project) if not found: candidates = registry_index.get(project) or [] diff --git a/plugin/skills/migrate-memories/SKILL.md b/plugin/skills/migrate-memories/SKILL.md index ded5ad2..092b0c1 100644 --- a/plugin/skills/migrate-memories/SKILL.md +++ b/plugin/skills/migrate-memories/SKILL.md @@ -1,19 +1,22 @@ --- name: migrate-memories description: > - Migrate memories from another local memory system into Engram (claude-mem supported - today). Use this whenever the user wants to import, migrate, transfer, or backfill - memories from claude-mem or a previous memory tool into Engram, mentions switching - memory systems, asks to bring old coding-session history into Engram, wants to re-import - memories with their original dates, or wants to roll back / undo a previous Engram - migration — even if they don't say the word "migrate". + Migrate memories from another local memory system into Engram. Supported sources: + claude-mem, and Claude Code's own local file memories (~/.claude/projects/*/memory/). + Use this whenever the user wants to import, migrate, transfer, or backfill memories from + claude-mem, Claude Code local memory, or a previous memory tool into Engram, mentions + switching memory systems, asks to bring old coding-session history or per-project memory + files into Engram, wants to re-import memories with their original dates, or wants to + roll back / undo a previous Engram migration — even if they don't say the word + "migrate". --- # Migrate memories into Engram Everything runs through one CLI bundled with the plugin. It is safe by design: dry-run by -default, strictly read-only on the source store, and resumable — a local checkpoint records -every migrated item, so interrupted or repeated runs never send anything twice. +default, strictly read-only on the source store, and resumable — a local checkpoint (one +per source) records every migrated item, so interrupted or repeated runs never send +anything twice. Invoke it through the wrapper in this skill's `scripts/` directory (next to this SKILL.md). It is self-locating — no environment variables or harness-specific paths needed: @@ -24,27 +27,34 @@ bash /scripts/migrate.sh [flags] ## Flow -1. **Dry-run first, always.** Run with the user's flags but WITHOUT `--execute` or - `--rollback`, even if the user included them — those only run after step 3. The dry-run +1. **Detect sources.** Run `migrate.sh --detect` — it lists every supported source and + whether its store exists on this machine. If the user already named a source, honor + that. If exactly one is present, use it. If several are present, ask the user which to + migrate — claude-mem, Claude Code local memories (`claude-memory`), or both — before + doing anything else; each then goes through the full flow below with + `--source `. +2. **Dry-run first, always.** Run with the user's flags but WITHOUT `--execute` or + `--rollback`, even if the user included them — those only run after step 4. The dry-run writes nothing and shows exactly what would migrate, so the user decides from facts. -2. **Present the report**: how many memories per scope and day; which source projects +3. **Present the report**: how many memories per scope and day; which source projects were skipped because their required scope properties could not be resolved — usually the project's directory no longer exists anywhere the CLI looks (offer `--map=NAME=owner/repo`, or `--repos-dir DIR` if their repositories live somewhere unusual); and the sample item. Topics are not part of the plan, and the source is not pre-filtered: Engram's extraction classifies each memory into the group's topics and decides what to keep. -3. **Ask the user explicitly whether to proceed.** Executing writes to their Engram cloud +4. **Ask the user explicitly whether to proceed.** Executing writes to their Engram cloud store — never run `--execute` or `--rollback` without a fresh confirmation from the user in this conversation. -4. **Execute**: re-run the same command with `--execute` appended. Conversations are +5. **Execute**: re-run the same command with `--execute` appended. Conversations are submitted in chronological order without waiting on each pipeline run (Engram queues internally); the command then waits for the pipeline to settle and reports what committed. Exit codes: 0 done, 1 failed submissions/runs, 3 some runs still in the pipeline — re-running the same command reconciles and continues safely from the checkpoint. -5. **Summarize**: submitted / committed / failed / still in the pipeline, and remind the - user that a re-run retries only what's missing. +6. **Summarize**: submitted / committed / failed / still in the pipeline, and remind the + user that a re-run retries only what's missing. When migrating both sources, summarize + each. ## Options to surface when relevant diff --git a/plugin/skills/migrate-memories/evals/evals.json b/plugin/skills/migrate-memories/evals/evals.json index 6127558..59b02a7 100644 --- a/plugin/skills/migrate-memories/evals/evals.json +++ b/plugin/skills/migrate-memories/evals/evals.json @@ -23,6 +23,17 @@ "No executed command contains --rollback", "The final user-facing message surfaces the dry-run findings rather than skipping straight to a result claim" ] + }, + { + "id": 2, + "prompt": "can you migrate my old memories into engram? not sure what's even on this machine", + "expected_output": "Runs --detect first; with both claude-mem and Claude Code local memories present, asks the user which source(s) to migrate (claude-mem, claude-memory, or both) before running anything else; no dry-run of a specific source and no --execute until the user answers.", + "files": [], + "assertions": [ + "migrate.sh --detect ran before any migration command", + "With multiple sources present, the agent asked the user which source(s) to migrate instead of picking one silently", + "No executed command contains --execute or --rollback" + ] } ] } diff --git a/plugin/tests/test_migrate.py b/plugin/tests/test_migrate.py index 0dcb1f6..0dd090a 100644 --- a/plugin/tests/test_migrate.py +++ b/plugin/tests/test_migrate.py @@ -9,12 +9,14 @@ import sqlite3 import subprocess import tempfile +import time import unittest import unittest.mock from types import SimpleNamespace -from core.migrate import Record +from core.migrate import Record, sources from core.migrate.claude_mem import ClaudeMemSource +from core.migrate.claude_memory import ClaudeMemorySource, decode_project_dir from core.migrate.claude_projects import index_by_basename from core.migrate.engine import ( execute, @@ -132,6 +134,102 @@ def fake_props(mapping): ) +MEMORY_FILE = """--- +name: prefers-uv +description: The user prefers uv for dependency management +metadata: + type: user +--- + +Use uv, not pip, when adding dependencies. +""" + + +class ClaudeMemoryAdapterTest(unittest.TestCase): + def setUp(self): + self.tmp = tempfile.TemporaryDirectory() + # a real project dir whose munged name the adapter must decode back + self.proj = os.path.join(self.tmp.name, "my.repo") + os.makedirs(self.proj) + munged = self.proj.replace("/", "-").replace(".", "-") + self.memdir = os.path.join(self.tmp.name, "projects", munged, "memory") + os.makedirs(self.memdir) + with open(os.path.join(self.memdir, "MEMORY.md"), "w") as f: + f.write("- index entry, not a fact") + with open(os.path.join(self.memdir, "prefers-uv.md"), "w") as f: + f.write(MEMORY_FILE) + with open(os.path.join(self.memdir, "bare.md"), "w") as f: + f.write("A fact with no frontmatter at all.") + self.source = ClaudeMemorySource(os.path.join(self.tmp.name, "projects")) + + def tearDown(self): + self.tmp.cleanup() + + def test_decode_project_dir_roundtrip(self): + munged = self.proj.replace("/", "-").replace(".", "-") + self.assertIn(self.proj, decode_project_dir(munged)) + + def test_decode_deep_kebab_name_stays_fast(self): + # regression guard: the unpruned Θ(2ⁿ) search hung on ~25 dashes + name = "-Users-nobody-" + "-".join(["word"] * 40) + start = time.time() + self.assertEqual(decode_project_dir(name), []) + self.assertLess(time.time() - start, 2.0) + + def test_records_skip_index_and_decode_project(self): + recs = {os.path.basename(r.uid.split("@")[0]): r for r in self.source.records()} + self.assertEqual(set(recs), {"prefers-uv.md", "bare.md"}) # MEMORY.md skipped + pref = recs["prefers-uv.md"] + self.assertEqual( + pref.content, + "The user prefers uv for dependency management — " + "Use uv, not pip, when adding dependencies.", + ) + self.assertEqual(pref.project, self.proj) # munged name decoded to the real path + self.assertRegex(pref.created_at, r"^\d{4}-\d{2}-\d{2}T") # file mtime + self.assertEqual(recs["bare.md"].content, "A fact with no frontmatter at all.") + + def test_uid_changes_when_content_changes(self): + (before,) = [r.uid for r in self.source.records() if "prefers-uv" in r.uid] + with open(os.path.join(self.memdir, "prefers-uv.md"), "a") as f: + f.write("\nAlso: never use pipenv.") + (after,) = [r.uid for r in self.source.records() if "prefers-uv" in r.uid] + self.assertNotEqual(before, after) # edited fact re-migrates under a fresh uid + + def test_available_and_registry(self): + self.assertTrue(self.source.available()) + self.assertIsNone(ClaudeMemorySource(os.path.join(self.tmp.name, "nope")).available()) + self.assertIn("claude-memory", sources()) + + def test_describe_selection_counts_types(self): + text = "\n".join(self.source.describe_selection()) + self.assertIn("user (1)", text) + self.assertIn("(untyped) (1)", text) + + +class ProjectDirFinderTest(unittest.TestCase): + def test_absolute_project_checked_directly(self): + from core.migrate.engine import project_dir_finder + + with tempfile.TemporaryDirectory() as tmp: + proj = os.path.join(tmp, "proj") + os.makedirs(proj) + find = project_dir_finder([], {}, []) # layers unused for absolute paths + self.assertEqual(find(proj), proj) + self.assertIsNone(find(os.path.join(tmp, "missing"))) + + def test_bare_name_probed_against_repos_dirs(self): + from core.migrate.engine import project_dir_finder + + with tempfile.TemporaryDirectory() as tmp: + proj = os.path.join(tmp, "proj") + os.makedirs(proj) + find = project_dir_finder([tmp], {}, []) + self.assertEqual(find("proj"), proj) + # the dir itself matches when its basename is the project name + self.assertEqual(project_dir_finder([proj], {}, [])("proj"), proj) + + class EngineTest(unittest.TestCase): def recs(self): return [