diff --git a/CLAUDE.md b/CLAUDE.md index 01d7bb37a4..43d0796f94 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -442,6 +442,7 @@ The manual-commit strategy (`manual_commit*.go`) does not modify the active bran - **Shadow branch migration** - if user does stash/pull/rebase (HEAD changes without commit), shadow branch is automatically moved to new base commit - **Orphaned branch cleanup** - if a shadow branch exists without a corresponding session state file, it is automatically reset when a new session starts - PrePush hook can push `entire/checkpoints/v1` branch alongside user pushes +- **OPF (OpenAI Privacy Filter) runs at pre-push, not post-commit**: when `redaction.openai_privacy_filter.enabled` is true, the PrePush hook re-redacts unpushed `entire/checkpoints/v1` commits with the OPF 8th layer, builds new commits carrying an `Entire-OPF-Applied: true` trailer, and atomically updates the local v1 ref before pushing. Per-commit condensation stays on the fast 7-layer pipeline. See `strategy/manual_commit_opf_rewrite.go` and `docs/security-and-privacy.md` for the full flow, including divergence detection, bootstrap caps, and CAS-on-conflict semantics. - Safe to use on main/master since it never modifies commit history #### Key Files @@ -450,6 +451,7 @@ The manual-commit strategy (`manual_commit*.go`) does not modify the active bran - `common.go` - Helpers for metadata extraction, tree building, rewind validation, `ListCheckpoints()` - `session.go` - Session/checkpoint data structures - `push_common.go` - PrePush logic for pushing `entire/checkpoints/v1` branch +- `manual_commit_opf_rewrite.go` - Pre-push OPF re-redaction: walks unpushed v1 commits, runs OPF over their blobs, rebuilds commits with `Entire-OPF-Applied: true` trailer, CAS-updates the local ref. Sentinel error types (use `errors.As`): `V1DivergedError`, `BootstrapTooLargeError`, `V1RefMovedError`, `OPFRuntimeFailedError`. - `manual_commit.go` - Manual-commit strategy main implementation - `manual_commit_types.go` - Type definitions: `SessionState`, `CheckpointInfo`, `CondenseResult` - `manual_commit_session.go` - Session state management (load/save/list session states) diff --git a/cmd/entire/cli/checkpoint/checkpoint.go b/cmd/entire/cli/checkpoint/checkpoint.go index 4d9d007cca..0e1b59cb1a 100644 --- a/cmd/entire/cli/checkpoint/checkpoint.go +++ b/cmd/entire/cli/checkpoint/checkpoint.go @@ -222,22 +222,11 @@ type WriteCommittedOptions struct { // Must be pre-redacted (via redact.JSONLBytes or redact.AlreadyRedacted for trusted sources). Transcript redact.RedactedBytes - // Prompts contains the raw user prompts from the session. These are NOT - // guaranteed to be redacted on entry — the writer always emits the typed - // PromptsRedacted blob below (running the safety-net pipeline if it is - // the zero value). Do not read Prompts independently for persistence; go - // through redactJoinedPrompts so the redaction guarantee is preserved. + // Prompts contains the raw user prompts from the session. Run through + // redactedJoinedPrompts before persisting — the writer does this + // inside writeSessionToSubdirectory. Prompts []string - // PromptsRedacted, when set, is the pre-redacted joined-prompts blob the - // writer uses verbatim instead of re-running the safety-net pipeline. - // Used by finalizeAllTurnCheckpoints to avoid running the OpenAI - // Privacy Filter once per checkpoint over identical joined-prompt - // strings. The typed wrapper makes the "this content was produced by - // the redaction pipeline" claim a compile-time invariant — callers - // cannot assign an arbitrary string. - PromptsRedacted redact.RedactedJoinedPrompts - // FilesTouched are files modified during the session FilesTouched []string @@ -366,16 +355,10 @@ type UpdateCommittedOptions struct { // Must be pre-redacted (via redact.JSONLBytes or redact.AlreadyRedacted for trusted sources). Transcript redact.RedactedBytes - // Prompts contains the raw user prompts (replaces existing). NOT - // guaranteed to be redacted on entry — see WriteCommittedOptions.Prompts - // for the relationship to PromptsRedacted. + // Prompts contains the raw user prompts (replaces existing). + // See WriteCommittedOptions.Prompts. Prompts []string - // PromptsRedacted, when set, is the pre-redacted joined-prompts blob - // the writer uses verbatim instead of re-running the safety-net - // pipeline. See WriteCommittedOptions.PromptsRedacted for rationale. - PromptsRedacted redact.RedactedJoinedPrompts - // Agent identifies the agent type (needed for transcript chunking) Agent types.AgentType diff --git a/cmd/entire/cli/checkpoint/checkpoint_test.go b/cmd/entire/cli/checkpoint/checkpoint_test.go index b40e6a404b..329084e4bc 100644 --- a/cmd/entire/cli/checkpoint/checkpoint_test.go +++ b/cmd/entire/cli/checkpoint/checkpoint_test.go @@ -4362,3 +4362,42 @@ func TestCheckpointSummary_HasReview(t *testing.T) { t.Errorf(`expected zero-value summary to omit "has_review" key, got %s`, string(bZero)) } } + +// TestRedactBlobBytes_JSONMetadata pins the .json branch of RedactBlobBytes: +// checkpoint metadata files (metadata.json) carry free-form fields like +// Summary.Intent and ReviewPrompt that previously bypassed redaction because +// the dispatcher only matched .jsonl. The PR 1236 fix extended the JSON-aware +// branch to .json. We assert via a low-entropy AWS-key shaped secret (catches +// the 7-layer pipeline) so the test stays deterministic without the OPF binary. +func TestRedactBlobBytes_JSONMetadata(t *testing.T) { + t.Parallel() + + meta := CommittedMetadata{ + Kind: "agent_review", + ReviewPrompt: "credential leak: key=AKIAYRWQG5EJLPZLBYNP", + Summary: &Summary{ + Intent: "leak: key=AKIAYRWQG5EJLPZLBYNP", + }, + } + b, err := json.Marshal(meta) + if err != nil { + t.Fatalf("marshal: %v", err) + } + + got := RedactBlobBytes(context.Background(), b, "metadata.json", false) + if strings.Contains(string(got), "AKIAYRWQG5EJLPZLBYNP") { + t.Errorf("expected AWS key redacted in metadata.json blob, got %s", string(got)) + } + if !strings.Contains(string(got), "REDACTED") { + t.Errorf("expected REDACTED placeholder in metadata.json blob, got %s", string(got)) + } + // JSON structure must survive — Kind is not redactable content, so it + // should round-trip through the JSON-aware redactor. + var roundTripped map[string]any + if err := json.Unmarshal(got, &roundTripped); err != nil { + t.Errorf("redacted .json blob must remain valid JSON, got parse err %v (content: %s)", err, string(got)) + } + if roundTripped["kind"] != "agent_review" { + t.Errorf(`expected "kind":"agent_review" preserved after redaction, got %v`, roundTripped["kind"]) + } +} diff --git a/cmd/entire/cli/checkpoint/committed.go b/cmd/entire/cli/checkpoint/committed.go index 0c91fa91e8..414812bf78 100644 --- a/cmd/entire/cli/checkpoint/committed.go +++ b/cmd/entire/cli/checkpoint/committed.go @@ -417,12 +417,10 @@ func (s *GitStore) writeSessionToSubdirectory(ctx context.Context, opts WriteCom filePaths.ContentHash = "/" + sessionPath + paths.ContentHashFileName } - // Write prompts. Uses the full 8-layer pipeline (including OPF) via - // redactedJoinedPrompts; the helper unwraps opts.PromptsRedacted when - // set so callers (finalizeAllTurnCheckpoints) that pre-redact once - // across multiple checkpoint writes don't pay OPF per checkpoint. + // Write prompts via the 7-layer pipeline. OPF runs only in the + // pre-push rewrite path (manual_commit_opf_rewrite.go). if len(opts.Prompts) > 0 { - promptContent := redactedJoinedPrompts(ctx, opts.Prompts, opts.PromptsRedacted) + promptContent := redactedJoinedPrompts(opts.Prompts) blobHash, err := CreateBlobFromContent(s.repo, []byte(promptContent)) if err != nil { return filePaths, err @@ -1403,10 +1401,9 @@ func (s *GitStore) UpdateCommitted(ctx context.Context, opts UpdateCommittedOpti } } - // Replace prompts (apply redaction as safety net; unwraps - // opts.PromptsRedacted when set). + // Replace prompts with 7-layer-redacted content. if len(opts.Prompts) > 0 { - promptContent := redactedJoinedPrompts(ctx, opts.Prompts, opts.PromptsRedacted) + promptContent := redactedJoinedPrompts(opts.Prompts) blobHash, err := CreateBlobFromContent(s.repo, []byte(promptContent)) if err != nil { return fmt.Errorf("failed to create prompt blob: %w", err) @@ -1725,11 +1722,14 @@ func (s *GitStore) copyMetadataDir(ctx context.Context, metadataDir, basePath st return fmt.Errorf("path traversal detected: %s", relPath) } - // Create blob from file with secrets redaction - // Committed-checkpoint write — run the full 8-layer pipeline - // including OPF. The per-turn temp-write path stays on plain - // redactors via the sibling createRedactedBlobFromFile. - blobHash, mode, err := createRedactedBlobFromFileWithPrivacyFilter(ctx, s.repo, path, relPath) + // Create blob from file with 7-layer secrets redaction. + // Post-commit emits 7-layer-only blobs; the pre-push rewrite + // (strategy/manual_commit_opf_rewrite.go) walks the resulting + // tree, re-redacts these blobs with OPF when enabled, and + // rewrites entire/checkpoints/v1 into 8-layer commits before + // they leave the local machine. + _ = ctx // ctx not needed by the 7-layer path; kept on caller signature for future use + blobHash, mode, err := createRedactedBlobFromFile(s.repo, path, relPath) if err != nil { return fmt.Errorf("failed to create blob for %s: %w", path, err) } @@ -1751,22 +1751,13 @@ func (s *GitStore) copyMetadataDir(ctx context.Context, metadataDir, basePath st } // createRedactedBlobFromFile reads a file, applies the 7-layer redaction -// pipeline, and creates a git blob. Used by per-turn temporary-checkpoint -// writes — the OpenAI Privacy Filter is intentionally NOT run here to -// keep per-turn latency inside the agent loop's budget. +// pipeline, and creates a git blob. Used by committed-checkpoint writes +// at post-commit time. The OpenAI Privacy Filter is intentionally NOT +// run here — OPF lives in the pre-push rewrite path +// (strategy/manual_commit_opf_rewrite.go), which re-redacts the 7-layer +// blobs into 8-layer commits before they leave the local machine. // JSONL files get JSONL-aware redaction; all other files get plain byte redaction. func createRedactedBlobFromFile(repo *git.Repository, filePath, treePath string) (plumbing.Hash, filemode.FileMode, error) { - return createRedactedBlobFromFileImpl(context.Background(), repo, filePath, treePath, false) -} - -// createRedactedBlobFromFileWithPrivacyFilter reads a file, applies the full -// 8-layer pipeline (including the OpenAI Privacy Filter), and creates a git -// blob. Used by committed-checkpoint writes — slower but more thorough. -func createRedactedBlobFromFileWithPrivacyFilter(ctx context.Context, repo *git.Repository, filePath, treePath string) (plumbing.Hash, filemode.FileMode, error) { - return createRedactedBlobFromFileImpl(ctx, repo, filePath, treePath, true) -} - -func createRedactedBlobFromFileImpl(ctx context.Context, repo *git.Repository, filePath, treePath string, usePrivacyFilter bool) (plumbing.Hash, filemode.FileMode, error) { info, err := os.Stat(filePath) if err != nil { return plumbing.ZeroHash, 0, fmt.Errorf("failed to stat file: %w", err) @@ -1793,7 +1784,7 @@ func createRedactedBlobFromFileImpl(ctx context.Context, repo *git.Repository, f return hash, mode, nil } - content = redactBytesForBlob(ctx, content, treePath, usePrivacyFilter) + content = RedactBlobBytes(context.Background(), content, treePath, false) hash, err := CreateBlobFromContent(repo, content) if err != nil { @@ -1802,15 +1793,24 @@ func createRedactedBlobFromFileImpl(ctx context.Context, repo *git.Repository, f return hash, mode, nil } -// redactBytesForBlob applies the appropriate redaction pipeline to file -// content for a checkpoint blob. JSONL files get JSONL-aware redaction -// (falling back to plain byte redaction on parse failure so the regex -// layers still apply); other files get plain byte redaction. -// usePrivacyFilter selects the lighter 7-layer pipeline (per-turn temp -// writes) versus the full 8-layer pipeline including OPF (committed -// writes). -func redactBytesForBlob(ctx context.Context, content []byte, treePath string, usePrivacyFilter bool) []byte { - if strings.HasSuffix(treePath, ".jsonl") { +// RedactBlobBytes redacts a single blob's content given its tree path. +// JSON-shaped files (.jsonl or .json) get JSON-aware redaction (falling +// back to plain bytes on parse failure so regex/credential layers +// still apply); other files get plain byte redaction. When +// usePrivacyFilter is true the full 8-layer pipeline (including OPF) +// runs; otherwise the 7-layer pipeline. +// +// .json is handled alongside .jsonl because checkpoint metadata files +// (metadata.json, per-session metadata.json) carry free-form fields +// like Summary.Intent / Summary.Outcome / ReviewPrompt that can +// contain PII the regex layers miss. The JSON-aware redactor extracts +// string leaves and applies OPF only to those, preserving the JSON +// structure. +// +// Post-commit condensation uses false (fast path). The pre-push rewrite +// (strategy/manual_commit_opf_rewrite.go) uses true. +func RedactBlobBytes(ctx context.Context, content []byte, treePath string, usePrivacyFilter bool) []byte { + if strings.HasSuffix(treePath, ".jsonl") || strings.HasSuffix(treePath, ".json") { var ( redacted redact.RedactedBytes err error @@ -1823,8 +1823,7 @@ func redactBytesForBlob(ctx context.Context, content []byte, treePath string, us if err == nil { return redacted.Bytes() } - // JSONL parse failed — fall through so regex/credential layers - // still apply via the plain byte path. + // JSONL parse failed — fall through to plain bytes. } if usePrivacyFilter { return redact.BytesWithPrivacyFilter(ctx, content) diff --git a/cmd/entire/cli/checkpoint/committed_opf_trailer_test.go b/cmd/entire/cli/checkpoint/committed_opf_trailer_test.go new file mode 100644 index 0000000000..b731455374 --- /dev/null +++ b/cmd/entire/cli/checkpoint/committed_opf_trailer_test.go @@ -0,0 +1,70 @@ +package checkpoint + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" + "github.com/entireio/cli/cmd/entire/cli/testutil" + "github.com/entireio/cli/cmd/entire/cli/trailers" + "github.com/entireio/cli/redact" + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/stretchr/testify/require" +) + +// TestWriteCommitted_DoesNotEmitOPFAppliedTrailer is the regression guard +// for the architectural promise: standard post-commit condensation writes +// 7-layer-only blobs and MUST NOT mark them with the Entire-OPF-Applied +// trailer. The trailer is emitted exclusively by the pre-push rewrite +// path; if a future change accidentally added it to the standard writer, +// the pre-push rewrite would skip those commits (HasOPFApplied true → +// reparent-only, no actual OPF run) and ship 7-layer content as if it +// were 8-layer. This test pins down that contract. +func TestWriteCommitted_DoesNotEmitOPFAppliedTrailer(t *testing.T) { + t.Parallel() + + tempDir := t.TempDir() + testutil.InitRepo(t, tempDir) + repo, err := git.PlainOpen(tempDir) + require.NoError(t, err) + + wt, err := repo.Worktree() + require.NoError(t, err) + readmeFile := filepath.Join(tempDir, "README.md") + require.NoError(t, os.WriteFile(readmeFile, []byte("# Test"), 0o644)) + _, err = wt.Add("README.md") + require.NoError(t, err) + _, err = wt.Commit("Initial commit", &git.CommitOptions{ + Author: &object.Signature{Name: "Test", Email: "test@test.com"}, + }) + require.NoError(t, err) + + store := NewGitStore(repo) + cpID := id.MustCheckpointID("a1b2c3d4e5f6") + + err = store.WriteCommitted(context.Background(), WriteCommittedOptions{ + CheckpointID: cpID, + SessionID: "regression-no-opf-trailer", + Strategy: "manual-commit", + Transcript: redact.AlreadyRedacted([]byte(`{"role":"user","content":"hello"}` + "\n")), + AuthorName: "Test", + AuthorEmail: "test@test.com", + }) + require.NoError(t, err) + + // Read the latest commit message on entire/checkpoints/v1 and assert + // HasOPFApplied is false. We resolve via the ref then walk back the + // single commit the writer just produced. + ref, err := repo.Reference(plumbing.NewBranchReferenceName("entire/checkpoints/v1"), true) + require.NoError(t, err, "writer should have created entire/checkpoints/v1") + commit, err := repo.CommitObject(ref.Hash()) + require.NoError(t, err) + + if trailers.HasOPFApplied(commit.Message) { + t.Errorf("standard WriteCommitted emitted Entire-OPF-Applied trailer; commit message:\n%s", commit.Message) + } +} diff --git a/cmd/entire/cli/checkpoint/prompts.go b/cmd/entire/cli/checkpoint/prompts.go index 83112ff2ce..6df096ce3f 100644 --- a/cmd/entire/cli/checkpoint/prompts.go +++ b/cmd/entire/cli/checkpoint/prompts.go @@ -1,7 +1,6 @@ package checkpoint import ( - "context" "strings" "github.com/entireio/cli/redact" @@ -29,16 +28,9 @@ func SplitPromptContent(content string) []string { return prompts } -// redactedJoinedPrompts returns the redacted prompt-blob content for the -// supplied prompts. When preRedacted is set it is unwrapped verbatim; -// otherwise the prompts are joined and run through the full 8-layer -// pipeline as a safety net. Callers that share the same prompts across -// multiple checkpoint writes (finalizeAllTurnCheckpoints) should compute -// the redacted blob once via redact.JoinedPrompts and pass it through to -// avoid running OPF repeatedly over identical input. -func redactedJoinedPrompts(ctx context.Context, prompts []string, preRedacted redact.RedactedJoinedPrompts) string { - if preRedacted.IsSet() { - return preRedacted.String() - } - return redact.JoinedPrompts(ctx, prompts, PromptSeparator).String() +// redactedJoinedPrompts joins prompts and runs the 7-layer redaction +// pipeline. OPF runs exclusively in the pre-push rewrite (not here), +// so the writer's hot path stays predictable. +func redactedJoinedPrompts(prompts []string) string { + return redact.String(strings.Join(prompts, PromptSeparator)) } diff --git a/cmd/entire/cli/checkpoint/prompts_test.go b/cmd/entire/cli/checkpoint/prompts_test.go index e4f6f4db91..93f4f31792 100644 --- a/cmd/entire/cli/checkpoint/prompts_test.go +++ b/cmd/entire/cli/checkpoint/prompts_test.go @@ -1,10 +1,8 @@ package checkpoint import ( - "context" "testing" - "github.com/entireio/cli/redact" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -16,7 +14,6 @@ func TestJoinAndSplitPrompts_RoundTrip(t *testing.T) { "first line\nwith newline", "second prompt", } - joined := JoinPrompts(original) split := SplitPromptContent(joined) @@ -26,34 +23,15 @@ func TestJoinAndSplitPrompts_RoundTrip(t *testing.T) { func TestSplitPromptContent_EmptyContent(t *testing.T) { t.Parallel() - assert.Nil(t, SplitPromptContent("")) } -// TestRedactedJoinedPrompts_PreRedactedIsTrustedVerbatim verifies that when -// the caller supplies a set RedactedJoinedPrompts the helper unwraps it -// untouched and never re-invokes the redaction pipeline. The pre-redacted -// path is what finalizeAllTurnCheckpoints relies on to avoid running OPF -// once per checkpoint over identical joined-prompt strings. -func TestRedactedJoinedPrompts_PreRedactedIsTrustedVerbatim(t *testing.T) { +// TestRedactedJoinedPrompts_AppliesSafetyNet verifies the helper joins +// prompts with the canonical separator and runs them through the 7-layer +// pipeline. OPF runs only in the pre-push rewrite path, never here. +func TestRedactedJoinedPrompts_AppliesSafetyNet(t *testing.T) { t.Parallel() - - const preRedacted = "[REDACTED_PERSON] asked about [REDACTED_EMAIL]" - got := redactedJoinedPrompts( - context.Background(), - []string{"raw prompt text"}, - redact.AlreadyRedactedJoinedPrompts(preRedacted), - ) - assert.Equal(t, preRedacted, got, "preRedacted should pass through verbatim") -} - -// TestRedactedJoinedPrompts_ZeroValueFallsBackToRedaction verifies that -// when the typed preRedacted is the zero value the helper joins the -// prompts and runs the full pipeline as a safety net. -func TestRedactedJoinedPrompts_ZeroValueFallsBackToRedaction(t *testing.T) { - t.Parallel() - - got := redactedJoinedPrompts(context.Background(), []string{"hello", "world"}, redact.RedactedJoinedPrompts{}) - assert.NotEmpty(t, got, "zero-value preRedacted should fall back to running the redaction pipeline") - assert.Contains(t, got, PromptSeparator, "fallback output should preserve the prompt separator") + got := redactedJoinedPrompts([]string{"hello", "world"}) + assert.NotEmpty(t, got) + assert.Contains(t, got, PromptSeparator) } diff --git a/cmd/entire/cli/checkpoint/v2_committed.go b/cmd/entire/cli/checkpoint/v2_committed.go index 7f04471168..b68f13d373 100644 --- a/cmd/entire/cli/checkpoint/v2_committed.go +++ b/cmd/entire/cli/checkpoint/v2_committed.go @@ -609,7 +609,7 @@ func (s *V2GitStore) updateCommittedMain(ctx context.Context, opts UpdateCommitt sessionPath := fmt.Sprintf("%s%d/", basePath, sessionIndex) if len(opts.Prompts) > 0 { - promptContent := redactedJoinedPrompts(ctx, opts.Prompts, opts.PromptsRedacted) + promptContent := redactedJoinedPrompts(opts.Prompts) blobHash, err := CreateBlobFromContent(s.repo, []byte(promptContent)) if err != nil { return 0, fmt.Errorf("failed to create prompt blob: %w", err) @@ -891,7 +891,7 @@ func (s *V2GitStore) writeMainCheckpointEntries(ctx context.Context, opts WriteC // and compact transcript to a session subdirectory (0/, 1/, 2/, … indexed by // session order within the checkpoint). The raw transcript (raw_transcript) and its // content hash (raw_transcript_hash.txt) go to /full/current, not here. -func (s *V2GitStore) writeMainSessionToSubdirectory(ctx context.Context, opts WriteCommittedOptions, sessionPath string, entries map[string]object.TreeEntry) (SessionFilePaths, error) { +func (s *V2GitStore) writeMainSessionToSubdirectory(_ context.Context, opts WriteCommittedOptions, sessionPath string, entries map[string]object.TreeEntry) (SessionFilePaths, error) { filePaths := SessionFilePaths{} // Clear existing entries at this session path @@ -903,7 +903,7 @@ func (s *V2GitStore) writeMainSessionToSubdirectory(ctx context.Context, opts Wr // Write prompts if len(opts.Prompts) > 0 { - promptContent := redactedJoinedPrompts(ctx, opts.Prompts, opts.PromptsRedacted) + promptContent := redactedJoinedPrompts(opts.Prompts) blobHash, err := CreateBlobFromContent(s.repo, []byte(promptContent)) if err != nil { return filePaths, err diff --git a/cmd/entire/cli/hooks_git_cmd.go b/cmd/entire/cli/hooks_git_cmd.go index 27ef45f281..f8729b94c3 100644 --- a/cmd/entire/cli/hooks_git_cmd.go +++ b/cmd/entire/cli/hooks_git_cmd.go @@ -2,6 +2,7 @@ package cli import ( "context" + "fmt" "log/slog" "time" @@ -231,6 +232,13 @@ func newHooksGitPrePushCmd() *cobra.Command { Use: "pre-push ", Short: "Handle pre-push git hook", Args: cobra.ExactArgs(1), + // SilenceUsage/Errors so non-zero exits from privacy-critical + // failures (OPF rewrite errors) print only the error message, + // not cobra's usage banner. The error message itself already + // includes user guidance (see ErrV1Diverged / ErrBootstrapTooLarge / + // ErrV1RefMoved in strategy/manual_commit_opf_rewrite.go). + SilenceUsage: true, + SilenceErrors: false, RunE: func(cmd *cobra.Command, args []string) error { if gitHooksDisabled { return nil @@ -245,7 +253,20 @@ func newHooksGitPrePushCmd() *cobra.Command { hookErr := g.strategy.PrePush(g.ctx, remote) g.logCompleted(hookErr) - return nil + // Propagate the error so the hook script exits non-zero and + // git push aborts the entire batch. PrePush itself only + // returns errors for privacy-critical failures (OPF rewrite — + // e.g., V1DivergedError, BootstrapTooLargeError, + // V1RefMovedError, OPFRuntimeFailedError); transient + // checkpoint-push failures are logged and swallowed before + // reaching this point. See strategy/manual_commit_push.go + // for the contract. We wrap with a short "pre-push:" prefix + // so the user sees the source of the abort without losing + // the underlying type (errors.As still finds the sentinels). + if hookErr == nil { + return nil + } + return fmt.Errorf("pre-push: %w", hookErr) }, } } diff --git a/cmd/entire/cli/settings/settings.go b/cmd/entire/cli/settings/settings.go index 11ffd3de0c..8a4b28631a 100644 --- a/cmd/entire/cli/settings/settings.go +++ b/cmd/entire/cli/settings/settings.go @@ -220,8 +220,22 @@ type OPFSettings struct { Categories map[string]bool `json:"categories,omitempty"` Command string `json:"command,omitempty"` TimeoutSeconds int `json:"timeout_seconds,omitempty"` + + // PromptDefault controls whether the pre-push hook asks the user + // before running OPF. "" (default) and "ask" both surface the + // interactive prompt; "always" runs without asking; "never" skips + // OPF and pushes 7-layer content. ENTIRE_OPF=yes|no on the push + // invocation overrides this setting per-push. + PromptDefault string `json:"prompt_default,omitempty"` } +// Valid PromptDefault values. Empty == OPFPromptAsk. +const ( + OPFPromptAsk = "ask" + OPFPromptAlways = "always" + OPFPromptNever = "never" +) + // GetCommitLinking returns the effective commit linking mode. // Returns the explicit value if set, otherwise defaults to "prompt" // to preserve existing user behavior. @@ -432,6 +446,25 @@ func SaveProjectRaw(path string, raw map[string]json.RawMessage) error { return nil } +// SaveLocalRaw writes a generic JSON object back to .entire/settings.local.json +// atomically (temp file + rename). Mirrors SaveProjectRaw for the per-developer +// overrides file; the only difference is the error wording, which says "local +// settings" so failure messages match the file actually being written. +// +// Pair with LoadLocalRaw for read-modify-write flows that target the local +// override (e.g. persisting an interactive prompt's "always" choice without +// touching the project-wide settings file). +func SaveLocalRaw(path string, raw map[string]json.RawMessage) error { + data, err := jsonutil.MarshalIndentWithNewline(raw, "", " ") + if err != nil { + return fmt.Errorf("marshal local settings: %w", err) + } + if err := jsonutil.WriteFileAtomic(path, data, 0o644); err != nil { + return fmt.Errorf("writing local settings: %w", err) + } + return nil +} + // ClonePreferencesPath returns the clone-local preferences path in the git common dir. func ClonePreferencesPath(ctx context.Context) (string, error) { commonDir, err := session.GetGitCommonDir(ctx) @@ -846,6 +879,13 @@ func validateOPFSettings(opf *OPFSettings) error { return fmt.Errorf("openai_privacy_filter.categories has unknown key %q (see docs/security-and-privacy.md for the supported set)", name) } } + switch opf.PromptDefault { + case "", OPFPromptAsk, OPFPromptAlways, OPFPromptNever: + // ok + default: + return fmt.Errorf("openai_privacy_filter.prompt_default must be one of %q, %q, %q (got %q)", + OPFPromptAsk, OPFPromptAlways, OPFPromptNever, opf.PromptDefault) + } return nil } @@ -884,6 +924,11 @@ func mergeOPFSettings(dst *OPFSettings, data json.RawMessage) error { return fmt.Errorf("parsing openai_privacy_filter.timeout_seconds: %w", err) } } + if v, ok := raw["prompt_default"]; ok { + if err := json.Unmarshal(v, &dst.PromptDefault); err != nil { + return fmt.Errorf("parsing openai_privacy_filter.prompt_default: %w", err) + } + } return validateOPFSettings(dst) } diff --git a/cmd/entire/cli/settings/settings_test.go b/cmd/entire/cli/settings/settings_test.go index df25c70192..377f818b8e 100644 --- a/cmd/entire/cli/settings/settings_test.go +++ b/cmd/entire/cli/settings/settings_test.go @@ -1133,6 +1133,47 @@ func TestLoadFromBytes_OPFSettings_RejectsOnFailureField(t *testing.T) { } } +// TestLoadFromBytes_OPFSettings_PromptDefault covers parsing + validation +// of the prompt_default field added for the pre-push prompt UX. Empty is +// allowed (treated as "ask"); ask/always/never are the only valid values. +func TestLoadFromBytes_OPFSettings_PromptDefault(t *testing.T) { + t.Parallel() + cases := []struct { + name string + value string + wantErr bool + wantVal string + }{ + {name: "ask", value: `"ask"`, wantVal: "ask"}, + {name: "always", value: `"always"`, wantVal: "always"}, + {name: "never", value: `"never"`, wantVal: "never"}, + {name: "empty_string_allowed_as_ask", value: `""`, wantVal: ""}, + {name: "bogus_value_rejected", value: `"sometimes"`, wantErr: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + body := []byte(`{"redaction":{"openai_privacy_filter":{"prompt_default":` + tc.value + `}}}`) + s, err := LoadFromBytes(body) + if tc.wantErr { + if err == nil { + t.Fatalf("expected error for %q, got nil", tc.value) + } + return + } + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if s.Redaction == nil || s.Redaction.OpenAIPrivacyFilter == nil { + t.Fatal("OPF settings not parsed") + } + if got := s.Redaction.OpenAIPrivacyFilter.PromptDefault; got != tc.wantVal { + t.Errorf("PromptDefault = %q, want %q", got, tc.wantVal) + } + }) + } +} + // TestLoadFromBytes_OPFSettings_Merge verifies override semantics for the // merge path (settings.local.json on top of settings.json): present fields // override, omitted fields preserve, categories merge per-key. diff --git a/cmd/entire/cli/strategy/cleanup.go b/cmd/entire/cli/strategy/cleanup.go index 8e43ef794f..7c234c5fa3 100644 --- a/cmd/entire/cli/strategy/cleanup.go +++ b/cmd/entire/cli/strategy/cleanup.go @@ -127,6 +127,77 @@ func ListShadowBranches(ctx context.Context) ([]string, error) { return shadowBranches, nil } +// CleanupPushedShadowBranches deletes shadow branches whose sessions +// have all ended cleanly (no active session referencing them, no +// pending turn-checkpoints awaiting finalization). Intended to be +// called only after a successful push so the caller knows any +// condensed checkpoint data already reached the remote. +// +// Returns the count of branches deleted. Failures (e.g., one branch +// fails to delete due to a stale lock) are logged but don't abort +// the operation — remaining branches are still attempted. +// +// Safety properties: +// - Skips any shadow branch referenced by a session with EndedAt +// == nil (still active). +// - Skips any shadow branch whose session has TurnCheckpointIDs +// pending (mid-finalize race window). +// - Multiple sessions can share the same shadow branch (same base +// commit + worktree); ALL must satisfy the criteria above. +// - Shadow branches with no associated session state are deleted +// (no session to lose data from). +func CleanupPushedShadowBranches(ctx context.Context) (int, error) { + branches, err := ListShadowBranches(ctx) + if err != nil { + return 0, fmt.Errorf("list shadow branches: %w", err) + } + if len(branches) == 0 { + return 0, nil + } + + states, err := ListSessionStates(ctx) + if err != nil { + return 0, fmt.Errorf("list session states: %w", err) + } + + // Build a set of shadow branch names that must be preserved + // because at least one session still depends on them. + protected := map[string]bool{} + for _, s := range states { + if s.EndedAt != nil && len(s.TurnCheckpointIDs) == 0 { + continue // safe — session ended cleanly and finalized + } + shadow := getShadowBranchNameForCommit(s.BaseCommit, s.WorktreeID) + protected[shadow] = true + } + + var toDelete []string + for _, b := range branches { + if !protected[b] { + toDelete = append(toDelete, b) + } + } + if len(toDelete) == 0 { + return 0, nil + } + + deleted, failed, delErr := DeleteShadowBranches(ctx, toDelete) + if delErr != nil { + // DeleteShadowBranches signature returns an error for future + // extensibility but currently always returns nil; log defensively. + logging.Warn(ctx, "shadow branch deletion reported error", + slog.String("error", delErr.Error()), + ) + } + if len(failed) > 0 { + logging.Warn(ctx, "some shadow branches failed to delete during post-push cleanup", + slog.Int("failed_count", len(failed)), + slog.Int("deleted_count", len(deleted)), + ) + } + return len(deleted), nil +} + // DeleteShadowBranches deletes the specified branches from the repository. // Returns two slices: successfully deleted branches and branches that failed to delete. // Individual branch deletion failures do not stop the operation - all branches are attempted. diff --git a/cmd/entire/cli/strategy/cleanup_pushed_shadow_test.go b/cmd/entire/cli/strategy/cleanup_pushed_shadow_test.go new file mode 100644 index 0000000000..c088440d4b --- /dev/null +++ b/cmd/entire/cli/strategy/cleanup_pushed_shadow_test.go @@ -0,0 +1,140 @@ +package strategy + +import ( + "context" + "testing" + "time" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/testutil" + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/stretchr/testify/require" +) + +// shadowCleanupEnv bundles the setup needed for testing post-push shadow +// branch cleanup: a git repo, a known base commit, and helpers to +// create shadow refs + matching session states. +type shadowCleanupEnv struct { + t *testing.T + repo *git.Repository + dir string + baseHash plumbing.Hash +} + +func newShadowCleanupEnv(t *testing.T) *shadowCleanupEnv { + t.Helper() + dir := t.TempDir() + testutil.InitRepo(t, dir) + repo, err := git.PlainOpen(dir) + require.NoError(t, err) + t.Chdir(dir) + + emptyTree := plumbing.NewHash("4b825dc642cb6eb9a060e54bf8d69288fbee4904") + baseHash, err := checkpoint.CreateCommit(context.Background(), repo, emptyTree, plumbing.ZeroHash, "initial commit", "test", "test@test.com") + require.NoError(t, err) + headRef := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.NewBranchReferenceName("main")) + require.NoError(t, repo.Storer.SetReference(headRef)) + require.NoError(t, repo.Storer.SetReference(plumbing.NewHashReference(plumbing.NewBranchReferenceName("main"), baseHash))) + return &shadowCleanupEnv{t: t, repo: repo, dir: dir, baseHash: baseHash} +} + +// addShadowBranch creates a shadow branch for the given (base, worktreeID) +// pair and returns its derived name. +func (e *shadowCleanupEnv) addShadowBranch(baseCommit, worktreeID string) string { + e.t.Helper() + name := getShadowBranchNameForCommit(baseCommit, worktreeID) + require.NoError(e.t, e.repo.Storer.SetReference( + plumbing.NewHashReference(plumbing.NewBranchReferenceName(name), e.baseHash))) + return name +} + +// addSessionState writes a session state file. If ended is non-nil the +// session is treated as ended; pendingCheckpoints simulates the +// mid-finalize race window. +func (e *shadowCleanupEnv) addSessionState(sessionID, baseCommit, worktreeID string, ended *time.Time, pendingCheckpoints []string) { + e.t.Helper() + state := &SessionState{ + SessionID: sessionID, + BaseCommit: baseCommit, + WorktreeID: worktreeID, + StartedAt: time.Now().Add(-time.Hour), + EndedAt: ended, + TurnCheckpointIDs: pendingCheckpoints, + } + require.NoError(e.t, SaveSessionState(context.Background(), state)) +} + +func (e *shadowCleanupEnv) branchExists(name string) bool { + e.t.Helper() + _, err := e.repo.Reference(plumbing.NewBranchReferenceName(name), false) + return err == nil +} + +// Predicate matrix: each shadow branch is paired with zero or more +// session states; the cleanup must respect the safety rules (active +// session OR pending turn checkpoints protect the branch; ended-clean +// or orphaned branches are deleted). +func TestCleanupPushedShadowBranches_Predicate(t *testing.T) { + ended := time.Now().Add(-time.Minute) + type sessionFixture struct { + id string + ended *time.Time + pendingCheckpoint []string + } + cases := []struct { + name string + sessions []sessionFixture + wantDeleted bool + }{ + {name: "ended_no_pending_deleted", sessions: []sessionFixture{{id: "s1", ended: &ended}}, wantDeleted: true}, + {name: "active_session_preserved", sessions: []sessionFixture{{id: "s1", ended: &ended}, {id: "s2", ended: nil}}, wantDeleted: false}, + {name: "pending_turn_checkpoints_preserved", sessions: []sessionFixture{{id: "s1", ended: &ended, pendingCheckpoint: []string{"a1b2c3d4e5f6"}}}, wantDeleted: false}, + {name: "orphaned_branch_no_sessions_deleted", sessions: nil, wantDeleted: true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + env := newShadowCleanupEnv(t) + shadow := env.addShadowBranch(env.baseHash.String(), "") + for _, s := range tc.sessions { + env.addSessionState(s.id, env.baseHash.String(), "", s.ended, s.pendingCheckpoint) + } + deleted, err := CleanupPushedShadowBranches(context.Background()) + require.NoError(t, err) + if tc.wantDeleted { + require.Equal(t, 1, deleted) + require.False(t, env.branchExists(shadow)) + } else { + require.Equal(t, 0, deleted) + require.True(t, env.branchExists(shadow)) + } + }) + } +} + +// Mixed: two shadow branches with different worktree IDs and different +// session statuses. The cleanup must delete only the safe one. +func TestCleanupPushedShadowBranches_MixedBranchesPartialDelete(t *testing.T) { + env := newShadowCleanupEnv(t) + preserved := env.addShadowBranch(env.baseHash.String(), "wt1") + deletable := env.addShadowBranch(env.baseHash.String(), "wt2") + ended := time.Now().Add(-time.Minute) + env.addSessionState("s-active", env.baseHash.String(), "wt1", nil, nil) + env.addSessionState("s-ended", env.baseHash.String(), "wt2", &ended, nil) + + deleted, err := CleanupPushedShadowBranches(context.Background()) + require.NoError(t, err) + require.Equal(t, 1, deleted) + require.True(t, env.branchExists(preserved)) + require.False(t, env.branchExists(deletable)) +} + +// No shadow branches → no-op, no error. +func TestCleanupPushedShadowBranches_NoBranches_NoOp(t *testing.T) { + env := newShadowCleanupEnv(t) + _ = env + + deleted, err := CleanupPushedShadowBranches(context.Background()) + require.NoError(t, err) + require.Equal(t, 0, deleted) +} diff --git a/cmd/entire/cli/strategy/hooks.go b/cmd/entire/cli/strategy/hooks.go index 394a45b65c..442427e8d7 100644 --- a/cmd/entire/cli/strategy/hooks.go +++ b/cmd/entire/cli/strategy/hooks.go @@ -171,7 +171,25 @@ func buildHookSpecs(cmdPrefix string) []hookSpec { commitMsgCmd := gitHookCommand(cmdPrefix, `commit-msg "$1" || true`, true) postCommitCmd := gitHookCommand(cmdPrefix, `post-commit 2>/dev/null || true`, false) postRewriteCmd := gitHookCommand(cmdPrefix, `post-rewrite "$1" 2>/dev/null || true`, false) - prePushCmd := gitHookCommand(cmdPrefix, `pre-push "$1" || true`, false) + // pre-push intentionally does NOT swallow exit codes — the OPF + // rewrite returns errors when it detects a privacy-critical + // condition (diverged remote, oversized bootstrap, CAS conflict, + // OPF runtime failure) and the user's git push must abort. + // Transient checkpoint-push failures (e.g. the + // entire/checkpoints/v1 push itself failing) are NOT returned + // from PrePush — they're logged and swallowed at the CLI level + // so they never reach this point as non-zero exits. + // + // Trade-off: an unrelated `entire` crash (segfault, panic in + // non-OPF code) ALSO aborts the user's push. This is the safer + // failure mode — we cannot distinguish from the shell's point of + // view whether a non-zero exit means "OPF declined to redact" or + // "entire crashed mid-rewrite", and silently letting potentially- + // unredacted content reach the remote would violate the contract + // the user opted into by enabling OPF. Users hit by unrelated + // bugs can `ENTIRE_OPF=no git push` for a one-off bypass while + // the bug is fixed. + prePushCmd := gitHookCommand(cmdPrefix, `pre-push "$1"`, false) return []hookSpec{ { diff --git a/cmd/entire/cli/strategy/manual_commit_condensation.go b/cmd/entire/cli/strategy/manual_commit_condensation.go index adc0214adc..474b820566 100644 --- a/cmd/entire/cli/strategy/manual_commit_condensation.go +++ b/cmd/entire/cli/strategy/manual_commit_condensation.go @@ -111,11 +111,18 @@ type condenseOpts struct { allAgentFiles map[string]struct{} // Union of all sessions' FilesTouched for cross-session exclusion (nil = single-session) } -// redactSessionJSONLBytes runs the full 8-layer redaction pipeline -// (including the OpenAI Privacy Filter when configured) over a session -// transcript. Exposed as a var so tests can inject deterministic -// success/error returns without spinning up the runtime. -var redactSessionJSONLBytes = redact.JSONLBytesWithPrivacyFilter +// redactSessionJSONLBytes runs the 7-layer redaction pipeline over a +// session transcript at post-commit condensation. OPF is intentionally +// NOT included here — it runs exclusively in the pre-push rewrite path +// (strategy/manual_commit_opf_rewrite.go), which re-redacts the +// 7-layer blobs and produces 8-layer commits before the push. +// +// Exposed as a var so tests can inject deterministic success/error +// returns. The signature still takes a context so the var can be +// re-wired to JSONLBytesWithPrivacyFilter from tests that need OPF. +var redactSessionJSONLBytes = func(_ context.Context, b []byte) (redact.RedactedBytes, error) { + return redact.JSONLBytes(b) +} // CondenseSession condenses a session's shadow branch to permanent storage. // checkpointID is the 12-hex-char value from the Entire-Checkpoint trailer. @@ -227,12 +234,8 @@ func (s *ManualCommitStrategy) CondenseSession(ctx context.Context, repo *git.Re summary = generateSummary(ctx, redactedTranscript, sessionData.FilesTouched, state) } - // Pre-redact joined prompts once so v1 and v2 writers (plus any - // subsequent UpdateCommitted within the same finalize) reuse the same - // result instead of each running the 8-layer pipeline over identical - // input. The typed return value carries a compile-time claim that the - // content has been through the pipeline. - redactedPrompts := redact.JoinedPrompts(ctx, sessionData.Prompts, cpkg.PromptSeparator) + // Post-commit emits 7-layer-only blobs. OPF runs later in the + // pre-push rewrite path, never here. // Build write options (shared by v1 and v2) writeOpts := cpkg.WriteCommittedOptions{ @@ -242,7 +245,6 @@ func (s *ManualCommitStrategy) CondenseSession(ctx context.Context, repo *git.Re Branch: branchName, Transcript: redactedTranscript, Prompts: sessionData.Prompts, - PromptsRedacted: redactedPrompts, FilesTouched: sessionData.FilesTouched, CheckpointsCount: state.StepCount, EphemeralBranch: shadowBranchName, diff --git a/cmd/entire/cli/strategy/manual_commit_hooks.go b/cmd/entire/cli/strategy/manual_commit_hooks.go index 279ec3ce5d..61ee0184de 100644 --- a/cmd/entire/cli/strategy/manual_commit_hooks.go +++ b/cmd/entire/cli/strategy/manual_commit_hooks.go @@ -2745,11 +2745,11 @@ func (s *ManualCommitStrategy) finalizeAllTurnCheckpoints(ctx context.Context, s // (attribution, files touched, prompts). Hooks run without user interaction // so there is no retry path — preserving partial metadata is better than // losing everything. Persisting an unredacted transcript would be worse. - // Run the full 8-layer pipeline (including OPF when configured) over - // the transcript — this is the condensation boundary where bytes are - // about to leave the local machine via push to entire/checkpoints/v1. + // Run the 7-layer pipeline over the transcript — OPF runs later in + // the pre-push rewrite path, which re-redacts these 7-layer blobs + // and produces 8-layer commits before the push goes out. _, redactSpan := perf.Start(logCtx, "redact_transcript") - redactedTranscript, redactErr := redact.JSONLBytesWithPrivacyFilter(logCtx, fullTranscript) + redactedTranscript, redactErr := redact.JSONLBytes(fullTranscript) redactSpan.End() if redactErr != nil { logging.Warn(logCtx, "finalize: transcript redaction failed, dropping transcript", @@ -2759,16 +2759,9 @@ func (s *ManualCommitStrategy) finalizeAllTurnCheckpoints(ctx context.Context, s redactedTranscript = redact.RedactedBytes{} } - // Pre-redact joined prompts ONCE so the checkpoint loop reuses the - // result instead of each iteration re-running the 8-layer pipeline - // (including OPF shell-out) over identical input. The original code - // here ran redact.String per prompt, which produced a per-prompt - // OPF call once OPF was wired up; pre-joining + one call collapses - // that to a single OPF invocation per finalize. The typed return - // value carries a compile-time claim that the content has been - // through the pipeline. - redactedPrompts := redact.JoinedPrompts(logCtx, prompts, checkpoint.PromptSeparator) - + // Post-commit emits 7-layer-only blobs; the writer joins + redacts + // via checkpoint.redactedJoinedPrompts. OPF runs later, once per + // push, in the pre-push rewrite path. store := checkpoint.NewGitStore(repo) v2 := settings.CheckpointsVersion(logCtx) == 2 @@ -2824,7 +2817,6 @@ func (s *ManualCommitStrategy) finalizeAllTurnCheckpoints(ctx context.Context, s SessionID: state.SessionID, Transcript: redactedTranscript, Prompts: prompts, - PromptsRedacted: redactedPrompts, Agent: state.AgentType, PrecomputedBlobs: precomputed, } diff --git a/cmd/entire/cli/strategy/manual_commit_opf_prompt.go b/cmd/entire/cli/strategy/manual_commit_opf_prompt.go new file mode 100644 index 0000000000..c7d12cd770 --- /dev/null +++ b/cmd/entire/cli/strategy/manual_commit_opf_prompt.go @@ -0,0 +1,186 @@ +package strategy + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "strings" + + "charm.land/huh/v2" + "github.com/entireio/cli/cmd/entire/cli/interactive" + "github.com/entireio/cli/cmd/entire/cli/logging" + "github.com/entireio/cli/cmd/entire/cli/settings" +) + +// OPFDecision is the resolved gate for a single pre-push OPF run. +type OPFDecision int + +const ( + OPFRun OPFDecision = iota // run the rewrite, push 8-layer + OPFSkip // skip the rewrite, push 7-layer + OPFAbort // cancel the push entirely (Ctrl-C / non-TTY abort) +) + +const envOPF = "ENTIRE_OPF" + +// resolveOPFDecision picks the run/skip/abort gate. Precedence (highest +// first): ENTIRE_OPF env var → settings.PromptDefault → interactive +// prompt → non-TTY fallback (run). +// +// Pure logic — prompter is only called when the user needs to decide. +// Tests inject a fake. +func resolveOPFDecision(env, promptDefault string, hasTTY bool, prompter func() (OPFDecision, error)) (OPFDecision, error) { + switch strings.ToLower(strings.TrimSpace(env)) { + case "yes": + return OPFRun, nil + case "no": + return OPFSkip, nil + } + switch strings.ToLower(strings.TrimSpace(promptDefault)) { + case settings.OPFPromptAlways: + return OPFRun, nil + case settings.OPFPromptNever: + return OPFSkip, nil + } + if !hasTTY { + // Non-interactive context: run OPF (matches the user's "if + // enabled, just run" preference). The caller emits a progress + // line at run time so scripted output isn't silent. + return OPFRun, nil + } + return prompter() +} + +// resolveOPFDecisionForPrePush is the production wiring: reads env + +// settings + TTY, calls askOPFPrompt when interactive, emits a stderr +// progress line on the non-TTY auto-run path. errOut receives the +// progress line (only printed when we'll actually run + the caller is +// non-interactive). +func resolveOPFDecisionForPrePush(ctx context.Context, opf *settings.OPFSettings, errOut io.Writer) (OPFDecision, error) { + hasTTY := interactive.CanPromptInteractively() + promptDefault := "" + if opf != nil { + promptDefault = opf.PromptDefault + } + d, err := resolveOPFDecision( + os.Getenv(envOPF), + promptDefault, + hasTTY, + func() (OPFDecision, error) { return askOPFPrompt(ctx, isAccessibleMode()) }, + ) + if err != nil { + return OPFAbort, err + } + if d == OPFRun && !hasTTY { + fmt.Fprintln(errOut, "→ OpenAI Privacy Filter: scanning checkpoints before push (may take ~30s)…") + } + return d, nil +} + +// askOPFPrompt shows the 3-option huh form. Ctrl-C / SIGINT returns +// OPFAbort. Selecting "Always" persists prompt_default=always to +// .entire/settings.local.json so future pushes don't ask. +// +// Style matches other entire CLI prompts: Dracula theme via +// huh.ThemeDracula (the same theme cli.NewAccessibleForm applies for +// callers in the cli package). Strategy can't import cli (cycle), so +// we apply the theme inline. +func askOPFPrompt(ctx context.Context, accessible bool) (OPFDecision, error) { + const ( + choiceYes = "yes" + choiceNo = "no" + choiceAlways = "always" + ) + choice := choiceYes + form := huh.NewForm( + huh.NewGroup( + huh.NewSelect[string](). + Title("Run OpenAI Privacy Filter on these checkpoints?"). + Description("Adds ~30s but redacts names/PII the regex layers can't catch. Ctrl-C to cancel the push."). + Options( + huh.NewOption("Yes — run OPF this push", choiceYes), + huh.NewOption("No — skip OPF, push as-is", choiceNo), + huh.NewOption("Always — run OPF on every push from now on", choiceAlways), + ). + Value(&choice), + ), + ).WithTheme(huh.ThemeFunc(huh.ThemeDracula)) + if accessible { + form = form.WithAccessible(true) + } + if err := form.RunWithContext(ctx); err != nil { + if errors.Is(err, huh.ErrUserAborted) { + return OPFAbort, nil + } + return OPFAbort, fmt.Errorf("opf prompt: %w", err) + } + if choice == choiceAlways { + if err := persistOPFPromptDefaultAlways(ctx); err != nil { + logging.Warn(ctx, "failed to persist OPF prompt_default=always", + slog.String("error", err.Error())) + } + } + if choice == choiceNo { + return OPFSkip, nil + } + return OPFRun, nil +} + +// persistOPFPromptDefaultAlways writes +// redaction.openai_privacy_filter.prompt_default = "always" to +// .entire/settings.local.json, preserving any unrelated fields by +// using a generic JSON-map round-trip. +func persistOPFPromptDefaultAlways(ctx context.Context) error { + path, raw, _, err := settings.LoadLocalRaw(ctx) + if err != nil { + return fmt.Errorf("load local settings: %w", err) + } + if raw == nil { + raw = map[string]json.RawMessage{} + } + redactionRaw := readSubObject(raw, "redaction") + opfRaw := readSubObject(redactionRaw, "openai_privacy_filter") + + val, err := json.Marshal(settings.OPFPromptAlways) + if err != nil { + return fmt.Errorf("marshal prompt_default: %w", err) + } + opfRaw["prompt_default"] = val + + if err := writeSubObject(redactionRaw, "openai_privacy_filter", opfRaw); err != nil { + return err + } + if err := writeSubObject(raw, "redaction", redactionRaw); err != nil { + return err + } + if err := settings.SaveLocalRaw(path, raw); err != nil { + return fmt.Errorf("save local settings: %w", err) + } + return nil +} + +func readSubObject(parent map[string]json.RawMessage, key string) map[string]json.RawMessage { + sub := map[string]json.RawMessage{} + data, ok := parent[key] + if !ok { + return sub + } + // Malformed sub-object → fresh map (we'll overwrite the slot). + if err := json.Unmarshal(data, &sub); err != nil { + return map[string]json.RawMessage{} + } + return sub +} + +func writeSubObject(parent map[string]json.RawMessage, key string, sub map[string]json.RawMessage) error { + data, err := json.Marshal(sub) + if err != nil { + return fmt.Errorf("marshal %s: %w", key, err) + } + parent[key] = data + return nil +} diff --git a/cmd/entire/cli/strategy/manual_commit_opf_prompt_test.go b/cmd/entire/cli/strategy/manual_commit_opf_prompt_test.go new file mode 100644 index 0000000000..38fb76a045 --- /dev/null +++ b/cmd/entire/cli/strategy/manual_commit_opf_prompt_test.go @@ -0,0 +1,150 @@ +package strategy + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/settings" + "github.com/stretchr/testify/require" +) + +// resolveOPFDecision is the pure-logic core: env > settings > prompt > +// non-TTY auto-run. Table-driven because every case is just a different +// (env, setting, tty, prompter) input combination. +func TestResolveOPFDecision_Precedence(t *testing.T) { + t.Parallel() + + const promptCalled = OPFDecision(-1) // sentinel; only used when prompter fires + promptYes := func() (OPFDecision, error) { return OPFRun, nil } + promptNo := func() (OPFDecision, error) { return OPFSkip, nil } + promptAbort := func() (OPFDecision, error) { return OPFAbort, nil } + promptErr := func() (OPFDecision, error) { return OPFAbort, errors.New("boom") } + promptNever := func() (OPFDecision, error) { + t.Helper() + t.Fatal("prompter must not be called") + return promptCalled, nil + } + + cases := []struct { + name string + env string + promptDefault string + hasTTY bool + prompter func() (OPFDecision, error) + want OPFDecision + wantErr bool + wantErrMessage string + }{ + // Env wins everywhere + {name: "env_yes_wins_over_setting_never", env: "yes", promptDefault: settings.OPFPromptNever, hasTTY: true, prompter: promptNever, want: OPFRun}, + {name: "env_no_wins_over_setting_always", env: "no", promptDefault: settings.OPFPromptAlways, hasTTY: true, prompter: promptNever, want: OPFSkip}, + {name: "env_yes_case_insensitive", env: "YES", promptDefault: "", hasTTY: false, prompter: promptNever, want: OPFRun}, + {name: "env_no_with_whitespace", env: " no ", promptDefault: "", hasTTY: false, prompter: promptNever, want: OPFSkip}, + // Setting wins over prompt + {name: "setting_always_skips_prompt", env: "", promptDefault: settings.OPFPromptAlways, hasTTY: true, prompter: promptNever, want: OPFRun}, + {name: "setting_never_skips_prompt", env: "", promptDefault: settings.OPFPromptNever, hasTTY: true, prompter: promptNever, want: OPFSkip}, + // Non-TTY fallback: run (matches the "if enabled, just run" semantics) + {name: "no_tty_auto_runs", env: "", promptDefault: "", hasTTY: false, prompter: promptNever, want: OPFRun}, + {name: "no_tty_ignores_ask_setting", env: "", promptDefault: settings.OPFPromptAsk, hasTTY: false, prompter: promptNever, want: OPFRun}, + // TTY + ask → prompter is called + {name: "tty_ask_user_chose_yes", env: "", promptDefault: settings.OPFPromptAsk, hasTTY: true, prompter: promptYes, want: OPFRun}, + {name: "tty_ask_user_chose_no", env: "", promptDefault: settings.OPFPromptAsk, hasTTY: true, prompter: promptNo, want: OPFSkip}, + {name: "tty_ask_user_aborted", env: "", promptDefault: settings.OPFPromptAsk, hasTTY: true, prompter: promptAbort, want: OPFAbort}, + // TTY + empty setting == ask + {name: "tty_empty_setting_treated_as_ask", env: "", promptDefault: "", hasTTY: true, prompter: promptYes, want: OPFRun}, + // Prompter errors propagate + {name: "prompter_error", env: "", promptDefault: "", hasTTY: true, prompter: promptErr, want: OPFAbort, wantErr: true, wantErrMessage: "boom"}, + // Unrecognized env values fall through to next layer + {name: "env_bogus_falls_through_to_setting", env: "maybe", promptDefault: settings.OPFPromptAlways, hasTTY: true, prompter: promptNever, want: OPFRun}, + {name: "env_empty_falls_through_to_setting", env: "", promptDefault: settings.OPFPromptAlways, hasTTY: true, prompter: promptNever, want: OPFRun}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + got, err := resolveOPFDecision(tc.env, tc.promptDefault, tc.hasTTY, tc.prompter) + if tc.wantErr { + require.Error(t, err) + if tc.wantErrMessage != "" { + require.Contains(t, err.Error(), tc.wantErrMessage) + } + } else { + require.NoError(t, err) + } + require.Equal(t, tc.want, got, "decision") + }) + } +} + +// TestPersistOPFPromptDefaultAlways_WritesNestedField verifies that the +// "Always" branch updates redaction.openai_privacy_filter.prompt_default +// in .entire/settings.local.json without disturbing other fields. +// +// Modifies process cwd (no t.Parallel), but uses t.Chdir so subsequent +// tests see the reverted cwd. +func TestPersistOPFPromptDefaultAlways_WritesNestedField(t *testing.T) { + tempDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(tempDir, paths.EntireDir), 0o755)) + // Seed an existing settings.local.json with some unrelated content + // so we can verify it survives the write. + existing := `{ + "enabled": true, + "redaction": { + "openai_privacy_filter": { + "categories": {"private_person": true} + } + } +}` + localPath := filepath.Join(tempDir, paths.EntireDir, "settings.local.json") + require.NoError(t, os.WriteFile(localPath, []byte(existing), 0o644)) + t.Chdir(tempDir) + + require.NoError(t, persistOPFPromptDefaultAlways(context.Background())) + + got, err := os.ReadFile(localPath) + require.NoError(t, err) + + // Parse and verify structure: enabled stays true, categories stay, + // new prompt_default key present with "always". + var parsed struct { + Enabled bool `json:"enabled"` + Redaction struct { + OPF struct { + Categories map[string]bool `json:"categories"` + PromptDefault string `json:"prompt_default"` + } `json:"openai_privacy_filter"` + } `json:"redaction"` + } + require.NoError(t, json.Unmarshal(got, &parsed)) + require.True(t, parsed.Enabled, "existing enabled field must survive") + require.True(t, parsed.Redaction.OPF.Categories["private_person"], "existing categories must survive") + require.Equal(t, settings.OPFPromptAlways, parsed.Redaction.OPF.PromptDefault) +} + +// TestPersistOPFPromptDefaultAlways_CreatesFileFromScratch covers the +// fresh-install path where .entire/settings.local.json doesn't exist yet. +func TestPersistOPFPromptDefaultAlways_CreatesFileFromScratch(t *testing.T) { + tempDir := t.TempDir() + require.NoError(t, os.MkdirAll(filepath.Join(tempDir, paths.EntireDir), 0o755)) + t.Chdir(tempDir) + + require.NoError(t, persistOPFPromptDefaultAlways(context.Background())) + + localPath := filepath.Join(tempDir, paths.EntireDir, "settings.local.json") + got, err := os.ReadFile(localPath) + require.NoError(t, err, "settings.local.json should be created") + + var parsed struct { + Redaction struct { + OPF struct { + PromptDefault string `json:"prompt_default"` + } `json:"openai_privacy_filter"` + } `json:"redaction"` + } + require.NoError(t, json.Unmarshal(got, &parsed)) + require.Equal(t, settings.OPFPromptAlways, parsed.Redaction.OPF.PromptDefault) +} diff --git a/cmd/entire/cli/strategy/manual_commit_opf_rewrite.go b/cmd/entire/cli/strategy/manual_commit_opf_rewrite.go new file mode 100644 index 0000000000..a48a0ba098 --- /dev/null +++ b/cmd/entire/cli/strategy/manual_commit_opf_rewrite.go @@ -0,0 +1,562 @@ +// Pre-push OPF rewrite for entire/checkpoints/v1. +// +// This is the ONLY production code path that runs the OPF-augmented +// redaction entry points. Post-commit condensation stays on 7-layer +// for predictable latency; OPF runs here, once per push, after the +// user opted in via settings. +package strategy + +import ( + "context" + "crypto/sha256" + "errors" + "fmt" + "io" + "log/slog" + "math" + "os" + "strconv" + "strings" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/checkpoint/remote" + "github.com/entireio/cli/cmd/entire/cli/logging" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/trailers" + "github.com/entireio/cli/redact" + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/go-git/go-git/v6/storage" +) + +// V1DivergedError: local entire/checkpoints/v1 has commits that aren't +// ancestors of the remote tip (force-push or another machine pushed). +// Rewriting under divergence would silently rebase rejected work, so +// we refuse. +type V1DivergedError struct { + Local, Remote, MergeBase plumbing.Hash +} + +func (e *V1DivergedError) Error() string { + return fmt.Sprintf("entire/checkpoints/v1 has diverged from remote (local=%s remote=%s merge_base=%s); "+ + "fetch the remote and either reset entire/checkpoints/v1 to /entire/checkpoints/v1 "+ + "or run `entire doctor --recover-v1` before pushing", + e.Local.String()[:7], e.Remote.String()[:7], e.MergeBase.String()[:7]) +} + +// BootstrapTooLargeError: first push to a remote with no v1 yet, but +// more unpushed commits than the safety cap. OPF inference is ~30s per +// commit, so unbounded bootstraps could take hours. +type BootstrapTooLargeError struct { + Count, Limit int +} + +func (e *BootstrapTooLargeError) Error() string { + return fmt.Sprintf("OPF bootstrap would rewrite %d entire/checkpoints/v1 commits "+ + "(limit %d). Set ENTIRE_OPF_BOOTSTRAP_LIMIT= or =unlimited to override, "+ + "or push without OPF (ENTIRE_OPF=no git push) to bring the remote into sync first", + e.Count, e.Limit) +} + +// V1RefMovedError: another worktree advanced the local ref during our +// rewrite (CAS conflict). Orphan rewritten objects sit in .git/objects +// until git gc --prune; no manual cleanup needed. +type V1RefMovedError struct { + Expected, Actual plumbing.Hash +} + +func (e *V1RefMovedError) Error() string { + return fmt.Sprintf("entire/checkpoints/v1 moved during OPF rewrite "+ + "(expected %s, found %s); another local worktree advanced the ref "+ + "mid-rewrite — re-run `git push` (no fetch needed; the move was local)", + e.Expected.String()[:7], e.Actual.String()[:7]) +} + +// OPFRuntimeFailedError: the OPF circuit breaker tripped mid-rewrite. +// Some blobs were silently downgraded to 7-layer; tagging those commits +// as Entire-OPF-Applied would be a privacy regression (future pushes +// would skip them while their content is 7-layer-only). Abort before +// CAS so the user fixes their OPF install and retries. +type OPFRuntimeFailedError struct { + OPFCommand string +} + +func (e *OPFRuntimeFailedError) Error() string { + return fmt.Sprintf("OPF runtime failed during pre-push rewrite (command=%q); "+ + "aborting push so 7-layer content isn't tagged as 8-layer-applied. "+ + "Run `%s --help` to verify your OPF install, then retry. Or set "+ + "ENTIRE_OPF=no on the push to skip OPF for this push only.", + e.OPFCommand, e.OPFCommand) +} + +const ( + // bootstrapDefaultLimit caps first-push history rewrites. Picked + // to bound worst-case wall-clock at ~50min @ 30s/commit. + bootstrapDefaultLimit = 100 + bootstrapEnvVar = "ENTIRE_OPF_BOOTSTRAP_LIMIT" +) + +func resolveBootstrapLimit() int { + v := strings.TrimSpace(os.Getenv(bootstrapEnvVar)) + switch { + case v == "": + return bootstrapDefaultLimit + case strings.EqualFold(v, "unlimited"): + return math.MaxInt32 + } + if n, err := strconv.Atoi(v); err == nil && n > 0 { + return n + } + return bootstrapDefaultLimit +} + +// RewriteUnpushedV1WithOPF re-redacts unpushed entire/checkpoints/v1 +// commits with OPF, builds new commits carrying Entire-OPF-Applied: +// true, and CAS-updates the local ref. Idempotent: already-applied +// commits are re-parented without re-running OPF. +// +// Caller checks redact.OPFEnabled() and skips this when OPF is off. +// Returns one of {V1DivergedError, BootstrapTooLargeError, +// V1RefMovedError, OPFRuntimeFailedError} for privacy-critical +// failures — the pre-push hook propagates these so git push aborts. +func RewriteUnpushedV1WithOPF(ctx context.Context, repo *git.Repository, target string) (plumbing.Hash, error) { + localTip, err := readV1Tip(repo, plumbing.NewBranchReferenceName(paths.MetadataBranchName)) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("read local v1: %w", err) + } + if localTip.IsZero() { + return plumbing.ZeroHash, nil // no checkpoints yet + } + remoteTip, err := resolveRemoteV1Tip(ctx, repo, target) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("read remote v1: %w", err) + } + + if !remoteTip.IsZero() { + mergeBase, mbErr := computeMergeBase(repo, localTip, remoteTip) + if mbErr != nil { + return plumbing.ZeroHash, fmt.Errorf("compute merge-base: %w", mbErr) + } + if mergeBase != remoteTip { + return plumbing.ZeroHash, &V1DivergedError{Local: localTip, Remote: remoteTip, MergeBase: mergeBase} + } + } + + unpushed, err := listUnpushedV1Commits(repo, localTip, remoteTip) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("list unpushed v1 commits: %w", err) + } + if len(unpushed) == 0 { + return localTip, nil + } + if remoteTip.IsZero() { + if limit := resolveBootstrapLimit(); len(unpushed) > limit { + return plumbing.ZeroHash, &BootstrapTooLargeError{Count: len(unpushed), Limit: limit} + } + } + + parent := remoteTip + for _, c := range unpushed { + newHash, err := rebuildV1Commit(ctx, repo, c, parent) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("rebuild commit %s: %w", c.Hash.String()[:7], err) + } + parent = newHash + } + + // Fail-closed: if OPF tripped its breaker mid-rewrite, some blobs + // got 7-layer fallback. Don't CAS — the orphan commits get GC'd. + if redact.OPFBreakerTripped() { + return plumbing.ZeroHash, &OPFRuntimeFailedError{OPFCommand: redact.OPFCommand()} + } + if err := atomicSetV1Ref(repo, localTip, parent); err != nil { + return plumbing.ZeroHash, err + } + return parent, nil +} + +func readV1Tip(repo *git.Repository, refName plumbing.ReferenceName) (plumbing.Hash, error) { + ref, err := repo.Reference(refName, true) + if err != nil { + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return plumbing.ZeroHash, nil + } + return plumbing.ZeroHash, fmt.Errorf("resolve ref %s: %w", refName, err) + } + return ref.Hash(), nil +} + +// opfRewriteFetchTmpRef is the temp ref used to stage the URL-fetched +// remote v1 tip during OPF rewrite. Cleaned up at the end of each +// resolveRemoteV1Tip call so the tracking is invisible to the user. +const opfRewriteFetchTmpRef = FetchTmpRefPrefix + "opf-rewrite-v1" + +// resolveRemoteV1Tip returns the hash of the remote's +// entire/checkpoints/v1 tip. +// +// When target is a remote name (e.g., "origin"), looks up the local +// tracking ref `refs/remotes//entire/checkpoints/v1`. When +// target is a URL (checkpoint_remote configured), fetches the v1 ref +// from the URL into a temporary local ref so the rewrite can see what's +// already on the remote — otherwise every push would re-redact the +// entire history as a "bootstrap" since URL-based remotes have no +// tracking refs locally. +// +// Returns ZeroHash with no error when the remote has no v1 yet (genuine +// bootstrap case). Fetch failures fall back to ZeroHash + a warning +// log; the rewrite then treats the push as bootstrap rather than +// blocking the user on a transient network issue. +func resolveRemoteV1Tip(ctx context.Context, repo *git.Repository, target string) (plumbing.Hash, error) { + if !remote.IsURL(target) { + return readV1Tip(repo, plumbing.NewRemoteReferenceName(target, paths.MetadataBranchName)) + } + srcRef := "refs/heads/" + paths.MetadataBranchName + if err := fetchURLIntoTmpRef(ctx, target, srcRef, opfRewriteFetchTmpRef, "v1 for OPF rewrite", true); err != nil { + logging.Warn(ctx, "OPF rewrite: failed to fetch remote v1 from URL; treating push as bootstrap", + slog.String("error", err.Error()), + ) + return plumbing.ZeroHash, nil + } + defer removeTempRefs(repo, []plumbing.ReferenceName{plumbing.ReferenceName(opfRewriteFetchTmpRef)}) + ref, err := repo.Reference(plumbing.ReferenceName(opfRewriteFetchTmpRef), true) + if err != nil { + if errors.Is(err, plumbing.ErrReferenceNotFound) { + return plumbing.ZeroHash, nil + } + return plumbing.ZeroHash, fmt.Errorf("resolve fetched v1 ref: %w", err) + } + return ref.Hash(), nil +} + +// computeMergeBase returns the merge-base commit hash. Multi-base +// (criss-cross) and unrelated-histories both return ZeroHash — +// caller treats those as diverged. +func computeMergeBase(repo *git.Repository, local, remote plumbing.Hash) (plumbing.Hash, error) { + lc, err := repo.CommitObject(local) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("load local commit: %w", err) + } + rc, err := repo.CommitObject(remote) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("load remote commit: %w", err) + } + bases, err := lc.MergeBase(rc) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("merge-base: %w", err) + } + if len(bases) != 1 { + return plumbing.ZeroHash, nil + } + return bases[0].Hash, nil +} + +// listUnpushedV1Commits returns commits reachable from localTip but not +// remoteTip, in graph order (oldest-first). Graph order matters more +// than timestamp order — commits made in rapid succession can share +// Author.When; the parent chain is the unambiguous truth. +// +// Optimization: the caller (RewriteUnpushedV1WithOPF) has already +// validated that remoteTip is the unique merge-base of local and +// remote, which means v1 is linear and remoteTip is an ancestor of +// localTip. So walking back from localTip, the FIRST commit we hit +// whose hash equals remoteTip is the boundary — no need to pre-build +// a full remote-reachability set. This drops the cost from +// O(local + remote history) to O(unpushed) per call. +func listUnpushedV1Commits(repo *git.Repository, localTip, remoteTip plumbing.Hash) ([]*object.Commit, error) { + iter, err := repo.Log(&git.LogOptions{From: localTip}) + if err != nil { + return nil, fmt.Errorf("log local tip: %w", err) + } + defer iter.Close() + + var unpushed []*object.Commit + if walkErr := iter.ForEach(func(c *object.Commit) error { + if !remoteTip.IsZero() && c.Hash == remoteTip { + return errStop + } + unpushed = append(unpushed, c) + return nil + }); walkErr != nil && !errors.Is(walkErr, errStop) { + return nil, fmt.Errorf("walk local v1 history: %w", walkErr) + } + // reverse for oldest-first + for i, j := 0, len(unpushed)-1; i < j; i, j = i+1, j-1 { + unpushed[i], unpushed[j] = unpushed[j], unpushed[i] + } + return unpushed, nil +} + +// rebuildV1Commit re-parents the commit onto parent. Already-applied +// commits keep their tree (idempotent); unapplied commits get an +// OPF-redacted tree + Entire-OPF-Applied: true trailer. +// +// Performance: we only redact files inside THIS commit's shard +// (sharded layout: //*). Files outside that shard live +// at the same tree because git trees accumulate parent content — they +// belong to other commits and either are already redacted (prior +// OPF-applied push) or never will be (this user opted out then in). +// Walking them every push is O(N×commits) work for no privacy gain. +func rebuildV1Commit(ctx context.Context, repo *git.Repository, oldCommit *object.Commit, parent plumbing.Hash) (plumbing.Hash, error) { + newTree := oldCommit.TreeHash + if !trailers.HasOPFApplied(oldCommit.Message) { + tree, err := repo.TreeObject(oldCommit.TreeHash) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("load tree: %w", err) + } + // Parse the shard path from the commit subject. Falls back to + // "" (walk everything) for bootstrap commits and unrecognized + // subjects — the conservative default still produces correct + // output, just slower. + shardPath := parseShardPathFromCommitMessage(oldCommit.Message) + newTree, err = rebuildTreeWithOPF(ctx, repo, tree, "", shardPath) + if err != nil { + return plumbing.ZeroHash, err + } + } + + parents := []plumbing.Hash{} + if !parent.IsZero() { + parents = append(parents, parent) + } + c := &object.Commit{ + Author: oldCommit.Author, + Committer: oldCommit.Committer, + Message: trailers.AppendOPFAppliedTrailer(oldCommit.Message), + TreeHash: newTree, + ParentHashes: parents, + } + // Sign the rewritten commit when commit signing is enabled, matching + // every other commit-construction site in this package (common.go, + // metadata_reconcile.go, push_common.go). Without this, a user who + // has signed checkpoint commits would see the rewrite produce + // unsigned commits — silently degrading their integrity story. + checkpoint.SignCommitBestEffort(ctx, c) + obj := repo.Storer.NewEncodedObject() + if err := c.Encode(obj); err != nil { + return plumbing.ZeroHash, fmt.Errorf("encode commit: %w", err) + } + hash, err := repo.Storer.SetEncodedObject(obj) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("store commit: %w", err) + } + return hash, nil +} + +// parseShardPathFromCommitMessage extracts the sharded path +// "/" from a "Checkpoint: " subject line. +// Returns "" when the subject doesn't match (bootstrap commits, or +// historical commits with a different format) — callers walk the +// whole tree in that case. +func parseShardPathFromCommitMessage(message string) string { + firstLine, _, _ := strings.Cut(message, "\n") + const prefix = "Checkpoint: " + if !strings.HasPrefix(firstLine, prefix) { + return "" + } + id := strings.TrimSpace(firstLine[len(prefix):]) + if len(id) != 12 { + return "" + } + for _, c := range id { + if (c < '0' || c > '9') && (c < 'a' || c > 'f') { + return "" + } + } + return id[:2] + "/" + id[2:] +} + +// rebuildTreeWithOPF walks a tree and produces a new tree with +// OPF-redacted file blobs. content_hash.txt files are recomputed in a +// second pass against the new full.jsonl in the same directory. +// +// shardPath scopes the walk: only files at paths starting with +// shardPath get redacted; other shards (and the root-level entries +// outside the shard) are copied verbatim. Empty shardPath means walk +// everything (used for bootstrap/unknown-subject commits). +// +// Path-specific behavior (when in the target shard): +// - content_hash.txt → SHA256 of the sibling full.jsonl's new bytes +// (deferred; not redacted itself) +// - everything else → redacted via checkpoint.RedactBlobBytes (OPF on). +// The fail-closed policy intentionally redacts ANY regular file +// inside the shard, not just a closed allowlist of suffixes — a +// future blob type (e.g. .md prose, agent dumps, no-extension +// transcript blobs) is redacted by default rather than slipping +// through. RedactBlobBytes itself dispatches: .jsonl/.json get +// JSON-aware leaf redaction (so free-form fields like Summary.Intent +// / ReviewPrompt are scrubbed); other files get byte redaction. The +// has-space gate inside OPF naturally excludes binary blobs from +// paying the model cost. +func rebuildTreeWithOPF(ctx context.Context, repo *git.Repository, tree *object.Tree, pathPrefix, shardPath string) (plumbing.Hash, error) { + entries := make([]object.TreeEntry, 0, len(tree.Entries)) + // deferredHashes records indexes of content_hash.txt entries we + // need to recompute after the full.jsonl in the same dir is built. + type deferred struct { + idx int + entryName string + entryMode filemode.FileMode + } + var deferredHashes []deferred + var newFullJSONLHash plumbing.Hash + + for _, e := range tree.Entries { + switch e.Mode { //nolint:exhaustive // non-tree/blob modes fall through to copy + case filemode.Dir: + subPath := e.Name + if pathPrefix != "" { + subPath = pathPrefix + "/" + e.Name + } + // Shard-scoping: only descend into directories that lead + // to the target shard, the shard itself, or its + // descendants. Other shard subtrees stay byte-identical. + if !shouldDescend(subPath, shardPath) { + entries = append(entries, e) + continue + } + subTree, err := repo.TreeObject(e.Hash) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("load subtree %s/%s: %w", pathPrefix, e.Name, err) + } + newSub, err := rebuildTreeWithOPF(ctx, repo, subTree, subPath, shardPath) + if err != nil { + return plumbing.ZeroHash, err + } + entries = append(entries, object.TreeEntry{Name: e.Name, Mode: e.Mode, Hash: newSub}) + + case filemode.Regular, filemode.Executable: + // Outside the target shard: copy verbatim. Inside (or when + // shardPath is empty for the bootstrap fallback): redact + // per file type. + if !insideShard(pathPrefix, shardPath) { + entries = append(entries, e) + continue + } + switch e.Name { + case paths.ContentHashFileName: + deferredHashes = append(deferredHashes, deferred{idx: len(entries), entryName: e.Name, entryMode: e.Mode}) + entries = append(entries, e) // placeholder; fixed in second pass + default: + content, err := readBlob(repo, e.Hash) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("read blob %s/%s: %w", pathPrefix, e.Name, err) + } + newBytes := checkpoint.RedactBlobBytes(ctx, content, e.Name, true) + newHash, err := checkpoint.CreateBlobFromContent(repo, newBytes) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("write redacted blob %s/%s: %w", pathPrefix, e.Name, err) + } + entries = append(entries, object.TreeEntry{Name: e.Name, Mode: e.Mode, Hash: newHash}) + if e.Name == paths.TranscriptFileName { + newFullJSONLHash = newHash + } + } + default: + entries = append(entries, e) + } + } + + for _, d := range deferredHashes { + if newFullJSONLHash.IsZero() { + continue // no transcript in this dir; keep original hash + } + jsonlBytes, err := readBlob(repo, newFullJSONLHash) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("read new transcript for content_hash: %w", err) + } + sum := sha256.Sum256(jsonlBytes) + hashBlob, err := checkpoint.CreateBlobFromContent(repo, []byte(fmt.Sprintf("sha256:%x", sum))) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("write content_hash: %w", err) + } + entries[d.idx] = object.TreeEntry{Name: d.entryName, Mode: d.entryMode, Hash: hashBlob} + } + + newTree := &object.Tree{Entries: entries} + obj := repo.Storer.NewEncodedObject() + if err := newTree.Encode(obj); err != nil { + return plumbing.ZeroHash, fmt.Errorf("encode tree: %w", err) + } + hash, err := repo.Storer.SetEncodedObject(obj) + if err != nil { + return plumbing.ZeroHash, fmt.Errorf("store tree: %w", err) + } + return hash, nil +} + +// shouldDescend reports whether the walker should recurse into a +// directory at path. With an empty shardPath we descend everywhere +// (bootstrap fallback). Otherwise we descend only into the target +// shard, its ancestors (so we can reach it), and its descendants. +func shouldDescend(path, shardPath string) bool { + if shardPath == "" || path == "" { + // shardPath="" means "no scoping" (bootstrap fallback); + // path=="" is the root, which is the ancestor of every shard. + return true + } + if path == shardPath { + return true + } + // ancestor of shardPath: shardPath starts with path + "/" + if strings.HasPrefix(shardPath+"/", path+"/") { + return true + } + // descendant of shardPath: path starts with shardPath + "/" + return strings.HasPrefix(path+"/", shardPath+"/") +} + +// insideShard reports whether file blobs at pathPrefix should be +// redacted. Empty shardPath means "redact everywhere"; otherwise the +// path must equal shardPath or be a descendant of it. +func insideShard(pathPrefix, shardPath string) bool { + if shardPath == "" { + return true + } + if pathPrefix == shardPath { + return true + } + return strings.HasPrefix(pathPrefix+"/", shardPath+"/") +} + +func readBlob(repo *git.Repository, hash plumbing.Hash) ([]byte, error) { + blob, err := repo.BlobObject(hash) + if err != nil { + return nil, fmt.Errorf("blob: %w", err) + } + r, err := blob.Reader() + if err != nil { + return nil, fmt.Errorf("blob reader: %w", err) + } + defer func() { _ = r.Close() }() + data, err := io.ReadAll(r) + if err != nil { + return nil, fmt.Errorf("blob read: %w", err) + } + return data, nil +} + +// atomicSetV1Ref CAS-updates the local v1 ref. A concrete +// ErrReferenceHasChanged from the storer means another worktree +// advanced the ref during our rewrite — return V1RefMovedError so the +// hook aborts the push. Other errors (I/O, packed-ref locks, storage +// bugs) get wrapped as-is so they aren't misreported as concurrency +// failures. +func atomicSetV1Ref(repo *git.Repository, expectedOld, newHash plumbing.Hash) error { + refName := plumbing.NewBranchReferenceName(paths.MetadataBranchName) + err := repo.Storer.CheckAndSetReference( + plumbing.NewHashReference(refName, newHash), + plumbing.NewHashReference(refName, expectedOld), + ) + if err == nil { + return nil + } + if errors.Is(err, storage.ErrReferenceHasChanged) { + actual := plumbing.ZeroHash + if cur, refErr := repo.Reference(refName, true); refErr == nil { + actual = cur.Hash() + } + return &V1RefMovedError{Expected: expectedOld, Actual: actual} + } + return fmt.Errorf("set v1 ref: %w", err) +} diff --git a/cmd/entire/cli/strategy/manual_commit_opf_rewrite_test.go b/cmd/entire/cli/strategy/manual_commit_opf_rewrite_test.go new file mode 100644 index 0000000000..358f4771cd --- /dev/null +++ b/cmd/entire/cli/strategy/manual_commit_opf_rewrite_test.go @@ -0,0 +1,477 @@ +package strategy + +import ( + "context" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/entireio/cli/cmd/entire/cli/checkpoint" + "github.com/entireio/cli/cmd/entire/cli/checkpoint/id" + "github.com/entireio/cli/cmd/entire/cli/paths" + "github.com/entireio/cli/cmd/entire/cli/testutil" + "github.com/entireio/cli/cmd/entire/cli/trailers" + "github.com/entireio/cli/redact" + "github.com/go-git/go-git/v6" + "github.com/go-git/go-git/v6/plumbing" + "github.com/go-git/go-git/v6/plumbing/filemode" + "github.com/go-git/go-git/v6/plumbing/object" + "github.com/stretchr/testify/require" +) + +// fakeOPFForRewrite tags any occurrence of "PERSONABC" as private_person. +// Deterministic + offline; real OPF inference is not needed to exercise +// the rewrite plumbing. +type fakeOPFForRewrite struct{} + +func (f *fakeOPFForRewrite) Redact(_ context.Context, text string, _ []string) ([]redact.Span, error) { + return findSentinelSpans(text), nil +} + +func (f *fakeOPFForRewrite) RedactBatch(_ context.Context, inputs []string, _ []string) ([][]redact.Span, error) { + out := make([][]redact.Span, len(inputs)) + for i, in := range inputs { + out[i] = findSentinelSpans(in) + } + return out, nil +} + +func findSentinelSpans(s string) []redact.Span { + const sentinel = "PERSONABC" + var spans []redact.Span + for idx := 0; ; { + hit := strings.Index(s[idx:], sentinel) + if hit < 0 { + break + } + start := idx + hit + end := start + len(sentinel) + spans = append(spans, redact.Span{Start: start, End: end, Label: "private_person"}) + idx = end + } + return spans +} + +// fakeRuntimeAlwaysFails trips the OPF circuit breaker on first call. +// Used to test the fail-closed assertion that breaker-trip during +// rewrite aborts before CAS. +type fakeRuntimeAlwaysFails struct{} + +func (f *fakeRuntimeAlwaysFails) Redact(_ context.Context, _ string, _ []string) ([]redact.Span, error) { + return nil, errors.New("simulated OPF runtime failure") +} +func (f *fakeRuntimeAlwaysFails) RedactBatch(_ context.Context, _ []string, _ []string) ([][]redact.Span, error) { + return nil, errors.New("simulated OPF runtime failure") +} + +// testOPFRuntime is the structural interface the redact package's +// ConfigurePrivacyFilterWithRuntime accepts. Mirrors redact.opfRuntime +// (unexported, can't be named directly from this package). +type testOPFRuntime interface { + Redact(ctx context.Context, text string, categories []string) ([]redact.Span, error) + RedactBatch(ctx context.Context, inputs []string, categories []string) ([][]redact.Span, error) +} + +// configureFakeOPF resets state and wires the given runtime as the +// process-global OPF. +func configureFakeOPF(t *testing.T, rt testOPFRuntime) { + t.Helper() + redact.ResetOPFConfigForTest() + t.Cleanup(redact.ResetOPFConfigForTest) + redact.ConfigurePrivacyFilterWithRuntime(redact.OPFConfig{ + Enabled: true, + Categories: map[string]bool{"private_person": true}, + Command: "/tmp/test-opf", + }, rt) +} + +// setupV1Repo creates a repo + one v1 checkpoint with "PERSONABC" in +// both the transcript and prompt. Returns the repo and the v1 tip. +func setupV1Repo(t *testing.T) (*git.Repository, plumbing.Hash) { + t.Helper() + tempDir := t.TempDir() + testutil.InitRepo(t, tempDir) + repo, err := git.PlainOpen(tempDir) + require.NoError(t, err) + + wt, err := repo.Worktree() + require.NoError(t, err) + require.NoError(t, os.WriteFile(filepath.Join(tempDir, "README.md"), []byte("# Test"), 0o644)) + _, err = wt.Add("README.md") + require.NoError(t, err) + _, err = wt.Commit("Initial commit", &git.CommitOptions{ + Author: &object.Signature{Name: "Test", Email: "test@test.com"}, + }) + require.NoError(t, err) + + store := checkpoint.NewGitStore(repo) + cpID := id.MustCheckpointID("a1b2c3d4e5f6") + require.NoError(t, store.WriteCommitted(context.Background(), checkpoint.WriteCommittedOptions{ + CheckpointID: cpID, + SessionID: "test-session", + Strategy: "manual-commit", + Transcript: redact.AlreadyRedacted([]byte(`{"role":"user","content":"Hello, PERSONABC asked"}` + "\n")), + Prompts: []string{"Look up PERSONABC"}, + AuthorName: "Test", + AuthorEmail: "test@test.com", + })) + ref, err := repo.Reference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), true) + require.NoError(t, err) + return repo, ref.Hash() +} + +// makeOrphanCommit writes a single v1 commit (no parents = orphan). +// Used by edge-case tests that need many cheap commits or commits +// with unrelated histories. +func makeOrphanCommit(t *testing.T, repo *git.Repository, treeHash plumbing.Hash, parents []plumbing.Hash, message string) plumbing.Hash { + t.Helper() + sig := &object.Signature{Name: "Test", Email: "test@test.com"} + c := &object.Commit{Author: *sig, Committer: *sig, Message: message, TreeHash: treeHash, ParentHashes: parents} + obj := repo.Storer.NewEncodedObject() + require.NoError(t, c.Encode(obj)) + hash, err := repo.Storer.SetEncodedObject(obj) + require.NoError(t, err) + return hash +} + +// emptyTreeHash writes (or resolves) git's well-known empty tree. +func emptyTreeHash(t *testing.T, repo *git.Repository) plumbing.Hash { + t.Helper() + obj := repo.Storer.NewEncodedObject() + require.NoError(t, (&object.Tree{}).Encode(obj)) + hash, err := repo.Storer.SetEncodedObject(obj) + require.NoError(t, err) + return hash +} + +// buildOrphanChain builds n linear orphan commits on v1 with the +// empty tree. Returns the tip. Useful for testing bootstrap/limit paths +// where the only thing that matters is commit count. +func buildOrphanChain(t *testing.T, repo *git.Repository, n int) plumbing.Hash { + t.Helper() + tree := emptyTreeHash(t, repo) + var parent, tip plumbing.Hash + for i := range n { + var parents []plumbing.Hash + if !parent.IsZero() { + parents = []plumbing.Hash{parent} + } + tip = makeOrphanCommit(t, repo, tree, parents, fmt.Sprintf("commit %d", i)) + parent = tip + } + require.NoError(t, repo.Storer.SetReference( + plumbing.NewHashReference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), tip))) + return tip +} + +// Happy path: a single unpushed unapplied commit gets rewritten, tagged +// applied, and its sentinel-bearing blobs no longer contain the sentinel. +func TestRewriteUnpushedV1WithOPF_HappyPath_RewritesAndTagsApplied(t *testing.T) { + configureFakeOPF(t, &fakeOPFForRewrite{}) + repo, originalTip := setupV1Repo(t) + + newTip, err := RewriteUnpushedV1WithOPF(context.Background(), repo, "origin") + require.NoError(t, err) + if newTip == originalTip { + t.Fatalf("rewrite returned same tip %s; expected new tip", newTip.String()[:7]) + } + + newCommit, err := repo.CommitObject(newTip) + require.NoError(t, err) + if !trailers.HasOPFApplied(newCommit.Message) { + t.Errorf("new commit missing Entire-OPF-Applied trailer:\n%s", newCommit.Message) + } + + ref, err := repo.Reference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), true) + require.NoError(t, err) + require.Equal(t, newTip, ref.Hash(), "local v1 ref should point to new tip") + + tree, err := newCommit.Tree() + require.NoError(t, err) + require.NoError(t, tree.Files().ForEach(func(f *object.File) error { + if !strings.HasSuffix(f.Name, ".jsonl") && !strings.HasSuffix(f.Name, ".txt") { + return nil + } + content, err := f.Contents() + if err != nil { + return err + } + if strings.Contains(content, "PERSONABC") { + t.Errorf("rewritten %s still contains sentinel 'PERSONABC'", f.Name) + } + return nil + })) +} + +// Idempotent re-run: a commit already tagged Entire-OPF-Applied is +// re-parented without re-redacting the tree and without duplicating +// the trailer. +func TestRewriteUnpushedV1WithOPF_SecondRun_IdempotentNoDuplicateTrailer(t *testing.T) { + configureFakeOPF(t, &fakeOPFForRewrite{}) + repo, _ := setupV1Repo(t) + + firstTip, err := RewriteUnpushedV1WithOPF(context.Background(), repo, "origin") + require.NoError(t, err) + firstCommit, err := repo.CommitObject(firstTip) + require.NoError(t, err) + require.True(t, trailers.HasOPFApplied(firstCommit.Message)) + firstTreeHash := firstCommit.TreeHash + + secondTip, err := RewriteUnpushedV1WithOPF(context.Background(), repo, "origin") + require.NoError(t, err) + + secondCommit, err := repo.CommitObject(secondTip) + require.NoError(t, err) + + wantTrailer := trailers.OPFAppliedTrailerKey + ": " + trailers.OPFAppliedTrailerValue + if count := strings.Count(secondCommit.Message, wantTrailer); count != 1 { + t.Errorf("trailer count = %d, want exactly 1\n%s", count, secondCommit.Message) + } + require.Equal(t, firstTreeHash, secondCommit.TreeHash, "applied commit tree should be preserved") +} + +// No v1 branch → no-op, no error. +func TestRewriteUnpushedV1WithOPF_NoV1Branch_ReturnsZeroHashNoError(t *testing.T) { + configureFakeOPF(t, &fakeOPFForRewrite{}) + tempDir := t.TempDir() + testutil.InitRepo(t, tempDir) + repo, err := git.PlainOpen(tempDir) + require.NoError(t, err) + + tip, err := RewriteUnpushedV1WithOPF(context.Background(), repo, "origin") + require.NoError(t, err) + require.True(t, tip.IsZero(), "expected zero hash for missing v1 ref") +} + +// Diverged remote: local has commits unreachable from remote. Refusal +// prevents silent rebase of work the remote already rejected. +func TestRewriteUnpushedV1WithOPF_DivergedRemote_ReturnsV1DivergedError(t *testing.T) { + configureFakeOPF(t, &fakeOPFForRewrite{}) + tempDir := t.TempDir() + testutil.InitRepo(t, tempDir) + repo, err := git.PlainOpen(tempDir) + require.NoError(t, err) + + tree := emptyTreeHash(t, repo) + localTip := makeOrphanCommit(t, repo, tree, nil, "local only") + remoteTip := makeOrphanCommit(t, repo, tree, nil, "remote only") + require.NoError(t, repo.Storer.SetReference( + plumbing.NewHashReference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), localTip))) + require.NoError(t, repo.Storer.SetReference( + plumbing.NewHashReference(plumbing.NewRemoteReferenceName("origin", paths.MetadataBranchName), remoteTip))) + + _, err = RewriteUnpushedV1WithOPF(context.Background(), repo, "origin") + var diverged *V1DivergedError + require.ErrorAs(t, err, &diverged) + require.Equal(t, localTip, diverged.Local) + require.Equal(t, remoteTip, diverged.Remote) +} + +// Bootstrap cap: a single table-driven test covers both the over-limit +// rejection and the unlimited-override pass paths since they share +// 90% of setup. +func TestRewriteUnpushedV1WithOPF_BootstrapCap(t *testing.T) { + cases := []struct { + name string + envLimit string + commits int + wantErr bool + wantCount int + wantLimit int + }{ + {name: "over_limit_rejected", envLimit: "2", commits: 3, wantErr: true, wantCount: 3, wantLimit: 2}, + {name: "unlimited_allows_any_size", envLimit: "unlimited", commits: 3, wantErr: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + configureFakeOPF(t, &fakeOPFForRewrite{}) + t.Setenv("ENTIRE_OPF_BOOTSTRAP_LIMIT", tc.envLimit) + + tempDir := t.TempDir() + testutil.InitRepo(t, tempDir) + repo, err := git.PlainOpen(tempDir) + require.NoError(t, err) + tip := buildOrphanChain(t, repo, tc.commits) + + newTip, err := RewriteUnpushedV1WithOPF(context.Background(), repo, "origin") + if !tc.wantErr { + require.NoError(t, err) + require.False(t, newTip.IsZero(), "expected new tip on success") + return + } + var tooLarge *BootstrapTooLargeError + require.ErrorAs(t, err, &tooLarge) + require.Equal(t, tc.wantCount, tooLarge.Count) + require.Equal(t, tc.wantLimit, tooLarge.Limit) + _ = tip // tip is the local v1 tip; on error we don't move the ref but we also don't assert here + }) + } +} + +// Shard scoping: the rewrite only touches files inside the current +// commit's own shard. Files belonging to other shards (sitting in the +// cumulative tree because git trees accumulate) are copied verbatim, +// so a push doesn't pay O(N) OPF cold-starts per commit. +func TestParseShardPathFromCommitMessage(t *testing.T) { + t.Parallel() + cases := []struct { + name, msg, want string + }{ + {name: "valid", msg: "Checkpoint: a1b2c3d4e5f6\n\nEntire-Session: s\n", want: "a1/b2c3d4e5f6"}, + {name: "trailing_space", msg: "Checkpoint: a1b2c3d4e5f6 \n", want: "a1/b2c3d4e5f6"}, + {name: "missing_prefix", msg: "Initialize sessions branch\n", want: ""}, + {name: "too_short", msg: "Checkpoint: abc123\n", want: ""}, + {name: "uppercase_rejected", msg: "Checkpoint: A1B2C3D4E5F6\n", want: ""}, + {name: "non_hex", msg: "Checkpoint: gghhiijjkkll\n", want: ""}, + {name: "empty_message", msg: "", want: ""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, tc.want, parseShardPathFromCommitMessage(tc.msg)) + }) + } +} + +// Shard-scoping invariants: descend into ancestors-of, the target, and +// descendants-of the target shard; copy everything else verbatim. +func TestShouldDescendAndInsideShard(t *testing.T) { + t.Parallel() + const shard = "a1/b2c3d4e5f6" + cases := []struct { + name string + path string + descend bool + inside bool + }{ + {name: "root_is_ancestor", path: "", descend: true, inside: false}, + {name: "shard_prefix_is_ancestor", path: "a1", descend: true, inside: false}, + {name: "shard_root_is_target", path: shard, descend: true, inside: true}, + {name: "session_subdir_is_descendant", path: shard + "/0", descend: true, inside: true}, + {name: "deeper_descendant", path: shard + "/0/tasks", descend: true, inside: true}, + {name: "sibling_shard_prefix", path: "b2", descend: false, inside: false}, + {name: "sibling_shard_full", path: "b2/c3d4e5f6a1a2", descend: false, inside: false}, + {name: "partial_overlap_not_ancestor", path: "a1/zzz", descend: false, inside: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + require.Equal(t, tc.descend, shouldDescend(tc.path, shard), "shouldDescend") + require.Equal(t, tc.inside, insideShard(tc.path, shard), "insideShard") + }) + } +} + +// TestRebuildTreeWithOPF_RedactsAllFileTypesInsideShard pins the +// fail-closed file-type policy. Previously the walker only redacted +// .jsonl / .txt / .json suffixes and copied everything else verbatim +// inside the shard — a privacy hole for any future blob type (.md +// prose, agent dumps, no-extension transcript files) that landed in a +// checkpoint shard. After P3 the policy is inverted: every regular +// file inside the shard except content_hash.txt flows through OPF. +// +// This test stands up a synthetic tree containing one .md and one +// no-extension file with the OPF sentinel, runs the walker, and +// asserts both files come out redacted. +func TestRebuildTreeWithOPF_RedactsAllFileTypesInsideShard(t *testing.T) { + configureFakeOPF(t, &fakeOPFForRewrite{}) + tempDir := t.TempDir() + testutil.InitRepo(t, tempDir) + repo, err := git.PlainOpen(tempDir) + require.NoError(t, err) + + writeBlob := func(content string) plumbing.Hash { + obj := repo.Storer.NewEncodedObject() + obj.SetType(plumbing.BlobObject) + w, err := obj.Writer() + require.NoError(t, err) + _, err = w.Write([]byte(content)) + require.NoError(t, err) + require.NoError(t, w.Close()) + hash, err := repo.Storer.SetEncodedObject(obj) + require.NoError(t, err) + return hash + } + mdHash := writeBlob("Agent transcript: PERSONABC reported the issue") + rawHash := writeBlob("PERSONABC also appeared in this no-extension blob") + hashTxtHash := writeBlob("sha256:abcd") + + // Entries must be sorted lexicographically per git's tree format. + tree := &object.Tree{Entries: []object.TreeEntry{ + {Name: paths.ContentHashFileName, Mode: filemode.Regular, Hash: hashTxtHash}, + {Name: "notes.md", Mode: filemode.Regular, Hash: mdHash}, + {Name: "transcript", Mode: filemode.Regular, Hash: rawHash}, + }} + + // Empty shardPath = "walk everything" (bootstrap fallback). With the + // inverted policy, both notes.md and transcript get redacted. + newTreeHash, err := rebuildTreeWithOPF(context.Background(), repo, tree, "", "") + require.NoError(t, err) + + newTree, err := repo.TreeObject(newTreeHash) + require.NoError(t, err) + + readEntry := func(name string) string { + for _, e := range newTree.Entries { + if e.Name != name { + continue + } + blob, err := repo.BlobObject(e.Hash) + require.NoError(t, err) + r, err := blob.Reader() + require.NoError(t, err) + defer func() { _ = r.Close() }() + data, err := io.ReadAll(r) + require.NoError(t, err) + return string(data) + } + t.Fatalf("entry %q not in rebuilt tree", name) + return "" + } + + if strings.Contains(readEntry("notes.md"), "PERSONABC") { + t.Error(".md blob inside the shard must be redacted — slipped through verbatim") + } + if strings.Contains(readEntry("transcript"), "PERSONABC") { + t.Error("no-extension blob inside the shard must be redacted — slipped through verbatim") + } + // content_hash.txt is on the deferred path: since there's no + // transcript file (named full.jsonl) in this synthetic tree, the + // deferred recomputation keeps the original hash. + if got := readEntry(paths.ContentHashFileName); got != "sha256:abcd" { + t.Errorf("content_hash.txt should be preserved when no transcript exists, got %q", got) + } +} + +// Empty shardPath = no scoping (bootstrap / unrecognized-subject +// fallback): descend everywhere, redact everywhere. +func TestShardScopeEmptyShardPathIsPermissive(t *testing.T) { + t.Parallel() + require.True(t, shouldDescend("anything/anywhere", "")) + require.True(t, insideShard("anything/anywhere", "")) + require.True(t, shouldDescend("", "")) + require.True(t, insideShard("", "")) +} + +// Fail-closed regression: when the OPF runtime fails and the breaker +// trips, the rewrite must NOT CAS the ref. Otherwise the new commits +// would carry Entire-OPF-Applied: true while their content is 7-layer +// only, and future pushes would skip them — silently shipping unredacted +// content to the remote. +func TestRewriteUnpushedV1WithOPF_BreakerTrippedMidRewrite_AbortsBeforeCAS(t *testing.T) { + configureFakeOPF(t, &fakeRuntimeAlwaysFails{}) + repo, originalTip := setupV1Repo(t) + + _, err := RewriteUnpushedV1WithOPF(context.Background(), repo, "origin") + var runtimeFail *OPFRuntimeFailedError + require.ErrorAs(t, err, &runtimeFail) + require.Contains(t, runtimeFail.OPFCommand, "test-opf", "OPFCommand should reflect configured command") + + ref, err := repo.Reference(plumbing.NewBranchReferenceName(paths.MetadataBranchName), true) + require.NoError(t, err) + require.Equal(t, originalTip, ref.Hash(), "local v1 ref must not move on OPF failure") +} diff --git a/cmd/entire/cli/strategy/manual_commit_push.go b/cmd/entire/cli/strategy/manual_commit_push.go index b80fce476c..7b07610a01 100644 --- a/cmd/entire/cli/strategy/manual_commit_push.go +++ b/cmd/entire/cli/strategy/manual_commit_push.go @@ -2,12 +2,22 @@ package strategy import ( "context" + "errors" + "log/slog" + "os" + "github.com/entireio/cli/cmd/entire/cli/logging" "github.com/entireio/cli/cmd/entire/cli/paths" "github.com/entireio/cli/cmd/entire/cli/settings" "github.com/entireio/cli/perf" + "github.com/entireio/cli/redact" ) +// errOPFAbortedByUser is returned when the user chose Abort (or pressed +// Ctrl-C) at the OPF prompt. PrePush returns it verbatim; the hook +// command propagates the non-zero exit code so git push aborts. +var errOPFAbortedByUser = errors.New("OPF prompt aborted by user; push cancelled") + // PrePush is called by the git pre-push hook before pushing to a remote. // It pushes the entire/checkpoints/v1 branch alongside the user's push (unless // v1 writes are disabled by checkpoints_version: 2), and pushes v2 refs whenever @@ -30,14 +40,84 @@ func (s *ManualCommitStrategy) PrePush(ctx context.Context, remote string) error return nil } - var err error if settings.CheckpointsVersion(ctx) != 2 { + // OPF pre-push rewrite: if OPF is configured, resolve the + // user's decision (env > settings > prompt > non-TTY auto-run), + // then re-redact unpushed v1 commits with the 8-layer pipeline + // before pushing. Skipped entirely when OPF is off, so the + // common-case fast path is unchanged. + if redact.OPFEnabled() { + cfg, _ := settings.Load(ctx) //nolint:errcheck // Load already failed at hook init; fall back to nil + var opfCfg *settings.OPFSettings + if cfg != nil && cfg.Redaction != nil { + opfCfg = cfg.Redaction.OpenAIPrivacyFilter + } + decision, decisionErr := resolveOPFDecisionForPrePush(ctx, opfCfg, os.Stderr) + if decisionErr != nil { + logging.Warn(ctx, "OPF pre-push decision failed; aborting push", + slog.String("error", decisionErr.Error()), + ) + return decisionErr + } + switch decision { + case OPFAbort: + return errOPFAbortedByUser + case OPFSkip: + // User opted out for this push (or settings/env say + // "never"). Push 7-layer content as-is. + logging.Info(ctx, "OPF skipped for this push (user choice or settings)") + case OPFRun: + _, opfSpan := perf.Start(ctx, "opf_pre_push_rewrite") + repo, repoErr := OpenRepository(ctx) + if repoErr != nil { + opfSpan.RecordError(repoErr) + opfSpan.End() + logging.Warn(ctx, "OPF pre-push: failed to open repo; aborting push", + slog.String("error", repoErr.Error()), + ) + return repoErr + } + if _, rewriteErr := RewriteUnpushedV1WithOPF(ctx, repo, ps.pushTarget()); rewriteErr != nil { + opfSpan.RecordError(rewriteErr) + opfSpan.End() + logging.Warn(ctx, "OPF pre-push rewrite failed; aborting push", + slog.String("error", rewriteErr.Error()), + ) + return rewriteErr + } + opfSpan.End() + } + } + + // Push the checkpoint branch. This is best-effort — failures here + // are logged but NOT propagated, so a transient checkpoint-push + // problem doesn't break the user's git push of their actual work. + // (OPF failures above are the exception — they're privacy-critical.) _, pushCheckpointsSpan := perf.Start(ctx, "push_checkpoints_branch") - err = pushBranchIfNeeded(ctx, ps.pushTarget(), paths.MetadataBranchName) - if err != nil { - pushCheckpointsSpan.RecordError(err) + pushErr := pushBranchIfNeeded(ctx, ps.pushTarget(), paths.MetadataBranchName) + if pushErr != nil { + pushCheckpointsSpan.RecordError(pushErr) + logging.Warn(ctx, "checkpoint branch push failed; user push continues", + slog.String("error", pushErr.Error()), + ) } pushCheckpointsSpan.End() + + // Post-push cleanup: only when the v1 push succeeded, so we + // know the condensed checkpoint data is on the remote. Failures + // here are non-fatal — shadow branches just accumulate until + // `entire clean` or the next successful push. + if pushErr == nil { + if deleted, cleanupErr := CleanupPushedShadowBranches(ctx); cleanupErr != nil { + logging.Warn(ctx, "post-push shadow branch cleanup failed", + slog.String("error", cleanupErr.Error()), + ) + } else if deleted > 0 { + logging.Info(ctx, "cleaned up vestigial shadow branches", + slog.Int("count", deleted), + ) + } + } } // Push v2 refs when enabled. @@ -47,5 +127,5 @@ func (s *ManualCommitStrategy) PrePush(ctx context.Context, remote string) error pushV2Span.End() } - return err + return nil } diff --git a/cmd/entire/cli/trailers/trailers.go b/cmd/entire/cli/trailers/trailers.go index a3c1b44036..93e4fe592d 100644 --- a/cmd/entire/cli/trailers/trailers.go +++ b/cmd/entire/cli/trailers/trailers.go @@ -48,6 +48,18 @@ const ( // AgentTrailerKey identifies the agent that created a checkpoint. // Format: human-readable agent name e.g. "Claude Code", "Cursor" AgentTrailerKey = "Entire-Agent" + + // OPFAppliedTrailerKey marks an entire/checkpoints/v1 commit whose blobs + // have been redacted by the OpenAI Privacy Filter (8-layer pipeline). + // Format: literal "true"; the trailer is omitted entirely when OPF was + // not applied. The pre-push rewrite path treats commits lacking this + // trailer as candidates to OPF-redact before they reach the remote. + OPFAppliedTrailerKey = "Entire-OPF-Applied" + + // OPFAppliedTrailerValue is the only value that means "OPF ran." Any + // other value (or trailer absence) is treated as "not applied" so a + // future "false" / "skipped" value never accidentally enables OPF. + OPFAppliedTrailerValue = "true" ) // Pre-compiled regexes for trailer parsing. @@ -59,6 +71,7 @@ var ( condensationTrailerRegex = regexp.MustCompile(CondensationTrailerKey + `:\s*(.+)`) sessionTrailerRegex = regexp.MustCompile(SessionTrailerKey + `:\s*(.+)`) checkpointTrailerRegex = regexp.MustCompile(CheckpointTrailerKey + `:\s*(` + checkpointID.Pattern + `)(?:\s|$)`) + opfAppliedTrailerRegex = regexp.MustCompile(OPFAppliedTrailerKey + `:\s*(\S+)`) ) // ParseStrategy extracts strategy from commit message. @@ -307,3 +320,31 @@ func AppendCheckpointTrailer(message, checkpointID string) string { trailer := fmt.Sprintf("%s: %s", CheckpointTrailerKey, checkpointID) return appendTrailerLine(message, trailer) } + +// HasOPFApplied reports whether the commit message carries an +// `Entire-OPF-Applied: true` trailer. Any other value (or absence) is +// treated as "OPF not applied" so the pre-push rewrite considers the +// commit a candidate for OPF redaction. Pinning the value to literal +// "true" — rather than just trailer presence — prevents a future +// "Entire-OPF-Applied: false" or "skipped" from accidentally meaning +// "yes, applied." +func HasOPFApplied(commitMessage string) bool { + matches := opfAppliedTrailerRegex.FindStringSubmatch(commitMessage) + if len(matches) < 2 { + return false + } + return strings.TrimSpace(matches[1]) == OPFAppliedTrailerValue +} + +// AppendOPFAppliedTrailer appends `Entire-OPF-Applied: true` in +// trailer-aware format. Idempotent: if the message already carries +// the trailer with value "true", the original message is returned +// unchanged so re-parenting an already-applied commit doesn't +// duplicate the trailer. +func AppendOPFAppliedTrailer(message string) string { + if HasOPFApplied(message) { + return message + } + trailer := fmt.Sprintf("%s: %s", OPFAppliedTrailerKey, OPFAppliedTrailerValue) + return appendTrailerLine(message, trailer) +} diff --git a/cmd/entire/cli/trailers/trailers_test.go b/cmd/entire/cli/trailers/trailers_test.go index 2fd0e89613..c50de8cfef 100644 --- a/cmd/entire/cli/trailers/trailers_test.go +++ b/cmd/entire/cli/trailers/trailers_test.go @@ -450,3 +450,70 @@ func TestParseCheckpoint(t *testing.T) { }) } } + +// TestHasOPFApplied covers the Entire-OPF-Applied trailer reader. The +// trailer marks a v1 commit whose blobs have been redacted by the +// OpenAI Privacy Filter (8-layer); commits without it carry 7-layer +// content and are eligible for the pre-push rewrite to add OPF. +func TestHasOPFApplied(t *testing.T) { + t.Parallel() + cases := []struct { + name string + message string + want bool + }{ + {"present_lowercase_true", "Checkpoint: a1b2c3d4e5f6\n\nEntire-OPF-Applied: true\n", true}, + {"absent", "Checkpoint: a1b2c3d4e5f6\n", false}, + {"present_among_other_trailers", "msg\n\nEntire-Session: 2026-01\nEntire-OPF-Applied: true\nEntire-Strategy: manual-commit\n", true}, + {"value_false_not_applied", "msg\n\nEntire-OPF-Applied: false\n", false}, + {"value_other_not_applied", "msg\n\nEntire-OPF-Applied: yes\n", false}, + {"empty_message", "", false}, + {"trailer_with_extra_spaces", "msg\n\nEntire-OPF-Applied: true \n", true}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + t.Parallel() + if got := HasOPFApplied(tc.message); got != tc.want { + t.Errorf("HasOPFApplied(%q) = %v, want %v", tc.message, got, tc.want) + } + }) + } +} + +// TestAppendOPFAppliedTrailer covers the formatter. Appending to a +// message without a trailer block inserts a blank line; appending to +// one with a trailer block joins directly. Idempotent — appending to +// a message that already has the trailer must not duplicate it. +func TestAppendOPFAppliedTrailer(t *testing.T) { + t.Parallel() + tests := []struct { + name string + msg string + want string + }{ + { + name: "no_existing_trailers", + msg: "Checkpoint: a1b2c3d4e5f6\n", + want: "Checkpoint: a1b2c3d4e5f6\n\nEntire-OPF-Applied: true\n", + }, + { + name: "existing_trailer_block", + msg: "Checkpoint: a1\n\nEntire-Session: s\nEntire-Strategy: manual-commit\n", + want: "Checkpoint: a1\n\nEntire-Session: s\nEntire-Strategy: manual-commit\nEntire-OPF-Applied: true\n", + }, + { + name: "idempotent_when_already_applied", + msg: "Checkpoint: a1\n\nEntire-OPF-Applied: true\n", + want: "Checkpoint: a1\n\nEntire-OPF-Applied: true\n", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := AppendOPFAppliedTrailer(tt.msg) + if got != tt.want { + t.Errorf("AppendOPFAppliedTrailer():\n got=%q\nwant=%q", got, tt.want) + } + }) + } +} diff --git a/docs/security-and-privacy.md b/docs/security-and-privacy.md index 74cdc393b8..82b2c28704 100644 --- a/docs/security-and-privacy.md +++ b/docs/security-and-privacy.md @@ -62,7 +62,7 @@ If a custom pattern itself reveals sensitive structure (e.g. an internal ID form ### Optional OpenAI Privacy Filter (`opf`) -A separate, **opt-in** layer that shells out to the [OpenAI Privacy Filter](https://github.com/openai/privacy-filter) (`opf`) — a 1.5B-parameter token-classification model that finds names, emails, phone numbers, addresses, dates, URLs, account numbers, and secrets that pure regex can miss. Disabled by default. Runs *in addition to* the seven built-in layers, only at the condensation and export boundaries (never per-turn), so the agent loop's hot path stays on the fast pipeline. +A separate, **opt-in** layer that shells out to the [OpenAI Privacy Filter](https://github.com/openai/privacy-filter) (`opf`) — a 1.5B-parameter token-classification model that finds names, emails, phone numbers, addresses, dates, URLs, account numbers, and secrets that pure regex can miss. Disabled by default. Runs *in addition to* the seven built-in layers, **only at push time** — never per-turn and never at commit time. Local commits stay on the fast 7-layer pipeline so per-commit latency is unchanged; OPF only re-redacts checkpoints right before they leave the machine via `git push`. Prerequisites: @@ -128,22 +128,101 @@ Full settings reference: - `command` — path or PATH-resolvable name of the `opf` binary. Defaults to `opf`. - `timeout_seconds` — per-invocation timeout. Defaults to `30`. +- `prompt_default` — `"ask"` (default), `"always"`, or `"never"`. Controls whether the pre-push hook surfaces an interactive prompt before running OPF. `ENTIRE_OPF=yes` or `ENTIRE_OPF=no` on a single `git push` invocation overrides this for that push only. -There is no `on_failure` setting; warn-on-failure is the only mode supported today. If OPF is not on PATH, fails to start, or times out, Entire prints a one-line `× OpenAI Privacy Filter unavailable …` notice and continues with the seven built-in layers. A per-process circuit breaker disables OPF for the remainder of the invocation after the first failure, so a broken install costs one warning rather than one timeout per redaction call. +The interactive prompt offers three options and reacts to **Ctrl-C** for cancellation: -Cost note: each shell-out loads the OPF model (~1.5B parameters on CPU). Condensation batches all eligible leaf strings into a single inference pass per scope (transcript + joined prompts), so a typical real-world commit adds ~25–30s of OPF inference rather than the multi-minute cost a per-leaf flow would incur. +``` +Run OpenAI Privacy Filter on these checkpoints? +Adds ~30s but redacts names/PII the regex layers can't catch. +Ctrl-C to cancel the push. + + ▸ Yes — run OPF this push + No — skip OPF, push as-is + Always — run OPF on every push from now on +``` + +- **Yes** runs OPF for this push only. +- **No** skips OPF for this push only; the 7-layer-redacted content reaches the remote. +- **Always** runs OPF this push AND writes `prompt_default: "always"` to `.entire/settings.local.json` so future pushes don't ask. +- **Ctrl-C** aborts the push entirely — `git push` exits non-zero, no refs go to the remote. + +Non-interactive contexts (CI, scripted pipes with no TTY) skip the prompt and run OPF automatically when enabled, printing `→ OpenAI Privacy Filter: scanning checkpoints…` to stderr so the wait isn't silent. Set `ENTIRE_OPF=no` to skip OPF in those contexts without disabling the feature globally. + +**CI consideration**: if you've enabled OPF locally and your CI runs `git push` (e.g. an agent-driven workflow), the CI push will attempt to run OPF too. If the `opf` binary isn't installed in CI, the push will abort with `OPFRuntimeFailedError` rather than silently shipping under-redacted content — by design, since "I enabled OPF" should mean "no content leaves my machines without OPF." The remedies are (a) install `opf` in CI, (b) set `ENTIRE_OPF=no` for CI pushes, or (c) set `prompt_default: "never"` if you only want OPF on interactive pushes. + +OPF failures at push time are **fail-closed**: if OPF is not on PATH, fails to start, or times out during the pre-push rewrite, the per-process circuit breaker trips and the rewrite aborts the push with `OPF runtime failed; aborting push`. Nothing reaches the remote. The intent is that "the user enabled OPF" means "I do not want unredacted content leaving this machine" — falling back to 7-layer silently on the push path would violate that contract. Fix the install or set `ENTIRE_OPF=no` for a one-off push. + +(The circuit breaker is per-process, so a broken install costs one warning instead of one timeout per blob — but the push still aborts.) + +Cost note: each shell-out loads the OPF model (~1.5B parameters on CPU). The pre-push rewrite batches all eligible leaf strings into a single inference pass per scope (transcript + joined prompts), so a typical real-world push adds ~25–30s of OPF inference rather than the multi-minute cost a per-leaf flow would incur. Per-commit latency is unaffected because OPF doesn't run at commit time. + +#### When OPF actually runs + +OPF execution lives in the pre-push hook. The flow: + +1. **Post-commit** writes the checkpoint with **7-layer-only** redaction to your local `entire/checkpoints/v1` branch. Fast, predictable, no OPF cost on the hot path. +2. **Pre-push** (`git push`): if OPF is enabled, the hook re-reads each unpushed `entire/checkpoints/v1` commit, runs the OpenAI Privacy Filter over its blobs to add the categories the regex layers don't catch (person names, addresses, etc.), and builds **new commits** carrying an `Entire-OPF-Applied: true` trailer. The local v1 ref fast-forwards atomically to the new tip, and the (now 8-layer-redacted) commits are what get pushed. +3. The original 7-layer-only commits become **unreachable** in the local git object database and eventually get swept by `git gc`. + +This means: + +- **The remote only ever sees 8-layer-redacted content** when OPF is enabled. +- **Local-only commits are 7-layer-redacted** until the moment you push. If you never push, OPF never runs. +- **Re-running pre-push is idempotent** — commits already carrying the trailer get re-parented into the chain but are not re-redacted. + +#### Force-pushed remote, bootstrap, and concurrent pushes + +The rewrite refuses to proceed and aborts the push when it detects a divergent state, so checkpoints on the remote are never silently rebased: + +- **Diverged remote**: if local `entire/checkpoints/v1` has commits that aren't ancestors of `/entire/checkpoints/v1`, the hook exits with a `entire/checkpoints/v1 has diverged from remote` error. Fetch the remote and either reset local v1 to the remote tip or resolve manually before pushing. +- **Bootstrap** (remote has no `entire/checkpoints/v1` yet, e.g. first push): the cap is `100` unpushed commits by default. Override via the env var on the push invocation: + + ```fish + set -x ENTIRE_OPF_BOOTSTRAP_LIMIT 500; git push + # or fully unbounded: + set -x ENTIRE_OPF_BOOTSTRAP_LIMIT unlimited; git push + ``` + +- **Concurrent push** from another worktree: the rewrite uses a CAS to update the local v1 ref. If another process moved the ref while OPF was running, the hook exits with a "concurrent push detected" error and `git push` aborts the whole batch. Fetch and retry. + +#### Persistence of un-redacted-by-OPF content + +Three places retain content that OPF *didn't* redact, with different lifetimes. Understanding them matters if your threat model goes beyond "what reaches the remote": + +| Location | Redaction level | Lifetime | Reaches remote? | +|---|---|---|---| +| `.entire/.jsonl` | **None — raw** | Until session is deleted (managed by the agent) | No | +| Shadow branch `entire/-` | 7-layer | Auto-deleted after the next successful push (only when its session has ended cleanly) | No | +| Unreachable git objects after pre-push rewrite | 7-layer | Until `git gc --prune` (default `gc.pruneExpire` is 2 weeks) | No | +| Reflog `git reflog show entire/checkpoints/v1` | 7-layer tips | Default `gc.reflogExpire` is 90 days | No | +| `/entire/checkpoints/v1` | 8-layer (after OPF rewrite) | Until you delete the branch on the remote | Yes | + +The `.entire/.jsonl` files are raw working state owned by the agent (Claude Code, etc.) — Entire reads from them but does not redact them in place, because the agent is reading and writing them continuously and editing under the agent's feet would corrupt the session. + +To aggressively scrub the unreachable git objects from the pre-push rewrite (instead of waiting for the 2-week GC window): + +```fish +git reflog expire --expire-unreachable=now refs/heads/entire/checkpoints/v1 +git gc --prune=now +``` + +This is I/O-heavy on large repositories; it's not run automatically. If you want it as part of your push workflow, wrap `git push` in a script that invokes it after a successful push. Verifying it's working: ```fish # After enabling OPF, run an agent turn that includes a name in the prompt, # e.g. "Create notes.txt with: Alice Johnson reviewed the proposal." -# Then commit and inspect the latest checkpoint: +# Commit (this stays on the fast 7-layer pipeline), then push. OPF runs +# during the pre-push step: +git commit -m "demo" +git push # → "→ OpenAI Privacy Filter: scanning N checkpoints (~30s)…" git log --oneline entire/checkpoints/v1 | head -2 entire checkpoint explain HEAD | grep -i 'REDACTED_PERSON' ``` -If `[REDACTED_PERSON]` appears in the prompt or transcript section, OPF is active. +If `[REDACTED_PERSON]` appears in the prompt or transcript section and the latest `entire/checkpoints/v1` commit carries `Entire-OPF-Applied: true`, OPF is active. ### Recommendations diff --git a/redact/export_test.go b/redact/export_test.go deleted file mode 100644 index 98bc31c120..0000000000 --- a/redact/export_test.go +++ /dev/null @@ -1,21 +0,0 @@ -package redact - -// This file exposes internal helpers to redact-package tests only. -// It is excluded from regular builds (the _test.go suffix is invisible to -// importers), so production code outside this package cannot reach the -// global OPF config. Tests in other packages that need to manipulate the -// global state can do so via thin re-exports declared in their own -// _test.go files, but those re-exports stay this side of the package -// boundary. - -// GetOPFConfigForTest returns the current OPF configuration, or nil if -// never configured. -func GetOPFConfigForTest() *OPFConfig { - return getOPFConfig() -} - -// ResetOPFConfigForTest clears configuration and the circuit breaker. -// Used to return to a "never configured" state between test cases. -func ResetOPFConfigForTest() { - resetOPFConfig() -} diff --git a/redact/opf.go b/redact/opf.go index 8b80fa0822..8b7fbf1d23 100644 --- a/redact/opf.go +++ b/redact/opf.go @@ -57,7 +57,7 @@ func ConfigurePrivacyFilter(cfg OPFConfig) { cfgCopy.Timeout = 30 } if cfgCopy.Command == "" { - cfgCopy.Command = "opf" + cfgCopy.Command = defaultOPFCommand } cfgCopy.runtime = newShellOut(cfgCopy.Command, cfgCopy.Timeout) opfConfigMu.Lock() @@ -86,6 +86,46 @@ func getOPFConfig() *OPFConfig { return opfConfig } +// OPFEnabled reports whether the OpenAI Privacy Filter is configured +// and turned on for this process. Callers gate pre-push rewrite work +// on this: when false, the pre-push hook pushes the local 7-layer +// checkpoint branch verbatim with no extra processing. Independent of +// the circuit breaker — a tripped breaker still reports Enabled=true +// because the runtime config didn't change; the rewrite logic itself +// handles the breaker by short-circuiting per-commit OPF calls. +func OPFEnabled() bool { + cfg := getOPFConfig() + return cfg != nil && cfg.Enabled +} + +// OPFBreakerTripped reports whether the per-process OPF circuit breaker +// has been tripped — i.e. an OPF invocation failed at some point during +// this process's lifetime. The pre-push rewrite uses this to detect +// when OPF silently fell back to 7-layer mid-rewrite and abort before +// CAS-ing the new ref; otherwise the rewritten commits would carry the +// Entire-OPF-Applied: true trailer despite containing only 7-layer +// content, and the next push would skip them. +func OPFBreakerTripped() bool { + return opfBreakerTripped.Load() +} + +// defaultOPFCommand is the binary name we resolve via $PATH when the +// user hasn't pinned a specific path in settings. Used as the fallback +// inside ConfigurePrivacyFilter and as the OPFCommand() default for +// error messages. +const defaultOPFCommand = "opf" + +// OPFCommand returns the configured OPF binary command, or the default +// when OPF is unconfigured. Used by error messages so the user sees the +// exact command they need to fix. +func OPFCommand() string { + cfg := getOPFConfig() + if cfg == nil || cfg.Command == "" { + return defaultOPFCommand + } + return cfg.Command +} + func resetOPFConfig() { opfConfigMu.Lock() opfConfig = nil diff --git a/redact/opf_test.go b/redact/opf_test.go index c9e8168521..98b7452c9e 100644 --- a/redact/opf_test.go +++ b/redact/opf_test.go @@ -479,3 +479,53 @@ func TestJSONLContentWithPrivacyFilter_FallsBackOnBatchError(t *testing.T) { t.Error("fallback should still return non-empty content") } } + +// shortReturnRuntime returns FEWER span slices than inputs — a runtime +// contract violation that would silently leave the tail leaves +// un-redacted under the old "log warning + proceed" behavior. The fix +// trips the circuit breaker so the pre-push rewrite aborts before +// commits are tagged Entire-OPF-Applied: true. +type shortReturnRuntime struct{} + +func (r *shortReturnRuntime) Redact(_ context.Context, _ string, _ []string) ([]Span, error) { + return nil, nil +} + +func (r *shortReturnRuntime) RedactBatch(_ context.Context, inputs []string, _ []string) ([][]Span, error) { + if len(inputs) == 0 { + return nil, nil + } + // Return exactly ONE span slice regardless of input count — the + // violation we're testing for. + return [][]Span{nil}, nil +} + +// TestJSONLContentWithPrivacyFilter_ShortReturnTripsBreaker pins the +// privacy contract: if the OPF runtime returns fewer span slices than +// inputs, we treat it as a runtime failure (trip the breaker + 7-layer +// fallback) rather than silently produce under-redacted output. The +// per-blob caller in the pre-push rewrite then catches the tripped +// breaker via OPFBreakerTripped() and aborts before CAS. +func TestJSONLContentWithPrivacyFilter_ShortReturnTripsBreaker(t *testing.T) { + resetOPFConfig() + t.Cleanup(resetOPFConfig) + origStderr := opfStderr + opfStderr = io.Discard + t.Cleanup(func() { opfStderr = origStderr }) + + ConfigurePrivacyFilterWithRuntime(OPFConfig{ + Enabled: true, + Categories: map[string]bool{"private_person": true}, + }, &shortReturnRuntime{}) + + // Three distinct leaves — the fake will return ONE span slice instead + // of three, triggering the short-return path. + content := `{"a":"Alice met Bob","b":"Charlie sat down","c":"Eve walked home"}` + _, err := JSONLContentWithPrivacyFilter(context.Background(), content) + if err != nil { + t.Fatalf("short return should fall back to 7-layer (no error), got %v", err) + } + if !opfBreakerTripped.Load() { + t.Error("short return must trip the OPF breaker so the rewrite's post-loop check aborts the push") + } +} diff --git a/redact/redact.go b/redact/redact.go index 4d32f8e057..aed74330fc 100644 --- a/redact/redact.go +++ b/redact/redact.go @@ -6,7 +6,6 @@ import ( "encoding/json" "fmt" "io" - "log/slog" "math" "net/url" "regexp" @@ -112,49 +111,6 @@ func AlreadyRedacted(data []byte) RedactedBytes { return RedactedBytes{data: data} } -// RedactedJoinedPrompts is the pre-redacted joined-prompts blob written to -// a checkpoint's prompt.txt. The private field can only be populated by -// RedactJoinedPrompts (which runs the full 8-layer pipeline) or -// AlreadyRedactedJoinedPrompts (trusted source) — callers cannot bypass -// redaction by assigning an arbitrary string to the field. The zero value -// signals "not pre-redacted" so writers fall back to the safety-net pass. -type RedactedJoinedPrompts struct { - content string -} - -// String returns the redacted blob content. Empty when the value is the -// zero value (no pre-redaction was performed). -func (r RedactedJoinedPrompts) String() string { - return r.content -} - -// IsSet reports whether a redaction pass actually populated this value. -// An empty content string is treated as "not set" so the writer's safety -// net can re-run rather than persist an empty prompt.txt. -func (r RedactedJoinedPrompts) IsSet() bool { - return r.content != "" -} - -// JoinedPrompts joins prompts with sep and runs the full 8-layer pipeline -// (including the OpenAI Privacy Filter when configured) over the result. -// This is the only standard way to produce a RedactedJoinedPrompts from -// raw prompt input — the resulting type carries a compile-time claim -// that the content has been through the pipeline. Mirrors the -// noun-named String/Bytes/JSONLBytes constructors in this package. -func JoinedPrompts(ctx context.Context, prompts []string, sep string) RedactedJoinedPrompts { - if len(prompts) == 0 { - return RedactedJoinedPrompts{} - } - return RedactedJoinedPrompts{content: StringWithPrivacyFilter(ctx, strings.Join(prompts, sep))} -} - -// AlreadyRedactedJoinedPrompts wraps content known to already be redacted -// by a prior write path or controlled test fixture. Use only for trusted -// sources; for fresh prompt input, use RedactJoinedPrompts. -func AlreadyRedactedJoinedPrompts(content string) RedactedJoinedPrompts { - return RedactedJoinedPrompts{content: content} -} - var ( betterleaksDetector *detect.Detector betterleaksDetectorOnce sync.Once @@ -704,17 +660,23 @@ func JSONLContentWithPrivacyFilter(ctx context.Context, content string) (string, return jsonlContentImpl(content, String) } fmt.Fprintf(opfStderr, "✓ OpenAI Privacy Filter: done (%.1fs)\n", time.Since(start).Seconds()) - if len(batched) < len(inputs) { - slog.Warn("OPF runtime returned fewer span slices than inputs", - slog.String("component", "redaction"), - slog.Int("inputs", len(inputs)), - slog.Int("returned", len(batched)), - ) + // A short return means the runtime gave us fewer span slices than + // inputs — the tail leaves would receive zero OPF spans and the + // caller would proceed as if OPF had found nothing. That silently + // produces under-redacted output and is indistinguishable from a + // "no PII present" result. Treat as a runtime contract violation: + // trip the breaker so the pre-push rewrite's post-loop + // OPFBreakerTripped() check aborts before the Entire-OPF-Applied + // trailer can be attached to under-redacted commits. The production + // shell-out always returns len(inputs), so this only fires for a + // misbehaving custom runtime — but the cost of leaving it dormant + // is too high for a privacy contract. + if len(batched) != len(inputs) { + shortErr := fmt.Errorf("opf runtime returned %d span slices for %d inputs", len(batched), len(inputs)) + handleOPFFailure(ctx, cfg, shortErr) + return jsonlContentImpl(content, String) } for i, in := range inputs { - if i >= len(batched) { - break - } spansByInput[in] = batched[i] } } diff --git a/redact/testhelpers.go b/redact/testhelpers.go new file mode 100644 index 0000000000..af375fcb19 --- /dev/null +++ b/redact/testhelpers.go @@ -0,0 +1,11 @@ +package redact + +// Cross-package test helpers. Lives in a regular .go file (not +// export_test.go) so tests in cmd/entire/cli/strategy can call it. +// The "ForTest" suffix is the production-code-must-not-call signal. + +// ResetOPFConfigForTest clears OPF configuration and the circuit +// breaker. Test-only. +func ResetOPFConfigForTest() { + resetOPFConfig() +}