From f4745d0572130a69a31e0d78ccdd96127d77bd12 Mon Sep 17 00:00:00 2001 From: Grivn Date: Tue, 27 Oct 2026 03:17:58 +0800 Subject: [PATCH] fix(memory): verify embedding models and report recall fallback Probe the selected model in embed --status while retaining the legacy provider availability field. Report failed query embeddings on stderr and continue recall without vector similarity, preserving JSON output and existing request counts. Cover missing and invalid models, provider failures, model overrides, stored coverage, and all recall projections with local HTTP regressions. Verified make test, the product build, and a synthetic CLI fallback/recovery smoke test. --- cmd/memory/embed.go | 11 ++- cmd/memory/embed_test.go | 101 +++++++++++++++++++++++++ cmd/memory/recall.go | 5 +- cmd/memory/recall_embedding_test.go | 111 ++++++++++++++++++++++++++++ docs/USAGE.md | 14 +++- docs/zh/USAGE.md | 9 +++ 6 files changed, 247 insertions(+), 4 deletions(-) create mode 100644 cmd/memory/embed_test.go create mode 100644 cmd/memory/recall_embedding_test.go diff --git a/cmd/memory/embed.go b/cmd/memory/embed.go index 5ce6c9523..337fdbe5c 100644 --- a/cmd/memory/embed.go +++ b/cmd/memory/embed.go @@ -46,12 +46,19 @@ Modes: return fmt.Errorf("embedding stats: %w", err) } available := ec.Available() + embeddingAvailable := false + if available { + // Discovery can succeed even when the configured model is missing + // or cannot embed. Keep this extra probe out of normal recall. + _, err := ec.Embed("availability probe") + embeddingAvailable = err == nil + } output := map[string]interface{}{ "total_insights": total, "embedded": embedded, "coverage": fmt.Sprintf("%.0f%%", float64(embedded)/float64(max(total, 1))*100), - "embedding_available": available, - "ollama_available": available, // Backward-compatible alias. + "embedding_available": embeddingAvailable, + "ollama_available": available, // Legacy provider-reachability field. "protocol": ec.Protocol(), "model": ec.Model(), } diff --git a/cmd/memory/embed_test.go b/cmd/memory/embed_test.go new file mode 100644 index 000000000..85492dd30 --- /dev/null +++ b/cmd/memory/embed_test.go @@ -0,0 +1,101 @@ +package memory + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "sync/atomic" + "testing" + + "github.com/mnemon-dev/mnemon/internal/memory/embed" + "github.com/mnemon-dev/mnemon/internal/memory/store" +) + +func TestEmbedStatusChecksConfiguredModel(t *testing.T) { + for _, tc := range []struct { + name, protocol, body string + discoveryStatus, embedStatus int + wantProvider, wantEmbedding bool + wantProbes int32 + }{ + {"ollama ready", "ollama", `{"embeddings":[[1,0]]}`, 200, 200, true, true, 1}, + {"ollama missing model", "ollama", `{"error":"model is missing"}`, 200, 404, true, false, 1}, + {"ollama empty vector", "ollama", `{"embeddings":[[]]}`, 200, 200, true, false, 1}, + {"ollama unavailable", "ollama", "", 503, 503, false, false, 0}, + {"openai ready", "openai", `{"data":[{"embedding":[1,0]}]}`, 200, 200, true, true, 1}, + {"openai missing model", "openai", `{"error":"model is missing"}`, 200, 404, true, false, 1}, + {"openai malformed vector", "openai", `{"data":[{"embedding":"invalid"}]}`, 200, 200, true, false, 1}, + {"openai without discovery", "openai", `{"data":[{"embedding":[1,0]}]}`, 404, 200, true, true, 2}, + } { + t.Run(tc.name, func(t *testing.T) { + oldDir, oldStore, oldReadOnly := dataDir, storeName, readOnly + oldStatus, oldModel := embedStatus, embedModel + t.Cleanup(func() { + dataDir, storeName, readOnly = oldDir, oldStore, oldReadOnly + embedStatus, embedModel = oldStatus, oldModel + }) + dataDir, storeName, readOnly = t.TempDir(), "embedding-status", false + embedStatus, embedModel = true, "selected-model" + t.Setenv("MNEMON_EMBED_PROTOCOL", tc.protocol) + t.Setenv("MNEMON_EMBED_DIMENSIONS", "") + t.Setenv("MNEMON_EMBED_API_KEY", "synthetic-key") + t.Setenv("MNEMON_EMBED_MODEL", "ignored-env-model") + var probes atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.Method == http.MethodGet { + w.WriteHeader(tc.discoveryStatus) + _, _ = w.Write([]byte(`{"models":[{"name":"different-model:latest"}]}`)) + return + } + probes.Add(1) + var request struct{ Model, Input string } + if err := json.NewDecoder(r.Body).Decode(&request); err != nil { + t.Errorf("decode embedding probe: %v", err) + } + if request.Model != "selected-model" || request.Input != "availability probe" { + t.Errorf("probe must use configured model and synthetic input: %+v", request) + } + w.WriteHeader(tc.embedStatus) + _, _ = w.Write([]byte(tc.body)) + })) + defer srv.Close() + t.Setenv("MNEMON_EMBED_ENDPOINT", srv.URL) + + db, err := store.Open(store.StoreDir(dataDir, storeName)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + insertTestInsight(t, db, "status-seed", "stored memory must not be sent by status", "test", "2026-01-01T00:00:00Z") + if err := db.UpdateEmbedding("status-seed", embed.SerializeVector([]float64{1, 0})); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + var runErr error + out := captureStdout(t, func() { runErr = embedCmd.RunE(embedCmd, nil) }) + if runErr != nil { + t.Fatal(runErr) + } + var result struct { + EmbeddingAvailable bool `json:"embedding_available"` + OllamaAvailable bool `json:"ollama_available"` + Model string `json:"model"` + Coverage string `json:"coverage"` + } + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("decode status: %v\n%s", err, out) + } + if result.EmbeddingAvailable != tc.wantEmbedding || result.OllamaAvailable != tc.wantProvider || result.Model != "selected-model" { + t.Errorf("unexpected embedding status: %s", out) + } + if result.Coverage != "100%" { + t.Errorf("model availability must not alter stored embedding coverage: %s", out) + } + if got := probes.Load(); got != tc.wantProbes { + t.Errorf("embedding probes = %d, want %d", got, tc.wantProbes) + } + }) + } +} diff --git a/cmd/memory/recall.go b/cmd/memory/recall.go index 7b0b88ab1..b19f6a76c 100644 --- a/cmd/memory/recall.go +++ b/cmd/memory/recall.go @@ -184,7 +184,10 @@ meta.intent and meta.intent_source (auto or override). --basic bypasses intent.` var queryVec []float64 ec := embed.NewClientWithModel(resolveEmbedModel()) if ec.Available() { - queryVec, _ = ec.Embed(keyword) + queryVec, err = ec.Embed(keyword) + if err != nil { + fmt.Fprintf(cmd.ErrOrStderr(), "warning: query embedding failed for model %q; continuing without vector similarity (check --embed-model or MNEMON_EMBED_MODEL and run 'mnemon embed --status')\n", ec.Model()) + } } // Extract query entities at cmd layer (avoid graph->search circular dep). diff --git a/cmd/memory/recall_embedding_test.go b/cmd/memory/recall_embedding_test.go new file mode 100644 index 000000000..8b72b43c9 --- /dev/null +++ b/cmd/memory/recall_embedding_test.go @@ -0,0 +1,111 @@ +package memory + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "sync/atomic" + "testing" + + "github.com/mnemon-dev/mnemon/internal/memory/embed" + "github.com/mnemon-dev/mnemon/internal/memory/search" + "github.com/mnemon-dev/mnemon/internal/memory/store" +) + +func TestRecallReportsEmbeddingFailure(t *testing.T) { + for _, tc := range []struct { + name, body string + status int + wantWarning bool + }{ + {"missing model", `{"error":"synthetic-secret and private-provider-echo"}`, 404, true}, + {"invalid vector", `{"embeddings":[[]]}`, 200, true}, + {"healthy model", `{"embeddings":[[1,0]]}`, 200, false}, + } { + t.Run(tc.name, func(t *testing.T) { + oldDir, oldStore, oldReadOnly, oldModel := dataDir, storeName, readOnly, embedModel + oldBasic, oldBrief, oldVerbose, oldLimit := recBasic, recBrief, recVerbose, recLimit + oldIntent, oldExcerpt := recIntent, recExcerpt + oldErr := recallCmd.ErrOrStderr() + t.Cleanup(func() { + dataDir, storeName, readOnly, embedModel = oldDir, oldStore, oldReadOnly, oldModel + recBasic, recBrief, recVerbose, recLimit = oldBasic, oldBrief, oldVerbose, oldLimit + recIntent, recExcerpt = oldIntent, oldExcerpt + recallCmd.SetErr(oldErr) + }) + dataDir, storeName, readOnly, embedModel = t.TempDir(), "embedding-recall", false, "selected-model" + recBasic, recLimit, recIntent, recExcerpt = false, 5, "GENERAL", 240 + t.Setenv("MNEMON_EMBED_PROTOCOL", "ollama") + t.Setenv("MNEMON_EMBED_DIMENSIONS", "") + t.Setenv("MNEMON_EMBED_API_KEY", "synthetic-secret") + var embeds atomic.Int32 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/api/tags" { + _, _ = w.Write([]byte(`{"models":[]}`)) + return + } + embeds.Add(1) + w.WriteHeader(tc.status) + _, _ = w.Write([]byte(tc.body)) + })) + defer srv.Close() + t.Setenv("MNEMON_EMBED_ENDPOINT", srv.URL) + db, err := store.Open(store.StoreDir(dataDir, storeName)) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + insertTestInsight(t, db, "embedding-seed", "SQLite snapshot memory", "test", "2026-01-01T00:00:00Z") + if err := db.UpdateEmbedding("embedding-seed", embed.SerializeVector([]float64{1, 0})); err != nil { + t.Fatal(err) + } + if err := db.Close(); err != nil { + t.Fatal(err) + } + for _, mode := range []string{"compact", "brief", "verbose"} { + t.Run(mode, func(t *testing.T) { + recBrief, recVerbose = mode == "brief", mode == "verbose" + var stderr bytes.Buffer + recallCmd.SetErr(&stderr) + var runErr error + out := captureStdout(t, func() { runErr = recallCmd.RunE(recallCmd, []string{"SQLite"}) }) + if runErr != nil { + t.Fatalf("recall must fall back successfully: %v", runErr) + } + var response struct{ Results []json.RawMessage } + if err := json.Unmarshal([]byte(out), &response); err != nil || len(response.Results) != 1 || !strings.Contains(string(response.Results[0]), "embedding-seed") { + t.Fatalf("recall lost its JSON result: %v\n%s", err, out) + } + warning := stderr.String() + if tc.wantWarning { + if strings.Count(warning, "\n") != 1 || !strings.Contains(warning, "without vector similarity") || !strings.Contains(warning, "selected-model") || !strings.Contains(warning, "--embed-model") { + t.Errorf("missing actionable one-line fallback warning: %q", warning) + } + } else if warning != "" { + t.Errorf("healthy embedding should be quiet: %q", warning) + } + for _, secret := range []string{"synthetic-secret", "private-provider-echo"} { + if strings.Contains(warning+out, secret) { + t.Errorf("provider content leaked: %q", secret) + } + } + if recVerbose { + var verbose search.RecallResponse + if err := json.Unmarshal([]byte(out), &verbose); err != nil { + t.Fatal(err) + } + similarity := verbose.Results[0].Signals.Similarity + if (tc.wantWarning && similarity != 0) || (!tc.wantWarning && similarity < 0.99) { + t.Errorf("unexpected vector similarity: %v", similarity) + } + } + }) + } + if got := embeds.Load(); got != 3 { + t.Errorf("recall made %d embedding requests, want one per query", got) + } + }) + } +} diff --git a/docs/USAGE.md b/docs/USAGE.md index 03c168a59..60fa114d4 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -437,6 +437,10 @@ Mnemon works fully without an embedding provider — all core features (remember When the configured provider is unavailable, the reranking system automatically redistributes similarity weight to keyword and graph signals — no configuration or degraded-mode flag is needed. Mnemon checks provider availability at runtime with a 2-second timeout. +If the provider responds but the query embedding fails (for example, because +the selected model is missing), `recall` prints one warning to stderr and +continues without vector similarity. JSON results remain on stdout. + ### Setup Ollama remains the default provider: @@ -465,6 +469,14 @@ Verify with: mnemon embed --status ``` +`embedding_available` verifies that the configured model returns a nonempty +vector, using a small synthetic input and the normal 30-second embedding +timeout. This explicit diagnostic can load the model and may incur a provider +request charge; it sends no stored memories. The legacy `ollama_available` +field keeps its provider-availability meaning for both protocols, so it can be +`true` while `embedding_available` is `false`. Coverage counts stored vectors +and does not establish that the current model can produce new ones. + ```json { "total_insights": 87, @@ -477,7 +489,7 @@ mnemon embed --status } ``` -`ollama_available` is retained as a compatibility alias for existing scripts; +`ollama_available` is retained for compatibility with existing scripts; new integrations should use `embedding_available` and `protocol`. ### Backfilling existing insights diff --git a/docs/zh/USAGE.md b/docs/zh/USAGE.md index fff9d05f7..fe39be1f9 100644 --- a/docs/zh/USAGE.md +++ b/docs/zh/USAGE.md @@ -400,6 +400,9 @@ Mnemon 无需嵌入服务即可完整运行 — 所有核心功能（remember、 配置的嵌入服务不可用时，重排序系统会自动将相似度权重重新分配给关键词和图信号 — 无需额外配置或降级模式标志。Mnemon 在运行时以 2 秒超时检测服务可用性。 +如果服务有响应，但查询嵌入生成失败（例如所选模型不存在），`recall` 会向 stderr +输出一行警告，并继续使用不含向量相似度的检索。JSON 结果仍输出到 stdout。 + ### 安装 Ollama 仍是默认服务： @@ -425,6 +428,12 @@ export MNEMON_EMBED_API_KEY=sk-... # 无需认证的本地服务器可省略 mnemon embed --status ``` +`embedding_available` 会使用一段简短的合成文本，验证当前配置的模型能否返回非空 +向量，采用正常嵌入请求的 30 秒超时。这个显式诊断可能加载模型，也可能产生服务商 +请求费用，但不会发送已保存的记忆。旧字段 `ollama_available` 对两种协议均保留 +服务可用性的含义，因此它可能为 `true`，而 `embedding_available` 为 `false`。 +覆盖率统计已存储的向量，不能证明当前模型仍能生成新向量。 + ```json { "total_insights": 87,