From 815c69a27b83dcfcdc652711415be637b6bef1ce Mon Sep 17 00:00:00 2001 From: Audrey Tang <20723+audreyt@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:53:03 +0800 Subject: [PATCH 1/5] feat(memory): add System One decision-model judge client Add internal/memory/judge, a small standard-library client for the System One API (POST {endpoint}/v1/systemone). Noul and Choice return typed, validated probabilities; questions are chunked at 16 per request with the same state resent in every chunk, requests share one deadline, response reads are bounded, and loopback endpoints bypass the proxy environment like the embed client. Configuration comes from MNEMON_JUDGE_MODEL, MNEMON_JUDGE_ENDPOINT, MNEMON_JUDGE_API_KEY, and MNEMON_JUDGE_TIMEOUT; an empty model disables it. Validated with an httptest fake covering request shape, Bearer auth, chunking, noul and choice parsing, JSON errors, malformed JSON, missing answers, and timeouts. The package is added to the deterministic make test list. --- Makefile | 1 + internal/memory/judge/judge.go | 264 +++++++++++++++++++++ internal/memory/judge/judge_test.go | 356 ++++++++++++++++++++++++++++ internal/memory/judge/wire.go | 76 ++++++ 4 files changed, 697 insertions(+) create mode 100644 internal/memory/judge/judge.go create mode 100644 internal/memory/judge/judge_test.go create mode 100644 internal/memory/judge/wire.go diff --git a/Makefile b/Makefile index dd20784d2..38051d1e7 100644 --- a/Makefile +++ b/Makefile @@ -32,6 +32,7 @@ DETERMINISTIC_PKGS := \ ./internal/memory/embed \ ./internal/memory/graph \ ./internal/memory/importdraft \ + ./internal/memory/judge \ ./internal/memory/internal/topk \ ./internal/memory/model \ ./internal/memory/search \ diff --git a/internal/memory/judge/judge.go b/internal/memory/judge/judge.go new file mode 100644 index 000000000..f5b4cceda --- /dev/null +++ b/internal/memory/judge/judge.go @@ -0,0 +1,264 @@ +// Package judge is a small client for the System One decision-model API +// (POST {endpoint}/v1/systemone). It turns yes/no and pick-one questions about +// a block of state into typed probabilities, which recall and remember use to +// drop irrelevant results and to classify semantic candidates. +// +// The judge is opt-in: NewClient returns nil unless MNEMON_JUDGE_MODEL is set, +// and every caller treats a failed judgment as advisory, never fatal. +package judge + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net" + "net/http" + "net/url" + "os" + "strings" + "time" +) + +const ( + // DefaultEndpoint is the default local Ollama endpoint. + DefaultEndpoint = "http://localhost:11434" + // DefaultTimeout bounds one whole judgment, across all of its chunks. + DefaultTimeout = 15 * time.Second + // MaxQuestionsPerRequest is the largest question count every supported + // provider accepts in one request (SubQ caps at 16; Ollama allows 64). + MaxQuestionsPerRequest = 16 + + systemOneRoute = "v1/systemone" + maxResponseBytes = 1 << 20 + maxErrorBytes = 4 << 10 +) + +// Client talks to a System One server (local Ollama or a cloud provider). +type Client struct { + endpoint string + model string + apiKey string + timeout time.Duration + http *http.Client +} + +// NewClient builds a judge client from MNEMON_JUDGE_MODEL, +// MNEMON_JUDGE_ENDPOINT, MNEMON_JUDGE_API_KEY, and MNEMON_JUDGE_TIMEOUT. +// It returns nil when MNEMON_JUDGE_MODEL is empty, which means the judge is +// not configured. +func NewClient() *Client { + model := strings.TrimSpace(os.Getenv("MNEMON_JUDGE_MODEL")) + if model == "" { + return nil + } + endpoint := os.Getenv("MNEMON_JUDGE_ENDPOINT") + if endpoint == "" { + endpoint = DefaultEndpoint + } + timeout := DefaultTimeout + if v := os.Getenv("MNEMON_JUDGE_TIMEOUT"); v != "" { + if d, err := time.ParseDuration(v); err == nil && d > 0 { + timeout = d + } else { + fmt.Fprintf(os.Stderr, "warning: invalid MNEMON_JUDGE_TIMEOUT %q, using %s\n", v, DefaultTimeout) + } + } + return &Client{ + endpoint: endpoint, + model: model, + apiKey: os.Getenv("MNEMON_JUDGE_API_KEY"), + timeout: timeout, + http: &http.Client{ + Transport: &http.Transport{ + // Remote endpoints follow the standard proxy environment; + // loopback servers never do, so a stray HTTPS_PROXY cannot + // capture local model traffic. + Proxy: proxyFunc(endpoint), + DialContext: (&net.Dialer{ + Timeout: 5 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + }, + }, + } +} + +// proxyFunc returns the proxy resolver for the judge endpoint: none for +// loopback, the standard environment resolution otherwise. +func proxyFunc(endpoint string) func(*http.Request) (*url.URL, error) { + if u, err := url.Parse(endpoint); err == nil { + if host := u.Hostname(); host == "localhost" || net.ParseIP(host).IsLoopback() { + return func(*http.Request) (*url.URL, error) { return nil, nil } + } + } + return http.ProxyFromEnvironment +} + +// Model returns the configured model name. +func (c *Client) Model() string { return c.model } + +// Question is a yes/no (noul) question. Name keys the answer. +type Question struct { + Name string + Instructions string +} + +// ChoiceQuestion is a pick-one question. Options maps each option to a +// description the model sees. +type ChoiceQuestion struct { + Name string + Instructions string + Options map[string]string +} + +// ChoiceAnswer is the validated answer to a ChoiceQuestion. +type ChoiceAnswer struct { + // Choice is one of the question's options. + Choice string + // Probability is the model's probability for Choice. + Probability float64 +} + +// Noul asks yes/no questions about state and returns P(yes) per question name. +// More than MaxQuestionsPerRequest questions are sent as several requests that +// each carry the same state. +func (c *Client) Noul(ctx context.Context, state any, questions []Question) (map[string]float64, error) { + wire := make([]wireQuestion, len(questions)) + for i, q := range questions { + wire[i] = wireQuestion{name: q.Name, Type: "noul", Instructions: q.Instructions} + } + answers, err := c.ask(ctx, state, wire) + if err != nil { + return nil, err + } + out := make(map[string]float64, len(questions)) + for _, q := range questions { + a := answers[q.Name] + if err := a.expect("noul"); err != nil { + return nil, fmt.Errorf("question %q: %w", q.Name, err) + } + if a.Noul == nil || !inUnit(*a.Noul) { + return nil, fmt.Errorf("question %q: missing or out-of-range noul value", q.Name) + } + out[q.Name] = *a.Noul + } + return out, nil +} + +// Choice asks pick-one questions about state and returns the validated answer +// per question name. Chunking follows the same rule as Noul. +func (c *Client) Choice(ctx context.Context, state any, questions []ChoiceQuestion) (map[string]ChoiceAnswer, error) { + wire := make([]wireQuestion, len(questions)) + for i, q := range questions { + wire[i] = wireQuestion{name: q.Name, Type: "choice", Instructions: q.Instructions, Criteria: q.Options} + } + answers, err := c.ask(ctx, state, wire) + if err != nil { + return nil, err + } + out := make(map[string]ChoiceAnswer, len(questions)) + for _, q := range questions { + a := answers[q.Name] + ans, err := a.choice(q.Options) + if err != nil { + return nil, fmt.Errorf("question %q: %w", q.Name, err) + } + out[q.Name] = ans + } + return out, nil +} + +// ask sends questions in chunks under one deadline and returns the answers +// keyed by question name. Every question must be answered. +func (c *Client) ask(ctx context.Context, state any, questions []wireQuestion) (map[string]wireAnswer, error) { + stateJSON, err := json.Marshal(state) + if err != nil { + return nil, fmt.Errorf("marshal judge state: %w", err) + } + seen := make(map[string]bool, len(questions)) + for _, q := range questions { + if q.name == "" || seen[q.name] { + return nil, fmt.Errorf("judge question name %q is empty or duplicated", q.name) + } + seen[q.name] = true + } + ctx, cancel := context.WithTimeout(ctx, c.timeout) + defer cancel() + + answers := make(map[string]wireAnswer, len(questions)) + for start := 0; start < len(questions); start += MaxQuestionsPerRequest { + end := min(start+MaxQuestionsPerRequest, len(questions)) + got, err := c.post(ctx, stateJSON, questions[start:end]) + if err != nil { + return nil, err + } + for _, q := range questions[start:end] { + a, ok := got[q.name] + if !ok { + return nil, fmt.Errorf("judge response is missing an answer for question %q", q.name) + } + answers[q.name] = a + } + } + return answers, nil +} + +// post sends one System One request and decodes its answers. +func (c *Client) post(ctx context.Context, state json.RawMessage, chunk []wireQuestion) (map[string]wireAnswer, error) { + questions := make(map[string]wireQuestion, len(chunk)) + for _, q := range chunk { + questions[q.name] = q + } + body, err := json.Marshal(wireRequest{Model: c.model, State: state, Questions: questions}) + if err != nil { + return nil, fmt.Errorf("marshal judge request: %w", err) + } + endpointURL, err := url.JoinPath(c.endpoint, systemOneRoute) + if err != nil { + return nil, fmt.Errorf("join judge endpoint: %w", err) + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, endpointURL, bytes.NewReader(body)) + if err != nil { + return nil, fmt.Errorf("build judge request: %w", err) + } + req.Header.Set("Content-Type", "application/json") + if c.apiKey != "" { + req.Header.Set("Authorization", "Bearer "+c.apiKey) + } + resp, err := c.http.Do(req) + if err != nil { + return nil, fmt.Errorf("judge request: %w", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return nil, statusError(resp) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, maxResponseBytes+1)) + if err != nil { + return nil, fmt.Errorf("read judge response: %w", err) + } + if len(data) > maxResponseBytes { + return nil, fmt.Errorf("judge response exceeds %d bytes", maxResponseBytes) + } + var decoded wireResponse + if err := json.Unmarshal(data, &decoded); err != nil { + return nil, fmt.Errorf("decode judge response: %w", err) + } + return decoded.Answers, nil +} + +// statusError turns a non-200 response into an error, preferring the server's +// {"error": "..."} message. +func statusError(resp *http.Response) error { + data, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBytes)) + var payload struct { + Error string `json:"error"` + } + if json.Unmarshal(data, &payload) == nil && payload.Error != "" { + return fmt.Errorf("judge server returned status %d: %s", resp.StatusCode, payload.Error) + } + return fmt.Errorf("judge server returned status %d", resp.StatusCode) +} diff --git a/internal/memory/judge/judge_test.go b/internal/memory/judge/judge_test.go new file mode 100644 index 000000000..529c5d80d --- /dev/null +++ b/internal/memory/judge/judge_test.go @@ -0,0 +1,356 @@ +package judge + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "strings" + "sync" + "testing" + "time" +) + +type recordedRequest struct { + Path string + Authorization string + Model string + State string + Questions map[string]wireQuestion +} + +// fakeServer is an httptest System One server. handle receives each decoded +// request and returns the HTTP status and body to send back. +type fakeServer struct { + mu sync.Mutex + requests []recordedRequest +} + +func (f *fakeServer) recorded() []recordedRequest { + f.mu.Lock() + defer f.mu.Unlock() + return append([]recordedRequest(nil), f.requests...) +} + +func startFake(t *testing.T, apiKey string, handle func(recordedRequest) (int, string)) (*Client, *fakeServer) { + t.Helper() + fake := &fakeServer{} + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var body struct { + Model string `json:"model"` + State json.RawMessage `json:"state"` + Questions map[string]wireQuestion `json:"questions"` + } + if err := json.NewDecoder(r.Body).Decode(&body); err != nil { + http.Error(w, `{"error":"bad body"}`, http.StatusBadRequest) + return + } + rec := recordedRequest{ + Path: r.URL.Path, + Authorization: r.Header.Get("Authorization"), + Model: body.Model, + State: string(body.State), + Questions: body.Questions, + } + fake.mu.Lock() + fake.requests = append(fake.requests, rec) + fake.mu.Unlock() + status, payload := handle(rec) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(status) + _, _ = w.Write([]byte(payload)) + })) + t.Cleanup(srv.Close) + t.Setenv("MNEMON_JUDGE_MODEL", "test-model") + t.Setenv("MNEMON_JUDGE_ENDPOINT", srv.URL) + t.Setenv("MNEMON_JUDGE_API_KEY", apiKey) + t.Setenv("MNEMON_JUDGE_TIMEOUT", "") + client := NewClient() + if client == nil { + t.Fatal("NewClient returned nil with MNEMON_JUDGE_MODEL set") + } + return client, fake +} + +// noulAnswers answers every asked question with the same noul value. +func noulAnswers(req recordedRequest, value float64) string { + answers := make([]string, 0, len(req.Questions)) + for name := range req.Questions { + answers = append(answers, fmt.Sprintf(`%q:{"type":"noul","noul":%v}`, name, value)) + } + return `{"model":"test-model","answers":{` + strings.Join(answers, ",") + `}}` +} + +func TestNewClient_nilWhenModelUnset(t *testing.T) { + // Given no judge model + t.Setenv("MNEMON_JUDGE_MODEL", "") + + // When the client is built + client := NewClient() + + // Then the judge is disabled + if client != nil { + t.Fatalf("NewClient() = %+v, want nil", client) + } +} + +func TestNewClient_envDefaultsAndOverrides(t *testing.T) { + // Given only a model + t.Setenv("MNEMON_JUDGE_MODEL", "clef") + t.Setenv("MNEMON_JUDGE_ENDPOINT", "") + t.Setenv("MNEMON_JUDGE_API_KEY", "") + t.Setenv("MNEMON_JUDGE_TIMEOUT", "") + + // When the client is built + client := NewClient() + + // Then the documented defaults apply + if client.endpoint != DefaultEndpoint || client.timeout != DefaultTimeout || client.apiKey != "" { + t.Fatalf("defaults = endpoint %q timeout %v key %q", client.endpoint, client.timeout, client.apiKey) + } + + // Given explicit settings and an unusable timeout + t.Setenv("MNEMON_JUDGE_ENDPOINT", "https://judge.example.test") + t.Setenv("MNEMON_JUDGE_API_KEY", "secret") + t.Setenv("MNEMON_JUDGE_TIMEOUT", "250ms") + override := NewClient() + t.Setenv("MNEMON_JUDGE_TIMEOUT", "soon") + invalid := NewClient() + + // Then overrides are honored and a bad timeout falls back to the default + if override.endpoint != "https://judge.example.test" || override.apiKey != "secret" || + override.timeout != 250*time.Millisecond { + t.Fatalf("override = %+v", override) + } + if invalid.timeout != DefaultTimeout { + t.Fatalf("invalid timeout resolved to %v, want %v", invalid.timeout, DefaultTimeout) + } +} + +func TestNoul_requestShapeAndParsing(t *testing.T) { + // Given a keyed server that answers 0.9 and 0.2 + client, fake := startFake(t, "k-123", func(recordedRequest) (int, string) { + return 200, `{"model":"test-model","answers":{ + "m0":{"type":"noul","noul":0.9},"m1":{"type":"noul","noul":0.2}}, + "usage":{"input_tokens":10,"output_tokens":2}}` + }) + state := map[string]any{"query": "q", "memories": map[string]string{"m0": "a", "m1": "b"}} + + // When two noul questions are asked + got, err := client.Noul(context.Background(), state, []Question{ + {Name: "m0", Instructions: "is m0 relevant?"}, + {Name: "m1", Instructions: "is m1 relevant?"}, + }) + + // Then the typed answers come back and the request carried the contract + if err != nil { + t.Fatalf("Noul: %v", err) + } + if got["m0"] != 0.9 || got["m1"] != 0.2 { + t.Fatalf("answers = %v", got) + } + reqs := fake.recorded() + if len(reqs) != 1 { + t.Fatalf("requests = %d, want 1", len(reqs)) + } + r := reqs[0] + if r.Path != "/v1/systemone" || r.Model != "test-model" || r.Authorization != "Bearer k-123" { + t.Fatalf("request = %+v", r) + } + if r.State != `{"memories":{"m0":"a","m1":"b"},"query":"q"}` { + t.Fatalf("state = %s", r.State) + } + if q := r.Questions["m1"]; q.Type != "noul" || q.Instructions != "is m1 relevant?" { + t.Fatalf("question m1 = %+v", q) + } +} + +func TestNoul_omitsAuthorizationWithoutKey(t *testing.T) { + // Given a server and no API key + client, fake := startFake(t, "", func(req recordedRequest) (int, string) { return 200, noulAnswers(req, 0.5) }) + + // When a question is asked + if _, err := client.Noul(context.Background(), "state", []Question{{Name: "a", Instructions: "?"}}); err != nil { + t.Fatalf("Noul: %v", err) + } + + // Then no Authorization header is sent + if got := fake.recorded()[0].Authorization; got != "" { + t.Fatalf("Authorization = %q, want none", got) + } +} + +func TestNoul_chunksAtSixteenWithIdenticalState(t *testing.T) { + // Given 35 questions + client, fake := startFake(t, "", func(req recordedRequest) (int, string) { return 200, noulAnswers(req, 0.75) }) + questions := make([]Question, 35) + for i := range questions { + questions[i] = Question{Name: fmt.Sprintf("m%d", i), Instructions: "?"} + } + + // When they are asked in one call + got, err := client.Noul(context.Background(), map[string]string{"query": "q"}, questions) + + // Then three requests carry the same state and every question is answered once + if err != nil { + t.Fatalf("Noul: %v", err) + } + reqs := fake.recorded() + if len(reqs) != 3 { + t.Fatalf("requests = %d, want 3", len(reqs)) + } + asked := map[string]int{} + for _, r := range reqs { + if len(r.Questions) > MaxQuestionsPerRequest { + t.Fatalf("chunk has %d questions", len(r.Questions)) + } + if r.State != reqs[0].State { + t.Fatalf("state differs across chunks: %s vs %s", r.State, reqs[0].State) + } + for name := range r.Questions { + asked[name]++ + } + } + if len(asked) != 35 || len(got) != 35 { + t.Fatalf("asked %d distinct questions, got %d answers; want 35", len(asked), len(got)) + } + for name, n := range asked { + if n != 1 { + t.Fatalf("question %s asked %d times", name, n) + } + } +} + +func TestChoice_parsesAnswerAndProbability(t *testing.T) { + // Given a server that picks "supersedes" and a second answer carrying only confidence + client, fake := startFake(t, "", func(recordedRequest) (int, string) { + return 200, `{"answers":{ + "c0":{"type":"choice","choice":"supersedes","probabilities":{"supersedes":0.8,"related":0.2},"confidence":0.8}, + "c1":{"type":"choice","choice":"related","confidence":0.6}}}` + }) + options := map[string]string{"supersedes": "replaces it", "related": "same subject"} + + // When two choice questions are asked + got, err := client.Choice(context.Background(), "state", []ChoiceQuestion{ + {Name: "c0", Instructions: "how?", Options: options}, + {Name: "c1", Instructions: "how?", Options: options}, + }) + + // Then probabilities come from the map, falling back to confidence, and criteria were sent + if err != nil { + t.Fatalf("Choice: %v", err) + } + if got["c0"] != (ChoiceAnswer{Choice: "supersedes", Probability: 0.8}) || + got["c1"] != (ChoiceAnswer{Choice: "related", Probability: 0.6}) { + t.Fatalf("answers = %+v", got) + } + q := fake.recorded()[0].Questions["c0"] + if q.Type != "choice" || q.Criteria["supersedes"] != "replaces it" { + t.Fatalf("question = %+v", q) + } +} + +func TestAsk_rejectsBadResponses(t *testing.T) { + cases := []struct { + name string + status int + body string + wantErr string + }{ + {"server error json", 400, `{"error":"unknown model"}`, "status 400: unknown model"}, + {"server error without json", 500, `oops`, "status 500"}, + {"malformed json", 200, `{"answers":`, "decode judge response"}, + {"missing answer", 200, `{"answers":{}}`, "missing an answer"}, + {"noul missing value", 200, `{"answers":{"a":{"type":"noul"}}}`, "missing or out-of-range"}, + {"noul out of range", 200, `{"answers":{"a":{"type":"noul","noul":1.5}}}`, "missing or out-of-range"}, + {"wrong answer type", 200, `{"answers":{"a":{"type":"choice","choice":"x"}}}`, "want \"noul\""}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + // Given a server that misbehaves + client, _ := startFake(t, "", func(recordedRequest) (int, string) { return tc.status, tc.body }) + + // When a question is asked + _, err := client.Noul(context.Background(), "state", []Question{{Name: "a", Instructions: "?"}}) + + // Then the call fails with a descriptive error + if err == nil || !strings.Contains(err.Error(), tc.wantErr) { + t.Fatalf("err = %v, want it to contain %q", err, tc.wantErr) + } + }) + } +} + +func TestChoice_rejectsInvalidAnswers(t *testing.T) { + cases := map[string]string{ + "unoffered choice": `{"answers":{"c":{"type":"choice","choice":"nonsense","confidence":0.9}}}`, + "no choice": `{"answers":{"c":{"type":"choice","confidence":0.9}}}`, + "probability range": `{"answers":{"c":{"type":"choice","choice":"related","probabilities":{"related":2}}}}`, + "no probability data": `{"answers":{"c":{"type":"choice","choice":"related"}}}`, + } + for name, body := range cases { + t.Run(name, func(t *testing.T) { + // Given a server returning an unusable choice answer + client, _ := startFake(t, "", func(recordedRequest) (int, string) { return 200, body }) + + // When the question is asked + _, err := client.Choice(context.Background(), "state", []ChoiceQuestion{ + {Name: "c", Instructions: "?", Options: map[string]string{"related": "x"}}, + }) + + // Then it is rejected + if err == nil { + t.Fatal("Choice accepted an invalid answer") + } + }) + } +} + +func TestAsk_rejectsDuplicateQuestionNames(t *testing.T) { + // Given a client and two questions sharing a name + client, fake := startFake(t, "", func(req recordedRequest) (int, string) { return 200, noulAnswers(req, 1) }) + + // When they are asked + _, err := client.Noul(context.Background(), "s", []Question{{Name: "a"}, {Name: "a"}}) + + // Then the call fails before any request is sent + if err == nil || len(fake.recorded()) != 0 { + t.Fatalf("err = %v, requests = %d", err, len(fake.recorded())) + } +} + +func TestAsk_timesOutOnSlowServer(t *testing.T) { + // Given a server that never answers and a short timeout + release := make(chan struct{}) + client, _ := startFake(t, "", func(recordedRequest) (int, string) { + <-release + return 200, `{}` + }) + t.Cleanup(func() { close(release) }) + client.timeout = 50 * time.Millisecond + + // When a question is asked + _, err := client.Noul(context.Background(), "s", []Question{{Name: "a", Instructions: "?"}}) + + // Then the deadline error surfaces + if err == nil || !strings.Contains(err.Error(), "deadline exceeded") { + t.Fatalf("err = %v, want a deadline error", err) + } +} + +func TestAsk_bypassesProxyForLoopback(t *testing.T) { + // Given a proxy environment pointing at a dead address + t.Setenv("HTTP_PROXY", "http://127.0.0.1:1") + t.Setenv("HTTPS_PROXY", "http://127.0.0.1:1") + t.Setenv("NO_PROXY", "") + client, _ := startFake(t, "", func(req recordedRequest) (int, string) { return 200, noulAnswers(req, 0.5) }) + + // When the judge reaches a loopback server + _, err := client.Noul(context.Background(), "s", []Question{{Name: "a", Instructions: "?"}}) + + // Then the proxy is not used + if err != nil { + t.Fatalf("loopback request went through the proxy: %v", err) + } +} diff --git a/internal/memory/judge/wire.go b/internal/memory/judge/wire.go new file mode 100644 index 000000000..db1ce01ec --- /dev/null +++ b/internal/memory/judge/wire.go @@ -0,0 +1,76 @@ +package judge + +import ( + "encoding/json" + "fmt" +) + +// wireQuestion is one entry of the request "questions" object. name is the +// map key and is not serialized into the question itself. +type wireQuestion struct { + name string + Type string `json:"type"` + Instructions string `json:"instructions"` + Criteria map[string]string `json:"criteria,omitempty"` +} + +type wireRequest struct { + Model string `json:"model"` + State json.RawMessage `json:"state"` + Questions map[string]wireQuestion `json:"questions"` +} + +type wireResponse struct { + Answers map[string]wireAnswer `json:"answers"` +} + +// wireAnswer holds the union of noul and choice answer fields. Pointers +// distinguish a missing field from a zero value. +type wireAnswer struct { + Type string `json:"type"` + Noul *float64 `json:"noul"` + Choice *string `json:"choice"` + Probabilities map[string]float64 `json:"probabilities"` + Confidence *float64 `json:"confidence"` +} + +// inUnit reports whether v is a finite probability. NaN fails every +// comparison, so it is rejected along with out-of-range values. +func inUnit(v float64) bool { return v >= 0 && v <= 1 } + +// expect rejects an answer whose declared type differs from the asked type. +func (a wireAnswer) expect(kind string) error { + if a.Type != "" && a.Type != kind { + return fmt.Errorf("answer type is %q, want %q", a.Type, kind) + } + return nil +} + +// choice validates a choice answer against the options that were offered. +func (a wireAnswer) choice(options map[string]string) (ChoiceAnswer, error) { + if err := a.expect("choice"); err != nil { + return ChoiceAnswer{}, err + } + if a.Choice == nil { + return ChoiceAnswer{}, fmt.Errorf("answer has no choice") + } + if _, ok := options[*a.Choice]; !ok { + return ChoiceAnswer{}, fmt.Errorf("choice %q is not an offered option", *a.Choice) + } + for option, p := range a.Probabilities { + if !inUnit(p) { + return ChoiceAnswer{}, fmt.Errorf("probability for %q is out of range", option) + } + } + p, ok := a.Probabilities[*a.Choice] + if !ok { + if a.Confidence == nil { + return ChoiceAnswer{}, fmt.Errorf("answer has no probability for choice %q", *a.Choice) + } + p = *a.Confidence + } + if !inUnit(p) { + return ChoiceAnswer{}, fmt.Errorf("probability for choice %q is out of range", *a.Choice) + } + return ChoiceAnswer{Choice: *a.Choice, Probability: p}, nil +} From 64252902c3681b2a47989f32dd9a090317b0d66b Mon Sep 17 00:00:00 2001 From: Audrey Tang <20723+audreyt@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:53:03 +0800 Subject: [PATCH 2/5] feat(memory): add --judge to recall and remember recall --judge scores every ranked result against the query with one noul question each, adds a relevance field to the compact, brief, and verbose shapes, and drops results below --judge-min (default 0.5). remember --judge classifies each semantic candidate, auto-linked ones included, as duplicate, supersedes, contradicts, related, or unrelated and reports it as judgment; it creates no edges. A missing or failing judge prints a one-line warning to stderr and leaves normal output untouched, and output without --judge is unchanged. RecallResult gains an optional Relevance field because the verbose shape serializes it directly. mnemon's architecture test now allows cmd/memory to import the standard-library-only internal/memory/judge package. Validated with command tests against a fake System One server for filtering, fail-open, the not-configured warning, and the remember annotation, plus make test. --- cmd/memory/brief.go | 1 + cmd/memory/judge.go | 140 ++++++ cmd/memory/judge_test.go | 412 ++++++++++++++++++ cmd/memory/recall.go | 17 + cmd/memory/remember.go | 11 +- internal/memory/search/recall.go | 3 + .../architecture/mnemond_structure_test.go | 9 +- 7 files changed, 589 insertions(+), 4 deletions(-) create mode 100644 cmd/memory/judge.go create mode 100644 cmd/memory/judge_test.go diff --git a/cmd/memory/brief.go b/cmd/memory/brief.go index d1945382a..d449be3f6 100644 --- a/cmd/memory/brief.go +++ b/cmd/memory/brief.go @@ -19,6 +19,7 @@ type briefResult struct { Score *float64 `json:"score,omitempty"` Confidence string `json:"confidence,omitempty"` Superseded bool `json:"superseded,omitempty"` + Relevance *float64 `json:"relevance,omitempty"` } type briefResponse struct { diff --git a/cmd/memory/judge.go b/cmd/memory/judge.go new file mode 100644 index 000000000..10535a26b --- /dev/null +++ b/cmd/memory/judge.go @@ -0,0 +1,140 @@ +package memory + +import ( + "context" + "fmt" + "os" + "strings" + + "github.com/mnemon-dev/mnemon/internal/memory/graph" + "github.com/mnemon-dev/mnemon/internal/memory/judge" + "github.com/mnemon-dev/mnemon/internal/memory/search" +) + +// judgeContentChars caps how much of one memory is sent to the judge, keeping +// a full request well under the 64 KiB body limit of a local server. +const judgeContentChars = 1200 + +// relationOptions are the choices offered when classifying how a new memory +// relates to an existing semantic candidate. +var relationOptions = map[string]string{ + "duplicate": "the new memory says the same thing as the existing one", + "supersedes": "the new memory updates or replaces the existing one", + "contradicts": "the new memory conflicts with the existing one without clearly replacing it", + "related": "same subject, different facts", + "unrelated": "different subjects", +} + +// judgment is the judge's classification of one semantic candidate. +type judgment struct { + Relation string `json:"relation"` + Probability float64 `json:"probability"` +} + +// judgedCandidate is a semantic candidate with an optional judgment. A +// candidate the judge did not classify serializes exactly as before. +type judgedCandidate struct { + graph.SemanticCandidate + Judgment *judgment `json:"judgment,omitempty"` +} + +// warnJudge prints a one-line advisory; the judge never fails a command. +func warnJudge(format string, args ...any) { + msg := strings.Join(strings.Fields(fmt.Sprintf(format, args...)), " ") + fmt.Fprintf(os.Stderr, "warning: %s\n", msg) +} + +// clipRunes truncates s to at most n runes. +func clipRunes(s string, n int) string { + runes := []rune(s) + if len(runes) <= n { + return s + } + return string(runes[:n]) +} + +// judgeRecall scores every result against the query and drops those below +// --judge-min. It fails open: when the judge is missing or errors, resp is +// left untouched. +func judgeRecall(query string, resp *search.RecallResponse) { + jc := judge.NewClient() + if jc == nil { + warnJudge("--judge ignored: MNEMON_JUDGE_MODEL is not set") + return + } + if len(resp.Results) == 0 { + return + } + memories := make(map[string]string, len(resp.Results)) + questions := make([]judge.Question, len(resp.Results)) + for i, r := range resp.Results { + name := fmt.Sprintf("m%d", i) + memories[name] = clipRunes(r.Insight.Content, judgeContentChars) + questions[i] = judge.Question{ + Name: name, + Instructions: fmt.Sprintf( + "Is memory `%s` about the same subject as `query`, so it could help answer it?", name), + } + } + state := map[string]any{"query": query, "memories": memories} + scores, err := jc.Noul(context.Background(), state, questions) + if err != nil { + warnJudge("judge unavailable, returning unjudged results: %v", err) + return + } + kept := make([]search.RecallResult, 0, len(resp.Results)) + for i, r := range resp.Results { + relevance := roundScore(scores[questions[i].Name]) + if relevance < recJudgeMin { + continue + } + r.Relevance = &relevance + kept = append(kept, r) + } + resp.Results = kept +} + +// judgeCandidates classifies each semantic candidate, auto-linked ones +// included: a near-identical candidate is the likeliest duplicate or +// superseded memory. +// It returns false, leaving the caller's output unchanged, when there is +// nothing to judge or the judge is missing or errors. +func judgeCandidates(content string, candidates []graph.SemanticCandidate) ([]judgedCandidate, bool) { + jc := judge.NewClient() + if jc == nil { + warnJudge("--judge ignored: MNEMON_JUDGE_MODEL is not set") + return nil, false + } + existing := make(map[string]string) + var questions []judge.ChoiceQuestion + for i, c := range candidates { + name := fmt.Sprintf("c%d", i) + existing[name] = clipRunes(c.Content, judgeContentChars) + questions = append(questions, judge.ChoiceQuestion{ + Name: name, + Instructions: fmt.Sprintf( + "How does `new_memory` relate to existing memory `%s`?", name), + Options: relationOptions, + }) + } + if len(questions) == 0 { + return nil, false + } + state := map[string]any{ + "new_memory": clipRunes(content, judgeContentChars), + "existing_memories": existing, + } + answers, err := jc.Choice(context.Background(), state, questions) + if err != nil { + warnJudge("judge unavailable, leaving candidates unjudged: %v", err) + return nil, false + } + out := make([]judgedCandidate, len(candidates)) + for i, c := range candidates { + out[i] = judgedCandidate{SemanticCandidate: c} + if a, ok := answers[fmt.Sprintf("c%d", i)]; ok { + out[i].Judgment = &judgment{Relation: a.Choice, Probability: roundScore(a.Probability)} + } + } + return out, true +} diff --git a/cmd/memory/judge_test.go b/cmd/memory/judge_test.go new file mode 100644 index 000000000..47efc2c69 --- /dev/null +++ b/cmd/memory/judge_test.go @@ -0,0 +1,412 @@ +package memory + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "os" + "strings" + "sync/atomic" + "testing" + + "github.com/mnemon-dev/mnemon/internal/memory/graph" + "github.com/mnemon-dev/mnemon/internal/memory/model" + "github.com/mnemon-dev/mnemon/internal/memory/store" +) + +const ( + zebraWest = "the deploy target for project zebra is staging-west" + zebraRunb = "zebra deploy runbook: promote the build to staging-west first" + zebraBread = "zebra bread recipe: deploy the oven at 220C for a crisp crust" +) + +func captureStderr(t *testing.T, fn func()) string { + t.Helper() + old := os.Stderr + r, w, err := os.Pipe() + if err != nil { + t.Fatalf("pipe stderr: %v", err) + } + os.Stderr = w + defer func() { os.Stderr = old }() + fn() + if err := w.Close(); err != nil { + t.Fatalf("close stderr writer: %v", err) + } + data, err := io.ReadAll(r) + if err != nil { + t.Fatalf("read stderr: %v", err) + } + return string(data) +} + +// startJudgeServer fakes System One. noul questions score a memory 0.05 when +// its content mentions bread and 0.9 otherwise; choice questions answer +// "supersedes" at 0.87. The returned counter is the number of requests served. +func startJudgeServer(t *testing.T, status int) (*atomic.Int32, *[]map[string]any) { + t.Helper() + var hits atomic.Int32 + var bodies []map[string]any + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + hits.Add(1) + var req struct { + State map[string]any `json:"state"` + Questions map[string]map[string]any `json:"questions"` + } + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, `{"error":"bad body"}`, http.StatusBadRequest) + return + } + bodies = append(bodies, map[string]any{"state": req.State, "questions": req.Questions}) + if status != http.StatusOK { + http.Error(w, `{"error":"model unavailable"}`, status) + return + } + memories, _ := req.State["memories"].(map[string]any) + answers := map[string]any{} + for name, q := range req.Questions { + if q["type"] == "choice" { + answers[name] = map[string]any{"type": "choice", "choice": "supersedes", "confidence": 0.87, + "probabilities": map[string]float64{"supersedes": 0.87, "related": 0.13}} + continue + } + score := 0.9 + if content, _ := memories[name].(string); strings.Contains(content, "bread") { + score = 0.05 + } + answers[name] = map[string]any{"type": "noul", "noul": score} + } + _ = json.NewEncoder(w).Encode(map[string]any{"model": "test", "answers": answers}) + })) + t.Cleanup(srv.Close) + t.Setenv("MNEMON_JUDGE_MODEL", "test-model") + t.Setenv("MNEMON_JUDGE_ENDPOINT", srv.URL) + t.Setenv("MNEMON_JUDGE_API_KEY", "") + t.Setenv("MNEMON_JUDGE_TIMEOUT", "") + return &hits, &bodies +} + +func configureJudgedRecall(t *testing.T) { + t.Helper() + oldDir, oldStore, oldReadOnly := dataDir, storeName, readOnly + oldBasic, oldBrief, oldVerbose, oldLimit := recBasic, recBrief, recVerbose, recLimit + oldCategory, oldSource, oldIntent, oldExcerpt := recCategory, recSource, recIntent, recExcerpt + oldJudge, oldJudgeMin := recJudge, recJudgeMin + t.Cleanup(func() { + dataDir, storeName, readOnly = oldDir, oldStore, oldReadOnly + recBasic, recBrief, recVerbose, recLimit = oldBasic, oldBrief, oldVerbose, oldLimit + recCategory, recSource, recIntent, recExcerpt = oldCategory, oldSource, oldIntent, oldExcerpt + recJudge, recJudgeMin = oldJudge, oldJudgeMin + }) + t.Setenv("MNEMON_EMBED_ENDPOINT", "http://127.0.0.1:1") + t.Setenv("MNEMON_EMBED_PROTOCOL", "ollama") + dataDir, storeName, readOnly = t.TempDir(), "judge-recall", true + recBasic, recBrief, recVerbose, recLimit = false, false, false, 10 + recCategory, recSource, recIntent, recExcerpt = "", "", "GENERAL", 240 + recJudge, recJudgeMin = false, 0.5 + + db, err := store.Open(store.StoreDir(dataDir, storeName)) + if err != nil { + t.Fatal(err) + } + insertTestInsight(t, db, "west", zebraWest, "test", "2026-01-01T00:00:00Z") + insertTestInsight(t, db, "runb", zebraRunb, "test", "2026-01-02T00:00:00Z") + insertTestInsight(t, db, "bread", zebraBread, "test", "2026-01-03T00:00:00Z") + if err := db.Close(); err != nil { + t.Fatal(err) + } +} + +func runRecall(t *testing.T) (stdout, stderr string) { + t.Helper() + var runErr error + stderr = captureStderr(t, func() { + stdout = captureStdout(t, func() { + runErr = recallCmd.RunE(recallCmd, []string{"where does project zebra deploy"}) + }) + }) + if runErr != nil { + t.Fatalf("recall: %v", runErr) + } + return stdout, stderr +} + +type judgedRow struct { + ID string `json:"id"` + Insight struct{ ID string } + Relevance *float64 `json:"relevance"` +} + +func recallIDs(t *testing.T, out string, verbose bool) (map[string]*float64, int) { + t.Helper() + var resp struct { + Results []judgedRow `json:"results"` + } + if err := json.Unmarshal([]byte(out), &resp); err != nil { + t.Fatalf("decode recall output: %v\n%s", err, out) + } + ids := map[string]*float64{} + for _, r := range resp.Results { + id := r.ID + if verbose { + id = r.Insight.ID + } + ids[id] = r.Relevance + } + return ids, len(resp.Results) +} + +func TestRecallJudgeAddsRelevanceAndDropsIrrelevant(t *testing.T) { + for _, mode := range []string{"compact", "brief", "verbose"} { + t.Run(mode, func(t *testing.T) { + // Given a store where bread is retrievable but unrelated + configureJudgedRecall(t) + recBrief, recVerbose = mode == "brief", mode == "verbose" + baseline, _ := runRecall(t) + before, _ := recallIDs(t, baseline, mode == "verbose") + if _, ok := before["bread"]; !ok || len(before) != 3 { + t.Fatalf("unjudged recall should return all three memories: %s", baseline) + } + if strings.Contains(baseline, "relevance") { + t.Fatalf("unjudged output must not mention relevance: %s", baseline) + } + + // When recall runs with the judge + hits, bodies := startJudgeServer(t, http.StatusOK) + recJudge = true + out, stderr := runRecall(t) + + // Then the bread memory is dropped, the rest carry relevance, and one request was sent + after, _ := recallIDs(t, out, mode == "verbose") + if _, ok := after["bread"]; ok || len(after) != 2 { + t.Fatalf("judged recall = %v, want west and runb only\n%s", after, out) + } + for id, rel := range after { + if rel == nil || *rel != 0.9 { + t.Fatalf("relevance for %s = %v, want 0.9", id, rel) + } + } + if hits.Load() != 1 || stderr != "" { + t.Fatalf("requests = %d, stderr = %q", hits.Load(), stderr) + } + state := (*bodies)[0]["state"].(map[string]any) + if state["query"] != "where does project zebra deploy" { + t.Fatalf("state query = %v", state["query"]) + } + }) + } +} + +func TestRecallJudgeMinRaisesTheBar(t *testing.T) { + // Given a judge that scores relevant memories 0.9 + configureJudgedRecall(t) + startJudgeServer(t, http.StatusOK) + recJudge, recJudgeMin = true, 0.95 + + // When the minimum is above that score + out, _ := runRecall(t) + + // Then nothing survives + if _, n := recallIDs(t, out, false); n != 0 { + t.Fatalf("results = %d, want 0\n%s", n, out) + } +} + +func TestRecallJudgeFailsOpenOnServerError(t *testing.T) { + // Given the unjudged output and a judge server that errors + configureJudgedRecall(t) + baseline, _ := runRecall(t) + startJudgeServer(t, http.StatusInternalServerError) + recJudge = true + + // When recall runs with --judge + out, stderr := runRecall(t) + + // Then the output is unchanged and one warning line is printed + if out != baseline { + t.Fatalf("fail-open output differs:\n%s\nvs\n%s", out, baseline) + } + if !strings.HasPrefix(stderr, "warning: judge unavailable") || strings.Count(stderr, "\n") != 1 { + t.Fatalf("stderr = %q, want a single warning line", stderr) + } +} + +func TestRecallJudgeWithoutConfigurationWarns(t *testing.T) { + // Given no judge model + configureJudgedRecall(t) + baseline, _ := runRecall(t) + t.Setenv("MNEMON_JUDGE_MODEL", "") + recJudge = true + + // When recall runs with --judge + out, stderr := runRecall(t) + + // Then the output is unchanged and the warning names the variable + if out != baseline { + t.Fatalf("unconfigured output differs from baseline") + } + if !strings.Contains(stderr, "MNEMON_JUDGE_MODEL") || strings.Count(stderr, "\n") != 1 { + t.Fatalf("stderr = %q", stderr) + } +} + +func TestRecallJudgeMinOutOfRangeIsRejected(t *testing.T) { + // Given --judge with an impossible threshold + configureJudgedRecall(t) + recJudge, recJudgeMin = true, 1.5 + + // When recall runs + err := recallCmd.RunE(recallCmd, []string{"anything"}) + + // Then it is rejected before any work + if err == nil || !strings.Contains(err.Error(), "--judge-min") { + t.Fatalf("err = %v", err) + } +} + +func configureJudgedRemember(t *testing.T) { + t.Helper() + configureRememberTest(t) + oldJudge := remJudge + t.Cleanup(func() { remJudge = oldJudge }) + t.Setenv("MNEMON_MAX_INSIGHTS", "1000") + remJudge = true + + db, err := store.Open(store.StoreDir(dataDir, store.DefaultStoreName)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + insertTestInsight(t, db, "west", zebraWest, "test", "2026-01-01T00:00:00Z") +} + +func runRemember(t *testing.T) (stdout, stderr string) { + t.Helper() + var runErr error + stderr = captureStderr(t, func() { + stdout = captureStdout(t, func() { + runErr = rememberCmd.RunE(rememberCmd, []string{"the deploy target for project zebra moved to staging-east"}) + }) + }) + if runErr != nil { + t.Fatalf("remember: %v", runErr) + } + return stdout, stderr +} + +type rememberOutput struct { + Candidates []struct { + ID string `json:"id"` + AutoLinked bool `json:"auto_linked"` + Judgment *struct { + Relation string `json:"relation"` + Probability float64 `json:"probability"` + } `json:"judgment"` + } `json:"semantic_candidates"` +} + +func TestRememberJudgeAnnotatesCandidatesWithoutCreatingEdges(t *testing.T) { + // Given an existing memory and a judge that calls the update a supersession + configureJudgedRemember(t) + hits, bodies := startJudgeServer(t, http.StatusOK) + + // When the updated memory is remembered with --judge + out, stderr := runRemember(t) + + // Then the candidate carries the judgment, and no supersedes edge was written + var result rememberOutput + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("decode: %v\n%s", err, out) + } + if len(result.Candidates) != 1 || result.Candidates[0].ID != "west" { + t.Fatalf("candidates = %+v\n%s", result.Candidates, out) + } + j := result.Candidates[0].Judgment + if j == nil || j.Relation != "supersedes" || j.Probability != 0.87 { + t.Fatalf("judgment = %+v", j) + } + if hits.Load() != 1 || stderr != "" { + t.Fatalf("requests = %d, stderr = %q", hits.Load(), stderr) + } + q := (*bodies)[0]["questions"].(map[string]map[string]any)["c0"] + if q["type"] != "choice" || len(q["criteria"].(map[string]any)) != 5 { + t.Fatalf("question = %v", q) + } + + db, err := store.Open(store.StoreDir(dataDir, store.DefaultStoreName)) + if err != nil { + t.Fatal(err) + } + defer db.Close() + edges, err := db.GetAllEdges() + if err != nil { + t.Fatal(err) + } + for _, e := range edges { + if e.EdgeType == model.EdgeSupersedes { + t.Fatalf("remember --judge created a supersedes edge: %+v", e) + } + } +} + +func TestRememberJudgeLeavesOutputUnchangedOnFailure(t *testing.T) { + // Given a judge server that errors + configureJudgedRemember(t) + startJudgeServer(t, http.StatusInternalServerError) + + // When remember runs with --judge + out, stderr := runRemember(t) + + // Then candidates have no judgment and a single warning is printed + var result rememberOutput + if err := json.Unmarshal([]byte(out), &result); err != nil { + t.Fatalf("decode: %v\n%s", err, out) + } + if len(result.Candidates) != 1 || result.Candidates[0].Judgment != nil || strings.Contains(out, `"judgment"`) { + t.Fatalf("output was altered by a failed judge: %s", out) + } + if !strings.HasPrefix(stderr, "warning: judge unavailable") || strings.Count(stderr, "\n") != 1 { + t.Fatalf("stderr = %q", stderr) + } +} + +func TestJudgeCandidatesIncludesAutoLinked(t *testing.T) { + // Given a near-identical auto-linked candidate and a review-zone candidate + hits, bodies := startJudgeServer(t, http.StatusOK) + candidates := []graph.SemanticCandidate{ + {ID: "west", Content: zebraWest, Similarity: 0.91, AutoLinked: true}, + {ID: "runbook", Content: zebraRunb, Similarity: 0.62}, + } + + // When the new memory's candidates are judged + judged, ok := judgeCandidates("the deploy target for project zebra moved to staging-east", candidates) + + // Then both carry a judgment: the auto-linked one is the likeliest supersession + if !ok || len(judged) != 2 { + t.Fatalf("judged = %+v, ok = %v", judged, ok) + } + for _, c := range judged { + if c.Judgment == nil || c.Judgment.Relation != "supersedes" { + t.Fatalf("candidate %s judgment = %+v", c.ID, c.Judgment) + } + } + if hits.Load() != 1 || len((*bodies)[0]["questions"].(map[string]map[string]any)) != 2 { + t.Fatalf("requests = %d, bodies = %v", hits.Load(), *bodies) + } +} + +func TestRememberWithoutJudgeNeverCallsIt(t *testing.T) { + // Given a configured judge server but no --judge flag + configureJudgedRemember(t) + remJudge = false + hits, _ := startJudgeServer(t, http.StatusOK) + + // When remember runs + out, _ := runRemember(t) + + // Then the server is untouched and the output has no judgment + if hits.Load() != 0 || strings.Contains(out, `"judgment"`) { + t.Fatalf("requests = %d, output = %s", hits.Load(), out) + } +} diff --git a/cmd/memory/recall.go b/cmd/memory/recall.go index b19f6a76c..390563394 100644 --- a/cmd/memory/recall.go +++ b/cmd/memory/recall.go @@ -24,6 +24,8 @@ var ( recVerbose bool recBrief bool recExcerpt int + recJudge bool + recJudgeMin float64 ) // compactResult is the LLM-friendly projection of a recall result. @@ -42,6 +44,8 @@ type compactResult struct { // It must survive the compact projection: an agent reading only this // shape would otherwise be handed corrected content with no signal. Superseded bool `json:"superseded,omitempty"` + // Relevance is the judge's relevance probability; absent unless --judge ran. + Relevance *float64 `json:"relevance,omitempty"` } // compactResponse wraps compact results with an optional hint. @@ -94,6 +98,7 @@ func toCompact(resp search.RecallResponse) compactResponse { Confidence: confidenceLabel(rounded), Score: rounded, Superseded: r.Superseded, + Relevance: r.Relevance, } results = append(results, cr) } @@ -130,6 +135,9 @@ meta.intent and meta.intent_source (auto or override). --basic bypasses intent.` if recBrief && recVerbose { return fmt.Errorf("--brief and --verbose cannot be used together") } + if recJudge && !(recJudgeMin >= 0 && recJudgeMin <= 1) { + return fmt.Errorf("--judge-min must be between 0 and 1, got %v", recJudgeMin) + } db, err := openDB() if err != nil { @@ -141,6 +149,9 @@ meta.intent and meta.intent_source (auto or override). --basic bypasses intent.` enc.SetIndent("", " ") if recBasic { + if recJudge { + warnJudge("--judge is ignored with --basic") + } // Legacy SQL LIKE recall. results, err := db.QueryInsights(store.QueryFilter{ Keyword: keyword, @@ -205,6 +216,9 @@ meta.intent and meta.intent_source (auto or override). --basic bypasses intent.` if err != nil { return fmt.Errorf("recall: %w", err) } + if recJudge { + judgeRecall(keyword, &resp) + } for _, r := range resp.Results { _ = db.IncrementAccessCount(r.Insight.ID) } @@ -224,6 +238,7 @@ meta.intent and meta.intent_source (auto or override). --basic bypasses intent.` Score: scorePointer(score), Confidence: confidenceLabel(score), Superseded: result.Superseded, + Relevance: result.Relevance, }) } return encodeBrief(os.Stdout, newBriefResponse(brief, resp.Meta.Hint)) @@ -243,5 +258,7 @@ func init() { recallCmd.Flags().BoolVar(&recVerbose, "verbose", false, "output full recall response (signals, meta, timestamps)") recallCmd.Flags().BoolVar(&recBrief, "brief", false, "output short excerpts for discovery; use 'mnemon show ' for full content") recallCmd.Flags().IntVar(&recExcerpt, "excerpt-chars", defaultBriefExcerptChars, "maximum characters per --brief excerpt") + recallCmd.Flags().BoolVar(&recJudge, "judge", false, "score results with the decision-model judge and drop irrelevant ones (env: MNEMON_JUDGE_MODEL)") + recallCmd.Flags().Float64Var(&recJudgeMin, "judge-min", 0.5, "minimum judge relevance (0-1) a result needs to be kept with --judge") rootCmd.AddCommand(recallCmd) } diff --git a/cmd/memory/remember.go b/cmd/memory/remember.go index ca744f522..0df515e05 100644 --- a/cmd/memory/remember.go +++ b/cmd/memory/remember.go @@ -24,6 +24,7 @@ var ( remEntities string remEntityMode string remNoDiff bool + remJudge bool ) var rememberCmd = &cobra.Command{ @@ -253,6 +254,13 @@ var rememberCmd = &cobra.Command{ causalCandidates = []graph.CausalCandidate{} } + var semanticOutput any = semanticCandidates + if remJudge { + if judged, ok := judgeCandidates(insight.Content, semanticCandidates); ok { + semanticOutput = judged + } + } + output := map[string]interface{}{ "id": insight.ID, "content": insight.Content, @@ -264,7 +272,7 @@ var rememberCmd = &cobra.Command{ "diff_suggestion": string(diffSuggestion), "created_at": insight.CreatedAt.Format(time.RFC3339), "edges_created": edgeStats, - "semantic_candidates": semanticCandidates, + "semantic_candidates": semanticOutput, "causal_candidates": causalCandidates, "embedded": embedded, "effective_importance": ei, @@ -285,5 +293,6 @@ func init() { rememberCmd.Flags().StringVar(&remEntities, "entities", "", "comma-separated entities (LLM-extracted, merged with auto-extraction)") rememberCmd.Flags().StringVar(&remEntityMode, "entity-mode", string(graph.EntityModeMerge), "entity handling mode (merge|provided|auto)") rememberCmd.Flags().BoolVar(&remNoDiff, "no-diff", false, "skip duplicate/conflict detection") + rememberCmd.Flags().BoolVar(&remJudge, "judge", false, "classify semantic candidates with the decision-model judge (env: MNEMON_JUDGE_MODEL)") rootCmd.AddCommand(rememberCmd) } diff --git a/internal/memory/search/recall.go b/internal/memory/search/recall.go index c15977431..fe3d306d2 100644 --- a/internal/memory/search/recall.go +++ b/internal/memory/search/recall.go @@ -117,6 +117,9 @@ type RecallResult struct { // Surfaced so a caller can see why a result ranked low, and so an agent // reading raw results is not silently handed stale content. Superseded bool `json:"superseded,omitempty"` + // Relevance is the decision-model judge's probability that this result + // answers the query. It is nil unless recall ran with --judge. + Relevance *float64 `json:"relevance,omitempty"` } // IntentAwareRecall performs MAGMA-aligned intent-aware retrieval: diff --git a/test/mnemond/architecture/mnemond_structure_test.go b/test/mnemond/architecture/mnemond_structure_test.go index 00984cde9..e4ec1fc6f 100644 --- a/test/mnemond/architecture/mnemond_structure_test.go +++ b/test/mnemond/architecture/mnemond_structure_test.go @@ -72,6 +72,7 @@ func assertMnemondPackageGraph(t *testing.T, root string) { "internal/localrpc": {}, "internal/retrieval": {}, "internal/memory/embed": {}, + "internal/memory/judge": {}, "internal/memory/internal/topk": {}, "internal/memory/model": {}, "internal/memory/service": {"internal/memory/graph", "internal/memory/search", "internal/memory/store"}, @@ -102,9 +103,11 @@ func assertMnemondPackageGraph(t *testing.T, root string) { }, "cmd/agency": {"internal/agency/attach", "internal/agency/client", "internal/daemon"}, "cmd/memory": { - "internal/memory/embed", "internal/memory/graph", "internal/memory/importdraft", - "internal/memory/model", "internal/memory/search", "internal/memory/setup", - "internal/memory/setup/assets", "internal/memory/store", "internal/memory/web", "internal/records", "internal/memory/service", "internal/memoryagent", "internal/aml", "internal/localrpc", "web", + "internal/aml", "internal/localrpc", "internal/memory/embed", "internal/memory/graph", + "internal/memory/importdraft", "internal/memory/judge", "internal/memory/model", + "internal/memory/search", "internal/memory/service", "internal/memory/setup", + "internal/memory/setup/assets", "internal/memory/store", "internal/memory/web", + "internal/memoryagent", "internal/records", "web", }, "web": {}, } From fd31b84b8cecc0c4751d917023c7c61d4edba905 Mon Sep 17 00:00:00 2001 From: Audrey Tang <20723+audreyt@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:53:03 +0800 Subject: [PATCH 3/5] docs: document the decision-model judge Describe MNEMON_JUDGE_MODEL, MNEMON_JUDGE_ENDPOINT, MNEMON_JUDGE_API_KEY, and MNEMON_JUDGE_TIMEOUT, the recall and remember --judge flags, and --judge-min in the README and usage guide, and state in both them and SECURITY.md that a cloud endpoint receives the query and memory text. Documentation only; the flag and variable defaults were checked against the code. --- README.md | 22 ++++++++++++++++++++++ SECURITY.md | 1 + docs/USAGE.md | 17 +++++++++++++++++ 3 files changed, 40 insertions(+) diff --git a/README.md b/README.md index 0952dbf5b..1e558b810 100644 --- a/README.md +++ b/README.md @@ -574,6 +574,28 @@ export MNEMON_EMBED_API_KEY=pa-... mnemon embed --status ``` +**Decision-model judge** (optional, opt-in per command): + +| Environment Variable | Default | Description | +|---|---|---| +| `MNEMON_JUDGE_MODEL` | *(empty = disabled)* | System One model name, e.g. `clef` or `clef-flash` (local Ollama), `jev-latest` (TypeSafe) | +| `MNEMON_JUDGE_ENDPOINT` | `http://localhost:11434` | Base URL of a System One server (`POST {endpoint}/v1/systemone`) | +| `MNEMON_JUDGE_API_KEY` | *(none)* | Bearer token for cloud endpoints such as `https://api.typesafe.ai` or `https://api.subq.ai` | +| `MNEMON_JUDGE_TIMEOUT` | `15s` | Total time budget for one judgment, across all of its requests (Go duration) | + +`mnemon recall --judge` asks the judge whether each ranked result is about the +same subject as the query, adds a `relevance` probability to every result, and +drops results below `--judge-min` (default `0.5`). `mnemon remember --judge` +classifies each semantic candidate as `duplicate`, +`supersedes`, `contradicts`, `related`, or `unrelated` and reports it as +`judgment` on that candidate; it never creates edges. Without `--judge`, output +is unchanged, and if the judge is missing or fails, mnemon prints a one-line +warning to stderr and returns normal output. + +> **Privacy:** the judge sends the query and the text of the judged memories to +> `MNEMON_JUDGE_ENDPOINT`. A cloud endpoint therefore receives that text; keep +> the default local Ollama endpoint if memories must not leave the machine. + ## Development ```bash diff --git a/SECURITY.md b/SECURITY.md index 49dcb9f56..fb8dcdffa 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -18,6 +18,7 @@ service; its model is under "Network service" below. Key security considerations - **SQLite database** — contains all stored insights; protected by filesystem permissions (`0644`). - **Hook scripts** — shell scripts executed by the LLM CLI at lifecycle events; written with `0755` permissions. - **Embedding provider connection** — optional requests send insight or query text to the configured Ollama or OpenAI-compatible server. The default local Ollama endpoint does not use TLS. If `MNEMON_EMBED_ENDPOINT` points outside a trusted local network, use HTTPS to protect content and any `MNEMON_EMBED_API_KEY` bearer token in transit. +- **Decision-model judge connection** — only when `--judge` is passed and `MNEMON_JUDGE_MODEL` is set, `recall` and `remember` send the query and the text of the judged memories to `MNEMON_JUDGE_ENDPOINT`. A cloud endpoint therefore receives that text. The default local Ollama endpoint does not use TLS; use HTTPS for any other endpoint to protect content and the `MNEMON_JUDGE_API_KEY` bearer token. ### Network service (memory cloud instances) diff --git a/docs/USAGE.md b/docs/USAGE.md index 3aa9430e8..15c3348b2 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -169,6 +169,9 @@ mnemon recall "why did we choose Qdrant" --intent WHY # Recall with category/source filter mnemon recall "auth" --cat decision --source agent +# Score results with the decision-model judge and drop irrelevant ones +MNEMON_JUDGE_MODEL=clef mnemon recall "vector database" --judge --judge-min 0.5 + # Simple SQL LIKE matching (faster, no graph traversal) mnemon recall "auth" --basic @@ -208,6 +211,7 @@ authorizes replacement. Capacity-based auto-pruning still applies separately. | `--entity-mode` | `merge` | Entity handling: `merge` (provided + auto), `provided` (only `--entities`), `auto` (only auto-extraction) | | `--source` | `user` | Source: `user`, `agent`, `external` | | `--no-diff` | `false` | Skip duplicate/conflict detection | +| `--judge` | `false` | Classify each semantic candidate with the decision-model judge (`judgment.relation`: `duplicate`, `supersedes`, `contradicts`, `related`, `unrelated`); creates no edges. Needs `MNEMON_JUDGE_MODEL` | **Recall flags:** @@ -220,6 +224,8 @@ authorizes replacement. Capacity-based auto-pruning still applies separately. | `--basic` | `false` | Use simple SQL LIKE matching instead of smart recall | | `--brief` | `false` | Emit compact JSON with short excerpts for discovery; fetch selected full content with `mnemon show ` | | `--excerpt-chars` | `240` | Maximum Unicode characters per `--brief` excerpt | +| `--judge` | `false` | Score results with the decision-model judge, add a `relevance` field, and drop results below `--judge-min`. Needs `MNEMON_JUDGE_MODEL`; on a missing or failing judge it warns on stderr and returns normal output | +| `--judge-min` | `0.5` | Minimum judge relevance (0-1) a result needs to be kept with `--judge` | | `--verbose` | `false` | Output full recall response (signals, meta, timestamps) | The default compact output is optimized for LLM/agent consumption. It includes @@ -412,9 +418,20 @@ Nodes are colored by category (decision, fact, insight, preference, context); ed | `MNEMON_EMBED_API_KEY` | (none) | Bearer token for OpenAI-compatible servers | | `MNEMON_EMBED_KEEP_ALIVE` | 30m | How long Ollama keeps the embedding model loaded after each request | | `MNEMON_EMBED_DIMENSIONS` | (native) | Embedding dimensions; set to truncate (e.g., `256` for Matryoshka models) | +| `MNEMON_JUDGE_MODEL` | (empty = disabled) | System One model for `--judge` (`clef`, `clef-flash`, `jev-latest`, ...) | +| `MNEMON_JUDGE_ENDPOINT` | `http://localhost:11434` | System One base URL; requests go to `{endpoint}/v1/systemone` | +| `MNEMON_JUDGE_API_KEY` | (none) | Bearer token for cloud System One endpoints | +| `MNEMON_JUDGE_TIMEOUT` | `15s` | Total time budget for one judgment (Go duration) | | `MNEMON_MAX_INSIGHTS` | `1000` | Active-insight ceiling before auto-pruning starts; `0` disables auto-pruning | | `MNEMON_AUTO_PRUNE_MIN_AGE` | `24h` | Minimum age before automatic pruning; accepts durations such as `24h`, integer days such as `7d`, or `0` to disable the grace period | +`--judge` talks to a [System One](https://docs.ollama.com/api/systemone) decision-model API. A +local Ollama (>= 0.35) serving `clef` or `clef-flash` keeps all text on the +machine. Pointing `MNEMON_JUDGE_ENDPOINT` at a cloud provider sends the recall +query and the text of the judged memories (each clipped to about 1200 +characters) to that provider. Questions are sent in requests of at most 16, each +repeating the same state. + Auto-prune may temporarily leave the active count above the ceiling when every eligible insight is still inside the grace period. Age is measured from local store insertion, so newly imported historical memories are protected too. Each deletion is soft, From 9d31cfba176e71fde8391b8dced68bab4d89ecc1 Mon Sep 17 00:00:00 2001 From: Audrey Tang <20723+audreyt@users.noreply.github.com> Date: Thu, 15 Oct 2026 12:53:03 +0800 Subject: [PATCH 4/5] docs(zh): document the decision-model judge Mirror the English README and usage guide in the Chinese docs: the MNEMON_JUDGE_* variables, the recall and remember --judge flags, and --judge-min, including the note that a cloud endpoint receives the query and memory text. Documentation only; defaults and flag names were compared with the English docs and the code. --- docs/zh/README.md | 4 ++++ docs/zh/USAGE.md | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/docs/zh/README.md b/docs/zh/README.md index 92c82e667..3a4ddb822 100644 --- a/docs/zh/README.md +++ b/docs/zh/README.md @@ -462,6 +462,10 @@ Sub-agent 委派是可选执行策略。当 runtime 支持时,主 agent 可以 | `MNEMON_EMBED_API_KEY` | *(无)* | OpenAI 兼容服务器(oMLX、vLLM 等)的 Bearer 令牌 | | `MNEMON_EMBED_KEEP_ALIVE` | `30m` | 每次请求后 Ollama 保持嵌入模型加载的时长(OpenAI 兼容服务器忽略此项) | | `MNEMON_EMBED_DIMENSIONS` | *(原生维度)* | 可选的 Matryoshka 维度截断 | +| `MNEMON_JUDGE_MODEL` | *(空 = 禁用)* | `--judge` 使用的 System One 模型(`clef`、`clef-flash`、`jev-latest` 等) | +| `MNEMON_JUDGE_ENDPOINT` | `http://localhost:11434` | System One 基础 URL,请求发往 `{endpoint}/v1/systemone`;云端端点会收到查询以及被判断记忆的文本 | +| `MNEMON_JUDGE_API_KEY` | *(无)* | 云端 System One 端点的 Bearer 令牌 | +| `MNEMON_JUDGE_TIMEOUT` | `15s` | 单次判断的总时间预算(Go duration) | 每次自动删除均为软删除,以 `prune` 操作记录到 oplog,并通过触发命令的 `auto_pruned_ids` 字段返回具体 ID。 diff --git a/docs/zh/USAGE.md b/docs/zh/USAGE.md index cd9f37833..61b4f0753 100644 --- a/docs/zh/USAGE.md +++ b/docs/zh/USAGE.md @@ -196,6 +196,7 @@ mnemon forget | `--entity-mode` | `merge` | 实体处理模式:`merge`(传入实体 + 自动抽取)、`provided`(只用 `--entities`)、`auto`(只用自动抽取) | | `--source` | `user` | 来源:`user`、`agent`、`external` | | `--no-diff` | `false` | 跳过重复/冲突检测 | +| `--judge` | `false` | 用决策模型 judge 对每个语义候选分类(`judgment.relation`:`duplicate`、`supersedes`、`contradicts`、`related`、`unrelated`);不会创建任何边。需要 `MNEMON_JUDGE_MODEL` | **Recall 标志:** @@ -208,6 +209,8 @@ mnemon forget | `--basic` | `false` | 使用简单 SQL LIKE 匹配代替智能召回 | | `--brief` | `false` | 输出仅含短摘要的紧凑 JSON;使用 `mnemon show ` 获取选中项全文 | | `--excerpt-chars` | `240` | 每条 `--brief` 摘要最多包含的 Unicode 字符数 | +| `--judge` | `false` | 用决策模型 judge 为结果打分,添加 `relevance` 字段,并丢弃低于 `--judge-min` 的结果。需要 `MNEMON_JUDGE_MODEL`;judge 缺失或失败时在 stderr 输出警告并返回正常结果 | +| `--judge-min` | `0.5` | 使用 `--judge` 时保留结果所需的最低 judge 相关度(0-1) | | `--verbose` | `false` | 输出完整召回响应(signals、meta、时间戳) | 默认紧凑输出针对 LLM/agent 消费优化,包含 `id`、`content`、`category`、 @@ -376,6 +379,10 @@ open graph.html | `MNEMON_EMBED_API_KEY` | (无) | OpenAI 兼容服务器的 Bearer 令牌 | | `MNEMON_EMBED_KEEP_ALIVE` | 30m | 每次请求后 Ollama 保持嵌入模型加载的时长 | | `MNEMON_EMBED_DIMENSIONS` | (原生维度) | 嵌入向量维度;可设置截断值(例如 Matryoshka 模型使用 `256`) | +| `MNEMON_JUDGE_MODEL` | (空 = 禁用) | `--judge` 使用的 System One 模型(`clef`、`clef-flash`、`jev-latest` 等) | +| `MNEMON_JUDGE_ENDPOINT` | `http://localhost:11434` | System One 基础 URL,请求发往 `{endpoint}/v1/systemone`;云端端点会收到查询以及被判断记忆的文本 | +| `MNEMON_JUDGE_API_KEY` | (无) | 云端 System One 端点的 Bearer 令牌 | +| `MNEMON_JUDGE_TIMEOUT` | `15s` | 单次判断的总时间预算(Go duration) | | `MNEMON_MAX_INSIGHTS` | `1000` | 触发自动清理的活跃洞察数量上限;设为 `0` 可关闭自动清理 | | `MNEMON_AUTO_PRUNE_MIN_AGE` | `24h` | 可被自动清理前的最短存活时间;支持 `24h`、整数天 `7d`,设为 `0` 可关闭保护期 | From 8dc931c6183d2c4a1db87f10ba22fe6b0c966879 Mon Sep 17 00:00:00 2001 From: Audrey Tang Date: Sun, 22 Nov 2026 08:10:15 +0800 Subject: [PATCH 5/5] feat(judge): support Karpo Decision endpoints and nested error responses Normalize MNEMON_JUDGE_ENDPOINT values that already end in /v1 or /v1/systemone, parse nested {"error":{"message":"..."}} error payloads returned by Karpo Decision and TypeSafe servers alongside Ollama's flat string format, and add wire-compatibility tests and docs. --- README.md | 2 +- docs/USAGE.md | 2 +- docs/zh/README.md | 2 +- docs/zh/USAGE.md | 2 +- internal/memory/judge/judge.go | 35 ++++++++++++++++--- internal/memory/judge/judge_test.go | 54 +++++++++++++++++++++++++++++ 6 files changed, 88 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index 1e558b810..b9bd67d76 100644 --- a/README.md +++ b/README.md @@ -578,7 +578,7 @@ mnemon embed --status | Environment Variable | Default | Description | |---|---|---| -| `MNEMON_JUDGE_MODEL` | *(empty = disabled)* | System One model name, e.g. `clef` or `clef-flash` (local Ollama), `jev-latest` (TypeSafe) | +| `MNEMON_JUDGE_MODEL` | *(empty = disabled)* | System One model name, e.g. `clef` or `clef-flash` (local Ollama), `karpo-decision` (self-hosted Karpo Decision), `jev-latest` (TypeSafe) | | `MNEMON_JUDGE_ENDPOINT` | `http://localhost:11434` | Base URL of a System One server (`POST {endpoint}/v1/systemone`) | | `MNEMON_JUDGE_API_KEY` | *(none)* | Bearer token for cloud endpoints such as `https://api.typesafe.ai` or `https://api.subq.ai` | | `MNEMON_JUDGE_TIMEOUT` | `15s` | Total time budget for one judgment, across all of its requests (Go duration) | diff --git a/docs/USAGE.md b/docs/USAGE.md index 15c3348b2..43d98fa72 100644 --- a/docs/USAGE.md +++ b/docs/USAGE.md @@ -418,7 +418,7 @@ Nodes are colored by category (decision, fact, insight, preference, context); ed | `MNEMON_EMBED_API_KEY` | (none) | Bearer token for OpenAI-compatible servers | | `MNEMON_EMBED_KEEP_ALIVE` | 30m | How long Ollama keeps the embedding model loaded after each request | | `MNEMON_EMBED_DIMENSIONS` | (native) | Embedding dimensions; set to truncate (e.g., `256` for Matryoshka models) | -| `MNEMON_JUDGE_MODEL` | (empty = disabled) | System One model for `--judge` (`clef`, `clef-flash`, `jev-latest`, ...) | +| `MNEMON_JUDGE_MODEL` | (empty = disabled) | System One model for `--judge` (`clef`, `clef-flash`, `karpo-decision`, `jev-latest`, ...) | | `MNEMON_JUDGE_ENDPOINT` | `http://localhost:11434` | System One base URL; requests go to `{endpoint}/v1/systemone` | | `MNEMON_JUDGE_API_KEY` | (none) | Bearer token for cloud System One endpoints | | `MNEMON_JUDGE_TIMEOUT` | `15s` | Total time budget for one judgment (Go duration) | diff --git a/docs/zh/README.md b/docs/zh/README.md index 3a4ddb822..ea15ded2d 100644 --- a/docs/zh/README.md +++ b/docs/zh/README.md @@ -462,7 +462,7 @@ Sub-agent 委派是可选执行策略。当 runtime 支持时,主 agent 可以 | `MNEMON_EMBED_API_KEY` | *(无)* | OpenAI 兼容服务器(oMLX、vLLM 等)的 Bearer 令牌 | | `MNEMON_EMBED_KEEP_ALIVE` | `30m` | 每次请求后 Ollama 保持嵌入模型加载的时长(OpenAI 兼容服务器忽略此项) | | `MNEMON_EMBED_DIMENSIONS` | *(原生维度)* | 可选的 Matryoshka 维度截断 | -| `MNEMON_JUDGE_MODEL` | *(空 = 禁用)* | `--judge` 使用的 System One 模型(`clef`、`clef-flash`、`jev-latest` 等) | +| `MNEMON_JUDGE_MODEL` | *(空 = 禁用)* | `--judge` 使用的 System One 模型(`clef`、`clef-flash`、`karpo-decision`、`jev-latest` 等) | | `MNEMON_JUDGE_ENDPOINT` | `http://localhost:11434` | System One 基础 URL,请求发往 `{endpoint}/v1/systemone`;云端端点会收到查询以及被判断记忆的文本 | | `MNEMON_JUDGE_API_KEY` | *(无)* | 云端 System One 端点的 Bearer 令牌 | | `MNEMON_JUDGE_TIMEOUT` | `15s` | 单次判断的总时间预算(Go duration) | diff --git a/docs/zh/USAGE.md b/docs/zh/USAGE.md index 61b4f0753..dbb84a6ed 100644 --- a/docs/zh/USAGE.md +++ b/docs/zh/USAGE.md @@ -379,7 +379,7 @@ open graph.html | `MNEMON_EMBED_API_KEY` | (无) | OpenAI 兼容服务器的 Bearer 令牌 | | `MNEMON_EMBED_KEEP_ALIVE` | 30m | 每次请求后 Ollama 保持嵌入模型加载的时长 | | `MNEMON_EMBED_DIMENSIONS` | (原生维度) | 嵌入向量维度;可设置截断值(例如 Matryoshka 模型使用 `256`) | -| `MNEMON_JUDGE_MODEL` | (空 = 禁用) | `--judge` 使用的 System One 模型(`clef`、`clef-flash`、`jev-latest` 等) | +| `MNEMON_JUDGE_MODEL` | (空 = 禁用) | `--judge` 使用的 System One 模型(`clef`、`clef-flash`、`karpo-decision`、`jev-latest` 等) | | `MNEMON_JUDGE_ENDPOINT` | `http://localhost:11434` | System One 基础 URL,请求发往 `{endpoint}/v1/systemone`;云端端点会收到查询以及被判断记忆的文本 | | `MNEMON_JUDGE_API_KEY` | (无) | 云端 System One 端点的 Bearer 令牌 | | `MNEMON_JUDGE_TIMEOUT` | `15s` | 单次判断的总时间预算(Go duration) | diff --git a/internal/memory/judge/judge.go b/internal/memory/judge/judge.go index f5b4cceda..cb7ad6abe 100644 --- a/internal/memory/judge/judge.go +++ b/internal/memory/judge/judge.go @@ -205,6 +205,21 @@ func (c *Client) ask(ctx context.Context, state any, questions []wireQuestion) ( return answers, nil } +// systemOneURL resolves the POST target from endpoint. A base URL gets +// /v1/systemone appended; an endpoint that already ends with /v1 or +// /v1/systemone is normalized without duplicating the path. +func systemOneURL(endpoint string) (string, error) { + trimmed := strings.TrimRight(endpoint, "/") + switch { + case strings.HasSuffix(trimmed, "/v1/systemone"): + return trimmed, nil + case strings.HasSuffix(trimmed, "/v1"): + return url.JoinPath(trimmed, "systemone") + default: + return url.JoinPath(endpoint, systemOneRoute) + } +} + // post sends one System One request and decodes its answers. func (c *Client) post(ctx context.Context, state json.RawMessage, chunk []wireQuestion) (map[string]wireAnswer, error) { questions := make(map[string]wireQuestion, len(chunk)) @@ -215,7 +230,7 @@ func (c *Client) post(ctx context.Context, state json.RawMessage, chunk []wireQu if err != nil { return nil, fmt.Errorf("marshal judge request: %w", err) } - endpointURL, err := url.JoinPath(c.endpoint, systemOneRoute) + endpointURL, err := systemOneURL(c.endpoint) if err != nil { return nil, fmt.Errorf("join judge endpoint: %w", err) } @@ -251,14 +266,24 @@ func (c *Client) post(ctx context.Context, state json.RawMessage, chunk []wireQu } // statusError turns a non-200 response into an error, preferring the server's -// {"error": "..."} message. +// {"error": "..."} (Ollama) or {"error": {"message": "..."}} (Karpo / TypeSafe) +// message. func statusError(resp *http.Response) error { data, _ := io.ReadAll(io.LimitReader(resp.Body, maxErrorBytes)) var payload struct { - Error string `json:"error"` + Error json.RawMessage `json:"error"` } - if json.Unmarshal(data, &payload) == nil && payload.Error != "" { - return fmt.Errorf("judge server returned status %d: %s", resp.StatusCode, payload.Error) + if json.Unmarshal(data, &payload) == nil && len(payload.Error) > 0 { + var flat string + if json.Unmarshal(payload.Error, &flat) == nil && flat != "" { + return fmt.Errorf("judge server returned status %d: %s", resp.StatusCode, flat) + } + var nested struct { + Message string `json:"message"` + } + if json.Unmarshal(payload.Error, &nested) == nil && nested.Message != "" { + return fmt.Errorf("judge server returned status %d: %s", resp.StatusCode, nested.Message) + } } return fmt.Errorf("judge server returned status %d", resp.StatusCode) } diff --git a/internal/memory/judge/judge_test.go b/internal/memory/judge/judge_test.go index 529c5d80d..a4b7926b3 100644 --- a/internal/memory/judge/judge_test.go +++ b/internal/memory/judge/judge_test.go @@ -259,6 +259,7 @@ func TestAsk_rejectsBadResponses(t *testing.T) { wantErr string }{ {"server error json", 400, `{"error":"unknown model"}`, "status 400: unknown model"}, + {"server error nested json", 422, `{"error":{"message":"question too long"}}`, "status 422: question too long"}, {"server error without json", 500, `oops`, "status 500"}, {"malformed json", 200, `{"answers":`, "decode judge response"}, {"missing answer", 200, `{"answers":{}}`, "missing an answer"}, @@ -354,3 +355,56 @@ func TestAsk_bypassesProxyForLoopback(t *testing.T) { t.Fatalf("loopback request went through the proxy: %v", err) } } + +func TestKarpoDecisionWireCompatibility(t *testing.T) { + // Karpo Decision (karpo_decision.serve.server) returns id, model, answers, + // and usage with output_tokens: 0 on POST /v1/systemone, and accepts + // endpoints configured as base URL, /v1, or /v1/systemone. + for _, suffix := range []string{"", "/v1", "/v1/systemone"} { + t.Run("endpoint"+suffix, func(t *testing.T) { + var gotPath string + var gotContentLength int64 + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + gotPath = r.URL.Path + gotContentLength = r.ContentLength + w.Header().Set("Content-Type", "application/json") + _, _ = w.Write([]byte(`{"id":"decision-abc123","model":"karpo-decision","answers":{"rel":{"type":"noul","noul":0.91},"edge":{"type":"choice","choice":"supersedes","probabilities":{"supersedes":0.88,"related":0.12},"confidence":0.76}},"usage":{"input_tokens":342,"output_tokens":0}}`)) + })) + t.Cleanup(srv.Close) + + t.Setenv("MNEMON_JUDGE_MODEL", "karpo-decision-31B") + t.Setenv("MNEMON_JUDGE_ENDPOINT", srv.URL+suffix) + t.Setenv("MNEMON_JUDGE_API_KEY", "") + client := NewClient() + if client == nil { + t.Fatal("NewClient returned nil") + } + + noul, err := client.Noul(context.Background(), map[string]string{"text": "switched to postgres"}, []Question{ + {Name: "rel", Instructions: "Is this memory relevant?"}, + }) + if err != nil { + t.Fatalf("Noul failed: %v", err) + } + if gotPath != "/v1/systemone" { + t.Fatalf("path = %q, want /v1/systemone", gotPath) + } + if gotContentLength <= 0 { + t.Fatalf("Content-Length = %d, want > 0 (required by Karpo server)", gotContentLength) + } + if noul["rel"] != 0.91 { + t.Fatalf("noul[rel] = %v, want 0.91", noul["rel"]) + } + + choice, err := client.Choice(context.Background(), "state", []ChoiceQuestion{ + {Name: "edge", Instructions: "Pick relation", Options: map[string]string{"supersedes": "replaces", "related": "related"}}, + }) + if err != nil { + t.Fatalf("Choice failed: %v", err) + } + if choice["edge"].Choice != "supersedes" || choice["edge"].Probability != 0.88 { + t.Fatalf("choice[edge] = %+v, want supersedes@0.88", choice["edge"]) + } + }) + } +}