diff --git a/apps/server/src/mcp/toolkits/knowledge/tools.ts b/apps/server/src/mcp/toolkits/knowledge/tools.ts index 055cf4ace5ee8..802f9565f59f6 100644 --- a/apps/server/src/mcp/toolkits/knowledge/tools.ts +++ b/apps/server/src/mcp/toolkits/knowledge/tools.ts @@ -109,7 +109,7 @@ export const KnowledgeToolkit = Toolkit.make( ...shared, description: MEMORY_WRITING_RULES.join(" ") + - " Search before adding/correcting. Create/edit/supersede require content, kind, entities, tags and sources (arrays may be empty); create needs a fresh factId. Cite the exact source message; only unsourced agent observations may omit citations (caller origin is recorded). Agents cannot claim direct user assertions. Saves immediately; facts are searchable and related ones appear with later user messages. Origin is not proof. Retry with the same commandId; updates need current revision. Prefer supersede to forget for changed facts: replacementFactId preserves the old content but excludes it from recall. Writes return duplicate/conflict hints. Users alone may restore set-aside or superseded facts; forgotten ones are irrecoverable. Use feedback for trust.", + " Search before adding or correcting. Create/edit/supersede require content, kind, entities, tags and sources (may be empty); create needs a new factId. Cite exact messages; only unsourced agent observations may omit citations. Caller origin is recorded. Agents cannot claim direct user assertions. Saves immediately for search and related turns. Origin is not proof. Reuse commandId on retries; updates need current revision. Prefer supersede to forget for changed facts: replacementFactId keeps old content out of recall. Writes flag duplicates/conflicts. Only users restore set-aside or superseded facts; forget is permanent. Feedback helpful: true if useful, false if misleading, irrelevant if off-task.", parameters: Schema.Struct({ mutation: ProjectKnowledgeMutation, environmentId: OrchestratorMcpEnvironmentTarget, diff --git a/apps/server/src/project/knowledge/MemoryCuration.test.ts b/apps/server/src/project/knowledge/MemoryCuration.test.ts index 69a2c623964b1..2e7bce588e59e 100644 --- a/apps/server/src/project/knowledge/MemoryCuration.test.ts +++ b/apps/server/src/project/knowledge/MemoryCuration.test.ts @@ -152,7 +152,53 @@ it.effect("curates only the selected project's automatic facts and protects user const beta = ProjectId.make("beta"); yield* createProject(alpha); yield* createProject(beta); - yield* createFact(alpha, "alpha-auto", "Deployment is limited to the current task."); + const automatic = yield* createFact( + alpha, + "alpha-auto", + "Deployment is limited to the current task.", + ); + const knowledge = yield* ProjectKnowledge.ProjectKnowledge; + const reviewedBy = automatic.origin; + const previous = yield* knowledge.mutate( + { + action: "feedback", + commandId: CommandId.make("rate-old"), + projectId: alpha, + factId: automatic.id, + expectedRevision: automatic.revision, + helpful: "irrelevant", + }, + reviewedBy, + ); + let rated = yield* knowledge.mutate( + { + action: "edit", + commandId: CommandId.make("revise"), + projectId: alpha, + factId: automatic.id, + expectedRevision: previous.revision, + kind: "decision", + content: "Deployment is limited to the current task on staging.", + entities: [], + tags: [], + sources: [], + }, + reviewedBy, + ); + for (const helpful of [true, false, "irrelevant"] as const) { + rated = yield* knowledge.mutate( + { + action: "feedback", + commandId: CommandId.make(`rate-${helpful}`), + projectId: alpha, + factId: automatic.id, + expectedRevision: rated.revision, + helpful, + }, + reviewedBy, + ); + } + assert.closeTo(rated.trust, 0.45, 1e-9); yield* createFact(alpha, "alpha-user", "Deployment approvals always require the owner.", true); yield* createFact( alpha, @@ -181,7 +227,9 @@ it.effect("curates only the selected project's automatic facts and protects user [`${owner}-auto`], ); } - const knowledge = yield* ProjectKnowledge.ProjectKnowledge; + const reviewed = calls.flatMap((call) => call.facts).find((fact) => fact.id === automatic.id)!; + assert.deepEqual(reviewed.feedback, { helpful: 1, unhelpful: 1, irrelevant: 1 }); + assert.equal(reviewed.trust, rated.trust); assert.equal( (yield* knowledge.readFact({ projectId: alpha, factId: ProjectFactId.make("alpha-auto") })) .fact.status, diff --git a/apps/server/src/project/knowledge/MemoryCuration.ts b/apps/server/src/project/knowledge/MemoryCuration.ts index 9a196747b7cf8..2a606c8cf5409 100644 --- a/apps/server/src/project/knowledge/MemoryCuration.ts +++ b/apps/server/src/project/knowledge/MemoryCuration.ts @@ -285,6 +285,28 @@ const make = Effect.gen(function* () { sourceBudget -= excerpt.length; } } + const feedbackRows = yield* sql<{ + readonly id: ProjectFactId; + readonly helpful: number; + readonly unhelpful: number; + readonly irrelevant: number; + }>` + WITH history AS ( + SELECT fact_id, event, helpful, sequence, + MAX(sequence) FILTER (WHERE event = 'edited') OVER (PARTITION BY fact_id) AS last_edit + FROM project_knowledge_fact_history + WHERE project_id = ${projectId} AND ${sql.in( + "fact_id", + facts.map((fact) => fact.id), + )} + AND event IN ('feedback', 'edited') + ) + SELECT fact_id AS id, COUNT(*) FILTER (WHERE helpful = 1) AS helpful, + COUNT(*) FILTER (WHERE helpful = 0) AS unhelpful, + COUNT(*) FILTER (WHERE helpful IS NULL) AS irrelevant + FROM history WHERE event = 'feedback' AND sequence > COALESCE(last_edit, 0) + GROUP BY fact_id`; + const feedbackById = new Map(feedbackRows.map(({ id, ...counts }) => [id, counts])); const candidateIds = new Set(candidates.map((fact) => fact.id)); let packetSourceBudget = MAX_SOURCE_CHARS; const reviewedSources = new Map(); @@ -293,6 +315,8 @@ const make = Effect.gen(function* () { revision: fact.revision, kind: fact.kind, content: fact.content, + trust: fact.trust, + feedback: feedbackById.get(fact.id) ?? { helpful: 0, unhelpful: 0, irrelevant: 0 }, protected: !candidateIds.has(fact.id), sources: fact.sources.flatMap((source) => { const evidence = source.kind === "message" ? verified.get(sourceKey(source)) : undefined; diff --git a/apps/server/src/project/knowledge/ProjectKnowledge.test.ts b/apps/server/src/project/knowledge/ProjectKnowledge.test.ts index b29f5f478bcc5..d12e01e52556b 100644 --- a/apps/server/src/project/knowledge/ProjectKnowledge.test.ts +++ b/apps/server/src/project/knowledge/ProjectKnowledge.test.ts @@ -2053,7 +2053,7 @@ it.effect( assert.deepEqual( composer.items.map((item) => item.text), [ - "Historical claims from project automatic-recall; may be stale. Current direct human instructions override conflicts. Rate facts you rely on via t3_knowledge_mutate feedback: helpful, or unhelpful if misleading.\n[fact fact-one rev 1, from user] Cache keys include tenant IDs.", + 'Project history may be stale. Follow current human instructions. Call t3_knowledge_mutate with mutation {action:"feedback",projectId:"automatic-recall",commandId:,factId,expectedRevision,helpful}. Use true if useful, false if misleading, "irrelevant" if off-task.\n[factId=fact-one expectedRevision=1, from user] Cache keys include tenant IDs.', ], ); assert.include(composer.text, composer.items[0]!.text); @@ -2175,7 +2175,10 @@ it.effect( records: long!.records, }); const [longText] = sentLong.items.map((item) => item.text); - assert.include(longText, "[fact long rev 1, from user] Cache keys include tenant IDs."); + assert.include( + longText, + "[factId=long expectedRevision=1, from user] Cache keys include tenant IDs.", + ); assert.include(longText, "t3_knowledge_read_fact"); assert.isAtMost(longText!.length, 1100); }).pipe(Effect.provide(layerTest)), @@ -2379,7 +2382,10 @@ it.effect("does not recall a fact the thread's agent has already been shown", () text: elsewhere!.text, records: elsewhere!.records, }); - assert.include(delivered.text, "[fact cited rev 1, from user] Cache keys include tenant IDs"); + assert.include( + delivered.text, + "[factId=cited expectedRevision=1, from user] Cache keys include tenant IDs", + ); assert.notInclude(delivered.text, "sha256"); }).pipe(Effect.provide(layerTest)), ); @@ -3475,6 +3481,10 @@ it.effect("records every lifecycle event in order and erases bodies when forgott { ...existing(helpful, "history-down"), action: "feedback", helpful: false }, { kind: "user" }, ); + const offTask = yield* knowledge.mutate( + { ...existing(unhelpful, "history-off-task"), action: "feedback", helpful: "irrelevant" }, + agent, + ); const detail = yield* knowledge.readFact({ projectId, factId: created.id }); assert.deepEqual( detail.history.map((entry) => [entry.event, entry.revision, entry.helpful]), @@ -3483,8 +3493,11 @@ it.effect("records every lifecycle event in order and erases bodies when forgott ["edited", 2, null], ["feedback", 3, true], ["feedback", 4, false], + ["feedback", 5, false], ], ); + assert.equal(detail.history.at(-1)?.feedbackScope, "task"); + assert.equal(detail.history.at(-1)?.reason, "Unrelated to this task."); assert.deepEqual(detail.history[0]?.actor, agent); assert.equal(detail.history[1]?.content, "Cache keys include tenant and region IDs."); assert.match(detail.history[1]?.contentDigest ?? "", /^[a-f0-9]{64}$/); @@ -3494,12 +3507,13 @@ it.effect("records every lifecycle event in order and erases bodies when forgott [ [0.5, 0.55], [0.55, unhelpful.trust], + [unhelpful.trust, offTask.trust], ], ); assert.isFalse(detail.historyTruncated); yield* knowledge.mutate( - { ...existing(unhelpful, "history-forget"), action: "forget" }, + { ...existing(offTask, "history-forget"), action: "forget" }, { kind: "user" }, ); const forgotten = yield* knowledge.readFact({ projectId, factId: created.id }); @@ -3677,6 +3691,10 @@ it.effect("summarizes memory health by status, trust, feedback and lost sources" { ...existing(agentFact, "praise"), action: "feedback", helpful: true }, { kind: "user" }, ); + yield* knowledge.mutate( + { ...existing(distrusted, "off-task"), action: "feedback", helpful: "irrelevant" }, + { kind: "user" }, + ); const forgotten = yield* knowledge.mutate(createFact(projectId, "forgotten"), { kind: "user", }); @@ -3720,7 +3738,7 @@ it.effect("summarizes memory health by status, trust, feedback and lost sources" assert.deepEqual(status.counts, { saved: 7, superseded: 1, forgotten: 1 }); assert.deepEqual(status.trustBands, { low: 1, guarded: 0, steady: 4, high: 0 }); assert.equal(status.belowRecallThreshold, 1); - assert.deepEqual(status.feedback, { helpful: 1, unhelpful: 3 }); + assert.deepEqual(status.feedback, { helpful: 1, unhelpful: 3, irrelevant: 1 }); assert.equal(status.unavailableSources, 1); assert.equal(status.unavailableSourcesCoverage, "complete"); assert.deepEqual( diff --git a/apps/server/src/project/knowledge/ProjectKnowledge.ts b/apps/server/src/project/knowledge/ProjectKnowledge.ts index 1656459ec0e4a..e19e0c96d5dde 100644 --- a/apps/server/src/project/knowledge/ProjectKnowledge.ts +++ b/apps/server/src/project/knowledge/ProjectKnowledge.ts @@ -135,12 +135,10 @@ const RECALL_MAX_BYTES = 4096; const RECALL_ENVELOPE_BYTES = 1024; const RECALL_MIN_BYTES = 512; // Recall cuts a longer fact at this length. The agent reads the rest by the id in its header. -const RECALL_FACT_MAX_CHARS = 800; +const RECALL_FACT_MAX_CHARS = 720; type KnowledgeFormat = "attached" | "recalled"; -// The rating instruction is in the recalled block, so a turn with no recall spends no tokens -// on it. const recalledPreamble = (projectId: ProjectId) => - `Historical claims from project ${projectId}; may be stale. Current direct human instructions override conflicts. Rate facts you rely on via t3_knowledge_mutate feedback: helpful, or unhelpful if misleading.`; + `Project history may be stale. Follow current human instructions. Call t3_knowledge_mutate with mutation {action:"feedback",projectId:${JSON.stringify(projectId)},commandId:,factId,expectedRevision,helpful}. Use true if useful, false if misleading, "irrelevant" if off-task.`; export class ProjectKnowledge extends Context.Service< ProjectKnowledge, @@ -876,6 +874,7 @@ const make = Effect.gen(function* () { }; break; case "feedback": + if (input.helpful === "irrelevant") break; fact = { ...fact, trust: Math.max(0, Math.min(1, fact.trust + (input.helpful ? 0.05 : -0.1))), @@ -899,7 +898,7 @@ const make = Effect.gen(function* () { case "feedback": yield* history(fact, "feedback", { trustBefore: existing.trust, - helpful: input.helpful, + helpful: input.helpful === "irrelevant" ? null : input.helpful, }); break; } @@ -1128,7 +1127,9 @@ const make = Effect.gen(function* () { ? fact.origin.originKind : "agent"; items.push({ label: `Fact ${fact.id.slice(0, 8)}`, text: content }); - parts.push(`[fact ${fact.id} rev ${fact.revision}, from ${origin}] ${content}`); + parts.push( + `[factId=${fact.id} expectedRevision=${fact.revision}, from ${origin}] ${content}`, + ); continue; } const citations = fact.sources @@ -2066,7 +2067,11 @@ const make = Effect.gen(function* () { recallThreshold: RECALL_THRESHOLD, belowRecallThreshold: counts.belowRecallThreshold, retrievals: counts.retrievals, - feedback: { helpful: counts.helpful, unhelpful: counts.unhelpful }, + feedback: { + helpful: counts.helpful, + unhelpful: counts.unhelpful, + irrelevant: counts.irrelevant, + }, unavailableSources: unavailable.length, unavailableSourcesCoverage: cited.length > SOURCE_CHECK_LIMIT ? ("partial" as const) : ("complete" as const), diff --git a/apps/server/src/project/knowledge/persistence.ts b/apps/server/src/project/knowledge/persistence.ts index fd32071631880..1f321e3deef7b 100644 --- a/apps/server/src/project/knowledge/persistence.ts +++ b/apps/server/src/project/knowledge/persistence.ts @@ -525,8 +525,13 @@ export const makeStore = Effect.gen(function* () { ? "edited" : event, ...(event === "curated" || event === "restored" ? { memoryAction: event } : {}), - helpful: row.helpful === null ? null : row.helpful !== 0, ...(reason === null ? {} : { reason }), + // Older clients label a feedback event with helpful=null as a positive vote. + ...(event === "feedback" + ? row.helpful === null + ? { helpful: false, feedbackScope: "task", reason: "Unrelated to this task." } + : { helpful: row.helpful !== 0, feedbackScope: "fact" } + : { helpful: null }), })) .toReversed(); }); @@ -575,13 +580,14 @@ export const makeStore = Effect.gen(function* () { (SELECT COUNT(*) FROM recallable WHERE trust < ${recallThreshold}) AS "belowRecallThreshold", (SELECT COALESCE(SUM(retrieval_count), 0) FROM project_knowledge_facts WHERE project_id = ${projectId} AND status != 'forgotten') AS retrievals`; - const feedback = yield* sql<{ readonly helpful: number; readonly count: number }>` + const feedback = yield* sql<{ readonly helpful: number | null; readonly count: number }>` SELECT helpful, COUNT(*) AS count FROM project_knowledge_fact_history WHERE project_id = ${projectId} AND event = 'feedback' GROUP BY helpful`; return { ...facts!, helpful: feedback.find((row) => row.helpful === 1)?.count ?? 0, unhelpful: feedback.find((row) => row.helpful === 0)?.count ?? 0, + irrelevant: feedback.find((row) => row.helpful === null)?.count ?? 0, }; }); /** Citations of active facts, for whole-project source checks. */ diff --git a/apps/server/src/provider/agentContextBudget.json b/apps/server/src/provider/agentContextBudget.json index 0ba390df50334..8e8be1e759ca3 100644 --- a/apps/server/src/provider/agentContextBudget.json +++ b/apps/server/src/provider/agentContextBudget.json @@ -1,5 +1,5 @@ { - "claudeSystemAppendBytes": 9572, - "codexAdditionalContextBytes": 10962, - "mcpToolListJsonBytes": 74595 + "claudeSystemAppendBytes": 9557, + "codexAdditionalContextBytes": 10947, + "mcpToolListJsonBytes": 74594 } diff --git a/docs/user/composer.md b/docs/user/composer.md index 8c5fce96fec6b..494ca209dbafb 100644 --- a/docs/user/composer.md +++ b/docs/user/composer.md @@ -231,12 +231,15 @@ has already found by searching or reading memory, or a fact that was saved from that thread. T3 sends recalled facts in a short form without their citations. It cuts a long fact short, and the agent reads the rest when it needs it. -T3 asks the agent to rate each recalled fact it relies on. Helpful ratings raise -the fact's trust and unhelpful ratings lower it, and T3 recalls only facts with -enough trust. To remove a fact permanently, forget it in Project memory. - -A fact's details show its history: when it was created, edited, superseded, and -rated helpful or unhelpful, with the trust change from each rating. **Memory health** counts facts by status. When facts lose their cited +T3 asks the agent to rate recalled facts it can judge. A helpful vote raises a +fact's trust, and a misleading fact gets a downvote that lowers trust. Marking a +fact unrelated to the task leaves its trust unchanged. T3 recalls only facts +with enough trust. During review, the curator sees each fact's trust and votes +since its last edit. It checks the fact and its sources before removing it. + +Open a fact's details to see its revisions, ratings and trust changes. To remove +it permanently, choose **Forget**. **Memory health** counts facts by status. +When facts lose their cited source, the **Lost source** filter lists them; on mobile, select the lost-source count in the memory summary. diff --git a/evals/mcp-tool-use/cases/project-intelligence.ndjson b/evals/mcp-tool-use/cases/project-intelligence.ndjson index d6b1c20dd5c4a..26f8b1618d481 100644 --- a/evals/mcp-tool-use/cases/project-intelligence.ndjson +++ b/evals/mcp-tool-use/cases/project-intelligence.ndjson @@ -9,7 +9,7 @@ {"id": "intelligence-summary-source-continuation", "area": "project-intelligence", "kind": "guard", "pattern": "bounded source expansion resumes within the same source", "experience": "The LCM audit found unbounded source expansion. Bounded pages now carry both a source offset and an offset within that source, and agents must use both to recover evidence after the first page.", "privacy": "commit-safe", "source": {"kind": "integration-audit", "commit": "ef873edeb5927d63e4e40920923b629616b95bbf", "path": "apps/server/src/project/conversationSummaries/ConversationSummaries.ts", "finding": "Expansion returned complete source messages with only source-count pagination."}, "setup": {"role": "top", "priorTurns": [{"role": "user", "text": "Read the original health-check output for project-app in thread-health-check."}, {"role": "assistant", "text": "The first page stops before the result; more source text remains.", "toolCalls": [{"tool": "t3_conversation_summary_expand", "input": {"projectId": "project-app", "threadId": "thread-health-check", "summaryId": "summary:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "offset": 0, "maxChars": 42}, "result": {"summary": {"state": "available", "summary": {"id": "summary:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "projectId": "project-app", "threadId": "thread-health-check", "sources": [{"kind": "turn_item", "threadId": "thread-health-check", "itemId": "item-health-output", "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}], "statements": [{"text": "The health check failed.", "citations": ["turn_item:item-health-output"]}], "modelSelection": {"instanceId": "codex", "model": "gpt-5"}, "createdAt": "2026-11-03T00:00:00.000Z"}}, "sources": [{"ref": {"kind": "turn_item", "threadId": "thread-health-check", "itemId": "item-health-output", "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}, "result": {"state": "available", "source": {"kind": "turn_item", "threadId": "thread-health-check", "itemId": "item-health-output", "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}, "text": "command: curl /health\nstatus: checking...\n", "title": "Health check", "role": "tool", "createdAt": "2026-11-03T00:00:00.000Z", "archived": false, "textOffset": 0, "totalChars": 91}}], "nextOffset": 0, "nextTextOffset": 42}}]}], "mocks": {"t3_conversation_summary_expand": {"summary": {"state": "available", "summary": {"id": "summary:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "projectId": "project-app", "threadId": "thread-health-check", "sources": [{"kind": "turn_item", "threadId": "thread-health-check", "itemId": "item-health-output", "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}], "statements": [{"text": "The health check failed.", "citations": ["turn_item:item-health-output"]}], "modelSelection": {"instanceId": "codex", "model": "gpt-5"}, "createdAt": "2026-11-03T00:00:00.000Z"}}, "sources": [{"ref": {"kind": "turn_item", "threadId": "thread-health-check", "itemId": "item-health-output", "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}, "result": {"state": "available", "source": {"kind": "turn_item", "threadId": "thread-health-check", "itemId": "item-health-output", "digest": "cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc"}, "text": "AssertionError: expected HTTP 200, received 503.\n", "title": "Health check", "role": "tool", "createdAt": "2026-11-03T00:00:00.000Z", "archived": false, "textOffset": 42, "totalChars": 91}}], "nextOffset": null, "nextTextOffset": null}}}, "prompt": "Continue reading from the returned cursor. What did the original health-check output actually report? Identify the source item.", "expect": {"calls": [{"tool": "t3_conversation_summary_expand", "args": {"projectId": "project-app", "threadId": "thread-health-check", "summaryId": "summary:dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd", "offset": 0, "textOffset": 42}}], "maxCalls": {"t3_conversation_summary_expand": 1}, "notCalls": ["t3_knowledge_mutate", "t3_project_code_query"], "textMust": ["503", "item-health-output"]}} {"id": "intelligence-remember-saves-directly", "area": "project-intelligence", "kind": "guard", "pattern": "remembering a fact saves it with no approval step", "experience": "Agents described saved facts as proposals a user had to confirm. Facts now save directly, and no confirm action exists.", "privacy": "commit-safe", "source": {"kind": "integration-audit", "path": "packages/provider-core/src/server/orchestrationInstructions.ts", "finding": "Project memory moved to a saved or forgotten lifecycle with no approval step."}, "setup": {"role": "top", "mocks": {"t3_knowledge_mutate": {"id": "fact-cache-ttl-saved", "projectId": "project-app", "content": "Keep the cache TTL at five minutes.", "entities": ["cache"], "tags": ["ttl"], "kind": "decision", "sources": [], "origin": {"kind": "agent-thread", "threadId": "thread-current", "providerInstanceId": "codex"}, "revision": 1, "trust": 0.5, "retrievalCount": 0, "createdAt": "2026-10-22T12:00:00.000Z", "updatedAt": "2026-10-22T12:00:00.000Z", "status": "saved", "forgottenAt": null, "hints": []}, "t3_knowledge_search": {"facts": {"state": "available", "matches": [], "coverage": "complete", "nextCursor": null}, "messages": {"matches": []}}, "t3_project_list": {"projects": [{"id": "project-app", "title": "Acme", "workspaceRoot": "/home/user/acme/app"}]}, "t3_project_read": {"project": {"id": "project-app", "title": "Acme", "workspaceRoot": "/home/user/acme/app"}}}}, "prompt": "For the current machine's registered project project-app, remember that we keep the cache TTL at five minutes. Tell me whether other agents can use it yet or whether I still need to do anything.", "expect": {"calls": [{"tool": "t3_knowledge_mutate", "args": {"mutation": {"action": "create", "projectId": "project-app", "content": "~five minutes"}}}], "notCalls": ["t3_knowledge_prepare_context"], "maxCalls": {"t3_knowledge_mutate": 1}, "finalTextMust": ["(?i:saved|remembered|recorded|stored)"], "textMustNot": ["(?i:(?:need|have|must)[^.]{0,40}\\b(?:confirm|approve|review)\\b|until (?:you|a user) (?:confirm|approve|review)|pending (?:confirmation|approval|review)|awaiting (?:confirmation|approval|review))"]}, "maxSteps": 10} {"id":"condensed-summary-drilldown-exact-source","area":"project-intelligence","kind":"guard","pattern":"a roll-up cites a child summary, not the original words","experience":"Handoffs of long threads now carry roll-ups: summaries of summaries. Quoting a roll-up, or its child summary, as the user's words loses the reason the user gave. The agent must follow the cited child down to the original message.","privacy":"commit-safe","source":{"kind":"design","path":"apps/server/src/project/conversationSummaries/ConversationSummaries.ts","finding":"Roll-up sources are child summaries, so exact text is two expand calls away."},"setup":{"role":"top","mocks":{"t3_conversation_summary_expand?summaryId=summary:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee":{"summary":{"state":"available","summary":{"id":"summary:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee","projectId":"project-app","threadId":"thread-parser-rollout","sources":[{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:1111111111111111111111111111111111111111111111111111111111111111"},{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:2222222222222222222222222222222222222222222222222222222222222222"},{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:3333333333333333333333333333333333333333333333333333333333333333"},{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:4444444444444444444444444444444444444444444444444444444444444444"}],"statements":[{"text":"The user ruled out new runtime dependencies; the agent kept the parser hand-written.","citations":["summary:2222222222222222222222222222222222222222222222222222222222222222"]}],"modelSelection":{"instanceId":"codex","model":"gpt-5"},"createdAt":"2026-10-22T12:00:00.000Z"}},"sources":[{"ref":{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:1111111111111111111111111111111111111111111111111111111111111111"},"result":{"state":"available","source":{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:1111111111111111111111111111111111111111111111111111111111111111"},"text":"- The user asked for a faster config parser. [#1]","title":"Parser rollout","role":"summary","createdAt":"2026-10-22T12:00:00.000Z","archived":false}},{"ref":{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:2222222222222222222222222222222222222222222222222222222222222222"},"result":{"state":"available","source":{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:2222222222222222222222222222222222222222222222222222222222222222"},"text":"- The user ruled out new runtime dependencies for the parser. [#3]","title":"Parser rollout","role":"summary","createdAt":"2026-10-22T12:00:00.000Z","archived":false}},{"ref":{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:3333333333333333333333333333333333333333333333333333333333333333"},"result":{"state":"available","source":{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:3333333333333333333333333333333333333333333333333333333333333333"},"text":"- The agent rewrote the tokenizer by hand and reported tests passing. [#2, #4]","title":"Parser rollout","role":"summary","createdAt":"2026-10-22T12:00:00.000Z","archived":false}},{"ref":{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:4444444444444444444444444444444444444444444444444444444444444444"},"result":{"state":"available","source":{"kind":"summary","threadId":"thread-parser-rollout","summaryId":"summary:4444444444444444444444444444444444444444444444444444444444444444"},"text":"- Benchmarks and the release note are still open. [#1]","title":"Parser rollout","role":"summary","createdAt":"2026-10-22T12:00:00.000Z","archived":false}}],"nextOffset":null,"nextTextOffset":null},"t3_conversation_summary_expand?summaryId=summary:2222222222222222222222222222222222222222222222222222222222222222":{"summary":{"state":"available","summary":{"id":"summary:2222222222222222222222222222222222222222222222222222222222222222","projectId":"project-app","threadId":"thread-parser-rollout","sources":[{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-0","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-1","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-2","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-3","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"}],"statements":[{"text":"The user ruled out new runtime dependencies for the parser.","citations":["message-parser-2"]}],"modelSelection":{"instanceId":"codex","model":"gpt-5"},"createdAt":"2026-10-22T12:00:00.000Z"}},"sources":[{"ref":{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-0","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"result":{"state":"available","source":{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-0","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"text":"Can you make the config parser faster? Startup spends 400 ms in it.","title":"Parser rollout","role":"user","createdAt":"2026-10-22T12:00:00.000Z","archived":false}},{"ref":{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-1","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"result":{"state":"available","source":{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-1","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"text":"I'll profile the tokenizer first and propose options, including a parser library.","title":"Parser rollout","role":"assistant","createdAt":"2026-10-22T12:00:00.000Z","archived":false}},{"ref":{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-2","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"result":{"state":"available","source":{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-2","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"text":"No new runtime dependencies, please. We ship to air-gapped customers and every new package needs a security review that takes six weeks.","title":"Parser rollout","role":"user","createdAt":"2026-10-22T12:00:00.000Z","archived":false}},{"ref":{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-3","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"result":{"state":"available","source":{"kind":"message","threadId":"thread-parser-rollout","messageId":"message-parser-3","digest":"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"},"text":"Understood. I rewrote the tokenizer by hand; the parser tests pass locally.","title":"Parser rollout","role":"assistant","createdAt":"2026-10-22T12:00:00.000Z","archived":false}}],"nextOffset":null,"nextTextOffset":null}}},"prompt":"The handoff for project project-app, thread thread-parser-rollout, includes conversation summary summary:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee: 'The user ruled out new runtime dependencies; the agent kept the parser hand-written. [#2]'. Why did the user rule them out? Quote the user's exact words.","expect":{"calls":[{"tool":"t3_conversation_summary_expand","args":{"projectId":"project-app","threadId":"thread-parser-rollout","summaryId":"summary:eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee"}},{"tool":"t3_conversation_summary_expand","args":{"projectId":"project-app","threadId":"thread-parser-rollout","summaryId":"summary:2222222222222222222222222222222222222222222222222222222222222222"}}],"textMust":["air-gapped","six weeks|six-week"],"notCalls":["t3_knowledge_mutate","t3_project_code_query"]}} -{"id":"intelligence-recalled-fact-rated-helpful","area":"project-intelligence","kind":"guard","pattern":"a recalled fact the agent relied on is rated helpful","experience":"Recalled project memory arrives with the user message. A fact's trust changes only when the agent rates the facts it relied on.","privacy":"commit-safe","source":{"kind":"integration-audit","path":"apps/server/src/project/knowledge/ProjectKnowledge.ts","finding":"Related saved facts arrive with user messages, and the recalled block tells the agent to rate the facts it relies on."},"setup":{"role":"top","world":{"repo":"acme/app","branch":"main","status":"","files":{"src/pricing.ts":"export async function getPrice(tenantId: string, sku: string): Promise {\n const response = await fetch(`https://prices.acme.test/${tenantId}/${sku}`);\n return (await response.json()).price;\n}\n"}},"mocks":{"t3_knowledge_mutate":{"id":"fact-tenant-cache-keys","projectId":"project-app","content":"Cache keys must include the tenant ID; a shared key leaked prices between tenants.","entities":[],"tags":[],"kind":"decision","sources":[{"kind":"user"}],"origin":{"kind":"user"},"revision":2,"trust":0.7,"retrievalCount":0,"createdAt":"2026-10-22T12:00:00.000Z","updatedAt":"2026-11-10T12:00:00.000Z","status":"saved","forgottenAt":null,"hints":[]}}},"prompt":"Add an in-memory cache for getPrice in src/pricing.ts.\n\n[Project knowledge: Recalled project memory; ref=recall]\n\n\n\nSaved facts from project project-app, for reference. Rate any you rely on with t3_knowledge_mutate feedback: helpful, or unhelpful if it misled you.\n[fact fact-tenant-cache-keys rev 1, from user] Cache keys must include the tenant ID; a shared key leaked prices between tenants.\n\n","expect":{"calls":[{"tool":"t3_knowledge_mutate","args":{"mutation":{"action":"feedback","factId":"fact-tenant-cache-keys","helpful":true,"projectId":"project-app","expectedRevision":1}}}],"maxCalls":{"t3_knowledge_mutate":1}}} -{"id":"intelligence-recalled-fact-rated-unhelpful","area":"project-intelligence","kind":"guard","pattern":"a recalled fact that misled the agent is rated unhelpful","experience":"A recalled fact can be wrong. When the repository contradicts it, an unhelpful rating lowers its trust so it stops being recalled.","privacy":"commit-safe","source":{"kind":"integration-audit","path":"apps/server/src/project/knowledge/ProjectKnowledge.ts","finding":"Related saved facts arrive with user messages, and the recalled block tells the agent to rate the facts it relies on."},"setup":{"role":"top","world":{"repo":"acme/app","branch":"main","status":"","files":{"src/api/pricingHandler.ts":"export function priceResponse(cents: number) {\n return { price: Math.floor(cents) / 100 };\n}\n"}},"shellMocks":[{"match":"src/legacy","output":"No such file or directory\n"}],"mocks":{"t3_knowledge_mutate":{"id":"fact-pricing-location","projectId":"project-app","content":"The pricing API handler lives in src/legacy/pricingHandler.ts.","entities":[],"tags":[],"kind":"decision","sources":[{"kind":"user"}],"origin":{"kind":"user"},"revision":2,"trust":0.55,"retrievalCount":0,"createdAt":"2026-10-22T12:00:00.000Z","updatedAt":"2026-11-10T12:00:00.000Z","status":"saved","forgottenAt":null,"hints":[]}}},"prompt":"Fix the rounding bug in the pricing API handler.\n\n[Project knowledge: Recalled project memory; ref=recall]\n\n\n\nSaved facts from project project-app, for reference. Rate any you rely on with t3_knowledge_mutate feedback: helpful, or unhelpful if it misled you.\n[fact fact-pricing-location rev 1, from user] The pricing API handler lives in src/legacy/pricingHandler.ts.\n\n","expect":{"calls":[{"tool":"t3_knowledge_mutate","args":{"mutation":{"action":"feedback","factId":"fact-pricing-location","helpful":false,"projectId":"project-app","expectedRevision":1}}}],"maxCalls":{"t3_knowledge_mutate":1}}} -{"id":"intelligence-recalled-fact-unused-control","area":"project-intelligence","kind":"control","pattern":"an unrelated recalled fact is not rated","experience":"Rating a fact the work never used would raise its trust for being retrieved, not for being right.","privacy":"commit-safe","source":{"kind":"integration-audit","path":"apps/server/src/project/knowledge/ProjectKnowledge.ts","finding":"Related saved facts arrive with user messages, and the recalled block tells the agent to rate the facts it relies on."},"setup":{"role":"top","world":{"repo":"acme/app","branch":"main","status":"","files":{"README.md":"# App\n\nAcme storefront.\n"}}},"prompt":"Rename the README title to Acme App.\n\n[Project knowledge: Recalled project memory; ref=recall]\n\n\n\nSaved facts from project project-app, for reference. Rate any you rely on with t3_knowledge_mutate feedback: helpful, or unhelpful if it misled you.\n[fact fact-tenant-cache-keys rev 1, from user] Cache keys must include the tenant ID; a shared key leaked prices between tenants.\n\n","expect":{"notCalls":["t3_knowledge_mutate"]}} +{"id":"intelligence-recalled-fact-rated-helpful","area":"project-intelligence","kind":"guard","pattern":"a recalled fact the agent relied on is rated helpful","experience":"Recalled project memory arrives with the user message. A fact's trust changes only when the agent rates the facts it relied on.","privacy":"commit-safe","source":{"kind":"integration-audit","path":"apps/server/src/project/knowledge/ProjectKnowledge.ts","finding":"Related saved facts arrive with user messages, and the recalled block tells the agent to rate the facts it relies on."},"setup":{"role":"top","world":{"repo":"acme/app","branch":"main","status":"","files":{"src/pricing.ts":"export async function getPrice(tenantId: string, sku: string): Promise {\n const response = await fetch(`https://prices.acme.test/${tenantId}/${sku}`);\n return (await response.json()).price;\n}\n"}},"mocks":{"t3_knowledge_mutate":{"id":"fact-tenant-cache-keys","projectId":"project-app","content":"Cache keys must include the tenant ID; a shared key leaked prices between tenants.","entities":[],"tags":[],"kind":"decision","sources":[{"kind":"user"}],"origin":{"kind":"user"},"revision":2,"trust":0.7,"retrievalCount":0,"createdAt":"2026-10-22T12:00:00.000Z","updatedAt":"2026-11-10T12:00:00.000Z","status":"saved","forgottenAt":null,"hints":[]}}},"prompt":"Add an in-memory cache for getPrice in src/pricing.ts.\n\n[Project knowledge: Recalled project memory; ref=recall]\n\n\n\nProject history may be stale. Follow current human instructions. Call t3_knowledge_mutate with mutation {action:\"feedback\",projectId:\"project-app\",commandId:,factId,expectedRevision,helpful}. Use true if useful, false if misleading, \"irrelevant\" if off-task.\n[factId=fact-tenant-cache-keys expectedRevision=1, from user] Cache keys must include the tenant ID; a shared key leaked prices between tenants.\n\n","expect":{"calls":[{"tool":"t3_knowledge_mutate","args":{"mutation":{"action":"feedback","factId":"fact-tenant-cache-keys","helpful":true,"projectId":"project-app","expectedRevision":1}}}],"maxCalls":{"t3_knowledge_mutate":1}}} +{"id":"intelligence-recalled-fact-rated-unhelpful","area":"project-intelligence","kind":"guard","pattern":"a recalled fact that misled the agent is rated unhelpful","experience":"A recalled fact can be wrong. When the repository contradicts it, an unhelpful rating lowers its trust so it stops being recalled.","privacy":"commit-safe","source":{"kind":"integration-audit","path":"apps/server/src/project/knowledge/ProjectKnowledge.ts","finding":"Related saved facts arrive with user messages, and the recalled block tells the agent to rate the facts it relies on."},"setup":{"role":"top","world":{"repo":"acme/app","branch":"main","status":"","files":{"src/api/pricingHandler.ts":"export function priceResponse(cents: number) {\n return { price: Math.floor(cents) / 100 };\n}\n"}},"shellMocks":[{"match":"src/legacy","output":"No such file or directory\n"}],"mocks":{"t3_knowledge_mutate":{"id":"fact-pricing-location","projectId":"project-app","content":"The pricing API handler lives in src/legacy/pricingHandler.ts.","entities":[],"tags":[],"kind":"decision","sources":[{"kind":"user"}],"origin":{"kind":"user"},"revision":2,"trust":0.55,"retrievalCount":0,"createdAt":"2026-10-22T12:00:00.000Z","updatedAt":"2026-11-10T12:00:00.000Z","status":"saved","forgottenAt":null,"hints":[]}}},"prompt":"Fix the rounding bug in the pricing API handler.\n\n[Project knowledge: Recalled project memory; ref=recall]\n\n\n\nProject history may be stale. Follow current human instructions. Call t3_knowledge_mutate with mutation {action:\"feedback\",projectId:\"project-app\",commandId:,factId,expectedRevision,helpful}. Use true if useful, false if misleading, \"irrelevant\" if off-task.\n[factId=fact-pricing-location expectedRevision=1, from user] The pricing API handler lives in src/legacy/pricingHandler.ts.\n\n","expect":{"calls":[{"tool":"t3_knowledge_mutate","args":{"mutation":{"action":"feedback","factId":"fact-pricing-location","helpful":false,"projectId":"project-app","expectedRevision":1}}}],"maxCalls":{"t3_knowledge_mutate":1}}} +{"id":"intelligence-recalled-fact-rated-irrelevant","area":"project-intelligence","kind":"control","pattern":"a poor match is rated irrelevant without reducing trust","experience":"An accurate fact can be irrelevant to the current task.","privacy":"commit-safe","source":{"kind":"integration-audit","path":"apps/server/src/project/knowledge/ProjectKnowledge.ts","finding":"Related saved facts arrive with user messages, and the recalled block tells the agent to rate the facts it relies on."},"setup":{"role":"top","world":{"repo":"acme/app","branch":"main","status":"","files":{"README.md":"# App\n\nAcme storefront.\n"}},"mocks":{"t3_knowledge_mutate":{"id":"fact-tenant-cache-keys","projectId":"project-app","content":"Cache keys must include the tenant ID; a shared key leaked prices between tenants.","entities":[],"tags":[],"kind":"decision","sources":[{"kind":"user"}],"origin":{"kind":"user"},"revision":2,"trust":0.65,"retrievalCount":0,"createdAt":"2026-10-22T12:00:00.000Z","updatedAt":"2026-11-10T12:00:00.000Z","status":"saved","forgottenAt":null,"hints":[]}}},"prompt":"Rename the README title to Acme App.\n\n[Project knowledge: Recalled project memory; ref=recall]\n\n\n\nProject history may be stale. Follow current human instructions. Call t3_knowledge_mutate with mutation {action:\"feedback\",projectId:\"project-app\",commandId:,factId,expectedRevision,helpful}. Use true if useful, false if misleading, \"irrelevant\" if off-task.\n[factId=fact-tenant-cache-keys expectedRevision=1, from user] Cache keys must include the tenant ID; a shared key leaked prices between tenants.\n\n","expect":{"calls":[{"tool":"t3_knowledge_mutate","args":{"mutation":{"action":"feedback","helpful":"irrelevant","factId":"fact-tenant-cache-keys","projectId":"project-app","expectedRevision":1}}}],"maxCalls":{"t3_knowledge_mutate":1}}} {"id": "intelligence-remember-reason-and-scope", "area": "project-intelligence", "kind": "bad", "pattern": "a saved decision preserves its scope and reason, without task limits", "experience": "The memory quality audit found one-off work limits recorded as standing constraints.", "privacy": "commit-safe", "source": {"kind": "memory-quality-audit", "finding": "2026-11 audit: task-scoped limits persisted as project rules."}, "setup": {"role": "top", "mocks": {"t3_knowledge_search": {"facts": {"state": "available", "matches": [], "coverage": "complete", "nextCursor": null}, "messages": {"matches": []}}, "t3_knowledge_mutate": {"id": "fact-invoice-units", "projectId": "project-app", "content": "Acme invoice amounts use integer minor units because floating-point rounding changed totals between clients.", "entities": ["Acme", "billing"], "tags": [], "kind": "decision", "sources": [], "origin": {"kind": "agent-thread", "threadId": "thread-current", "providerInstanceId": "codex"}, "revision": 1, "trust": 0.5, "retrievalCount": 0, "createdAt": "2026-10-22T12:00:00.000Z", "updatedAt": "2026-10-22T12:00:00.000Z", "status": "saved", "forgottenAt": null, "hints": []}, "t3_project_list": {"projects": [{"id": "project-app", "title": "Acme", "workspaceRoot": "/home/user/acme/app"}]}, "t3_project_read": {"project": {"id": "project-app", "title": "Acme", "workspaceRoot": "/home/user/acme/app"}}}}, "prompt": "For registered project project-app, remember our standing decision: Acme invoice amounts use integer minor units because floating-point rounding changed totals between clients. For this patch only, do not migrate the storage tables. Save the durable decision for future billing work.", "expect": {"calls": [{"tool": "t3_knowledge_search"}, {"tool": "t3_knowledge_mutate", "args": {"mutation": {"action": "create", "projectId": "project-app", "content": "~rounding"}}}], "callsMustNot": [{"tool": "t3_knowledge_mutate", "args": {"mutation": {"content": "~this patch"}}}, {"tool": "t3_knowledge_mutate", "args": {"mutation": {"content": "~do not migrate"}}}, {"tool": "t3_knowledge_mutate", "args": {"mutation": {"content": "~never migrate"}}}, {"tool": "t3_knowledge_mutate", "args": {"mutation": {"content": "~leave storage"}}}, {"tool": "t3_knowledge_mutate", "args": {"mutation": {"content": "~storage tables"}}}], "maxCalls": {"t3_knowledge_mutate": 1}, "notCalls": ["AskUserQuestion", "request_user_input"]}, "maxSteps": 10} diff --git a/evals/memory-curation/cases.ts b/evals/memory-curation/cases.ts index 32cc12d0657df..97d714f8af265 100644 --- a/evals/memory-curation/cases.ts +++ b/evals/memory-curation/cases.ts @@ -35,6 +35,7 @@ export const cases: ReadonlyArray = [ fact( "old", "Acme has required copying production databases before development work since 2020. Never write directly to production data.", + { trust: 0.2, feedback: { helpful: 0, unhelpful: 3, irrelevant: 12 } }, ), ], expected: [{ factId: "old", action: "keep" }], diff --git a/packages/contracts/src/projectKnowledge.ts b/packages/contracts/src/projectKnowledge.ts index 2e57cf2e8aaed..b6042e7879d76 100644 --- a/packages/contracts/src/projectKnowledge.ts +++ b/packages/contracts/src/projectKnowledge.ts @@ -159,7 +159,11 @@ const ProjectKnowledgeMutationSchema = Schema.Union([ Schema.Struct({ action: Schema.Literal("edit"), ...ExistingFact, ...ProjectFactFields.fields }), Schema.Struct({ action: Schema.Literal("forget"), ...ExistingFact }), Schema.Struct({ action: Schema.Literal("restore"), ...ExistingFact }), - Schema.Struct({ action: Schema.Literal("feedback"), ...ExistingFact, helpful: Schema.Boolean }), + Schema.Struct({ + action: Schema.Literal("feedback"), + ...ExistingFact, + helpful: Schema.Union([Schema.Boolean, Schema.Literal("irrelevant")]), + }), // Creates replacementFactId and links it to factId, which keeps its content (unlike forget). Schema.Struct({ action: Schema.Literal("supersede"), @@ -264,6 +268,7 @@ export const ProjectFactHistoryEntry = Schema.Struct({ trustBefore: Schema.NullOr(Schema.Number), trustAfter: Schema.Number, helpful: Schema.NullOr(Schema.Boolean), + feedbackScope: Schema.optionalKey(Schema.Literals(["fact", "task"])), // The replacement for "superseded"; the replaced fact for a superseding "created". relatedFactId: Schema.NullOr(ProjectFactId), reason: Schema.optionalKey(Schema.String), @@ -323,7 +328,11 @@ export const ProjectKnowledgeStatus = Schema.Struct({ recallThreshold: Schema.Number, belowRecallThreshold: NonNegativeInt, retrievals: NonNegativeInt, - feedback: Schema.Struct({ helpful: NonNegativeInt, unhelpful: NonNegativeInt }), + feedback: Schema.Struct({ + helpful: NonNegativeInt, + unhelpful: NonNegativeInt, + irrelevant: Schema.optionalKey(NonNegativeInt), + }), unavailableSources: NonNegativeInt, unavailableSourcesCoverage: Schema.Literals(["complete", "partial"]), // Optional so older servers still decode. diff --git a/packages/provider-core/src/server/orchestrationInstructions.ts b/packages/provider-core/src/server/orchestrationInstructions.ts index eaaed374dca73..b21d8c7fb1e2f 100644 --- a/packages/provider-core/src/server/orchestrationInstructions.ts +++ b/packages/provider-core/src/server/orchestrationInstructions.ts @@ -2,9 +2,9 @@ import type { ProviderInteractionMode } from "@t3tools/contracts"; export const T3_CODE_PROJECT_INTELLIGENCE_INSTRUCTIONS = `### Project intelligence -- Use intelligence when the task needs project history or relationships; do not prefetch it for a routine edit. Resolve an unknown project with \`t3_project_list\` and \`t3_project_read\`. -- Prior decisions, conventions, and constraints: \`t3_knowledge_search\`, then verify message citations with \`t3_knowledge_read_source\`. When asked to remember project knowledge, use \`t3_knowledge_mutate\` with exact message citations when available. It saves immediately, with no approval step; a saved fact records its origin, not proof that it is true. -- When writing memory, search for an existing fact first; update or supersede a correction instead of adding a competing copy. +- Use intelligence for project history or relationships. Skip prefetch on routine edits. Resolve unknown projects with \`t3_project_list\` and \`t3_project_read\`. +- Prior decisions, conventions and constraints: \`t3_knowledge_search\`; verify citations with \`t3_knowledge_read_source\`. To remember, search first, then \`t3_knowledge_mutate\` with exact citations if available. Update or supersede corrections. Saves are immediate; claims need verification. +- For recalled facts, load \`t3_knowledge_mutate\` via tool search if deferred. Call it once per judged fact with action feedback: helpful=true if useful, false if misleading, "irrelevant" if off-task. - Symbols, callers, dependencies, affected code, or reaching tests: \`t3_project_code_query\`. Pass the thread id for its registered worktree. Check coverage, freshness, and edge evidence; static candidates are not proof of runtime behavior. - For a file's checkpoint history, use \`t3_project_file_history\`. A checkpoint does not establish authorship. - For compressed project history, use \`t3_conversation_summary_search\` with a few distinctive terms, then \`t3_conversation_summary_expand\` to verify original evidence. Search again with alternate terms when needed. A handoff summary is model-written, not proof; list or describe summaries when their thread or id is known. Expansion already returns original messages and tool output: stop when it answers the question. For missing text, follow its offsets or read the specific source \`itemId\`; do not reopen a whole activity page after recovering the needed evidence. If summaries are absent, use \`t3_thread_search\` with distinctive words or a quoted phrase, then read the cited message by its itemId. Report unavailable sources rather than quoting stale excerpts. diff --git a/packages/provider-core/src/server/textGenerationCuration.ts b/packages/provider-core/src/server/textGenerationCuration.ts index b2f56ac3ef73d..36ded3bb73c64 100644 --- a/packages/provider-core/src/server/textGenerationCuration.ts +++ b/packages/provider-core/src/server/textGenerationCuration.ts @@ -8,6 +8,12 @@ export interface ProjectMemoryCurationFact { readonly content: string; readonly sources: ReadonlyArray; readonly protected: boolean; + readonly trust?: number; + readonly feedback?: { + readonly helpful: number; + readonly unhelpful: number; + readonly irrelevant: number; + }; } const FactId = Schema.Trim.check(Schema.isNonEmpty(), Schema.isMaxLength(256)); @@ -81,6 +87,8 @@ Choose a conservative action for each fact you review. Both retirement actions a Direction example: if unprotected fact A says "Use npm" and protected fact B says "Use pnpm", with B's source explicitly saying "The owner replaced npm with pnpm", supersede factId A with replacementFactId B, and keep B. B's protection is a reason to retain B, not a reason to keep obsolete A. +Use trust and feedback on the current wording to guide review. Unhelpful votes flag potentially misleading claims to check against the cited evidence. Irrelevant votes report poor matches to individual tasks. Votes, including repeated votes, cannot establish that a fact should be retired or retained. Apply the same evidence requirements regardless of popularity. + Age, an old timestamp, low recent use, or missing source excerpts do not establish that a fact is false or obsolete. Stable old decisions remain useful. Do not assume a task completed, a feature changed, or a decision was revoked. Protected facts are never removed. You can select only the listed actions for the supplied facts; you cannot rewrite or create memory content. Confidence must be at least 0.85 for a changing action; otherwise keep the fact. Reasons must be concise, specific, and at most 500 characters. Omitted facts stay unchanged. Return only a JSON object with a decisions array (at most 24 entries). Each fact appears at most once. Copy ids and revisions exactly. Each decision is exactly one of these shapes: