Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion apps/server/src/mcp/toolkits/knowledge/tools.ts
Original file line number Diff line number Diff line change
Expand Up @@ -109,7 +109,7 @@ export const KnowledgeToolkit = Toolkit.make(
...shared,
description:
MEMORY_WRITING_RULES.join(" ") +
" Search before adding/correcting. Create/edit/supersede require content, kind, entities, tags and sources (arrays may be empty); create needs a fresh factId. Cite the exact source message; only unsourced agent observations may omit citations (caller origin is recorded). Agents cannot claim direct user assertions. Saves immediately; facts are searchable and related ones appear with later user messages. Origin is not proof. Retry with the same commandId; updates need current revision. Prefer supersede to forget for changed facts: replacementFactId preserves the old content but excludes it from recall. Writes return duplicate/conflict hints. Users alone may restore set-aside or superseded facts; forgotten ones are irrecoverable. Use feedback for trust.",
" Search before adding or correcting. Create/edit/supersede require content, kind, entities, tags and sources (may be empty); create needs a new factId. Cite exact messages; only unsourced agent observations may omit citations. Caller origin is recorded. Agents cannot claim direct user assertions. Saves immediately for search and related turns. Origin is not proof. Reuse commandId on retries; updates need current revision. Prefer supersede to forget for changed facts: replacementFactId keeps old content out of recall. Writes flag duplicates/conflicts. Only users restore set-aside or superseded facts; forget is permanent. Feedback helpful: true if useful, false if misleading, irrelevant if off-task.",
parameters: Schema.Struct({
mutation: ProjectKnowledgeMutation,
environmentId: OrchestratorMcpEnvironmentTarget,
Expand Down
52 changes: 50 additions & 2 deletions apps/server/src/project/knowledge/MemoryCuration.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -152,7 +152,53 @@ it.effect("curates only the selected project's automatic facts and protects user
const beta = ProjectId.make("beta");
yield* createProject(alpha);
yield* createProject(beta);
yield* createFact(alpha, "alpha-auto", "Deployment is limited to the current task.");
const automatic = yield* createFact(
alpha,
"alpha-auto",
"Deployment is limited to the current task.",
);
const knowledge = yield* ProjectKnowledge.ProjectKnowledge;
const reviewedBy = automatic.origin;
const previous = yield* knowledge.mutate(
{
action: "feedback",
commandId: CommandId.make("rate-old"),
projectId: alpha,
factId: automatic.id,
expectedRevision: automatic.revision,
helpful: "irrelevant",
},
reviewedBy,
);
let rated = yield* knowledge.mutate(
{
action: "edit",
commandId: CommandId.make("revise"),
projectId: alpha,
factId: automatic.id,
expectedRevision: previous.revision,
kind: "decision",
content: "Deployment is limited to the current task on staging.",
entities: [],
tags: [],
sources: [],
},
reviewedBy,
);
for (const helpful of [true, false, "irrelevant"] as const) {
rated = yield* knowledge.mutate(
{
action: "feedback",
commandId: CommandId.make(`rate-${helpful}`),
projectId: alpha,
factId: automatic.id,
expectedRevision: rated.revision,
helpful,
},
reviewedBy,
);
}
assert.closeTo(rated.trust, 0.45, 1e-9);
yield* createFact(alpha, "alpha-user", "Deployment approvals always require the owner.", true);
yield* createFact(
alpha,
Expand Down Expand Up @@ -181,7 +227,9 @@ it.effect("curates only the selected project's automatic facts and protects user
[`${owner}-auto`],
);
}
const knowledge = yield* ProjectKnowledge.ProjectKnowledge;
const reviewed = calls.flatMap((call) => call.facts).find((fact) => fact.id === automatic.id)!;
assert.deepEqual(reviewed.feedback, { helpful: 1, unhelpful: 1, irrelevant: 1 });
assert.equal(reviewed.trust, rated.trust);
assert.equal(
(yield* knowledge.readFact({ projectId: alpha, factId: ProjectFactId.make("alpha-auto") }))
.fact.status,
Expand Down
24 changes: 24 additions & 0 deletions apps/server/src/project/knowledge/MemoryCuration.ts
Original file line number Diff line number Diff line change
Expand Up @@ -285,6 +285,28 @@ const make = Effect.gen(function* () {
sourceBudget -= excerpt.length;
}
}
const feedbackRows = yield* sql<{
readonly id: ProjectFactId;
readonly helpful: number;
readonly unhelpful: number;
readonly irrelevant: number;
}>`
WITH history AS (
SELECT fact_id, event, helpful, sequence,
MAX(sequence) FILTER (WHERE event = 'edited') OVER (PARTITION BY fact_id) AS last_edit
FROM project_knowledge_fact_history
WHERE project_id = ${projectId} AND ${sql.in(
"fact_id",
facts.map((fact) => fact.id),
)}
AND event IN ('feedback', 'edited')
)
SELECT fact_id AS id, COUNT(*) FILTER (WHERE helpful = 1) AS helpful,
COUNT(*) FILTER (WHERE helpful = 0) AS unhelpful,
COUNT(*) FILTER (WHERE helpful IS NULL) AS irrelevant
FROM history WHERE event = 'feedback' AND sequence > COALESCE(last_edit, 0)
GROUP BY fact_id`;
const feedbackById = new Map(feedbackRows.map(({ id, ...counts }) => [id, counts]));
const candidateIds = new Set(candidates.map((fact) => fact.id));
let packetSourceBudget = MAX_SOURCE_CHARS;
const reviewedSources = new Map<string, KnowledgeMessageRef>();
Expand All @@ -293,6 +315,8 @@ const make = Effect.gen(function* () {
revision: fact.revision,
kind: fact.kind,
content: fact.content,
trust: fact.trust,
feedback: feedbackById.get(fact.id) ?? { helpful: 0, unhelpful: 0, irrelevant: 0 },
protected: !candidateIds.has(fact.id),
sources: fact.sources.flatMap((source) => {
const evidence = source.kind === "message" ? verified.get(sourceKey(source)) : undefined;
Expand Down
28 changes: 23 additions & 5 deletions apps/server/src/project/knowledge/ProjectKnowledge.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -2053,7 +2053,7 @@ it.effect(
assert.deepEqual(
composer.items.map((item) => item.text),
[
"Historical claims from project automatic-recall; may be stale. Current direct human instructions override conflicts. Rate facts you rely on via t3_knowledge_mutate feedback: helpful, or unhelpful if misleading.\n[fact fact-one rev 1, from user] Cache keys include tenant IDs.",
'Project history may be stale. Follow current human instructions. Call t3_knowledge_mutate with mutation {action:"feedback",projectId:"automatic-recall",commandId:<new id>,factId,expectedRevision,helpful}. Use true if useful, false if misleading, "irrelevant" if off-task.\n[factId=fact-one expectedRevision=1, from user] Cache keys include tenant IDs.',
],
);
assert.include(composer.text, composer.items[0]!.text);
Expand Down Expand Up @@ -2175,7 +2175,10 @@ it.effect(
records: long!.records,
});
const [longText] = sentLong.items.map((item) => item.text);
assert.include(longText, "[fact long rev 1, from user] Cache keys include tenant IDs.");
assert.include(
longText,
"[factId=long expectedRevision=1, from user] Cache keys include tenant IDs.",
);
assert.include(longText, "t3_knowledge_read_fact");
assert.isAtMost(longText!.length, 1100);
}).pipe(Effect.provide(layerTest)),
Expand Down Expand Up @@ -2379,7 +2382,10 @@ it.effect("does not recall a fact the thread's agent has already been shown", ()
text: elsewhere!.text,
records: elsewhere!.records,
});
assert.include(delivered.text, "[fact cited rev 1, from user] Cache keys include tenant IDs");
assert.include(
delivered.text,
"[factId=cited expectedRevision=1, from user] Cache keys include tenant IDs",
);
assert.notInclude(delivered.text, "sha256");
}).pipe(Effect.provide(layerTest)),
);
Expand Down Expand Up @@ -3475,6 +3481,10 @@ it.effect("records every lifecycle event in order and erases bodies when forgott
{ ...existing(helpful, "history-down"), action: "feedback", helpful: false },
{ kind: "user" },
);
const offTask = yield* knowledge.mutate(
{ ...existing(unhelpful, "history-off-task"), action: "feedback", helpful: "irrelevant" },
agent,
);
const detail = yield* knowledge.readFact({ projectId, factId: created.id });
assert.deepEqual(
detail.history.map((entry) => [entry.event, entry.revision, entry.helpful]),
Expand All @@ -3483,8 +3493,11 @@ it.effect("records every lifecycle event in order and erases bodies when forgott
["edited", 2, null],
["feedback", 3, true],
["feedback", 4, false],
["feedback", 5, false],
],
);
assert.equal(detail.history.at(-1)?.feedbackScope, "task");
assert.equal(detail.history.at(-1)?.reason, "Unrelated to this task.");
assert.deepEqual(detail.history[0]?.actor, agent);
assert.equal(detail.history[1]?.content, "Cache keys include tenant and region IDs.");
assert.match(detail.history[1]?.contentDigest ?? "", /^[a-f0-9]{64}$/);
Expand All @@ -3494,12 +3507,13 @@ it.effect("records every lifecycle event in order and erases bodies when forgott
[
[0.5, 0.55],
[0.55, unhelpful.trust],
[unhelpful.trust, offTask.trust],
],
);
assert.isFalse(detail.historyTruncated);

yield* knowledge.mutate(
{ ...existing(unhelpful, "history-forget"), action: "forget" },
{ ...existing(offTask, "history-forget"), action: "forget" },
{ kind: "user" },
);
const forgotten = yield* knowledge.readFact({ projectId, factId: created.id });
Expand Down Expand Up @@ -3677,6 +3691,10 @@ it.effect("summarizes memory health by status, trust, feedback and lost sources"
{ ...existing(agentFact, "praise"), action: "feedback", helpful: true },
{ kind: "user" },
);
yield* knowledge.mutate(
{ ...existing(distrusted, "off-task"), action: "feedback", helpful: "irrelevant" },
{ kind: "user" },
);
const forgotten = yield* knowledge.mutate(createFact(projectId, "forgotten"), {
kind: "user",
});
Expand Down Expand Up @@ -3720,7 +3738,7 @@ it.effect("summarizes memory health by status, trust, feedback and lost sources"
assert.deepEqual(status.counts, { saved: 7, superseded: 1, forgotten: 1 });
assert.deepEqual(status.trustBands, { low: 1, guarded: 0, steady: 4, high: 0 });
assert.equal(status.belowRecallThreshold, 1);
assert.deepEqual(status.feedback, { helpful: 1, unhelpful: 3 });
assert.deepEqual(status.feedback, { helpful: 1, unhelpful: 3, irrelevant: 1 });
assert.equal(status.unavailableSources, 1);
assert.equal(status.unavailableSourcesCoverage, "complete");
assert.deepEqual(
Expand Down
19 changes: 12 additions & 7 deletions apps/server/src/project/knowledge/ProjectKnowledge.ts
Original file line number Diff line number Diff line change
Expand Up @@ -135,12 +135,10 @@ const RECALL_MAX_BYTES = 4096;
const RECALL_ENVELOPE_BYTES = 1024;
const RECALL_MIN_BYTES = 512;
// Recall cuts a longer fact at this length. The agent reads the rest by the id in its header.
const RECALL_FACT_MAX_CHARS = 800;
const RECALL_FACT_MAX_CHARS = 720;
type KnowledgeFormat = "attached" | "recalled";
// The rating instruction is in the recalled block, so a turn with no recall spends no tokens
// on it.
const recalledPreamble = (projectId: ProjectId) =>
`Historical claims from project ${projectId}; may be stale. Current direct human instructions override conflicts. Rate facts you rely on via t3_knowledge_mutate feedback: helpful, or unhelpful if misleading.`;
`Project history may be stale. Follow current human instructions. Call t3_knowledge_mutate with mutation {action:"feedback",projectId:${JSON.stringify(projectId)},commandId:<new id>,factId,expectedRevision,helpful}. Use true if useful, false if misleading, "irrelevant" if off-task.`;

export class ProjectKnowledge extends Context.Service<
ProjectKnowledge,
Expand Down Expand Up @@ -876,6 +874,7 @@ const make = Effect.gen(function* () {
};
break;
case "feedback":
if (input.helpful === "irrelevant") break;
fact = {
...fact,
trust: Math.max(0, Math.min(1, fact.trust + (input.helpful ? 0.05 : -0.1))),
Expand All @@ -899,7 +898,7 @@ const make = Effect.gen(function* () {
case "feedback":
yield* history(fact, "feedback", {
trustBefore: existing.trust,
helpful: input.helpful,
helpful: input.helpful === "irrelevant" ? null : input.helpful,
});
break;
}
Expand Down Expand Up @@ -1128,7 +1127,9 @@ const make = Effect.gen(function* () {
? fact.origin.originKind
: "agent";
items.push({ label: `Fact ${fact.id.slice(0, 8)}`, text: content });
parts.push(`[fact ${fact.id} rev ${fact.revision}, from ${origin}] ${content}`);
parts.push(
`[factId=${fact.id} expectedRevision=${fact.revision}, from ${origin}] ${content}`,
);
continue;
}
const citations = fact.sources
Expand Down Expand Up @@ -2066,7 +2067,11 @@ const make = Effect.gen(function* () {
recallThreshold: RECALL_THRESHOLD,
belowRecallThreshold: counts.belowRecallThreshold,
retrievals: counts.retrievals,
feedback: { helpful: counts.helpful, unhelpful: counts.unhelpful },
feedback: {
helpful: counts.helpful,
unhelpful: counts.unhelpful,
irrelevant: counts.irrelevant,
},
unavailableSources: unavailable.length,
unavailableSourcesCoverage:
cited.length > SOURCE_CHECK_LIMIT ? ("partial" as const) : ("complete" as const),
Expand Down
10 changes: 8 additions & 2 deletions apps/server/src/project/knowledge/persistence.ts
Original file line number Diff line number Diff line change
Expand Up @@ -525,8 +525,13 @@ export const makeStore = Effect.gen(function* () {
? "edited"
: event,
...(event === "curated" || event === "restored" ? { memoryAction: event } : {}),
helpful: row.helpful === null ? null : row.helpful !== 0,
...(reason === null ? {} : { reason }),
// Older clients label a feedback event with helpful=null as a positive vote.
...(event === "feedback"
? row.helpful === null
? { helpful: false, feedbackScope: "task", reason: "Unrelated to this task." }
: { helpful: row.helpful !== 0, feedbackScope: "fact" }
: { helpful: null }),
}))
.toReversed();
});
Expand Down Expand Up @@ -575,13 +580,14 @@ export const makeStore = Effect.gen(function* () {
(SELECT COUNT(*) FROM recallable WHERE trust < ${recallThreshold}) AS "belowRecallThreshold",
(SELECT COALESCE(SUM(retrieval_count), 0) FROM project_knowledge_facts
WHERE project_id = ${projectId} AND status != 'forgotten') AS retrievals`;
const feedback = yield* sql<{ readonly helpful: number; readonly count: number }>`
const feedback = yield* sql<{ readonly helpful: number | null; readonly count: number }>`
SELECT helpful, COUNT(*) AS count FROM project_knowledge_fact_history
WHERE project_id = ${projectId} AND event = 'feedback' GROUP BY helpful`;
return {
...facts!,
helpful: feedback.find((row) => row.helpful === 1)?.count ?? 0,
unhelpful: feedback.find((row) => row.helpful === 0)?.count ?? 0,
irrelevant: feedback.find((row) => row.helpful === null)?.count ?? 0,
};
});
/** Citations of active facts, for whole-project source checks. */
Expand Down
6 changes: 3 additions & 3 deletions apps/server/src/provider/agentContextBudget.json
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
{
"claudeSystemAppendBytes": 9572,
"codexAdditionalContextBytes": 10962,
"mcpToolListJsonBytes": 74595
"claudeSystemAppendBytes": 9557,
"codexAdditionalContextBytes": 10947,
"mcpToolListJsonBytes": 74594
}
15 changes: 9 additions & 6 deletions docs/user/composer.md
Original file line number Diff line number Diff line change
Expand Up @@ -231,12 +231,15 @@ has already found by searching or reading memory, or a fact that was saved from
that thread. T3 sends recalled facts in a short form without their citations. It
cuts a long fact short, and the agent reads the rest when it needs it.

T3 asks the agent to rate each recalled fact it relies on. Helpful ratings raise
the fact's trust and unhelpful ratings lower it, and T3 recalls only facts with
enough trust. To remove a fact permanently, forget it in Project memory.

A fact's details show its history: when it was created, edited, superseded, and
rated helpful or unhelpful, with the trust change from each rating. **Memory health** counts facts by status. When facts lose their cited
T3 asks the agent to rate recalled facts it can judge. A helpful vote raises a
fact's trust, and a misleading fact gets a downvote that lowers trust. Marking a
fact unrelated to the task leaves its trust unchanged. T3 recalls only facts
with enough trust. During review, the curator sees each fact's trust and votes
since its last edit. It checks the fact and its sources before removing it.

Open a fact's details to see its revisions, ratings and trust changes. To remove
it permanently, choose **Forget**. **Memory health** counts facts by status.
When facts lose their cited
source, the **Lost source** filter lists them; on mobile, select the lost-source
count in the memory summary.

Expand Down
Loading
Loading