diff --git a/apps/server/scripts/migrate-dev-db.test.ts b/apps/server/scripts/migrate-dev-db.test.ts index 7f84036473c11..759411a49c301 100644 --- a/apps/server/scripts/migrate-dev-db.test.ts +++ b/apps/server/scripts/migrate-dev-db.test.ts @@ -211,6 +211,34 @@ it.layer(NodeServices.layer)("migrate-dev-db", (it) => { yield* sql`UPDATE orchestration_v2_tool_search_state SET cursor = (SELECT max(rowid) FROM orchestration_v2_projection_turn_items), backfill_end = (SELECT max(rowid) FROM orchestration_v2_projection_turn_items) WHERE id = 1`; + for (const [summaryId, projectId, threadId, text, scrubbed] of [ + ["summary-kept", "project-kept", "stopped-thread", "keptsummaryterm", false], + [ + "summary-dropped", + "project-deleted", + "deleted-project-thread", + "droppedsummaryterm", + false, + ], + ["summary-scrubbed", "project-kept", "stopped-thread", "scrubbedsummaryterm", true], + [ + "summary-legacy-scrubbed", + "project-kept", + "stopped-thread", + "[Source unavailable: removed or changed.]", + false, + ], + ] as const) { + yield* sql`INSERT INTO conversation_summaries + (summary_id, project_id, thread_id, position, created_at, payload_json) + VALUES (${summaryId}, ${projectId}, ${threadId}, '0', '2026-08-01', + ${JSON.stringify({ statements: [{ text }], scrubbed })})`; + } + assert.deepEqual( + yield* sql`SELECT text FROM conversation_summary_fts + WHERE conversation_summary_fts MATCH 'keptsummaryterm'`, + [{ text: "keptsummaryterm" }], + ); }), ); @@ -233,6 +261,18 @@ it.layer(NodeServices.layer)("migrate-dev-db", (it) => { toolHits.map((hit) => hit.id), ["tool-copy"], ); + const summaryHits = yield* sql<{ id: string }>` + SELECT summary.summary_id AS id FROM conversation_summary_fts + JOIN conversation_summary_search_ids AS ids ON ids.id = conversation_summary_fts.rowid + JOIN conversation_summaries AS summary USING (summary_id) + WHERE conversation_summary_fts MATCH 'keptsummaryterm'`; + assert.deepEqual( + summaryHits.map((hit) => hit.id), + ["summary-kept"], + ); + assert.deepEqual(yield* sql`SELECT text FROM conversation_summary_fts`, [ + { text: "keptsummaryterm" }, + ]); const threads = yield* sql<{ thread_id: string }>` SELECT thread_id FROM orchestration_v2_projection_threads ORDER BY thread_id`; const events = yield* sql<{ stream_id: string }>` diff --git a/apps/server/scripts/migrate-dev-db.ts b/apps/server/scripts/migrate-dev-db.ts index 3ef5ca2b4ecba..5eeefa1b46635 100644 --- a/apps/server/scripts/migrate-dev-db.ts +++ b/apps/server/scripts/migrate-dev-db.ts @@ -322,6 +322,16 @@ const copySourceSlice = Effect.fn("copyDevDbSourceSlice")(function* (sourcePath: if (virtualTables.includes("project_knowledge_fts")) { yield* sql`INSERT INTO project_knowledge_fts(project_knowledge_fts) VALUES ('rebuild')`; } + if (virtualTables.includes("conversation_summary_fts")) { + yield* sql`INSERT INTO conversation_summary_fts (rowid, text) + SELECT ids.id, (SELECT group_concat(json_extract(value, '$.text'), ' ') + FROM json_each(summary.payload_json, '$.statements')) + FROM conversation_summaries AS summary + JOIN conversation_summary_search_ids AS ids USING (summary_id) + WHERE coalesce(json_extract(summary.payload_json, '$.scrubbed'), 0) = 0 + AND coalesce(json_extract(summary.payload_json, '$.statements[0].text'), '') + != '[Source unavailable: removed or changed.]'`; + } }), ); yield* sql`DETACH DATABASE src`; diff --git a/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts b/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts index 7c3ec79d56fdc..dc81ed64200b1 100644 --- a/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts +++ b/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts @@ -101,6 +101,7 @@ const optedIn = { const decodeSettings = Schema.decodeUnknownEffect(ServerSettingsSchema); const decodeSummarySearchInput = Schema.decodeEffect(ConversationSummarySearchInput); const encodeSummary = Schema.encodeEffect(Schema.fromJsonString(ConversationSummary)); +const encodeModelSelection = Schema.encodeSync(Schema.fromJsonString(ModelSelection)); // Freeze the protocol-2 summary codec from before ranged/tool sources. An older client // compiles these shapes into RpcClient; merely accepting our current schema proves nothing. const LegacySummary = Schema.Struct({ @@ -239,13 +240,28 @@ const settingsLayer = (overrides: Parameters[0] }), ).pipe(Layer.provide(ServerSettings.layerTest(overrides))); -const makeLayer = (generate: Generate, settings: Parameters[0]) => +const makeLayer = ( + generate: Generate, + settings: Parameters[0], + afterSourceRead: ( + input: Parameters[0], + ) => Effect.Effect = () => Effect.void, +) => Layer.merge(ConversationSummaries.layer, ContextHandoffService.layer).pipe( Layer.provide( Layer.mock(TextGeneration.TextGeneration)({ generateConversationSummary: generate }), ), Layer.provide(IdAllocator.layer), - Layer.provideMerge(ProjectKnowledge.layer.pipe(Layer.provide(HolographicMemory.layer))), + Layer.provideMerge( + Layer.effect( + ProjectKnowledge.ProjectKnowledge, + Effect.map(ProjectKnowledge.ProjectKnowledge, (knowledge) => ({ + ...knowledge, + readSources: (input) => + knowledge.readSources(input).pipe(Effect.tap(() => afterSourceRead(input))), + })), + ).pipe(Layer.provide(ProjectKnowledge.layer.pipe(Layer.provide(HolographicMemory.layer)))), + ), Layer.provideMerge( Layer.mergeAll( ProjectStore.layer, @@ -474,10 +490,39 @@ const onlySummary = Effect.gen(function* () { return read.summary; }); +const adoptThreadProject = Effect.fn("test.adoptSummaryThreadProject")(function* () { + const sql = yield* SqlClient.SqlClient; + const adoptedProjectId = ProjectId.make("adopted-project"); + yield* sql`INSERT INTO projection_projects + (project_id, title, workspace_root, scripts_json, created_at, updated_at) + SELECT ${adoptedProjectId}, title, workspace_root, scripts_json, created_at, updated_at + FROM projection_projects WHERE project_id = ${projectId}`; + const projections = yield* ProjectionStore.ProjectionStoreV2; + yield* projections.apply({ + id: EventId.make("thread-project-adopted"), + type: "thread.metadata-updated", + threadId, + providerInstanceId: instanceId, + occurredAt: at, + payload: { ...threadPayload, projectId: adoptedProjectId }, + }); + const settings = yield* ServerSettings.ServerSettingsService; + yield* settings.updateSettings({ + projectSettingsOverrides: { + ...optedIn.projectSettingsOverrides, + [adoptedProjectId]: { enableConversationSummaries: true }, + }, + }); + return adoptedProjectId; +}); + it.effect( "summarizes the settled range once with exact citations and no stored source text", () => { const model = citing(); + let sourceRead: + | { entered: Deferred.Deferred; release: Deferred.Deferred } + | undefined; return Effect.gen(function* () { yield* seed; yield* summarize; @@ -507,7 +552,44 @@ it.effect( SELECT payload_json FROM conversation_summaries `; assert.isTrue(rows.every((row) => !row.payload_json.includes("Detail. Detail."))); - }).pipe(Effect.provide(makeLayer(model.generate, optedIn))); + + const adoptedProjectId = yield* adoptThreadProject(); + const service = yield* ConversationSummaries.ConversationSummaries; + const gate = { entered: yield* Deferred.make(), release: yield* Deferred.make() }; + sourceRead = gate; + const oldRead = yield* service + .describe({ ...scope, summaryId: summary.id }) + .pipe(Effect.forkChild); + yield* Deferred.await(gate.entered); + yield* Effect.gen(function* () { + yield* summarize; + const adopted = yield* service.list({ projectId: adoptedProjectId, threadId }); + assert.equal(adopted.summaries.length, 1); + assert.deepInclude(adopted.summaries[0], { + state: "available", + summary: { ...summary, projectId: adoptedProjectId }, + }); + }).pipe(Effect.ensuring(Deferred.succeed(gate.release, undefined))); + assert.equal((yield* Fiber.join(oldRead)).state, "stale"); + assert.equal( + (yield* service.describe({ projectId: adoptedProjectId, threadId, summaryId: summary.id })) + .state, + "available", + ); + yield* summarize; + assert.equal(model.calls.length, 2); + }).pipe( + Effect.provide( + makeLayer(model.generate, optedIn, (input) => { + if (sourceRead === undefined || input.projectId !== projectId) return Effect.void; + const gate = sourceRead; + sourceRead = undefined; + return Deferred.succeed(gate.entered, undefined).pipe( + Effect.andThen(Deferred.await(gate.release)), + ); + }), + ), + ); }, ); @@ -871,6 +953,44 @@ it.effect.each([ yield* sql`UPDATE projection_projects SET deleted_at = '2026-01-01T00:00:00.000Z' WHERE project_id = ${projectId}`; assert.deepEqual(yield* stalled(), []); + + const adoptedProjectId = yield* adoptThreadProject(); + assert.deepInclude((yield* stalled())[0], { threadId, runId: freshRunId, retry: 1 }); + const adoptedModel = { instanceId, model: "gpt-5-mini" }; + const adoptedModelKey = encodeModelSelection(adoptedModel); + const adoptedSettings = { + projectSettingsOverrides: { + [projectId]: { enableConversationSummaries: false }, + [adoptedProjectId]: { + enableConversationSummaries: true, + textGenerationModelSelection: adoptedModel, + }, + }, + }; + yield* settings.updateSettings(adoptedSettings); + assert.deepEqual(yield* stalled(), []); + assert.deepInclude((yield* stalled(adoptedModelKey))[0], { + threadId, + runId: freshRunId, + retry: 1, + }); + yield* sql` + UPDATE conversation_summary_passes + SET payload_json = json_set(payload_json, '$.retries', 6) WHERE thread_id = ${threadId} + `; + yield* settings.updateSettings({ + projectSettingsOverrides: { + ...adoptedSettings.projectSettingsOverrides, + [adoptedProjectId]: { enableConversationSummaries: false }, + }, + }); + assert.deepEqual(yield* stalled(adoptedModelKey), []); + yield* settings.updateSettings(adoptedSettings); + assert.deepInclude((yield* stalled(adoptedModelKey))[0], { + threadId, + runId: freshRunId, + retry: 1, + }); }).pipe(Effect.provide(makeLayer(model.generate, optedIn))); }, ); @@ -1344,26 +1464,60 @@ it.effect( }, ); -it.effect( - "keeps scrubbed summaries stale after a source reverts and regenerates their text", - () => { +it.effect.each([false, true])( + "keeps scrubbed summaries stale after a source reverts and regenerates their text (legacy: %s)", + (legacy) => { const model = citing(); + let sourceRead: + | { entered: Deferred.Deferred; release: Deferred.Deferred } + | undefined; return Effect.gen(function* () { yield* seed; yield* summarize; const summary = yield* onlySummary; + if (legacy) { + const sql = yield* SqlClient.SqlClient; + yield* sql`UPDATE conversation_summaries + SET payload_json = json_remove(payload_json, '$._revision') + WHERE summary_id = ${summary.id}`; + } const service = yield* ConversationSummaries.ConversationSummaries; const projections = yield* ProjectionStore.ProjectionStoreV2; yield* projections.apply(messageEvent(0, "Changed source")); - assert.equal((yield* service.describe({ ...scope, summaryId: summary.id })).state, "stale"); - yield* projections.apply(messageEvent(0, rawText(0))); - assert.equal((yield* service.describe({ ...scope, summaryId: summary.id })).state, "stale"); - yield* summarize; - assert.equal(model.calls.length, 2); - const regenerated = yield* onlySummary; - assert.equal(regenerated.statements[0]!.text, summary.statements[0]!.text); - assert.notInclude(regenerated.statements[0]!.text, "Source unavailable"); - }).pipe(Effect.provide(makeLayer(model.generate, optedIn))); + const gate = { entered: yield* Deferred.make(), release: yield* Deferred.make() }; + sourceRead = gate; + const oldRead = yield* service + .describe({ ...scope, summaryId: summary.id }) + .pipe(Effect.forkChild({ startImmediately: true })); + yield* Deferred.await(gate.entered); + yield* Effect.gen(function* () { + assert.equal((yield* service.describe({ ...scope, summaryId: summary.id })).state, "stale"); + yield* projections.apply(messageEvent(0, rawText(0))); + assert.equal((yield* service.describe({ ...scope, summaryId: summary.id })).state, "stale"); + yield* summarize; + assert.equal(model.calls.length, 2); + const regenerated = yield* onlySummary; + // A source revert can reproduce the entire public payload, even createdAt on this clock. + assert.deepEqual(regenerated, summary); + assert.notInclude(regenerated.statements[0]!.text, "Source unavailable"); + }).pipe(Effect.ensuring(Deferred.succeed(gate.release, undefined))); + assert.equal((yield* Fiber.join(oldRead)).state, "stale"); + assert.equal( + (yield* service.describe({ ...scope, summaryId: summary.id })).state, + "available", + ); + }).pipe( + Effect.provide( + makeLayer(model.generate, optedIn, () => { + if (sourceRead === undefined) return Effect.void; + const gate = sourceRead; + sourceRead = undefined; + return Deferred.succeed(gate.entered, undefined).pipe( + Effect.andThen(Deferred.await(gate.release)), + ); + }), + ), + ); }, ); @@ -2321,6 +2475,58 @@ it.effect("hands off almost all omitted history of a 600-item thread as summarie rawText(index), "the six newest messages arrive word for word", ); + + const sql = yield* SqlClient.SqlClient; + const originalIds = yield* sql<{ id: string }>`SELECT summary_id AS id + FROM conversation_summaries WHERE thread_id = ${threadId} ORDER BY summary_id`; + const adoptedProjectId = yield* adoptThreadProject(); + for (let pass = 0, hasMore = true; hasMore; pass++) { + assert.isBelow(pass, 60, "the adopted thread converges"); + hasMore = (yield* summarize).hasMore; + } + const stored = yield* sql<{ id: string; projectId: string; payloadProjectId: string }>` + SELECT summary_id AS id, project_id AS "projectId", + json_extract(payload_json, '$.projectId') AS "payloadProjectId" + FROM conversation_summaries WHERE thread_id = ${threadId} ORDER BY summary_id`; + assert.deepEqual( + stored.map((row) => row.id), + originalIds.map((row) => row.id), + ); + assert.isTrue( + stored.every( + (row) => row.projectId === adoptedProjectId && row.payloadProjectId === adoptedProjectId, + ), + ); + const adoptedScope = { projectId: adoptedProjectId, threadId }; + const adopted = yield* service.list(adoptedScope); + assert.isTrue(adopted.summaries.every((read) => read.state === "available")); + const first = adopted.summaries[0]; + if (first?.state !== "available") throw new Error("Expected an adopted root summary"); + let hop = first.summary; + assert.equal(hop.sources[0]?.kind, "summary"); + while (hop.sources[0]?.kind === "summary") { + const page = yield* service.expand({ ...adoptedScope, summaryId: hop.id, limit: 1 }); + assert.equal(page.sources[0]?.result.state, "available"); + const child = yield* service.describe({ + ...adoptedScope, + summaryId: hop.sources[0].summaryId, + }); + if (child.state !== "available") throw new Error("Expected an adopted child summary"); + hop = child.summary; + } + const exact = yield* service.expand({ ...adoptedScope, summaryId: hop.id, limit: 1 }); + assert.equal( + exact.sources[0]?.result.state === "available" ? exact.sources[0].result.text : null, + rawText(0), + ); + assert.isAbove( + (yield* service.search({ ...adoptedScope, query: "Statement" })).summaries.length, + 0, + ); + assert.deepEqual((yield* service.search({ ...scope, query: "Statement" })).summaries, []); + const adoptedSpent = model.calls.length; + yield* summarize; + assert.equal(model.calls.length, adoptedSpent); }).pipe(Effect.provide(makeLayer(model.generate, optedIn))); }); diff --git a/apps/server/src/project/conversationSummaries/ConversationSummaries.ts b/apps/server/src/project/conversationSummaries/ConversationSummaries.ts index a0de2683bccb7..7de3aebeb1b1e 100644 --- a/apps/server/src/project/conversationSummaries/ConversationSummaries.ts +++ b/apps/server/src/project/conversationSummaries/ConversationSummaries.ts @@ -113,6 +113,15 @@ const HANDOFF_CANDIDATE_LIMIT = 40; const CLEANUP_LIMIT = 100; const SCRUBBED_TEXT = "[Source unavailable: removed or changed.]"; +interface StoredSummaryRow extends StoredSummary { + readonly revision: string | null; +} + +interface LoadedSummary { + readonly summary: ConversationSummary; + readonly revision: string | null; +} + export class ConversationSummaries extends Context.Service< ConversationSummaries, { @@ -309,7 +318,7 @@ const make = Effect.gen(function* () { projectId: ProjectId, threadId: ThreadId, ) { - const summaries: Array = []; + const summaries: Array = []; let after: SummaryCursor | undefined; while (true) { const rows = yield* sql<{ @@ -317,9 +326,11 @@ const make = Effect.gen(function* () { readonly summary_id: string; readonly sources_json: string; readonly scrubbed: number; + readonly revision: string | null; }>` SELECT position, summary_id, json_extract(payload_json, '$.sources') AS sources_json, + json_extract(payload_json, '$._revision') AS revision, (coalesce(json_extract(payload_json, '$.scrubbed'), 0) = 1 OR json_extract(payload_json, '$.statements[0].text') = ${SCRUBBED_TEXT}) AS scrubbed FROM conversation_summaries @@ -334,6 +345,7 @@ const make = Effect.gen(function* () { sources: yield* decodeSources(row.sources_json), scrubbed: row.scrubbed === 1, position: row.position, + revision: row.revision, }); if (rows.length < THREAD_SUMMARY_PAGE_SIZE) return summaries; after = rows.at(-1)!; @@ -358,8 +370,12 @@ const make = Effect.gen(function* () { const findSummary = Effect.fn("ConversationSummaries.findSummary")(function* ( input: ConversationSummaryDescribeInput, ) { - const rows = yield* sql<{ readonly payload_json: string }>` - SELECT payload_json FROM conversation_summaries + const rows = yield* sql<{ + readonly payload_json: string; + readonly revision: string | null; + }>` + SELECT payload_json, json_extract(payload_json, '$._revision') AS revision + FROM conversation_summaries WHERE project_id = ${input.projectId} AND thread_id = ${input.threadId} AND summary_id = ${input.summaryId} `.pipe(Effect.mapError(storageError(input.projectId))); @@ -368,9 +384,10 @@ const make = Effect.gen(function* () { reason: "missing_summary", projectId: input.projectId, }); - return yield* decodeSummary(rows[0].payload_json).pipe( + const summary = yield* decodeSummary(rows[0].payload_json).pipe( Effect.mapError(storageError(input.projectId)), ); + return { summary, revision: rows[0].revision }; }); const invisibleMessageIds = Effect.fn("ConversationSummaries.invisibleMessageIds")(function* ( @@ -548,10 +565,10 @@ const make = Effect.gen(function* () { const verify = Effect.fn("ConversationSummaries.verify")(function* ( projectId: ProjectId, threadId: ThreadId, - rows: ReadonlyMap, + rows: ReadonlyMap, ids: ReadonlyArray, ) { - const leaves: Array = []; + const leaves: Array = []; const visited = new Set(); const collect = (id: string) => { const row = rows.get(id); @@ -608,46 +625,50 @@ const make = Effect.gen(function* () { const row = rows.get(id); if (row === undefined || reason === null || reason === "archived" || row.scrubbed) continue; // Handoff references keep the summary skeleton, not text derived from a removed source. + // A regeneration can reuse this id, even with identical text and the same timestamp. + // Scrub only the publication whose sources this read verified, in its original project. yield* sql`UPDATE conversation_summaries SET payload_json = json_set(payload_json, '$.scrubbed', json('true'), '$.statements', json(${encodeStatements([ { text: SCRUBBED_TEXT, citations: [conversationSummarySourceKey(row.sources[0]!)] }, - ])})) WHERE summary_id = ${id}`; + ])})) WHERE summary_id = ${id} AND project_id = ${projectId} + AND json_extract(payload_json, '$._revision') IS ${row.revision}`; } return reasons; }); - const storedOf = (summary: ConversationSummary): StoredSummary => ({ + const storedOf = ({ summary, revision }: LoadedSummary): StoredSummaryRow => ({ id: summary.id, sources: summary.sources, scrubbed: isScrubbed(summary), position: "", + revision, }); /** Rows a verification of these summaries needs: themselves, or the thread for a roll-up. */ const rowsFor = Effect.fn("ConversationSummaries.rowsFor")(function* ( projectId: ProjectId, threadId: ThreadId, - summaries: ReadonlyArray, + summaries: ReadonlyArray, ) { return summaries.some((summary) => childIds(storedOf(summary)).length > 0) ? new Map((yield* loadThreadSummaries(projectId, threadId)).map((row) => [row.id, row])) - : new Map(summaries.map((summary) => [summary.id, storedOf(summary)])); + : new Map(summaries.map((stored) => [stored.summary.id, storedOf(stored)])); }); const verifyAndReadAll = Effect.fn("ConversationSummaries.readAll")(function* ( projectId: ProjectId, threadId: ThreadId, - summaries: ReadonlyArray, + summaries: ReadonlyArray, ) { if (summaries.length === 0) return []; const reasons = yield* verify( projectId, threadId, yield* rowsFor(projectId, threadId, summaries), - summaries.map((summary) => summary.id), + summaries.map(({ summary }) => summary.id), ); - return summaries.map((summary): ConversationSummaryRead => { + return summaries.map(({ summary }): ConversationSummaryRead => { const reason = reasons.get(summary.id); return reason === null ? { state: "available", summary } @@ -661,8 +682,8 @@ const make = Effect.gen(function* () { }); }); - const read = (summary: ConversationSummary) => - verifyAndReadAll(summary.projectId, summary.threadId, [summary]).pipe( + const read = (stored: LoadedSummary) => + verifyAndReadAll(stored.summary.projectId, stored.summary.threadId, [stored]).pipe( Effect.map((reads) => reads[0]!), ); @@ -746,7 +767,7 @@ const make = Effect.gen(function* () { } const limit = input.limit ?? 20; const query = input.query?.trim().toLowerCase() ?? ""; - const matching: Array = []; + const matching: Array = []; const scanLimit = query === "" ? limit + 1 : THREAD_SUMMARY_PAGE_SIZE; // Fold Unicode in application code; SQLite lower() only folds ASCII. Each read is // bounded, and the cursor remains the last returned summary, even across sparse matches. @@ -755,8 +776,11 @@ const make = Effect.gen(function* () { readonly payload_json: string; readonly summary_id: string; readonly position: string; + readonly revision: string | null; }>` - SELECT payload_json, summary_id, position FROM conversation_summaries + SELECT payload_json, summary_id, position, + json_extract(payload_json, '$._revision') AS revision + FROM conversation_summaries WHERE project_id = ${input.projectId} AND thread_id = ${input.threadId} ${afterSummary(after)} ORDER BY position, summary_id LIMIT ${scanLimit} @@ -767,7 +791,7 @@ const make = Effect.gen(function* () { query === "" || summary.statements.some((statement) => statement.text.toLowerCase().includes(query)) ) - matching.push(summary); + matching.push({ summary, revision: row.revision }); if (matching.length > limit) break; } if (rows.length < scanLimit) break; @@ -780,7 +804,7 @@ const make = Effect.gen(function* () { .enableConversationSummaries, lastPass, summaries: yield* verifyAndReadAll(input.projectId, input.threadId, page), - nextCursor: matching.length > limit ? (page.at(-1)?.id ?? null) : null, + nextCursor: matching.length > limit ? (page.at(-1)?.summary.id ?? null) : null, }; }, (effect, input) => effect.pipe(Effect.mapError(storageError(input.projectId))), @@ -894,8 +918,10 @@ const make = Effect.gen(function* () { readonly summary_id: string; readonly payload_json: string; readonly excerpt: string; + readonly revision: string | null; }>` SELECT summary.summary_id, summary.payload_json, + json_extract(summary.payload_json, '$._revision') AS revision, snippet(conversation_summary_fts, 0, char(1), char(2), ' … ', 64) AS excerpt FROM conversation_summary_fts JOIN conversation_summary_search_ids AS ids ON ids.id = conversation_summary_fts.rowid @@ -912,12 +938,16 @@ const make = Effect.gen(function* () { .slice(index, index + limit - summaries.length) .flatMap((rank) => loaded.get(rank.summaryId) ?? []); index += Math.min(limit - summaries.length, candidates.length - index); - const decoded = yield* Effect.forEach(batch, (row) => decodeSummary(row.payload_json)); - const byThread = Map.groupBy(decoded, (summary) => summary.threadId); + const decoded = yield* Effect.forEach(batch, (row) => + decodeSummary(row.payload_json).pipe( + Effect.map((summary): LoadedSummary => ({ summary, revision: row.revision })), + ), + ); + const byThread = Map.groupBy(decoded, ({ summary }) => summary.threadId); const verifiedById = new Map(); for (const [threadId, group] of byThread) { const reads = yield* verifyAndReadAll(input.projectId, threadId, group); - group.forEach((summary, position) => verifiedById.set(summary.id, reads[position]!)); + group.forEach(({ summary }, position) => verifiedById.set(summary.id, reads[position]!)); } for (const row of batch) { const verified = verifiedById.get(row.summary_id)!; @@ -976,7 +1006,8 @@ const make = Effect.gen(function* () { )( function* (input) { yield* requireProject(input.projectId); - const summary = yield* findSummary(input); + const stored = yield* findSummary(input); + const { summary } = stored; const offset = input.offset ?? 0; const textOffset = input.textOffset ?? 0; if ( @@ -1043,7 +1074,7 @@ const make = Effect.gen(function* () { } } return { - summary: yield* read(summary), + summary: yield* read(stored), sources, nextOffset, nextTextOffset, @@ -1108,9 +1139,10 @@ const make = Effect.gen(function* () { const enabled = (settings: ServerSettingsValue, projectId: ProjectId) => resolveProjectSettings(settings, projectId).settings.enableConversationSummaries; const rows = yield* sql<{ readonly project_id: ProjectId }>` - SELECT DISTINCT project_id FROM conversation_summary_passes - WHERE json_extract(payload_json, '$.chunkFailures') IS NOT NULL - OR json_extract(payload_json, '$.retries') IS NOT NULL + SELECT DISTINCT thread.project_id FROM conversation_summary_passes AS pass + JOIN orchestration_v2_projection_threads AS thread ON thread.thread_id = pass.thread_id + WHERE json_extract(pass.payload_json, '$.chunkFailures') IS NOT NULL + OR json_extract(pass.payload_json, '$.retries') IS NOT NULL `; const reenabled = rows .map((row) => row.project_id) @@ -1119,7 +1151,10 @@ const make = Effect.gen(function* () { yield* sql` UPDATE conversation_summary_passes SET payload_json = json_remove(payload_json, '$.chunkFailures', '$.retries') - WHERE ${sql.in("project_id", reenabled)} + WHERE thread_id IN ( + SELECT thread_id FROM orchestration_v2_projection_threads + WHERE ${sql.in("project_id", reenabled)} + ) `; }, ); @@ -1463,21 +1498,46 @@ const make = Effect.gen(function* () { modelSelection: settings.textGenerationModelSelection, createdAt: DateTime.formatIso(yield* DateTime.now), }; - yield* sql` + // This private publication token is excluded by the public summary codec and id digest. + const published = yield* sql<{ readonly revision: string }>` INSERT INTO conversation_summaries ( summary_id, project_id, thread_id, position, created_at, payload_json ) VALUES ( ${id}, ${projectId}, ${threadId}, ${input.position}, - ${summary.createdAt}, ${encodeSummary(summary)} + ${summary.createdAt}, + json_set(${encodeSummary(summary)}, '$._revision', lower(hex(randomblob(16)))) ) ON CONFLICT (summary_id) DO UPDATE SET + project_id = excluded.project_id, created_at = excluded.created_at, payload_json = excluded.payload_json - WHERE coalesce(json_extract(conversation_summaries.payload_json, '$.scrubbed'), 0) = 1 + WHERE conversation_summaries.project_id != excluded.project_id + OR coalesce(json_extract(conversation_summaries.payload_json, '$.scrubbed'), 0) = 1 OR json_extract(conversation_summaries.payload_json, '$.statements[0].text') = ${SCRUBBED_TEXT} + RETURNING json_extract(payload_json, '$._revision') AS revision `; + // A no-op conflict retains the existing publication, including a legacy null revision. + const stored = + published[0] ?? + (yield* sql<{ readonly revision: string | null }>` + SELECT json_extract(payload_json, '$._revision') AS revision + FROM conversation_summaries WHERE summary_id = ${id} + AND project_id = ${projectId} AND thread_id = ${threadId} + `)[0]; + if (stored === undefined) + return { + ok: false as const, + reason: "The summary was removed while it was being saved.", + unusableReply: false, + }; return { ok: true as const, - row: { id, sources: input.sources, scrubbed: false, position: input.position }, + row: { + id, + sources: input.sources, + scrubbed: false, + position: input.position, + revision: stored.revision, + }, }; }); @@ -1910,8 +1970,8 @@ const make = Effect.gen(function* () { ); const eligibleOverrides = overriddenProjects.filter(usesRecoveredModel); const eligibleProjects = usesRecoveredModel(null) - ? sql`(NOT ${sql.in("passes.project_id", overriddenProjects)} OR ${sql.in("passes.project_id", eligibleOverrides)})` - : sql.in("passes.project_id", eligibleOverrides); + ? sql`(NOT ${sql.in("thread.project_id", overriddenProjects)} OR ${sql.in("thread.project_id", eligibleOverrides)})` + : sql.in("thread.project_id", eligibleOverrides); const rows = yield* sql<{ readonly threadId: ThreadId; readonly payload: string }>` SELECT passes.thread_id AS "threadId", passes.payload_json AS payload FROM conversation_summary_passes AS passes @@ -1921,7 +1981,7 @@ const make = Effect.gen(function* () { AND ${eligibleProjects} AND EXISTS ( SELECT 1 FROM projection_projects AS project - WHERE project.project_id = passes.project_id AND project.deleted_at IS NULL + WHERE project.project_id = thread.project_id AND project.deleted_at IS NULL ) AND coalesce(json_extract(passes.payload_json, '$.retries'), 0) < ${STALLED_RETRY_LIMIT} AND thread.deleted_at IS NULL