From f4d8c930e830a53064ee113ce5fc57bff5d8d2de Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:44:33 -0800 Subject: [PATCH] fix(server): repair summary search after snapshots and project adoption --- apps/server/scripts/migrate-dev-db.test.ts | 40 +++++++++ apps/server/scripts/migrate-dev-db.ts | 10 +++ .../ConversationSummaries.test.ts | 90 +++++++++++++++++++ .../ConversationSummaries.ts | 4 +- 4 files changed, 143 insertions(+), 1 deletion(-) diff --git a/apps/server/scripts/migrate-dev-db.test.ts b/apps/server/scripts/migrate-dev-db.test.ts index 7f84036473c11..759411a49c301 100644 --- a/apps/server/scripts/migrate-dev-db.test.ts +++ b/apps/server/scripts/migrate-dev-db.test.ts @@ -211,6 +211,34 @@ it.layer(NodeServices.layer)("migrate-dev-db", (it) => { yield* sql`UPDATE orchestration_v2_tool_search_state SET cursor = (SELECT max(rowid) FROM orchestration_v2_projection_turn_items), backfill_end = (SELECT max(rowid) FROM orchestration_v2_projection_turn_items) WHERE id = 1`; + for (const [summaryId, projectId, threadId, text, scrubbed] of [ + ["summary-kept", "project-kept", "stopped-thread", "keptsummaryterm", false], + [ + "summary-dropped", + "project-deleted", + "deleted-project-thread", + "droppedsummaryterm", + false, + ], + ["summary-scrubbed", "project-kept", "stopped-thread", "scrubbedsummaryterm", true], + [ + "summary-legacy-scrubbed", + "project-kept", + "stopped-thread", + "[Source unavailable: removed or changed.]", + false, + ], + ] as const) { + yield* sql`INSERT INTO conversation_summaries + (summary_id, project_id, thread_id, position, created_at, payload_json) + VALUES (${summaryId}, ${projectId}, ${threadId}, '0', '2026-08-01', + ${JSON.stringify({ statements: [{ text }], scrubbed })})`; + } + assert.deepEqual( + yield* sql`SELECT text FROM conversation_summary_fts + WHERE conversation_summary_fts MATCH 'keptsummaryterm'`, + [{ text: "keptsummaryterm" }], + ); }), ); @@ -233,6 +261,18 @@ it.layer(NodeServices.layer)("migrate-dev-db", (it) => { toolHits.map((hit) => hit.id), ["tool-copy"], ); + const summaryHits = yield* sql<{ id: string }>` + SELECT summary.summary_id AS id FROM conversation_summary_fts + JOIN conversation_summary_search_ids AS ids ON ids.id = conversation_summary_fts.rowid + JOIN conversation_summaries AS summary USING (summary_id) + WHERE conversation_summary_fts MATCH 'keptsummaryterm'`; + assert.deepEqual( + summaryHits.map((hit) => hit.id), + ["summary-kept"], + ); + assert.deepEqual(yield* sql`SELECT text FROM conversation_summary_fts`, [ + { text: "keptsummaryterm" }, + ]); const threads = yield* sql<{ thread_id: string }>` SELECT thread_id FROM orchestration_v2_projection_threads ORDER BY thread_id`; const events = yield* sql<{ stream_id: string }>` diff --git a/apps/server/scripts/migrate-dev-db.ts b/apps/server/scripts/migrate-dev-db.ts index 3ef5ca2b4ecba..5eeefa1b46635 100644 --- a/apps/server/scripts/migrate-dev-db.ts +++ b/apps/server/scripts/migrate-dev-db.ts @@ -322,6 +322,16 @@ const copySourceSlice = Effect.fn("copyDevDbSourceSlice")(function* (sourcePath: if (virtualTables.includes("project_knowledge_fts")) { yield* sql`INSERT INTO project_knowledge_fts(project_knowledge_fts) VALUES ('rebuild')`; } + if (virtualTables.includes("conversation_summary_fts")) { + yield* sql`INSERT INTO conversation_summary_fts (rowid, text) + SELECT ids.id, (SELECT group_concat(json_extract(value, '$.text'), ' ') + FROM json_each(summary.payload_json, '$.statements')) + FROM conversation_summaries AS summary + JOIN conversation_summary_search_ids AS ids USING (summary_id) + WHERE coalesce(json_extract(summary.payload_json, '$.scrubbed'), 0) = 0 + AND coalesce(json_extract(summary.payload_json, '$.statements[0].text'), '') + != '[Source unavailable: removed or changed.]'`; + } }), ); yield* sql`DETACH DATABASE src`; diff --git a/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts b/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts index 7c3ec79d56fdc..d5882aba0e680 100644 --- a/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts +++ b/apps/server/src/project/conversationSummaries/ConversationSummaries.test.ts @@ -474,6 +474,32 @@ const onlySummary = Effect.gen(function* () { return read.summary; }); +const adoptThreadProject = Effect.fn("test.adoptSummaryThreadProject")(function* () { + const sql = yield* SqlClient.SqlClient; + const adoptedProjectId = ProjectId.make("adopted-project"); + yield* sql`INSERT INTO projection_projects + (project_id, title, workspace_root, scripts_json, created_at, updated_at) + SELECT ${adoptedProjectId}, title, workspace_root, scripts_json, created_at, updated_at + FROM projection_projects WHERE project_id = ${projectId}`; + const projections = yield* ProjectionStore.ProjectionStoreV2; + yield* projections.apply({ + id: EventId.make("thread-project-adopted"), + type: "thread.metadata-updated", + threadId, + providerInstanceId: instanceId, + occurredAt: at, + payload: { ...threadPayload, projectId: adoptedProjectId }, + }); + const settings = yield* ServerSettings.ServerSettingsService; + yield* settings.updateSettings({ + projectSettingsOverrides: { + ...optedIn.projectSettingsOverrides, + [adoptedProjectId]: { enableConversationSummaries: true }, + }, + }); + return adoptedProjectId; +}); + it.effect( "summarizes the settled range once with exact citations and no stored source text", () => { @@ -507,6 +533,18 @@ it.effect( SELECT payload_json FROM conversation_summaries `; assert.isTrue(rows.every((row) => !row.payload_json.includes("Detail. Detail."))); + + const adoptedProjectId = yield* adoptThreadProject(); + yield* summarize; + const service = yield* ConversationSummaries.ConversationSummaries; + const adopted = yield* service.list({ projectId: adoptedProjectId, threadId }); + assert.equal(adopted.summaries.length, 1); + assert.deepInclude(adopted.summaries[0], { + state: "available", + summary: { ...summary, projectId: adoptedProjectId }, + }); + yield* summarize; + assert.equal(model.calls.length, 2); }).pipe(Effect.provide(makeLayer(model.generate, optedIn))); }, ); @@ -2321,6 +2359,58 @@ it.effect("hands off almost all omitted history of a 600-item thread as summarie rawText(index), "the six newest messages arrive word for word", ); + + const sql = yield* SqlClient.SqlClient; + const originalIds = yield* sql<{ id: string }>`SELECT summary_id AS id + FROM conversation_summaries WHERE thread_id = ${threadId} ORDER BY summary_id`; + const adoptedProjectId = yield* adoptThreadProject(); + for (let pass = 0, hasMore = true; hasMore; pass++) { + assert.isBelow(pass, 60, "the adopted thread converges"); + hasMore = (yield* summarize).hasMore; + } + const stored = yield* sql<{ id: string; projectId: string; payloadProjectId: string }>` + SELECT summary_id AS id, project_id AS "projectId", + json_extract(payload_json, '$.projectId') AS "payloadProjectId" + FROM conversation_summaries WHERE thread_id = ${threadId} ORDER BY summary_id`; + assert.deepEqual( + stored.map((row) => row.id), + originalIds.map((row) => row.id), + ); + assert.isTrue( + stored.every( + (row) => row.projectId === adoptedProjectId && row.payloadProjectId === adoptedProjectId, + ), + ); + const adoptedScope = { projectId: adoptedProjectId, threadId }; + const adopted = yield* service.list(adoptedScope); + assert.isTrue(adopted.summaries.every((read) => read.state === "available")); + const first = adopted.summaries[0]; + if (first?.state !== "available") throw new Error("Expected an adopted root summary"); + let hop = first.summary; + assert.equal(hop.sources[0]?.kind, "summary"); + while (hop.sources[0]?.kind === "summary") { + const page = yield* service.expand({ ...adoptedScope, summaryId: hop.id, limit: 1 }); + assert.equal(page.sources[0]?.result.state, "available"); + const child = yield* service.describe({ + ...adoptedScope, + summaryId: hop.sources[0].summaryId, + }); + if (child.state !== "available") throw new Error("Expected an adopted child summary"); + hop = child.summary; + } + const exact = yield* service.expand({ ...adoptedScope, summaryId: hop.id, limit: 1 }); + assert.equal( + exact.sources[0]?.result.state === "available" ? exact.sources[0].result.text : null, + rawText(0), + ); + assert.isAbove( + (yield* service.search({ ...adoptedScope, query: "Statement" })).summaries.length, + 0, + ); + assert.deepEqual((yield* service.search({ ...scope, query: "Statement" })).summaries, []); + const adoptedSpent = model.calls.length; + yield* summarize; + assert.equal(model.calls.length, adoptedSpent); }).pipe(Effect.provide(makeLayer(model.generate, optedIn))); }); diff --git a/apps/server/src/project/conversationSummaries/ConversationSummaries.ts b/apps/server/src/project/conversationSummaries/ConversationSummaries.ts index a0de2683bccb7..dc17908d1b38b 100644 --- a/apps/server/src/project/conversationSummaries/ConversationSummaries.ts +++ b/apps/server/src/project/conversationSummaries/ConversationSummaries.ts @@ -1471,8 +1471,10 @@ const make = Effect.gen(function* () { ${summary.createdAt}, ${encodeSummary(summary)} ) ON CONFLICT (summary_id) DO UPDATE SET + project_id = excluded.project_id, created_at = excluded.created_at, payload_json = excluded.payload_json - WHERE coalesce(json_extract(conversation_summaries.payload_json, '$.scrubbed'), 0) = 1 + WHERE conversation_summaries.project_id != excluded.project_id + OR coalesce(json_extract(conversation_summaries.payload_json, '$.scrubbed'), 0) = 1 OR json_extract(conversation_summaries.payload_json, '$.statements[0].text') = ${SCRUBBED_TEXT} `; return {