From 0350d87c0da9e55bbff84b2df164b41aad2ed12a Mon Sep 17 00:00:00 2001 From: Navid Nadali Date: Wed, 18 Nov 2026 22:40:25 +0000 Subject: [PATCH 1/2] fix(server): turn starts no longer rescan every attempt per ended run A turn start looks for provider turns left running by earlier runs that ended without completing. For each such run it scanned every attempt of the thread, so the search grew with ended runs times attempts and held the event loop on threads with a long history. The attempts are now grouped by run once, and each ended run reads only its own. Co-Authored-By: Claude Opus 5.5 --- .../ProviderTurnStartService.test.ts | 57 +++++++++++++++++++ .../ProviderTurnStartService.ts | 57 +++++++++++-------- 2 files changed, 90 insertions(+), 24 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 66fc7c97c7fa1..2714656361cf4 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -2158,6 +2158,63 @@ effectIt.effect( }), ); +it("finds abandoned turns in a long history reading each record a bounded number of times", () => { + let reads = 0; + const counted = (record: T): T => + new Proxy(record, { + get: (target, key, receiver) => { + reads += 1; + return Reflect.get(target, key, receiver); + }, + }); + const providerThreadId = ProviderThreadId.make("provider-thread-long-history"); + const endings = ["failed", "interrupted", "cancelled"] as const; + const runs = []; + const attempts = []; + const providerTurns = []; + for (let index = 0; index < 2_000; index += 1) { + // Every tenth run ended without completing, after a steering restart. + const status = index % 10 === 0 ? endings[(index / 10) % endings.length]! : "completed"; + const attemptIds = status === "completed" ? [`a${index}`] : [`a${index}-steered`, `a${index}`]; + runs.push({ + id: `r${index}`, + ordinal: index + 1, + status, + providerThreadId, + activeAttemptId: attemptIds.at(-1), + }); + for (const attemptId of attemptIds) { + attempts.push({ id: attemptId, runId: `r${index}` }); + const running = attemptId === "a10-steered" || attemptId === "a20" || attemptId === "a21"; + providerTurns.push({ + id: `t-${attemptId}`, + providerThreadId, + runAttemptId: attemptId, + status: running ? "running" : "completed", + }); + } + } + const projection = { + runs: runs.map(counted), + attempts: attempts.map(counted), + providerTurns: providerTurns.map(counted), + } as unknown as Parameters[0]; + + const abandoned = ProviderTurnStart.abandonedProviderTurns( + projection, + { ordinal: 2_001 }, + providerThreadId, + ); + const searchReads = reads; + + expect(abandoned.map(({ turn, run }) => [turn.id, run.id])).toEqual([ + ["t-a10-steered", "r10"], + ["t-a20", "r20"], + ]); + const records = runs.length + attempts.length + providerTurns.length; + expect(searchReads).toBeLessThanOrEqual(4 * records); +}); + effectIt.effect("sends new steering text when restarting a manual continuation", () => Effect.gen(function* () { const harness = makeLocalCommandHarness({ diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 43c5e663f90d1..baf62ac2a1515 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -93,6 +93,38 @@ const NON_COMPLETING_TERMINAL_RUN_STATUSES: ReadonlySet, + run: Pick, + providerThreadId: OrchestrationV2ProviderThread["id"], +) { + const attemptsByRun = Map.groupBy(projection.attempts, (attempt) => attempt.runId); + const abandonedRuns = new Map(); + for (const previous of projection.runs) { + if ( + previous.ordinal >= run.ordinal || + previous.providerThreadId !== providerThreadId || + !NON_COMPLETING_TERMINAL_RUN_STATUSES.has(previous.status) + ) { + continue; + } + if (previous.activeAttemptId !== null) { + abandonedRuns.set(previous.activeAttemptId, previous); + } + for (const attempt of attemptsByRun.get(previous.id) ?? []) { + abandonedRuns.set(attempt.id, previous); + } + } + return projection.providerTurns.flatMap((turn) => { + const previous = abandonedRuns.get(turn.runAttemptId); + return turn.providerThreadId === providerThreadId && + turn.status === "running" && + previous !== undefined + ? [{ turn, run: previous }] + : []; + }); +} + export const isProviderTurnStartError = Schema.is(ProviderTurnStartError); const isProviderAdapterProtocolError = Schema.is(ProviderAdapter.ProviderAdapterProtocolError); const isProviderAdapterOpenSessionError = Schema.is( @@ -770,30 +802,7 @@ export const layer: Layer.Layer< // running when the agent ignored Stop or an interrupt timed out. Every attempt // of such a run owns a possible abandoned turn, including attempts a steering // restart superseded, and the session cannot be reused until those turns stop. - const abandonedRuns = new Map(); - for (const previous of projection.runs) { - if ( - previous.ordinal >= run.ordinal || - previous.providerThreadId !== providerThread.id || - !NON_COMPLETING_TERMINAL_RUN_STATUSES.has(previous.status) - ) { - continue; - } - if (previous.activeAttemptId !== null) { - abandonedRuns.set(previous.activeAttemptId, previous); - } - for (const attempt of projection.attempts) { - if (attempt.runId === previous.id) abandonedRuns.set(attempt.id, previous); - } - } - const abandonedTurns = projection.providerTurns.flatMap((turn) => { - const previous = abandonedRuns.get(turn.runAttemptId); - return turn.providerThreadId === providerThread.id && - turn.status === "running" && - previous !== undefined - ? [{ turn, run: previous }] - : []; - }); + const abandonedTurns = abandonedProviderTurns(projection, run, providerThread.id); const abandonedTurn = abandonedTurns.at(-1)?.turn; if (abandonedTurn !== undefined) { const liveSession = yield* providerSessions.get(providerSessionId); From dc85de6f9701743f5cef2d03a5a797ff0270be17 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Thu, 19 Nov 2026 02:36:36 +0000 Subject: [PATCH 2/2] test(server): assert on read count directly in abandoned-turn scan test Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/ProviderTurnStartService.test.ts | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index e811fbd4fad49..cd31c2a9cd33c 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -2211,14 +2211,13 @@ it("finds abandoned turns in a long history reading each record a bounded number { ordinal: 2_001 }, providerThreadId, ); - const searchReads = reads; expect(abandoned.map(({ turn, run }) => [turn.id, run.id])).toEqual([ ["t-a10-steered", "r10"], ["t-a20", "r20"], ]); const records = runs.length + attempts.length + providerTurns.length; - expect(searchReads).toBeLessThanOrEqual(4 * records); + expect(reads).toBeLessThanOrEqual(4 * records); }); effectIt.effect("sends new steering text when restarting a manual continuation", () =>