diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index ad4325a6679f6..e5c2e6e7cfd3b 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -2164,6 +2164,62 @@ effectIt.effect( }), ); +it("finds abandoned turns in a long history reading each record a bounded number of times", () => { + let reads = 0; + const counted = (record: T): T => + new Proxy(record, { + get: (target, key, receiver) => { + reads += 1; + return Reflect.get(target, key, receiver); + }, + }); + const providerThreadId = ProviderThreadId.make("provider-thread-long-history"); + const endings = ["failed", "interrupted", "cancelled"] as const; + const runs = []; + const attempts = []; + const providerTurns = []; + for (let index = 0; index < 2_000; index += 1) { + // Every tenth run ended without completing, after a steering restart. + const status = index % 10 === 0 ? endings[(index / 10) % endings.length]! : "completed"; + const attemptIds = status === "completed" ? [`a${index}`] : [`a${index}-steered`, `a${index}`]; + runs.push({ + id: `r${index}`, + ordinal: index + 1, + status, + providerThreadId, + activeAttemptId: attemptIds.at(-1), + }); + for (const attemptId of attemptIds) { + attempts.push({ id: attemptId, runId: `r${index}` }); + const running = attemptId === "a10-steered" || attemptId === "a20" || attemptId === "a21"; + providerTurns.push({ + id: `t-${attemptId}`, + providerThreadId, + runAttemptId: attemptId, + status: running ? "running" : "completed", + }); + } + } + const projection = { + runs: runs.map(counted), + attempts: attempts.map(counted), + providerTurns: providerTurns.map(counted), + } as unknown as Parameters[0]; + + const abandoned = ProviderTurnStart.abandonedProviderTurns( + projection, + { ordinal: 2_001 }, + providerThreadId, + ); + + expect(abandoned.map(({ turn, run }) => [turn.id, run.id])).toEqual([ + ["t-a10-steered", "r10"], + ["t-a20", "r20"], + ]); + const records = runs.length + attempts.length + providerTurns.length; + expect(reads).toBeLessThanOrEqual(4 * records); +}); + effectIt.effect("sends new steering text when restarting a manual continuation", () => Effect.gen(function* () { const harness = makeLocalCommandHarness({ diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index b0e4b4fce918c..5110a25f7d432 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -95,6 +95,38 @@ const NON_COMPLETING_TERMINAL_RUN_STATUSES: ReadonlySet, + run: Pick, + providerThreadId: OrchestrationV2ProviderThread["id"], +) { + const attemptsByRun = Map.groupBy(projection.attempts, (attempt) => attempt.runId); + const abandonedRuns = new Map(); + for (const previous of projection.runs) { + if ( + previous.ordinal >= run.ordinal || + previous.providerThreadId !== providerThreadId || + !NON_COMPLETING_TERMINAL_RUN_STATUSES.has(previous.status) + ) { + continue; + } + if (previous.activeAttemptId !== null) { + abandonedRuns.set(previous.activeAttemptId, previous); + } + for (const attempt of attemptsByRun.get(previous.id) ?? []) { + abandonedRuns.set(attempt.id, previous); + } + } + return projection.providerTurns.flatMap((turn) => { + const previous = abandonedRuns.get(turn.runAttemptId); + return turn.providerThreadId === providerThreadId && + turn.status === "running" && + previous !== undefined + ? [{ turn, run: previous }] + : []; + }); +} + export const isProviderTurnStartError = Schema.is(ProviderTurnStartError); const isProviderAdapterProtocolError = Schema.is(ProviderAdapter.ProviderAdapterProtocolError); const isProviderAdapterOpenSessionError = Schema.is( @@ -799,30 +831,7 @@ export const layer: Layer.Layer< // running when the agent ignored Stop or an interrupt timed out. Every attempt // of such a run owns a possible abandoned turn, including attempts a steering // restart superseded, and the session cannot be reused until those turns stop. - const abandonedRuns = new Map(); - for (const previous of projection.runs) { - if ( - previous.ordinal >= run.ordinal || - previous.providerThreadId !== providerThread.id || - !NON_COMPLETING_TERMINAL_RUN_STATUSES.has(previous.status) - ) { - continue; - } - if (previous.activeAttemptId !== null) { - abandonedRuns.set(previous.activeAttemptId, previous); - } - for (const attempt of projection.attempts) { - if (attempt.runId === previous.id) abandonedRuns.set(attempt.id, previous); - } - } - const abandonedTurns = projection.providerTurns.flatMap((turn) => { - const previous = abandonedRuns.get(turn.runAttemptId); - return turn.providerThreadId === providerThread.id && - turn.status === "running" && - previous !== undefined - ? [{ turn, run: previous }] - : []; - }); + const abandonedTurns = abandonedProviderTurns(projection, run, providerThread.id); const abandonedTurn = abandonedTurns.at(-1)?.turn; if (abandonedTurn !== undefined) { const liveSession = yield* providerSessions.get(providerSessionId);