From ef47de44b032b298238b95609d8f234f12378e3d Mon Sep 17 00:00:00 2001 From: Navid Nadali Date: Thu, 19 Nov 2026 04:10:41 +0000 Subject: [PATCH] fix(server): Codex runs sharing a session no longer fall behind together Codex runs every thread of an instance on one provider session, so each run receives every thread's events. A run dropped another thread's events only when it processed them: their snapshots were queued as commit jobs and their usage reports waited for those commits, so with several runs live each one waited for every thread's commits and all fell behind. A run now routes its events as they arrive and drops, before the commit queue, the ones it would reject without a routing change. Past the root turn every event still reaches the run. Co-Authored-By: Claude Opus 5.5 --- .../RunExecutionService.test.ts | 97 ++++++++++++++++++- .../orchestration-v2/RunExecutionService.ts | 11 +++ 2 files changed, 106 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index bbac6607820d..6e4b3ff60d59 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -31,6 +31,7 @@ import { } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Clock from "effect/Clock"; +import * as Context from "effect/Context"; import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as DateTime from "effect/DateTime"; @@ -41,6 +42,7 @@ import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import { CodexAppServerRequestError } from "effect-codex-app-server/errors"; @@ -4602,6 +4604,8 @@ function runBackgroundItemScenario( ) => Effect.Effect; // Consumes through the real commit coordinator, as the event sink does. readonly batched?: boolean; + // A commit coordinator shared with other runs, instead of the run's own. + readonly coordinator?: Effect.Success>; readonly keepEventStreamOpen?: boolean; readonly hasPendingBackgroundWorkForThread?: ( providerThread: OrchestrationV2ProviderThread, @@ -4631,7 +4635,8 @@ function runBackgroundItemScenario( ); let inSharedCommit = false; const coordinator = - options?.batched === true + options?.coordinator ?? + (options?.batched === true ? yield* ProviderCommitBatch.make( (work) => Effect.suspend(() => { @@ -4646,7 +4651,7 @@ function runBackgroundItemScenario( ), true, ) - : undefined; + : undefined); const layerTest = RunExecutionService.layer.pipe( Layer.provide( Layer.mergeAll( @@ -5235,3 +5240,91 @@ it.effect("drops a child's held text once the child is handed to another run", ( assert.deepEqual(yield* Ref.get(staleWrites), []); }), ); + +const InSharedCommit = Context.Reference("test/RunExecutionService/InSharedCommit", { + defaultValue: () => false, +}); + +it.effect("keeps every run on a shared provider session up with its own turn", () => + Effect.gen(function* () { + // Nine runs share one provider session, so each receives every thread's events. Ten other + // writers keep the writer busy (a pass ~1 s). Every 10 s each thread reports usage, writes + // two items and reports usage again. + const runs = 9; + const writer = yield* Semaphore.make(1); + const coordinator = yield* ProviderCommitBatch.make( + (work) => writer.withPermits(1)(Effect.provideService(work, InSharedCommit, true)), + true, + ); + for (let other = 0; other < 10; other++) + yield* coordinator + .barrier(writer.withPermits(1)(Effect.sleep("100 millis")), [`thread:writer-${other}`]) + .pipe(Effect.forever, Effect.forkChild); + const ids = Array.from({ length: runs }, (_, run) => + backgroundScenarioIds(`shared-session-${run}`), + ); + const rootTurn = (own: BackgroundScenarioIds): ProviderAdapter.ProviderAdapterV2Event => + ({ + type: "provider_turn.updated", + driver, + providerTurn: { + id: own.rootProviderTurnId, + providerThreadId: own.providerThreadId, + nodeId: own.rootNodeId, + runAttemptId: own.attemptId, + status: "running", + }, + }) as ProviderAdapter.ProviderAdapterV2Event; + const subscriptions = yield* Effect.forEach(ids, () => + Queue.unbounded(), + ); + // The last round of its own items each run has ingested. + const ingestedRound = Array.from({ length: runs }, () => 0); + for (let run = 0; run < runs; run++) { + const own = ids[run]!; + yield* Queue.offer(subscriptions[run]!, rootTurn(own)); + yield* runBackgroundItemScenario( + `shared-session-${run}`, + () => Stream.fromQueue(subscriptions[run]!), + { + coordinator: { + ...coordinator, + // As the event sink's writes: after this thread's queued snapshots, on the writer. + barrier: (effect: Effect.Effect) => + Effect.flatMap(InSharedCommit, (inCommit) => + inCommit + ? effect + : coordinator.barrier(writer.withPermits(1)(effect), [own.threadId]), + ), + }, + onIngest: (event) => + Effect.sync(() => { + if (event.type === "turn_item.updated") + ingestedRound[run] = Math.max( + ingestedRound[run]!, + Math.floor(event.turnItem.ordinal / 10), + ); + }), + }, + ).pipe(Effect.forkChild); + } + let round = 0; + yield* Effect.gen(function* () { + for (;;) { + yield* Effect.sleep("10 seconds"); + round++; + for (const own of ids) + for (const event of [ + rootTurn(own), + backgroundTurnItemEvent(own, "dynamic_tool", "running", round * 10 + 1), + backgroundTurnItemEvent(own, "dynamic_tool", "completed", round * 10 + 2), + rootTurn(own), + ]) + for (const subscription of subscriptions) yield* Queue.offer(subscription, event); + } + }).pipe(Effect.forkChild); + yield* TestClock.adjust("5 minutes"); + + assert.isAtMost(Math.max(...ingestedRound.map((ingested) => round - ingested)), 2); + }), +); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index f1d21f4a6bad..6f74566f1e85 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1972,9 +1972,20 @@ export const layer: Layer.Layer< yield* trackChildLifecycle(event, deliveredEvent !== null); yield* scheduleIdleFlush; }); + // A shared provider session hands every run every thread's events. Routing them as they + // arrive drops, before the commit queue, those this run would reject without a routing + // change; past the root turn every event still rechecks whether ingestion can stop. + let routedAhead = yield* Ref.get(eventRouting); const providerEventFiber = yield* eventSink .consumeProviderEvents( eventSubscription.events.pipe( + Stream.filter((event) => { + if (routedAhead.rootTurnEnded) return true; + const [accepted, routing] = routeProviderEvent(event, routeIdentity, routedAhead); + const keep = accepted || routing !== routedAhead; + routedAhead = routing; + return keep; + }), Stream.merge(Stream.fromQueue(idleFlushTicks), { haltStrategy: "left" }), ), (event) =>