From 59702a15239ca685858ec1bcb47a0b8892e73af4 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Mon, 12 Oct 2026 15:42:13 -0400 Subject: [PATCH 1/7] fix(server): Codex runs fail and recover instead of hanging when the app-server dies or stops answering Every Codex thread shares one app-server process. When it stopped answering or exited mid-turn, T3 never noticed: running turns stayed "running" forever, delegated tasks never woke their parent, and new Codex threads hung at "starting" until T3 restarted. The adapter now fails its session when the app-server connection ends, and when the app-server does not answer a once-a-minute thread/loaded/list probe within a minute. The session manager then fails the session's runs and the next turn starts a fresh app-server. Events Codex sent before the failure are kept. The protocol also ends the connection 2 s after the process exits, since a command Codex started can keep its stdout open, and fails requests still waiting for a reply when the client closes. A release no longer overwrites the status of a replacement session that opened under the same id meanwhile. Co-Authored-By: Claude Opus 5.5 --- .../Adapters/CodexAdapterV2.test.ts | 67 +++++++++++++++++ .../Adapters/CodexAdapterV2.ts | 47 +++++++++++- .../ProviderSessionManager.test.ts | 75 +++++++++++++++++++ .../ProviderSessionManager.ts | 17 ++++- .../effect-codex-app-server/src/client.ts | 11 ++- .../src/protocol.test.ts | 59 +++++++++++++++ .../effect-codex-app-server/src/protocol.ts | 28 +++++++ 7 files changed, 301 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 6e4a6494fb78..454c30c5832e 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -49,6 +49,8 @@ import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; +import * as Sink from "effect/Sink"; +import * as Stdio from "effect/Stdio"; import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -1943,6 +1945,71 @@ describe("CodexAdapterV2 session initialize", () => { ); }); +describe("CodexAdapterV2 lost app-server", () => { + const sessionFailureOver = (stdin: Stream.Stream) => + Effect.gen(function* () { + const adapter = yield* CodexAdapterV2.makeCodexAdapterV2({ + instanceId: CodexAdapterV2.CODEX_DEFAULT_INSTANCE_ID, + settings: DEFAULT_CODEX_SETTINGS, + environment: {}, + clientFactory: { + open: () => + CodexClient.make( + Stdio.make({ + args: Effect.succeed([]), + stdin, + stdout: () => Sink.drain, + stderr: () => Sink.drain, + }), + ), + }, + crypto: yield* Crypto.Crypto, + fileSystem: yield* FileSystem.FileSystem, + idAllocator: yield* IdAllocator.IdAllocatorV2, + serverConfig: yield* makeReplayServerConfig("codex-lost-app-server").pipe(Effect.orDie), + }); + const runtime = yield* adapter.openSession({ + threadId: ThreadId.make("thread-codex-lost-app-server"), + providerSessionId: ProviderSessionId.make("provider-session-codex-lost-app-server"), + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimePolicy: CODEX_TEST_RUNTIME_POLICY, + }); + return yield* runtime.events.pipe( + Stream.runDrain, + Effect.flip, + Effect.forkChild({ startImmediately: true }), + ); + }); + + it.effect("fails the session when the app-server stops answering", () => + Effect.gen(function* () { + const failure = yield* sessionFailureOver(Stream.never); + yield* TestClock.adjust("1 minute"); + yield* TestClock.adjust("1 minute"); + + const error = yield* Fiber.join(failure); + assert.instanceOf(error, ProviderAdapter.ProviderAdapterEventStreamError); + assert.include(String(error.cause), "Codex stopped responding"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); + + it.effect("fails the session when the app-server connection ends", () => + Effect.gen(function* () { + const error = yield* Fiber.join(yield* sessionFailureOver(Stream.empty)); + assert.instanceOf(error, ProviderAdapter.ProviderAdapterEventStreamError); + assert.include(String(error.cause), "Codex app-server connection ended"); + }).pipe( + Effect.provide( + Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), + ), + ), + ); +}); + describe("CodexAdapterV2 post-settle continuation", () => { const awaitUntil = (predicate: () => boolean, label: string): Effect.Effect => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 19d1e7b7ffe2..cf736578cfb8 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -471,6 +471,11 @@ function codexMcpToolOutput( // Reading an app's resource is one MCP round trip; a server that hangs must not // hold the item (and the run's ingestion) open. const MCP_APP_CAPTURE_TIMEOUT = "20 seconds"; +// A wedged app-server stays alive but stops answering. The probe, +// thread/loaded/list, only reads Codex's in-memory thread list, and any reply +// to it, even an error, proves the app-server is still reading. +const CODEX_LIVENESS_PROBE_INTERVAL = "1 minute"; +const CODEX_LIVENESS_PROBE_TIMEOUT = "1 minute"; function codexMcpAppResourceUri(item: CodexDynamicToolItem): string | undefined { if (item.type !== "mcpToolCall") return undefined; @@ -1791,7 +1796,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( model: input.modelSelection.model, now, }); - const events = yield* Queue.unbounded(); + const events = yield* Queue.unbounded< + ProviderAdapter.ProviderAdapterV2Event, + ProviderAdapter.ProviderAdapterEventStreamError + >(); const rateLimitSnapshot = yield* Ref.make(undefined); const limitedTurnItems = yield* Ref.make( new Map>(), @@ -6400,6 +6408,43 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.mapError(mcpAppsError("Codex MCP resource read failed."))), }; + // Every thread on this instance shares the app-server, so losing it + // fails the session after the events it already sent: its runs fail + // and the next turn starts a new app-server. + const unresponsive = Effect.gen(function* () { + while (true) { + yield* Effect.sleep(CODEX_LIVENESS_PROBE_INTERVAL); + const answered = yield* client + .request("thread/loaded/list", { limit: 1 }) + .pipe( + Effect.catchTags({ CodexAppServerRequestError: () => Effect.void }), + Effect.timeoutOption(CODEX_LIVENESS_PROBE_TIMEOUT), + ); + if (Option.isNone(answered)) { + return `Codex stopped responding: it did not answer a liveness check within ${CODEX_LIVENESS_PROBE_TIMEOUT}.`; + } + } + }); + yield* Effect.raceFirst( + client.terminated.pipe( + Effect.map((error) => `Codex app-server connection ended: ${error.message}`), + ), + // A probe that fails with a transport error is the termination above. + unresponsive.pipe(Effect.catch(() => Effect.never)), + ).pipe( + Effect.flatMap((detail) => + Queue.fail( + events, + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: CODEX_PROVIDER, + providerSessionId: input.providerSessionId, + cause: detail, + }), + ), + ), + Effect.forkIn(scope), + ); + const runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime = { instanceId: adapterOptions.instanceId, driver: CODEX_PROVIDER, diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 56db053920a2..923d705f6157 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3744,6 +3744,81 @@ it.effect("ProviderSessionManagerV2 release retries leave a replacement session }), ); +it.effect( + "ProviderSessionManagerV2 leaves a replacement's status to it when a failed release ends", + () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const scopeCloseReached = yield* Deferred.make(); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread-provider-session-manager-failed-replacement"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + const now = yield* DateTime.now; + // The release settles this request right after recording the session. + const request = yield* makePendingRuntimeRequestEvents({ + idAllocator, + threadId, + providerSessionId, + providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }), + now, + }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), + ...request.events, + ], + }); + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + + // The runtime's events end unexpectedly, and its release stalls in scope close. + const events = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(events); + yield* Queue.end(events!); + yield* Deferred.await(scopeCloseReached); + + // Another thread on the same runtime reopens it meanwhile; then the + // time-boxed close lets the release write its records. + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + const settled = yield* eventSink + .stream({ + threadId, + afterSequence: yield* eventSink.latestSequence({ threadId }), + eventType: "runtime-request.updated", + }) + .pipe(Stream.runHead, Effect.forkScoped); + yield* TestClock.adjust("30 seconds"); + yield* Fiber.join(settled); + const status = (yield* projectionStore.getThreadProjection(threadId)).providerSessions.at( + -1, + )?.status; + + // The replacement's close hangs as well; release it while the clock can move. + const stopping = yield* manager.shutdown.pipe(Effect.forkChild); + yield* TestClock.adjust("30 seconds"); + yield* Fiber.join(stopping); + + assert.isDefined(status); + assert.notEqual(status, "error"); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + hangSessionScopeClose: true, + scopeCloseReached, + }), + ), + ); + }), +); + it.effect("ProviderSessionManagerV2 keeps each failed release's cleanup", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index edcaecfa8848..dca8ed2c6798 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -989,6 +989,8 @@ export const layerWithOptions = ( readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; readonly gracefulSubscribers?: boolean; + /** Set by callers inside `open`, which already hold this session's open lock. */ + readonly openLockHeld?: boolean; }) => Effect.acquireUseRelease( removeLiveEntry(input), @@ -1028,8 +1030,19 @@ export const layerWithOptions = ( releasedBackgroundTasks, releasedAt, }; + // A thread of a shared runtime can reopen it under the same + // id while this closes. The open lock orders the check and + // write against that, so a replacement keeps its status. + const writeRecords = Effect.gen(function* () { + const replaced = (yield* Ref.get(sessions)).has( + sessionKey(input.providerSessionId), + ); + yield* writeReleaseRecords({ ...records, replaced }); + }); const recorded = yield* Effect.exit( - writeReleaseRecords({ ...records, replaced: false }), + input.openLockHeld === true + ? writeRecords + : sessionOpen.withLock(input.providerSessionId, writeRecords), ); if (Exit.isFailure(recorded)) { yield* retryReleaseRecords(records); @@ -2230,11 +2243,13 @@ export const layerWithOptions = ( providerSessionId: input.providerSessionId, reason: "manual_shutdown", detail: "The provider session start was interrupted.", + openLockHeld: true, } : { providerSessionId: input.providerSessionId, reason: "runtime_error", detail: "Failed to persist the provider-session attachment.", + openLockHeld: true, }, ).pipe(logReleaseFailure(input.providerSessionId)), ), diff --git a/packages/effect-codex-app-server/src/client.ts b/packages/effect-codex-app-server/src/client.ts index e7c5f50b5704..5f8fddad9551 100644 --- a/packages/effect-codex-app-server/src/client.ts +++ b/packages/effect-codex-app-server/src/client.ts @@ -1,4 +1,5 @@ import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; @@ -75,6 +76,8 @@ export class CodexAppServerClient extends Context.Service< params: unknown, ) => Effect.Effect, ) => Effect.Effect; + /** Resolves with the reason once the app-server connection ends, such as the process exiting. */ + readonly terminated: Effect.Effect; } >()("effect-codex-app-server/client/CodexAppServerClient") {} @@ -98,6 +101,7 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make stdio: Stdio.Stdio, options: CodexAppServerClientOptions = {}, terminationError?: Effect.Effect, + exited?: Effect.Effect, ): Effect.fn.Return { const requestHandlers = new Map(); const notificationHandlers = new Map>(); @@ -195,9 +199,12 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make : Effect.fail(CodexError.CodexAppServerRequestError.methodNotFound(request.method)); }; + const termination = yield* Deferred.make(); const transport = yield* CodexProtocol.makeCodexAppServerPatchedProtocol({ stdio, + onTermination: (error) => Deferred.succeed(termination, error).pipe(Effect.asVoid), ...(terminationError ? { terminationError } : {}), + ...(exited ? { exited } : {}), ...(options.logIncoming !== undefined ? { logIncoming: options.logIncoming } : {}), ...(options.logOutgoing !== undefined ? { logOutgoing: options.logOutgoing } : {}), ...(options.logger ? { logger: options.logger } : {}), @@ -258,6 +265,7 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make Effect.sync(() => { unknownNotificationHandler = handler; }), + terminated: Deferred.await(termination), }); }); @@ -271,5 +279,6 @@ const makeChildProcessClient = Effect.fn( "effect-codex-app-server/CodexAppServerClient.makeChildProcessClient", )(function* (handle: ChildProcessSpawner.ChildProcessHandle, options: CodexAppServerClientOptions) { yield* Stream.runDrain(handle.stderr).pipe(Effect.ignore, Effect.forkScoped); - return yield* make(makeChildStdio(handle), options, makeTerminationError(handle)); + const terminationError = makeTerminationError(handle); + return yield* make(makeChildStdio(handle), options, terminationError, terminationError); }); diff --git a/packages/effect-codex-app-server/src/protocol.test.ts b/packages/effect-codex-app-server/src/protocol.test.ts index 7249afff1071..64b64cdad69f 100644 --- a/packages/effect-codex-app-server/src/protocol.test.ts +++ b/packages/effect-codex-app-server/src/protocol.test.ts @@ -1,10 +1,13 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; import * as Stdio from "effect/Stdio"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import * as NodeServices from "@effect/platform-node/NodeServices"; import { assert, it } from "@effect/vitest"; @@ -21,6 +24,10 @@ const encoder = new TextEncoder(); const encodeJsonl = (value: unknown) => encoder.encode(`${encodeUnknownJsonString(value)}\n`); const decodeJson = Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown)); +const requestFailure = { + onFailure: (error: CodexError.CodexAppServerError) => error, + onSuccess: () => assert.fail("Expected the request to fail"), +}; const decodeAccountTokenUsageResponse = Schema.decodeUnknownEffect( CodexRpc.CLIENT_REQUEST_RESPONSES["account/usage/read"], ); @@ -804,4 +811,56 @@ it.layer(NodeServices.layer)("effect-codex-app-server protocol", (it) => { assert.equal("cause" in error, false); }), ); + + it.effect("terminates when the process exits while its output stays open", () => + Effect.gen(function* () { + const { stdio, input, output } = yield* makeInMemoryStdio(); + const exited = yield* Deferred.make(); + const termination = yield* Deferred.make(); + const transport = yield* CodexProtocol.makeCodexAppServerPatchedProtocol({ + stdio, + exited: Deferred.await(exited), + onTermination: (error) => Deferred.succeed(termination, error).pipe(Effect.asVoid), + }); + const answered = yield* transport.request("thread/read", {}).pipe(Effect.forkScoped); + yield* Queue.take(output); + const unanswered = yield* transport.request("thread/start", {}).pipe(Effect.forkScoped); + yield* Queue.take(output); + yield* TestClock.adjust("1 minute"); + + // A reply written just before the exit still arrives. + const exitError = new CodexError.CodexAppServerProcessExitedError({ code: 1 }); + yield* Deferred.succeed(exited, exitError); + yield* Queue.offer(input, encodeJsonl({ id: 1, result: {} })); + assert.deepEqual(yield* Fiber.join(answered), {}); + yield* TestClock.adjust("1 second"); + assert.isFalse(yield* Deferred.isDone(termination)); + + yield* TestClock.adjust("1 second"); + assert.strictEqual(yield* Deferred.await(termination), exitError); + assert.strictEqual( + yield* Fiber.join(unanswered).pipe(Effect.match(requestFailure)), + exitError, + ); + }), + ); + + it.effect("fails requests still waiting for a reply when the client closes", () => + Effect.gen(function* () { + const { stdio, output } = yield* makeInMemoryStdio(); + const clientScope = yield* Scope.make(); + const transport = yield* CodexProtocol.makeCodexAppServerPatchedProtocol({ stdio }).pipe( + Scope.provide(clientScope), + ); + const response = yield* transport.request("thread/start", {}).pipe(Effect.forkScoped); + yield* Queue.take(output); + + yield* Scope.close(clientScope, Exit.void); + + assert.instanceOf( + yield* Fiber.join(response).pipe(Effect.match(requestFailure)), + CodexError.CodexAppServerInputStreamEndedError, + ); + }), + ); }); diff --git a/packages/effect-codex-app-server/src/protocol.ts b/packages/effect-codex-app-server/src/protocol.ts index 4a32973a988b..0fa41228bd8f 100644 --- a/packages/effect-codex-app-server/src/protocol.ts +++ b/packages/effect-codex-app-server/src/protocol.ts @@ -16,6 +16,7 @@ const isJsonRpcId = Schema.is(JsonRpcId); const isJsonRpcResponseEnvelope = Schema.is(JsonRpcResponseEnvelope); const isCodexAppServerError = Schema.is(CodexError.CodexAppServerError); const MAX_BUFFERED_RAW_MESSAGES = 32; +const EXITED_OUTPUT_GRACE = "2 seconds"; export interface CodexAppServerProtocolLogEvent { readonly direction: "incoming" | "outgoing"; @@ -37,6 +38,11 @@ export interface CodexAppServerIncomingRequest { export interface CodexAppServerPatchedProtocolOptions { readonly stdio: Stdio.Stdio; readonly terminationError?: Effect.Effect; + /** + * Completes when the app-server process exits. Its stdout can stay open + * after that, while a command it started still holds the pipe. + */ + readonly exited?: Effect.Effect; readonly logIncoming?: boolean; readonly logOutgoing?: boolean; readonly logger?: (event: CodexAppServerProtocolLogEvent) => Effect.Effect; @@ -447,6 +453,28 @@ export const makeCodexAppServerPatchedProtocol = Effect.fn("makeCodexAppServerPa yield* Stream.fromQueue(outgoing).pipe(Stream.run(options.stdio.stdout()), Effect.forkScoped); + if (options.exited) { + yield* options.exited.pipe( + // Output written just before exit still arrives; the pipe normally + // closes with the process and terminates the transport first. + Effect.tap(() => Effect.sleep(EXITED_OUTPUT_GRACE)), + Effect.flatMap((error) => handleTermination(() => Effect.succeed(error))), + Effect.forkScoped, + ); + } + + // Closing the client fails the requests still waiting on a reply, which + // would otherwise wait forever. + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + const closed = new CodexError.CodexAppServerInputStreamEndedError({}); + yield* Ref.update(terminationFailure, (current) => + Option.isSome(current) ? current : Option.some(closed), + ); + yield* failAllPending(closed); + }), + ); + const request = (method: string, payload?: unknown) => Effect.gen(function* () { const requestId = yield* Ref.modify( From 54051a84db212ab492194eb77ebc6d9b066a80b6 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Mon, 12 Oct 2026 15:52:52 -0400 Subject: [PATCH 2/7] test(server): delegated-task MCP replay drops the recorder's trailing exit The recorder stops Codex after its last frame, but this test keeps the parent turn running past the recording. Codex sessions now fail when the app-server exits, so replaying that exit failed the parent run. Co-Authored-By: Claude Opus 5.5 --- .../mcp/OrchestratorMcpToolkit.integration.test.ts | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index 1e7ce5a4ff52..b15b64e62fbf 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -3766,9 +3766,17 @@ describe("orchestrator MCP toolkit", () => { Effect.gen(function* () { const cwd = yield* checkpointWorkspace("delegated-task-status"); const rawTranscript = yield* readDelegatedTaskStatusTranscript(); - const transcript = yield* CodexOrchestratorReplayHarness.decodeTranscript( - materializeReplayTranscriptWorkspace(rawTranscript, cwd), - ); + const workspaceTranscript = materializeReplayTranscriptWorkspace(rawTranscript, cwd); + // The recorder stopped Codex after its last frame, but this test keeps + // the parent turn running past the recording. Replaying that exit would + // end the session and fail the parent run. + const transcript = yield* CodexOrchestratorReplayHarness.decodeTranscript({ + ...workspaceTranscript, + entries: + workspaceTranscript.entries.at(-1)?.type === "runtime_exit" + ? workspaceTranscript.entries.slice(0, -1) + : workspaceTranscript.entries, + }); const layerOrchestrator = ProviderReplayHarness.layerProviderReplay( { name: "delegated-task-status/codex", From da00922823da67673253974a29c590bb28248201 Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Mon, 12 Oct 2026 16:04:40 -0400 Subject: [PATCH 3/7] fix(server): a release leaves a replacement's status alone even after the replacement closes Checking whether a replacement is live missed one that opened and closed while the failed session was still closing; the old failure then landed after the replacement's own status. The manager now tracks the latest runtime opened under each id until its release is recorded, and only that runtime writes its release status. Co-Authored-By: Claude Opus 5.5 --- .../ProviderSessionManager.test.ts | 73 ++++++++++++++++++- .../ProviderSessionManager.ts | 51 ++++++++----- 2 files changed, 104 insertions(+), 20 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 923d705f6157..33ebccca8c37 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -332,7 +332,8 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( readonly hasPendingBackgroundWork?: Effect.Effect; readonly hasPendingBackgroundWorkForThread?: Effect.Effect; readonly backgroundTasksEndWithProcess?: boolean; - readonly hangSessionScopeClose?: boolean; + /** "first" wedges only the first session opened. */ + readonly hangSessionScopeClose?: boolean | "first"; readonly startTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; readonly spawnBeforeOpen?: boolean; @@ -395,7 +396,11 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( }); if (options.spawnBeforeOpen !== true) { yield* countClose; - if (options.hangSessionScopeClose === true) yield* hangClose; + if ( + options.hangSessionScopeClose === true || + (options.hangSessionScopeClose === "first" && (yield* Ref.get(state)).openCount === 1) + ) + yield* hangClose; } return { @@ -481,7 +486,7 @@ function layerTest(input: { readonly hasPendingBackgroundWork?: Effect.Effect; readonly hasPendingBackgroundWorkForThread?: Effect.Effect; readonly backgroundTasksEndWithProcess?: boolean; - readonly hangSessionScopeClose?: boolean; + readonly hangSessionScopeClose?: boolean | "first"; readonly startTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; readonly spawnBeforeOpen?: boolean; @@ -3819,6 +3824,68 @@ it.effect( }), ); +it.effect("ProviderSessionManagerV2 leaves a closed replacement's status to it", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const scopeCloseReached = yield* Deferred.make(); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const threadId = ThreadId.make("thread-provider-session-manager-closed-replacement"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }), + ], + }); + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + + // The first runtime's events end unexpectedly, and its release stalls in scope close. + const events = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(events); + yield* Queue.end(events!); + yield* Deferred.await(scopeCloseReached); + + // A replacement opens and closes meanwhile. + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + yield* manager.close(providerSessionId); + const statuses = yield* eventSink + .stream({ threadId, afterSequence: yield* eventSink.latestSequence({ threadId }) }) + .pipe( + Stream.filter( + ({ event }) => + event.type === "provider-session.updated" || + event.type === "provider-session.attached", + ), + Stream.takeUntil(({ event }) => event.type === "provider-session.attached"), + Stream.map(({ event }) => event.type), + Stream.runCollect, + Effect.forkScoped, + ); + + // The stalled close ends; the next open waits on the open lock until the + // first release has written its records. + yield* TestClock.adjust("30 seconds"); + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + + assert.deepEqual(Array.from(yield* Fiber.join(statuses)), ["provider-session.attached"]); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + hangSessionScopeClose: "first", + scopeCloseReached, + }), + ), + ); + }), +); + it.effect("ProviderSessionManagerV2 keeps each failed release's cleanup", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index dca8ed2c6798..bcde30821f08 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -388,6 +388,10 @@ export const layerWithOptions = ( }), ); const sessions = yield* Ref.make(new Map()); + // The latest session opened under each id, kept until its release records it. + const latestOpened = yield* Ref.make( + new Map(), + ); // One retry per released entry, so a later release with the same id // cannot drop cleanup for threads only the earlier session served. const releaseRecordRetries = yield* FiberSet.make(); @@ -853,6 +857,27 @@ export const layerWithOptions = ( { concurrency: 1 }, ).pipe(Effect.flatMap(Exit.asVoidAll)); + // A replacement under the same id, like the next thread of a shared + // runtime, owns the session's status from its open on, even after it + // closes. Only the latest session opened under an id writes its release + // status; the entry is dropped once that is written. Callers hold the + // open lock, so no replacement opens mid-write. + const writeOwnReleaseRecords = ( + input: Omit[0], "replaced">, + ) => + Effect.gen(function* () { + const key = sessionKey(input.entry.runtime.providerSessionId); + const replaced = (yield* Ref.get(latestOpened)).get(key) !== input.entry.runtime; + yield* writeReleaseRecords({ ...input, replaced }); + if (replaced) return; + yield* Ref.update(latestOpened, (current) => { + if (current.get(key) !== input.entry.runtime) return current; + const updated = new Map(current); + updated.delete(key); + return updated; + }); + }); + // The session already left the live map, so a later release finds nothing // to do. Without a retry the UI would keep a ready session and answerable // approvals until a server restart. @@ -862,13 +887,7 @@ export const layerWithOptions = ( const providerSessionId = input.entry.runtime.providerSessionId; const attempt = Effect.gen(function* () { const exit = yield* Effect.exit( - sessionOpen.withLock( - providerSessionId, - Effect.gen(function* () { - const replaced = (yield* Ref.get(sessions)).has(sessionKey(providerSessionId)); - yield* writeReleaseRecords({ ...input, replaced }); - }), - ), + sessionOpen.withLock(providerSessionId, writeOwnReleaseRecords(input)), ); if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit; yield* Effect.logWarning("orchestration-v2.provider-session-release-records-failed", { @@ -1031,18 +1050,15 @@ export const layerWithOptions = ( releasedAt, }; // A thread of a shared runtime can reopen it under the same - // id while this closes. The open lock orders the check and - // write against that, so a replacement keeps its status. - const writeRecords = Effect.gen(function* () { - const replaced = (yield* Ref.get(sessions)).has( - sessionKey(input.providerSessionId), - ); - yield* writeReleaseRecords({ ...records, replaced }); - }); + // id while this closes; the open lock orders that against + // this write. const recorded = yield* Effect.exit( input.openLockHeld === true - ? writeRecords - : sessionOpen.withLock(input.providerSessionId, writeRecords), + ? writeOwnReleaseRecords(records) + : sessionOpen.withLock( + input.providerSessionId, + writeOwnReleaseRecords(records), + ), ); if (Exit.isFailure(recorded)) { yield* retryReleaseRecords(records); @@ -2224,6 +2240,7 @@ export const layerWithOptions = ( updated.set(key, entry); return updated; }); + yield* Ref.update(latestOpened, (current) => new Map(current).set(key, runtime)); yield* dropReservation; yield* withActivityError( input.providerSessionId, From 8a200256c7cbcc59076329620a050640236c690e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 15:07:15 +0000 Subject: [PATCH 4/7] test(server): cover shared Codex runtime recovery --- .../Adapters/CodexAdapterV2.test.ts | 292 ++++++++++++++---- .../ProviderSessionManager.test.ts | 120 ++----- .../src/protocol.test.ts | 13 +- 3 files changed, 255 insertions(+), 170 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index bb88b162a4c5..ab93a657150a 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -39,6 +39,8 @@ import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Option from "effect/Option"; import * as Predicate from "effect/Predicate"; import * as FileSystem from "effect/FileSystem"; import { MCP_APP_OUTPUT_KEY, readMcpAppReference } from "@t3tools/shared/mcpApp"; @@ -49,8 +51,6 @@ import * as Path from "effect/Path"; import * as PlatformError from "effect/PlatformError"; import * as Ref from "effect/Ref"; import * as Schema from "effect/Schema"; -import * as Sink from "effect/Sink"; -import * as Stdio from "effect/Stdio"; import * as Stream from "effect/Stream"; import { TestClock } from "effect/testing"; import { ChildProcess, ChildProcessSpawner } from "effect/process"; @@ -61,6 +61,9 @@ import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderS import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as EffectWorker from "../EffectWorker.ts"; +import * as EventSink from "../EventSink.ts"; +import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; +import * as ProviderSessionManager from "../ProviderSessionManager.ts"; import * as Orchestrator from "../Orchestrator.ts"; import * as ProviderReplayHarness from "../testkit/ProviderReplayHarness.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; @@ -1946,67 +1949,240 @@ describe("CodexAdapterV2 session initialize", () => { }); describe("CodexAdapterV2 lost app-server", () => { - const sessionFailureOver = (stdin: Stream.Stream) => - Effect.gen(function* () { - const adapter = yield* CodexAdapterV2.makeCodexAdapterV2({ - instanceId: CodexAdapterV2.CODEX_DEFAULT_INSTANCE_ID, - settings: DEFAULT_CODEX_SETTINGS, - environment: {}, - clientFactory: { - open: () => - CodexClient.make( - Stdio.make({ - args: Effect.succeed([]), - stdin, - stdout: () => Sink.drain, - stderr: () => Sink.drain, - }), - ), + it.effect.each(["SIGKILL", "exit code", "silence"] as const)( + "fails active turns and replaces the shared app-server after %s", + (termination) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-codex-session-recovery-" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const handles: Array = []; + const testSpawner = ChildProcessSpawner.make(() => + spawner + .spawn( + ChildProcess.make( + process.execPath, + [ + "--input-type=module", + "-e", + `import * as NodeFS from "node:fs"; +import * as NodeReadline from "node:readline"; + +const fixture = JSON.parse( + NodeFS.readFileSync(process.argv[1], "utf8"), +); +let nextThreadId = 0; +let nextTurnId = 0; +const write = (message) => process.stdout.write(\`\${JSON.stringify(message)}\\n\`); + +NodeReadline.createInterface({ input: process.stdin }).on("line", (line) => { + const { id, method, params } = JSON.parse(line); + switch (method) { + case "initialize": + write({ + id, + result: { + userAgent: "codex-session-recovery-mock", + codexHome: process.cwd(), + platformFamily: "unix", + platformOs: "linux", }, - crypto: yield* Crypto.Crypto, - fileSystem: yield* FileSystem.FileSystem, - idAllocator: yield* IdAllocator.IdAllocatorV2, - serverConfig: yield* makeReplayServerConfig("codex-lost-app-server").pipe(Effect.orDie), - }); - const runtime = yield* adapter.openSession({ - threadId: ThreadId.make("thread-codex-lost-app-server"), - providerSessionId: ProviderSessionId.make("provider-session-codex-lost-app-server"), - modelSelection: CODEX_TEST_MODEL_SELECTION, - runtimePolicy: CODEX_TEST_RUNTIME_POLICY, }); - return yield* runtime.events.pipe( - Stream.runDrain, - Effect.flip, - Effect.forkChild({ startImmediately: true }), - ); - }); + break; + case "thread/start": { + const thread = { + ...fixture.responses.threadStart.thread, + id: \`thread-\${process.pid}-\${nextThreadId++}\`, + }; + write({ id, result: { ...fixture.responses.threadStart, thread } }); + break; + } + case "thread/resume": { + const thread = { + ...fixture.responses.threadStart.thread, + id: params.threadId, + }; + write({ id, result: { ...fixture.responses.threadStart, thread } }); + break; + } + case "turn/start": { + const turn = { + ...fixture.responses.turnStart.turn, + id: \`turn-\${process.pid}-\${nextTurnId++}\`, + }; + write({ id, result: { turn } }); + write({ method: "turn/started", params: { threadId: params.threadId, turn } }); + break; + } + case "thread/loaded/list": + break; + case "test/exit": + process.exit(7); + break; + default: + if (id !== undefined) write({ id, result: {} }); + } +});`, + path.join( + import.meta.dirname, + "../../provider/testFixtures/codexMultiAgentWire.json", + ), + ], + { cwd }, + ), + ) + .pipe(Effect.tap((handle) => Effect.sync(() => handles.push(handle)))), + ); + const layerFactory = CodexAdapterV2.layerAppServerClientFactory.pipe( + Layer.provide(Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, testSpawner)), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ); + const layerRegistry = ProviderAdapterRegistry.layerFromAdaptersEffect( + Effect.gen(function* () { + const adapter = yield* CodexAdapterV2.makeCodexAdapterV2({ + instanceId: CODEX_TEST_MODEL_SELECTION.instanceId, + settings: DEFAULT_CODEX_SETTINGS, + environment: {}, + clientFactory: yield* CodexAdapterV2.CodexAppServerClientFactory, + crypto: yield* Crypto.Crypto, + fileSystem: yield* FileSystem.FileSystem, + idAllocator: yield* IdAllocator.IdAllocatorV2, + serverConfig: yield* ServerConfig.ServerConfig, + }); + return [adapter]; + }), + ).pipe( + Layer.provide( + Layer.mergeAll( + layerFactory, + IdAllocator.layer, + ServerConfig.layerTest(cwd, { prefix: "t3-codex-session-recovery-config-" }), + ).pipe(Layer.provideMerge(NodeServices.layer)), + ), + ); - it.effect("fails the session when the app-server stops answering", () => - Effect.gen(function* () { - const failure = yield* sessionFailureOver(Stream.never); - yield* TestClock.adjust("1 minute"); - yield* TestClock.adjust("1 minute"); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const eventSink = yield* EventSink.EventSinkV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const createAndStart = (threadId: ThreadId) => + Effect.gen(function* () { + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`create:${threadId}`), + threadId, + projectId: ProjectId.make("project:codex-session-recovery"), + title: threadId, + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + yield* send(threadId, "start"); + }); + const send = (threadId: ThreadId, text: string) => + Effect.gen(function* () { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`send:${threadId}:${text}`), + threadId, + messageId: MessageId.make(`message:${threadId}:${text}`), + text, + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "running", + ); + }); + const threadA = ThreadId.make("thread:codex-recovery-a"); + const threadB = ThreadId.make("thread:codex-recovery-b"); + yield* createAndStart(threadA); + yield* createAndStart(threadB); + assert.equal(handles.length, 1); + const sessionA = (yield* orchestrator.getThreadProjection(threadA)).providerSessions.at( + -1, + )!; + const sessionB = (yield* orchestrator.getThreadProjection(threadB)).providerSessions.at( + -1, + )!; + assert.equal(sessionA.id, sessionB.id); + const beforeTermination = yield* eventSink.latestSequence(); + const handle = handles[0]!; + if (termination === "silence") { + yield* TestClock.adjust("1 minute"); + yield* TestClock.adjust("1 minute"); + } else if (termination === "SIGKILL") { + yield* handle.kill({ killSignal: "SIGKILL" }); + } else { + const encoder = new TextEncoder(); + yield* Stream.make(encoder.encode('{"id":999,"method":"test/exit"}\n')).pipe( + Stream.run(handle.stdin), + ); + } + if (termination !== "silence") { + const exit = yield* Effect.exit(handle.exitCode); + assert.equal(Exit.isFailure(exit), termination === "SIGKILL"); + if (Exit.isSuccess(exit)) assert.equal(Number(exit.value), 7); + } - const error = yield* Fiber.join(failure); - assert.instanceOf(error, ProviderAdapter.ProviderAdapterEventStreamError); - assert.include(String(error.cause), "Codex stopped responding"); - }).pipe( - Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), - ), - ), - ); + // Await committed failures, rather than guessing when the event pump ran. + yield* eventSink + .stream({ afterSequence: beforeTermination, eventType: "provider-session.updated" }) + .pipe( + Stream.filter( + ({ event }) => + event.type === "provider-session.updated" && event.payload.status === "error", + ), + Stream.take(2), + Stream.runDrain, + ); + yield* eventSink + .stream({ afterSequence: beforeTermination, eventType: "run.updated" }) + .pipe( + Stream.filter( + ({ event }) => event.type === "run.updated" && event.payload.status === "failed", + ), + Stream.take(2), + Stream.runDrain, + ); + yield* worker.drain(); + assert.isTrue(Option.isNone(yield* manager.get(sessionA.id))); + for (const threadId of [threadA, threadB]) { + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(projection.runs.at(-1)?.status, "failed"); + assert.equal(projection.providerSessions.at(-1)?.status, "error"); + } - it.effect("fails the session when the app-server connection ends", () => - Effect.gen(function* () { - const error = yield* Fiber.join(yield* sessionFailureOver(Stream.empty)); - assert.instanceOf(error, ProviderAdapter.ProviderAdapterEventStreamError); - assert.include(String(error.cause), "Codex app-server connection ended"); - }).pipe( - Effect.provide( - Layer.mergeAll(IdAllocator.layer, McpProviderSessions.layer, NodeServices.layer), - ), - ), + yield* createAndStart(ThreadId.make("thread:codex-recovery-new")); + assert.equal(handles.length, 2); + assert.notEqual(handles[0]!.pid, handles[1]!.pid); + yield* send(threadA, "retry"); + assert.equal(handles.length, 2); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: `codex-session-recovery-${termination}`, runtimePolicyOverride: { cwd } }, + layerRegistry, + { runEffectWorker: false }, + ), + ), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); }); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 33ebccca8c37..1c8024a53481 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -3749,9 +3749,9 @@ it.effect("ProviderSessionManagerV2 release retries leave a replacement session }), ); -it.effect( - "ProviderSessionManagerV2 leaves a replacement's status to it when a failed release ends", - () => +it.effect.each(["open", "closed"] as const)( + "ProviderSessionManagerV2 preserves a %s replacement's status after a late release", + (replacement) => Effect.gen(function* () { const state = yield* Ref.make(emptyState); const scopeCloseReached = yield* Deferred.make(); @@ -3765,58 +3765,38 @@ it.effect( providerInstanceId: modelSelection.instanceId, threadId, }); - const now = yield* DateTime.now; - // The release settles this request right after recording the session. - const request = yield* makePendingRuntimeRequestEvents({ - idAllocator, - threadId, - providerSessionId, - providerThread: makeProviderThread({ idAllocator, threadId, providerSessionId, now }), - now, - }); yield* eventSink.write({ events: [ - yield* makeThreadCreatedEvent({ idAllocator, threadId, now }), - ...request.events, + yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }), ], }); yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); - // The runtime's events end unexpectedly, and its release stalls in scope close. - const events = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); - assert.isDefined(events); - yield* Queue.end(events!); + const release = yield* manager + .release({ + providerSessionId, + reason: "runtime_error", + }) + .pipe(Effect.forkChild); yield* Deferred.await(scopeCloseReached); - // Another thread on the same runtime reopens it meanwhile; then the - // time-boxed close lets the release write its records. yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); - const settled = yield* eventSink - .stream({ - threadId, - afterSequence: yield* eventSink.latestSequence({ threadId }), - eventType: "runtime-request.updated", - }) - .pipe(Stream.runHead, Effect.forkScoped); - yield* TestClock.adjust("30 seconds"); - yield* Fiber.join(settled); - const status = (yield* projectionStore.getThreadProjection(threadId)).providerSessions.at( + if (replacement === "closed") yield* manager.close(providerSessionId); + const expected = (yield* projectionStore.getThreadProjection(threadId)).providerSessions.at( -1, - )?.status; - - // The replacement's close hangs as well; release it while the clock can move. - const stopping = yield* manager.shutdown.pipe(Effect.forkChild); + ); yield* TestClock.adjust("30 seconds"); - yield* Fiber.join(stopping); - - assert.isDefined(status); - assert.notEqual(status, "error"); + yield* Fiber.join(release); + assert.deepEqual( + (yield* projectionStore.getThreadProjection(threadId)).providerSessions.at(-1), + expected, + ); }).pipe( Effect.provide( layerTest({ state, idleTimeoutMs: 60_000, - hangSessionScopeClose: true, + hangSessionScopeClose: "first", scopeCloseReached, }), ), @@ -3824,68 +3804,6 @@ it.effect( }), ); -it.effect("ProviderSessionManagerV2 leaves a closed replacement's status to it", () => - Effect.gen(function* () { - const state = yield* Ref.make(emptyState); - const scopeCloseReached = yield* Deferred.make(); - yield* Effect.gen(function* () { - const eventSink = yield* EventSink.EventSinkV2; - const idAllocator = yield* IdAllocator.IdAllocatorV2; - const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; - const threadId = ThreadId.make("thread-provider-session-manager-closed-replacement"); - const providerSessionId = yield* idAllocator.allocate.providerSession({ - providerInstanceId: modelSelection.instanceId, - threadId, - }); - yield* eventSink.write({ - events: [ - yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }), - ], - }); - yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); - - // The first runtime's events end unexpectedly, and its release stalls in scope close. - const events = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); - assert.isDefined(events); - yield* Queue.end(events!); - yield* Deferred.await(scopeCloseReached); - - // A replacement opens and closes meanwhile. - yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); - yield* manager.close(providerSessionId); - const statuses = yield* eventSink - .stream({ threadId, afterSequence: yield* eventSink.latestSequence({ threadId }) }) - .pipe( - Stream.filter( - ({ event }) => - event.type === "provider-session.updated" || - event.type === "provider-session.attached", - ), - Stream.takeUntil(({ event }) => event.type === "provider-session.attached"), - Stream.map(({ event }) => event.type), - Stream.runCollect, - Effect.forkScoped, - ); - - // The stalled close ends; the next open waits on the open lock until the - // first release has written its records. - yield* TestClock.adjust("30 seconds"); - yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); - - assert.deepEqual(Array.from(yield* Fiber.join(statuses)), ["provider-session.attached"]); - }).pipe( - Effect.provide( - layerTest({ - state, - idleTimeoutMs: 60_000, - hangSessionScopeClose: "first", - scopeCloseReached, - }), - ), - ); - }), -); - it.effect("ProviderSessionManagerV2 keeps each failed release's cleanup", () => Effect.gen(function* () { const state = yield* Ref.make(emptyState); diff --git a/packages/effect-codex-app-server/src/protocol.test.ts b/packages/effect-codex-app-server/src/protocol.test.ts index 64b64cdad69f..fda588ef109f 100644 --- a/packages/effect-codex-app-server/src/protocol.test.ts +++ b/packages/effect-codex-app-server/src/protocol.test.ts @@ -24,10 +24,6 @@ const encoder = new TextEncoder(); const encodeJsonl = (value: unknown) => encoder.encode(`${encodeUnknownJsonString(value)}\n`); const decodeJson = Schema.decodeEffect(Schema.fromJsonString(Schema.Unknown)); -const requestFailure = { - onFailure: (error: CodexError.CodexAppServerError) => error, - onSuccess: () => assert.fail("Expected the request to fail"), -}; const decodeAccountTokenUsageResponse = Schema.decodeUnknownEffect( CodexRpc.CLIENT_REQUEST_RESPONSES["account/usage/read"], ); @@ -826,8 +822,6 @@ it.layer(NodeServices.layer)("effect-codex-app-server protocol", (it) => { yield* Queue.take(output); const unanswered = yield* transport.request("thread/start", {}).pipe(Effect.forkScoped); yield* Queue.take(output); - yield* TestClock.adjust("1 minute"); - // A reply written just before the exit still arrives. const exitError = new CodexError.CodexAppServerProcessExitedError({ code: 1 }); yield* Deferred.succeed(exited, exitError); @@ -838,10 +832,7 @@ it.layer(NodeServices.layer)("effect-codex-app-server protocol", (it) => { yield* TestClock.adjust("1 second"); assert.strictEqual(yield* Deferred.await(termination), exitError); - assert.strictEqual( - yield* Fiber.join(unanswered).pipe(Effect.match(requestFailure)), - exitError, - ); + assert.strictEqual(yield* Fiber.join(unanswered).pipe(Effect.flip), exitError); }), ); @@ -858,7 +849,7 @@ it.layer(NodeServices.layer)("effect-codex-app-server protocol", (it) => { yield* Scope.close(clientScope, Exit.void); assert.instanceOf( - yield* Fiber.join(response).pipe(Effect.match(requestFailure)), + yield* Fiber.join(response).pipe(Effect.flip), CodexError.CodexAppServerInputStreamEndedError, ); }), From 355acfd7d06683f67d55cf95b245136783f471ca Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 19 Oct 2026 15:14:11 +0000 Subject: [PATCH 5/7] fix(server): preserve Codex failure causes and simplify release ordering --- .../Adapters/CodexAdapterV2.ts | 16 ++--- .../ProviderSessionManager.ts | 61 +++++++------------ .../effect-codex-app-server/src/client.ts | 2 +- .../src/protocol.test.ts | 17 ++++-- .../effect-codex-app-server/src/protocol.ts | 3 +- 5 files changed, 41 insertions(+), 58 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 7070a74c93b1..43a4ef1e2106 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -471,9 +471,7 @@ function codexMcpToolOutput( // Reading an app's resource is one MCP round trip; a server that hangs must not // hold the item (and the run's ingestion) open. const MCP_APP_CAPTURE_TIMEOUT = "20 seconds"; -// A wedged app-server stays alive but stops answering. The probe, -// thread/loaded/list, only reads Codex's in-memory thread list, and any reply -// to it, even an error, proves the app-server is still reading. +// Reading the in-memory thread list is cheap; even an error reply proves liveness. const CODEX_LIVENESS_PROBE_INTERVAL = "1 minute"; const CODEX_LIVENESS_PROBE_TIMEOUT = "1 minute"; @@ -6416,9 +6414,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }).pipe(Effect.mapError(mcpAppsError("Codex MCP resource read failed."))), }; - // Every thread on this instance shares the app-server, so losing it - // fails the session after the events it already sent: its runs fail - // and the next turn starts a new app-server. + // Drain buffered events before failing every run sharing this process. const unresponsive = Effect.gen(function* () { while (true) { yield* Effect.sleep(CODEX_LIVENESS_PROBE_INTERVAL); @@ -6434,19 +6430,17 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } }); yield* Effect.raceFirst( - client.terminated.pipe( - Effect.map((error) => `Codex app-server connection ended: ${error.message}`), - ), + client.terminated, // A probe that fails with a transport error is the termination above. unresponsive.pipe(Effect.catch(() => Effect.never)), ).pipe( - Effect.flatMap((detail) => + Effect.flatMap((cause) => Queue.fail( events, new ProviderAdapter.ProviderAdapterEventStreamError({ driver: CODEX_PROVIDER, providerSessionId: input.providerSessionId, - cause: detail, + cause, }), ), ), diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index bcde30821f08..bd4b3ee00ff2 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -840,38 +840,27 @@ export const layerWithOptions = ( readonly detail?: string; readonly releasedBackgroundTasks: ReadonlyArray; readonly releasedAt: DateTime.Utc; - readonly replaced: boolean; }) => - Effect.all( - [ - input.replaced - ? Effect.succeed(Exit.void) - : Effect.exit(writeReleasedSessionEvents(input)), - Effect.exit( - writeReleasedRuntimeRequestEvents(input).pipe( - input.entry.requestEventPermit.withPermits(1), - ), - ), - Effect.exit(writeReleasedBackgroundWorkEvents(input.releasedBackgroundTasks)), - ], - { concurrency: 1 }, - ).pipe(Effect.flatMap(Exit.asVoidAll)); - - // A replacement under the same id, like the next thread of a shared - // runtime, owns the session's status from its open on, even after it - // closes. Only the latest session opened under an id writes its release - // status; the entry is dropped once that is written. Callers hold the - // open lock, so no replacement opens mid-write. - const writeOwnReleaseRecords = ( - input: Omit[0], "replaced">, - ) => Effect.gen(function* () { + // The open lock protects status ownership, including after a replacement closes. const key = sessionKey(input.entry.runtime.providerSessionId); - const replaced = (yield* Ref.get(latestOpened)).get(key) !== input.entry.runtime; - yield* writeReleaseRecords({ ...input, replaced }); - if (replaced) return; + const ownsStatus = (yield* Ref.get(latestOpened)).get(key) === input.entry.runtime; + yield* Effect.all( + [ + ownsStatus + ? Effect.exit(writeReleasedSessionEvents(input)) + : Effect.succeed(Exit.void), + Effect.exit( + writeReleasedRuntimeRequestEvents(input).pipe( + input.entry.requestEventPermit.withPermits(1), + ), + ), + Effect.exit(writeReleasedBackgroundWorkEvents(input.releasedBackgroundTasks)), + ], + { concurrency: 1 }, + ).pipe(Effect.flatMap(Exit.asVoidAll)); + if (!ownsStatus) return; yield* Ref.update(latestOpened, (current) => { - if (current.get(key) !== input.entry.runtime) return current; const updated = new Map(current); updated.delete(key); return updated; @@ -881,13 +870,11 @@ export const layerWithOptions = ( // The session already left the live map, so a later release finds nothing // to do. Without a retry the UI would keep a ready session and answerable // approvals until a server restart. - const retryReleaseRecords = ( - input: Omit[0], "replaced">, - ) => { + const retryReleaseRecords = (input: Parameters[0]) => { const providerSessionId = input.entry.runtime.providerSessionId; const attempt = Effect.gen(function* () { const exit = yield* Effect.exit( - sessionOpen.withLock(providerSessionId, writeOwnReleaseRecords(input)), + sessionOpen.withLock(providerSessionId, writeReleaseRecords(input)), ); if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit; yield* Effect.logWarning("orchestration-v2.provider-session-release-records-failed", { @@ -1049,16 +1036,10 @@ export const layerWithOptions = ( releasedBackgroundTasks, releasedAt, }; - // A thread of a shared runtime can reopen it under the same - // id while this closes; the open lock orders that against - // this write. const recorded = yield* Effect.exit( input.openLockHeld === true - ? writeOwnReleaseRecords(records) - : sessionOpen.withLock( - input.providerSessionId, - writeOwnReleaseRecords(records), - ), + ? writeReleaseRecords(records) + : sessionOpen.withLock(input.providerSessionId, writeReleaseRecords(records)), ); if (Exit.isFailure(recorded)) { yield* retryReleaseRecords(records); diff --git a/packages/effect-codex-app-server/src/client.ts b/packages/effect-codex-app-server/src/client.ts index 5f8fddad9551..eadadab2873b 100644 --- a/packages/effect-codex-app-server/src/client.ts +++ b/packages/effect-codex-app-server/src/client.ts @@ -76,7 +76,7 @@ export class CodexAppServerClient extends Context.Service< params: unknown, ) => Effect.Effect, ) => Effect.Effect; - /** Resolves with the reason once the app-server connection ends, such as the process exiting. */ + /** Reports unexpected termination; deliberate scope closure interrupts its waiters. */ readonly terminated: Effect.Effect; } >()("effect-codex-app-server/client/CodexAppServerClient") {} diff --git a/packages/effect-codex-app-server/src/protocol.test.ts b/packages/effect-codex-app-server/src/protocol.test.ts index fda588ef109f..96dd1ef9165d 100644 --- a/packages/effect-codex-app-server/src/protocol.test.ts +++ b/packages/effect-codex-app-server/src/protocol.test.ts @@ -832,7 +832,13 @@ it.layer(NodeServices.layer)("effect-codex-app-server protocol", (it) => { yield* TestClock.adjust("1 second"); assert.strictEqual(yield* Deferred.await(termination), exitError); - assert.strictEqual(yield* Fiber.join(unanswered).pipe(Effect.flip), exitError); + const failure = yield* Fiber.join(unanswered).pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => assert.fail("Expected the request to fail"), + }), + ); + assert.strictEqual(failure, exitError); }), ); @@ -848,10 +854,13 @@ it.layer(NodeServices.layer)("effect-codex-app-server protocol", (it) => { yield* Scope.close(clientScope, Exit.void); - assert.instanceOf( - yield* Fiber.join(response).pipe(Effect.flip), - CodexError.CodexAppServerInputStreamEndedError, + const failure = yield* Fiber.join(response).pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => assert.fail("Expected the request to fail"), + }), ); + assert.instanceOf(failure, CodexError.CodexAppServerInputStreamEndedError); }), ); }); diff --git a/packages/effect-codex-app-server/src/protocol.ts b/packages/effect-codex-app-server/src/protocol.ts index 0fa41228bd8f..1c69eaeeba4a 100644 --- a/packages/effect-codex-app-server/src/protocol.ts +++ b/packages/effect-codex-app-server/src/protocol.ts @@ -463,8 +463,7 @@ export const makeCodexAppServerPatchedProtocol = Effect.fn("makeCodexAppServerPa ); } - // Closing the client fails the requests still waiting on a reply, which - // would otherwise wait forever. + // Deliberate closure fails pending requests without reporting a runtime failure. yield* Effect.addFinalizer(() => Effect.gen(function* () { const closed = new CodexError.CodexAppServerInputStreamEndedError({}); From 86a878718626bd9b9278ed4c8b548f6ebdbb8c39 Mon Sep 17 00:00:00 2001 From: Spencer Morris Date: Mon, 19 Oct 2026 15:44:34 -0400 Subject: [PATCH 6/7] fix(server): commands a lost Codex app-server never finished no longer show as running MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit When the app-server dies or stops answering, the adapter now fails every command, dynamic tool, and MCP app capture it still tracks — for active turns and for settled turns with background work — before failing the event stream. Previously the run failed but its command item and node stayed running, so the composer kept a Running chip with a Stop button for work that no longer existed. The terminalization is the same triplet the interrupt path already runs, so both now go through one `terminalizeTrackedTurnWork` helper instead of repeating the three calls (and the opaque persistent-tools flag) at each site. The test that used a Codex exit to leave a stale command for Stop now asserts that the exit itself fails the command. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 0146592a7fa7af88683715618538d2698a378496) --- .../Adapters/CodexAdapterV2.test.ts | 54 +++++++++++++------ .../Adapters/CodexAdapterV2.ts | 54 ++++++++++++++----- 2 files changed, 77 insertions(+), 31 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index ce7b41333c01..ad552fded6b1 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -963,6 +963,23 @@ NodeReadline.createInterface({ input: process.stdin }).on("line", (line) => { }; write({ id, result: { turn } }); write({ method: "turn/started", params: { threadId: params.threadId, turn } }); + write({ + method: "item/started", + params: { + threadId: params.threadId, + turnId: turn.id, + startedAtMs: Date.now(), + item: { + type: "commandExecution", + id: \`command-\${turn.id}\`, + command: "sleep 600", + cwd: process.cwd(), + status: "inProgress", + commandActions: [], + aggregatedOutput: null, + }, + }, + }); break; } case "thread/loaded/list": @@ -1116,6 +1133,13 @@ NodeReadline.createInterface({ input: process.stdin }).on("line", (line) => { const projection = yield* orchestrator.getThreadProjection(threadId); assert.equal(projection.runs.at(-1)?.status, "failed"); assert.equal(projection.providerSessions.at(-1)?.status, "error"); + // Nothing will complete the command the process took with it. + const command = projection.turnItems.find((item) => item.type === "command_execution"); + assert.equal(command?.status, "failed"); + assert.equal( + projection.nodes.find((node) => node.id === command?.nodeId)?.status, + "failed", + ); } yield* createAndStart(ThreadId.make("thread:codex-recovery-new")); @@ -3842,7 +3866,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { ); }); - it.effect("Stop ends a background command no Codex process tracks any more", () => + it.effect("fails a background command its exited Codex process never finished", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -3904,6 +3928,16 @@ describe("CodexAdapterV2 post-settle continuation", () => { Stream.runHead, Effect.forkChild({ startImmediately: true }), ); + const commandFailed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "turn-item.updated" && + event.payload.type === "command_execution" && + event.payload.status === "failed", + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); yield* orchestrator.dispatch({ type: "message.dispatch", commandId: CommandId.make("start-background-stop-untracked"), @@ -3917,28 +3951,14 @@ describe("CodexAdapterV2 post-settle continuation", () => { }); yield* worker.drain(); yield* Fiber.join(settled); - yield* worker.drain(); - const before = yield* orchestrator.getThreadShell(threadId); - assert.deepEqual( - before?.pendingBackgroundTasks?.map((task) => task.kind), - ["command"], - "the thread still shows the command the gone process never finished", - ); - const run = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; - yield* orchestrator.dispatch({ - type: "run.interrupt", - commandId: CommandId.make("stop-background-untracked"), - threadId, - runId: run.id, - holdQueue: true, - }); + yield* Fiber.join(commandFailed); yield* worker.drain(); const projection = yield* orchestrator.getThreadProjection(threadId); assert.deepEqual( projection.turnItems.flatMap((item) => item.type === "command_execution" ? [item.status] : [], ), - ["interrupted"], + ["failed"], ); assert.equal(projection.runs.at(-1)?.status, "completed"); assert.deepEqual( diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index dad45e20f552..1073712cc00c 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -3841,6 +3841,31 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }, ); + // Shared by interrupt and lost app-server paths. Persistent dynamic tools + // end too, because the turn itself is ending. + const terminalizeTrackedTurnWork = Effect.fn("CodexAdapterV2.terminalizeTrackedTurnWork")( + function* ( + context: ActiveCodexTurnContext, + status: "interrupted" | "failed", + completedAt: DateTime.Utc, + ) { + yield* terminalizeRunningCommandItems( + context, + context.nativeTurnId, + status, + completedAt, + ); + yield* terminalizeRunningDynamicTools( + context, + context.nativeTurnId, + status, + completedAt, + true, + ); + yield* terminalizeMcpAppCaptures(context, context.nativeTurnId); + }, + ); + // The plan coalescer builds these artifacts on every streamed update. const buildProposedPlanArtifacts = Effect.fnUntraced(function* (input: { readonly context: ActiveCodexTurnContext; @@ -6744,11 +6769,25 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } } }); + // No item/completed will arrive for work the process took with it, + // including background commands of turns that already completed. + const failLostTurnWork = turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + const completedAt = yield* DateTime.now; + for (const context of [ + ...(yield* Ref.get(activeTurns)).values(), + ...(yield* Ref.get(settledTurns)).values(), + ]) { + yield* terminalizeTrackedTurnWork(context, "failed", completedAt); + } + }), + ); yield* Effect.raceFirst( client.terminated, // A probe that fails with a transport error is the termination above. unresponsive.pipe(Effect.catch(() => Effect.never)), ).pipe( + Effect.tap(() => failLostTurnWork), Effect.flatMap((cause) => Queue.fail( events, @@ -7436,20 +7475,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const context = target.context; if ((yield* Ref.get(settledTurns)).get(context.nativeTurnId) !== context) continue; - yield* terminalizeRunningCommandItems( - context, - context.nativeTurnId, - "interrupted", - completedAt, - ); - yield* terminalizeRunningDynamicTools( - context, - context.nativeTurnId, - "interrupted", - completedAt, - true, - ); - yield* terminalizeMcpAppCaptures(context, context.nativeTurnId); + yield* terminalizeTrackedTurnWork(context, "interrupted", completedAt); yield* Ref.update(runningCommandItemsByTurn, (current) => { const updated = new Map(current); updated.delete(context.nativeTurnId); From d18883483d664abf01b4cbee57c0bdfdfeecf77d Mon Sep 17 00:00:00 2001 From: Dara Adedeji Date: Tue, 27 Oct 2026 17:47:35 -0400 Subject: [PATCH 7/7] test(server): keep the liveness probe out of Codex replay tests Replays follow a recorded conversation, so the replay harness answers the probe itself. The compaction tests now set their clock before the session opens; jumping decades afterwards made the probe catch up minute by minute. Also restores the Path and ServerConfig imports the merge with main dropped, and writes writeReleaseRecords with Effect.fn for main's new lint rule. --- .../Adapters/CodexAdapterV2.test.ts | 14 ++++++++++---- .../ProviderSessionManager.ts | 19 ++++++++++--------- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index ad552fded6b1..89779c0f22d0 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -40,6 +40,7 @@ import * as Exit from "effect/Exit"; import * as Option from "effect/Option"; import * as Predicate from "effect/Predicate"; import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; import { MCP_APP_OUTPUT_KEY, readMcpAppReference } from "@t3tools/shared/mcpApp"; import { resolveAttachmentPathById } from "../../attachmentStore.ts"; import * as Fiber from "effect/Fiber"; @@ -64,6 +65,7 @@ import * as Orchestrator from "../Orchestrator.ts"; import * as ProviderReplayHarness from "../testkit/ProviderReplayHarness.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; +import * as ServerConfig from "../../config.ts"; import * as CodexAdapterV2 from "./CodexAdapterV2.ts"; import { makeReplayServerConfig, withCodexReplayChildMetadata } from "./CodexAdapterV2.testkit.ts"; import * as CodexAdapterV2Testkit from "./CodexAdapterV2.testkit.ts"; @@ -1210,10 +1212,13 @@ describe("CodexAdapterV2 post-settle continuation", () => { (client) => ({ ...client, + // Liveness probes follow the clock, not the recorded conversation. request: (method, params) => - onRequest(method, params).pipe( - Effect.andThen(client.request(method, params)), - ), + method === "thread/loaded/list" + ? Effect.succeed({ data: [] } as never) + : onRequest(method, params).pipe( + Effect.andThen(client.request(method, params)), + ), }) satisfies CodexClient.CodexAppServerClient["Service"], ), Effect.provide(context), @@ -2235,6 +2240,8 @@ describe("CodexAdapterV2 post-settle continuation", () => { ], }); const compactionStarted = yield* Deferred.make(); + // Before the session opens, so its liveness timer does not replay the jump. + yield* TestClock.setTime(1782622445000); const harness = yield* makeCodexReplayHarness(transcript, (event) => event.type === "turn_item.updated" && event.turnItem.nativeItemRef?.nativeId === "unfinished-item" && @@ -2243,7 +2250,6 @@ describe("CodexAdapterV2 post-settle continuation", () => { ? Deferred.succeed(compactionStarted, event.turnItem.providerTurnId) : Effect.void, ); - yield* TestClock.setTime(1782622445000); yield* harness.runtime.startTurn( makeCodexTestTurnInput({ threadId: harness.threadId, diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 84cdd08911ef..0c3a0e786627 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -819,14 +819,14 @@ export const layerWithOptions = ( { discard: true }, ); - const writeReleaseRecords = (input: { - readonly entry: LiveSessionEntry; - readonly reason: ProviderSessionReleaseReason; - readonly detail?: string; - readonly releasedBackgroundTasks: ReadonlyArray; - readonly releasedAt: DateTime.Utc; - }) => - Effect.gen(function* () { + const writeReleaseRecords = Effect.fn("ProviderSessionManager.writeReleaseRecords")( + function* (input: { + readonly entry: LiveSessionEntry; + readonly reason: ProviderSessionReleaseReason; + readonly detail?: string; + readonly releasedBackgroundTasks: ReadonlyArray; + readonly releasedAt: DateTime.Utc; + }) { // The open lock protects status ownership, including after a replacement closes. const key = sessionKey(input.entry.runtime.providerSessionId); const ownsStatus = (yield* Ref.get(latestOpened)).get(key) === input.entry.runtime; @@ -850,7 +850,8 @@ export const layerWithOptions = ( updated.delete(key); return updated; }); - }); + }, + ); // Retry after removal from the live map; later releases cannot clear stale ready sessions and approvals. const retryReleaseRecords = (input: Parameters[0]) => {