diff --git a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts index dde77d484094..0e9032c58d2a 100644 --- a/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts +++ b/apps/server/src/mcp/OrchestratorMcpToolkit.integration.test.ts @@ -190,9 +190,17 @@ describe("orchestrator MCP toolkit", () => { Effect.gen(function* () { const cwd = yield* checkpointWorkspace("delegated-task-status"); const rawTranscript = yield* readDelegatedTaskStatusTranscript(result); - const transcript = yield* CodexOrchestratorReplayHarness.decodeTranscript( - materializeReplayTranscriptWorkspace(rawTranscript, cwd), - ); + const workspaceTranscript = materializeReplayTranscriptWorkspace(rawTranscript, cwd); + // The recorder stopped Codex after its last frame, but this test keeps + // the parent turn running past the recording. Replaying that exit would + // end the session and fail the parent run. + const transcript = yield* CodexOrchestratorReplayHarness.decodeTranscript({ + ...workspaceTranscript, + entries: + workspaceTranscript.entries.at(-1)?.type === "runtime_exit" + ? workspaceTranscript.entries.slice(0, -1) + : workspaceTranscript.entries, + }); const layerOrchestrator = ProviderReplayHarness.layerProviderReplay( { name: "delegated-task-status/codex", diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index 1e762543a13a..89779c0f22d0 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -36,8 +36,11 @@ import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Crypto from "effect/Crypto"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; +import * as Option from "effect/Option"; import * as Predicate from "effect/Predicate"; import * as FileSystem from "effect/FileSystem"; +import * as Path from "effect/Path"; import { MCP_APP_OUTPUT_KEY, readMcpAppReference } from "@t3tools/shared/mcpApp"; import { resolveAttachmentPathById } from "../../attachmentStore.ts"; import * as Fiber from "effect/Fiber"; @@ -55,10 +58,14 @@ import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderS import * as ProviderEventLoggers from "@t3tools/provider-core/server/ProviderEventLoggers"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as EffectWorker from "../EffectWorker.ts"; +import * as EventSink from "../EventSink.ts"; +import * as ProviderAdapterRegistry from "../ProviderAdapterRegistry.ts"; +import * as ProviderSessionManager from "../ProviderSessionManager.ts"; import * as Orchestrator from "../Orchestrator.ts"; import * as ProviderReplayHarness from "../testkit/ProviderReplayHarness.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import type * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; +import * as ServerConfig from "../../config.ts"; import * as CodexAdapterV2 from "./CodexAdapterV2.ts"; import { makeReplayServerConfig, withCodexReplayChildMetadata } from "./CodexAdapterV2.testkit.ts"; import * as CodexAdapterV2Testkit from "./CodexAdapterV2.testkit.ts"; @@ -893,6 +900,268 @@ describe("CodexAdapterV2 session initialize", () => { ); }); +describe("CodexAdapterV2 lost app-server", () => { + it.effect.each(["SIGKILL", "exit code", "silence"] as const)( + "fails active turns and replaces the shared app-server after %s", + (termination) => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-codex-session-recovery-" }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + const handles: Array = []; + const testSpawner = ChildProcessSpawner.make(() => + spawner + .spawn( + ChildProcess.make( + process.execPath, + [ + "--input-type=module", + "-e", + `import * as NodeFS from "node:fs"; +import * as NodeReadline from "node:readline"; + +const fixture = JSON.parse( + NodeFS.readFileSync(process.argv[1], "utf8"), +); +let nextThreadId = 0; +let nextTurnId = 0; +const write = (message) => process.stdout.write(\`\${JSON.stringify(message)}\\n\`); + +NodeReadline.createInterface({ input: process.stdin }).on("line", (line) => { + const { id, method, params } = JSON.parse(line); + switch (method) { + case "initialize": + write({ + id, + result: { + userAgent: "codex-session-recovery-mock", + codexHome: process.cwd(), + platformFamily: "unix", + platformOs: "linux", + }, + }); + break; + case "thread/start": { + const thread = { + ...fixture.responses.threadStart.thread, + id: \`thread-\${process.pid}-\${nextThreadId++}\`, + }; + write({ id, result: { ...fixture.responses.threadStart, thread } }); + break; + } + case "thread/resume": { + const thread = { + ...fixture.responses.threadStart.thread, + id: params.threadId, + }; + write({ id, result: { ...fixture.responses.threadStart, thread } }); + break; + } + case "turn/start": { + const turn = { + ...fixture.responses.turnStart.turn, + id: \`turn-\${process.pid}-\${nextTurnId++}\`, + }; + write({ id, result: { turn } }); + write({ method: "turn/started", params: { threadId: params.threadId, turn } }); + write({ + method: "item/started", + params: { + threadId: params.threadId, + turnId: turn.id, + startedAtMs: Date.now(), + item: { + type: "commandExecution", + id: \`command-\${turn.id}\`, + command: "sleep 600", + cwd: process.cwd(), + status: "inProgress", + commandActions: [], + aggregatedOutput: null, + }, + }, + }); + break; + } + case "thread/loaded/list": + break; + case "test/exit": + process.exit(7); + break; + default: + if (id !== undefined) write({ id, result: {} }); + } +});`, + path.join( + import.meta.dirname, + "../../provider/testFixtures/codexMultiAgentWire.json", + ), + ], + { cwd }, + ), + ) + .pipe(Effect.tap((handle) => Effect.sync(() => handles.push(handle)))), + ); + const layerFactory = CodexAdapterV2.layerAppServerClientFactory.pipe( + Layer.provide(Layer.succeed(ChildProcessSpawner.ChildProcessSpawner, testSpawner)), + Layer.provide( + Layer.succeed( + ProviderEventLoggers.ProviderEventLoggers, + ProviderEventLoggers.NoOpProviderEventLoggers, + ), + ), + ); + const layerRegistry = ProviderAdapterRegistry.layerFromAdaptersEffect( + Effect.gen(function* () { + const adapter = yield* CodexAdapterV2.makeCodexAdapterV2({ + instanceId: CODEX_TEST_MODEL_SELECTION.instanceId, + settings: DEFAULT_CODEX_SETTINGS, + environment: {}, + clientFactory: yield* CodexAdapterV2.CodexAppServerClientFactory, + crypto: yield* Crypto.Crypto, + fileSystem: yield* FileSystem.FileSystem, + idAllocator: yield* IdAllocator.IdAllocatorV2, + serverConfig: yield* ServerConfig.ServerConfig, + }); + return [adapter]; + }), + ).pipe( + Layer.provide( + Layer.mergeAll( + layerFactory, + IdAllocator.layer, + ServerConfig.layerTest(cwd, { prefix: "t3-codex-session-recovery-config-" }), + ).pipe(Layer.provideMerge(NodeServices.layer)), + ), + ); + + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const eventSink = yield* EventSink.EventSinkV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const createAndStart = (threadId: ThreadId) => + Effect.gen(function* () { + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make(`create:${threadId}`), + threadId, + projectId: ProjectId.make("project:codex-session-recovery"), + title: threadId, + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + yield* send(threadId, "start"); + }); + const send = (threadId: ThreadId, text: string) => + Effect.gen(function* () { + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(`send:${threadId}:${text}`), + threadId, + messageId: MessageId.make(`message:${threadId}:${text}`), + text, + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* worker.drain(); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "running", + ); + }); + const threadA = ThreadId.make("thread:codex-recovery-a"); + const threadB = ThreadId.make("thread:codex-recovery-b"); + yield* createAndStart(threadA); + yield* createAndStart(threadB); + assert.equal(handles.length, 1); + const sessionA = (yield* orchestrator.getThreadProjection(threadA)).providerSessions.at( + -1, + )!; + const sessionB = (yield* orchestrator.getThreadProjection(threadB)).providerSessions.at( + -1, + )!; + assert.equal(sessionA.id, sessionB.id); + const beforeTermination = yield* eventSink.latestSequence(); + const handle = handles[0]!; + if (termination === "silence") { + yield* TestClock.adjust("1 minute"); + yield* TestClock.adjust("1 minute"); + } else if (termination === "SIGKILL") { + yield* handle.kill({ killSignal: "SIGKILL" }); + } else { + const encoder = new TextEncoder(); + yield* Stream.make(encoder.encode('{"id":999,"method":"test/exit"}\n')).pipe( + Stream.run(handle.stdin), + ); + } + if (termination !== "silence") { + const exit = yield* Effect.exit(handle.exitCode); + assert.equal(Exit.isFailure(exit), termination === "SIGKILL"); + if (Exit.isSuccess(exit)) assert.equal(Number(exit.value), 7); + } + + // Await committed failures, rather than guessing when the event pump ran. + yield* eventSink + .stream({ afterSequence: beforeTermination, eventType: "provider-session.updated" }) + .pipe( + Stream.filter( + ({ event }) => + event.type === "provider-session.updated" && event.payload.status === "error", + ), + Stream.take(2), + Stream.runDrain, + ); + yield* eventSink + .stream({ afterSequence: beforeTermination, eventType: "run.updated" }) + .pipe( + Stream.filter( + ({ event }) => event.type === "run.updated" && event.payload.status === "failed", + ), + Stream.take(2), + Stream.runDrain, + ); + yield* worker.drain(); + assert.isTrue(Option.isNone(yield* manager.get(sessionA.id))); + for (const threadId of [threadA, threadB]) { + const projection = yield* orchestrator.getThreadProjection(threadId); + assert.equal(projection.runs.at(-1)?.status, "failed"); + assert.equal(projection.providerSessions.at(-1)?.status, "error"); + // Nothing will complete the command the process took with it. + const command = projection.turnItems.find((item) => item.type === "command_execution"); + assert.equal(command?.status, "failed"); + assert.equal( + projection.nodes.find((node) => node.id === command?.nodeId)?.status, + "failed", + ); + } + + yield* createAndStart(ThreadId.make("thread:codex-recovery-new")); + assert.equal(handles.length, 2); + assert.notEqual(handles[0]!.pid, handles[1]!.pid); + yield* send(threadA, "retry"); + assert.equal(handles.length, 2); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: `codex-session-recovery-${termination}`, runtimePolicyOverride: { cwd } }, + layerRegistry, + { runEffectWorker: false }, + ), + ), + ); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); +}); + describe("CodexAdapterV2 post-settle continuation", () => { // The harness notifies after every recorded event or continuation, so a wait // re-checks exactly when the state it reads changes. Subscribing before the @@ -943,10 +1212,13 @@ describe("CodexAdapterV2 post-settle continuation", () => { (client) => ({ ...client, + // Liveness probes follow the clock, not the recorded conversation. request: (method, params) => - onRequest(method, params).pipe( - Effect.andThen(client.request(method, params)), - ), + method === "thread/loaded/list" + ? Effect.succeed({ data: [] } as never) + : onRequest(method, params).pipe( + Effect.andThen(client.request(method, params)), + ), }) satisfies CodexClient.CodexAppServerClient["Service"], ), Effect.provide(context), @@ -1968,6 +2240,8 @@ describe("CodexAdapterV2 post-settle continuation", () => { ], }); const compactionStarted = yield* Deferred.make(); + // Before the session opens, so its liveness timer does not replay the jump. + yield* TestClock.setTime(1782622445000); const harness = yield* makeCodexReplayHarness(transcript, (event) => event.type === "turn_item.updated" && event.turnItem.nativeItemRef?.nativeId === "unfinished-item" && @@ -1976,7 +2250,6 @@ describe("CodexAdapterV2 post-settle continuation", () => { ? Deferred.succeed(compactionStarted, event.turnItem.providerTurnId) : Effect.void, ); - yield* TestClock.setTime(1782622445000); yield* harness.runtime.startTurn( makeCodexTestTurnInput({ threadId: harness.threadId, @@ -3599,7 +3872,7 @@ describe("CodexAdapterV2 post-settle continuation", () => { ); }); - it.effect("Stop ends a background command no Codex process tracks any more", () => + it.effect("fails a background command its exited Codex process never finished", () => Effect.scoped( Effect.gen(function* () { const fs = yield* FileSystem.FileSystem; @@ -3661,6 +3934,16 @@ describe("CodexAdapterV2 post-settle continuation", () => { Stream.runHead, Effect.forkChild({ startImmediately: true }), ); + const commandFailed = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "turn-item.updated" && + event.payload.type === "command_execution" && + event.payload.status === "failed", + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); yield* orchestrator.dispatch({ type: "message.dispatch", commandId: CommandId.make("start-background-stop-untracked"), @@ -3674,28 +3957,14 @@ describe("CodexAdapterV2 post-settle continuation", () => { }); yield* worker.drain(); yield* Fiber.join(settled); - yield* worker.drain(); - const before = yield* orchestrator.getThreadShell(threadId); - assert.deepEqual( - before?.pendingBackgroundTasks?.map((task) => task.kind), - ["command"], - "the thread still shows the command the gone process never finished", - ); - const run = (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)!; - yield* orchestrator.dispatch({ - type: "run.interrupt", - commandId: CommandId.make("stop-background-untracked"), - threadId, - runId: run.id, - holdQueue: true, - }); + yield* Fiber.join(commandFailed); yield* worker.drain(); const projection = yield* orchestrator.getThreadProjection(threadId); assert.deepEqual( projection.turnItems.flatMap((item) => item.type === "command_execution" ? [item.status] : [], ), - ["interrupted"], + ["failed"], ); assert.equal(projection.runs.at(-1)?.status, "completed"); assert.deepEqual( diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 74c4b46646e3..1073712cc00c 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -488,6 +488,9 @@ function codexMcpToolOutput( // A hung MCP resource read must not hold run ingestion open. const MCP_APP_CAPTURE_TIMEOUT = "20 seconds"; +// Reading the in-memory thread list is cheap; even an error reply proves liveness. +const CODEX_LIVENESS_PROBE_INTERVAL = "1 minute"; +const CODEX_LIVENESS_PROBE_TIMEOUT = "1 minute"; function codexMcpAppResourceUri(item: CodexDynamicToolItem): string | undefined { if (item.type !== "mcpToolCall") return undefined; @@ -1746,7 +1749,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( model: input.modelSelection.model, now, }); - const events = yield* Queue.unbounded(); + const events = yield* Queue.unbounded< + ProviderAdapter.ProviderAdapterV2Event, + ProviderAdapter.ProviderAdapterEventStreamError + >(); const rateLimitSnapshot = yield* Ref.make(undefined); const limitedTurnItems = yield* Ref.make( new Map>(), @@ -3835,6 +3841,31 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }, ); + // Shared by interrupt and lost app-server paths. Persistent dynamic tools + // end too, because the turn itself is ending. + const terminalizeTrackedTurnWork = Effect.fn("CodexAdapterV2.terminalizeTrackedTurnWork")( + function* ( + context: ActiveCodexTurnContext, + status: "interrupted" | "failed", + completedAt: DateTime.Utc, + ) { + yield* terminalizeRunningCommandItems( + context, + context.nativeTurnId, + status, + completedAt, + ); + yield* terminalizeRunningDynamicTools( + context, + context.nativeTurnId, + status, + completedAt, + true, + ); + yield* terminalizeMcpAppCaptures(context, context.nativeTurnId); + }, + ); + // The plan coalescer builds these artifacts on every streamed update. const buildProposedPlanArtifacts = Effect.fnUntraced(function* (input: { readonly context: ActiveCodexTurnContext; @@ -6723,6 +6754,53 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ), }; + // Drain buffered events before failing every run sharing this process. + const unresponsive = Effect.gen(function* () { + while (true) { + yield* Effect.sleep(CODEX_LIVENESS_PROBE_INTERVAL); + const answered = yield* client + .request("thread/loaded/list", { limit: 1 }) + .pipe( + Effect.catchTags({ CodexAppServerRequestError: () => Effect.void }), + Effect.timeoutOption(CODEX_LIVENESS_PROBE_TIMEOUT), + ); + if (Option.isNone(answered)) { + return `Codex stopped responding: it did not answer a liveness check within ${CODEX_LIVENESS_PROBE_TIMEOUT}.`; + } + } + }); + // No item/completed will arrive for work the process took with it, + // including background commands of turns that already completed. + const failLostTurnWork = turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + const completedAt = yield* DateTime.now; + for (const context of [ + ...(yield* Ref.get(activeTurns)).values(), + ...(yield* Ref.get(settledTurns)).values(), + ]) { + yield* terminalizeTrackedTurnWork(context, "failed", completedAt); + } + }), + ); + yield* Effect.raceFirst( + client.terminated, + // A probe that fails with a transport error is the termination above. + unresponsive.pipe(Effect.catch(() => Effect.never)), + ).pipe( + Effect.tap(() => failLostTurnWork), + Effect.flatMap((cause) => + Queue.fail( + events, + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver: CODEX_PROVIDER, + providerSessionId: input.providerSessionId, + cause, + }), + ), + ), + Effect.forkIn(scope), + ); + const runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime = { instanceId: adapterOptions.instanceId, driver: CODEX_PROVIDER, @@ -7397,20 +7475,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const context = target.context; if ((yield* Ref.get(settledTurns)).get(context.nativeTurnId) !== context) continue; - yield* terminalizeRunningCommandItems( - context, - context.nativeTurnId, - "interrupted", - completedAt, - ); - yield* terminalizeRunningDynamicTools( - context, - context.nativeTurnId, - "interrupted", - completedAt, - true, - ); - yield* terminalizeMcpAppCaptures(context, context.nativeTurnId); + yield* terminalizeTrackedTurnWork(context, "interrupted", completedAt); yield* Ref.update(runningCommandItemsByTurn, (current) => { const updated = new Map(current); updated.delete(context.nativeTurnId); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 3c635be6753d..9fcfada5240e 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -331,7 +331,8 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( readonly hasPendingBackgroundWork?: Effect.Effect; readonly hasPendingBackgroundWorkForThread?: Effect.Effect; readonly backgroundTasksEndWithProcess?: boolean; - readonly hangSessionScopeClose?: boolean; + /** "first" wedges only the first session opened. */ + readonly hangSessionScopeClose?: boolean | "first"; readonly startTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; readonly spawnBeforeOpen?: boolean; @@ -392,7 +393,11 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( }); if (options.spawnBeforeOpen !== true) { yield* countClose; - if (options.hangSessionScopeClose === true) yield* hangClose; + if ( + options.hangSessionScopeClose === true || + (options.hangSessionScopeClose === "first" && (yield* Ref.get(state)).openCount === 1) + ) + yield* hangClose; } return { @@ -478,7 +483,7 @@ function layerTest(input: { readonly hasPendingBackgroundWork?: Effect.Effect; readonly hasPendingBackgroundWorkForThread?: Effect.Effect; readonly backgroundTasksEndWithProcess?: boolean; - readonly hangSessionScopeClose?: boolean; + readonly hangSessionScopeClose?: boolean | "first"; readonly startTurn?: Effect.Effect; readonly beforeUnload?: Effect.Effect; readonly spawnBeforeOpen?: boolean; @@ -2747,6 +2752,61 @@ it.effect("ProviderSessionManagerV2 release retries leave a replacement session }), ); +it.effect.each(["open", "closed"] as const)( + "ProviderSessionManagerV2 preserves a %s replacement's status after a late release", + (replacement) => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const scopeCloseReached = yield* Deferred.make(); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread-provider-session-manager-failed-replacement"); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId, + }); + yield* eventSink.write({ + events: [ + yield* makeThreadCreatedEvent({ idAllocator, threadId, now: yield* DateTime.now }), + ], + }); + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + + const release = yield* manager + .release({ + providerSessionId, + reason: "runtime_error", + }) + .pipe(Effect.forkChild); + yield* Deferred.await(scopeCloseReached); + + yield* manager.open({ threadId, providerSessionId, modelSelection, runtimePolicy }); + if (replacement === "closed") yield* manager.close(providerSessionId); + const expected = (yield* projectionStore.getThreadProjection(threadId)).providerSessions.at( + -1, + ); + yield* TestClock.adjust("30 seconds"); + yield* Fiber.join(release); + assert.deepEqual( + (yield* projectionStore.getThreadProjection(threadId)).providerSessions.at(-1), + expected, + ); + }).pipe( + Effect.provide( + layerTest({ + state, + idleTimeoutMs: 60_000, + hangSessionScopeClose: "first", + scopeCloseReached, + }), + ), + ); + }), +); + // regression: #15048 runs got stuck after a failed session release it.effect("ProviderSessionManagerV2 keeps each failed release's cleanup", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 00b42d510bd9..0c3a0e786627 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -375,6 +375,10 @@ export const layerWithOptions = ( }), ); const sessions = yield* Ref.make(new Map()); + // The latest session opened under each id, kept until its release records it. + const latestOpened = yield* Ref.make( + new Map(), + ); // Retry per released entry; reusing ids must not lose cleanup for earlier threads. const releaseRecordRetries = yield* FiberSet.make(); const nextSubscriberId = yield* Ref.make(0); @@ -815,43 +819,46 @@ export const layerWithOptions = ( { discard: true }, ); - const writeReleaseRecords = (input: { - readonly entry: LiveSessionEntry; - readonly reason: ProviderSessionReleaseReason; - readonly detail?: string; - readonly releasedBackgroundTasks: ReadonlyArray; - readonly releasedAt: DateTime.Utc; - readonly replaced: boolean; - }) => - Effect.all( - [ - input.replaced - ? Effect.succeed(Exit.void) - : Effect.exit(writeReleasedSessionEvents(input)), - Effect.exit( - writeReleasedRuntimeRequestEvents(input).pipe( - input.entry.requestEventPermit.withPermits(1), + const writeReleaseRecords = Effect.fn("ProviderSessionManager.writeReleaseRecords")( + function* (input: { + readonly entry: LiveSessionEntry; + readonly reason: ProviderSessionReleaseReason; + readonly detail?: string; + readonly releasedBackgroundTasks: ReadonlyArray; + readonly releasedAt: DateTime.Utc; + }) { + // The open lock protects status ownership, including after a replacement closes. + const key = sessionKey(input.entry.runtime.providerSessionId); + const ownsStatus = (yield* Ref.get(latestOpened)).get(key) === input.entry.runtime; + yield* Effect.all( + [ + ownsStatus + ? Effect.exit(writeReleasedSessionEvents(input)) + : Effect.succeed(Exit.void), + Effect.exit( + writeReleasedRuntimeRequestEvents(input).pipe( + input.entry.requestEventPermit.withPermits(1), + ), ), - ), - Effect.exit(writeReleasedBackgroundWorkEvents(input.releasedBackgroundTasks)), - ], - { concurrency: 1 }, - ).pipe(Effect.flatMap(Exit.asVoidAll)); + Effect.exit(writeReleasedBackgroundWorkEvents(input.releasedBackgroundTasks)), + ], + { concurrency: 1 }, + ).pipe(Effect.flatMap(Exit.asVoidAll)); + if (!ownsStatus) return; + yield* Ref.update(latestOpened, (current) => { + const updated = new Map(current); + updated.delete(key); + return updated; + }); + }, + ); // Retry after removal from the live map; later releases cannot clear stale ready sessions and approvals. - const retryReleaseRecords = ( - input: Omit[0], "replaced">, - ) => { + const retryReleaseRecords = (input: Parameters[0]) => { const providerSessionId = input.entry.runtime.providerSessionId; const attempt = Effect.gen(function* () { const exit = yield* Effect.exit( - sessionOpen.withLock( - providerSessionId, - Effect.gen(function* () { - const replaced = (yield* Ref.get(sessions)).has(sessionKey(providerSessionId)); - yield* writeReleaseRecords({ ...input, replaced }); - }), - ), + sessionOpen.withLock(providerSessionId, writeReleaseRecords(input)), ); if (Exit.isSuccess(exit) || Cause.hasInterruptsOnly(exit.cause)) return yield* exit; yield* Effect.logWarning("orchestration-v2.provider-session-release-records-failed", { @@ -971,6 +978,8 @@ export const layerWithOptions = ( readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; readonly gracefulSubscribers?: boolean; + /** Set by callers inside `open`, which already hold this session's open lock. */ + readonly openLockHeld?: boolean; }) => Effect.acquireUseRelease( removeLiveEntry(input), @@ -1011,7 +1020,9 @@ export const layerWithOptions = ( releasedAt, }; const recorded = yield* Effect.exit( - writeReleaseRecords({ ...records, replaced: false }), + input.openLockHeld === true + ? writeReleaseRecords(records) + : sessionOpen.withLock(input.providerSessionId, writeReleaseRecords(records)), ); if (Exit.isFailure(recorded)) { yield* retryReleaseRecords(records); @@ -2191,6 +2202,7 @@ export const layerWithOptions = ( updated.set(key, entry); return updated; }); + yield* Ref.update(latestOpened, (current) => new Map(current).set(key, runtime)); yield* dropReservation; yield* withActivityError( input.providerSessionId, @@ -2209,11 +2221,13 @@ export const layerWithOptions = ( providerSessionId: input.providerSessionId, reason: "manual_shutdown", detail: "The provider session start was interrupted.", + openLockHeld: true, } : { providerSessionId: input.providerSessionId, reason: "runtime_error", detail: "Failed to persist the provider-session attachment.", + openLockHeld: true, }, ).pipe(logReleaseFailure(input.providerSessionId)), ), diff --git a/packages/effect-codex-app-server/src/client.ts b/packages/effect-codex-app-server/src/client.ts index d872b9fa8473..59bf11e6b177 100644 --- a/packages/effect-codex-app-server/src/client.ts +++ b/packages/effect-codex-app-server/src/client.ts @@ -1,4 +1,5 @@ import * as Context from "effect/Context"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Schema from "effect/Schema"; @@ -75,6 +76,8 @@ export class CodexAppServerClient extends Context.Service< params: unknown, ) => Effect.Effect, ) => Effect.Effect; + /** Reports unexpected termination; deliberate scope closure interrupts its waiters. */ + readonly terminated: Effect.Effect; } >()("effect-codex-app-server/client/CodexAppServerClient") {} @@ -98,6 +101,7 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make stdio: Stdio.Stdio, options: CodexAppServerClientOptions = {}, terminationError?: Effect.Effect, + exited?: Effect.Effect, ): Effect.fn.Return { const requestHandlers = new Map(); const notificationHandlers = new Map>(); @@ -195,9 +199,12 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make : Effect.fail(CodexError.CodexAppServerRequestError.methodNotFound(request.method)); }; + const termination = yield* Deferred.make(); const transport = yield* CodexProtocol.makeCodexAppServerPatchedProtocol({ stdio, + onTermination: (error) => Deferred.succeed(termination, error).pipe(Effect.asVoid), ...(terminationError ? { terminationError } : {}), + ...(exited ? { exited } : {}), ...(options.logIncoming !== undefined ? { logIncoming: options.logIncoming } : {}), ...(options.logOutgoing !== undefined ? { logOutgoing: options.logOutgoing } : {}), ...(options.logger ? { logger: options.logger } : {}), @@ -270,6 +277,7 @@ export const make = Effect.fn("effect-codex-app-server/CodexAppServerClient.make unknownNotificationHandler = handler; }); }), + terminated: Deferred.await(termination), }); }); @@ -283,5 +291,6 @@ const makeChildProcessClient = Effect.fn( "effect-codex-app-server/CodexAppServerClient.makeChildProcessClient", )(function* (handle: ChildProcessSpawner.ChildProcessHandle, options: CodexAppServerClientOptions) { yield* Stream.runDrain(handle.stderr).pipe(Effect.ignore, Effect.forkScoped); - return yield* make(makeChildStdio(handle), options, makeTerminationError(handle)); + const terminationError = makeTerminationError(handle); + return yield* make(makeChildStdio(handle), options, terminationError, terminationError); }); diff --git a/packages/effect-codex-app-server/src/protocol.test.ts b/packages/effect-codex-app-server/src/protocol.test.ts index f470d0b25bfa..9f96d84d1c3e 100644 --- a/packages/effect-codex-app-server/src/protocol.test.ts +++ b/packages/effect-codex-app-server/src/protocol.test.ts @@ -1,8 +1,10 @@ import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; +import * as Exit from "effect/Exit"; import * as Fiber from "effect/Fiber"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; +import * as Scope from "effect/Scope"; import * as Stdio from "effect/Stdio"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; @@ -690,4 +692,60 @@ it.layer(NodeServices.layer)("effect-codex-app-server protocol", (it) => { assert.strictEqual(yield* Deferred.await(termination), exitError); }), ); + + it.effect("terminates when the process exits while its output stays open", () => + Effect.gen(function* () { + const { stdio, input, output } = yield* makeInMemoryStdio(); + const exited = yield* Deferred.make(); + const termination = yield* Deferred.make(); + const transport = yield* CodexProtocol.makeCodexAppServerPatchedProtocol({ + stdio, + exited: Deferred.await(exited), + onTermination: (error) => Deferred.succeed(termination, error).pipe(Effect.asVoid), + }); + const answered = yield* transport.request("thread/read", {}).pipe(Effect.forkScoped); + yield* Queue.take(output); + const unanswered = yield* transport.request("thread/start", {}).pipe(Effect.forkScoped); + yield* Queue.take(output); + // A reply written just before the exit still arrives. + const exitError = new CodexError.CodexAppServerProcessExitedError({ code: 1 }); + yield* Deferred.succeed(exited, exitError); + yield* Queue.offer(input, encodeJsonl({ id: 1, result: {} })); + assert.deepEqual(yield* Fiber.join(answered), {}); + yield* TestClock.adjust("1 second"); + assert.isFalse(yield* Deferred.isDone(termination)); + + yield* TestClock.adjust("1 second"); + assert.strictEqual(yield* Deferred.await(termination), exitError); + const failure = yield* Fiber.join(unanswered).pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => assert.fail("Expected the request to fail"), + }), + ); + assert.strictEqual(failure, exitError); + }), + ); + + it.effect("fails requests still waiting for a reply when the client closes", () => + Effect.gen(function* () { + const { stdio, output } = yield* makeInMemoryStdio(); + const clientScope = yield* Scope.make(); + const transport = yield* CodexProtocol.makeCodexAppServerPatchedProtocol({ stdio }).pipe( + Scope.provide(clientScope), + ); + const response = yield* transport.request("thread/start", {}).pipe(Effect.forkScoped); + yield* Queue.take(output); + + yield* Scope.close(clientScope, Exit.void); + + const failure = yield* Fiber.join(response).pipe( + Effect.match({ + onFailure: (error) => error, + onSuccess: () => assert.fail("Expected the request to fail"), + }), + ); + assert.instanceOf(failure, CodexError.CodexAppServerInputStreamEndedError); + }), + ); }); diff --git a/packages/effect-codex-app-server/src/protocol.ts b/packages/effect-codex-app-server/src/protocol.ts index b0ff5436e9e7..8370c836dafe 100644 --- a/packages/effect-codex-app-server/src/protocol.ts +++ b/packages/effect-codex-app-server/src/protocol.ts @@ -19,6 +19,7 @@ const MAX_BUFFERED_RAW_MESSAGES = 32; // Bound JSONL lines before joining and parsing; the LF delimiter is excluded. const MAX_INCOMING_MESSAGE_BYTES = 128 * 1024 * 1024; const EXIT_STATUS_GRACE = "250 millis"; +const EXITED_OUTPUT_GRACE = "2 seconds"; export interface CodexAppServerProtocolLogEvent { readonly direction: "incoming" | "outgoing"; @@ -40,6 +41,11 @@ export interface CodexAppServerIncomingRequest { export interface CodexAppServerPatchedProtocolOptions { readonly stdio: Stdio.Stdio; readonly terminationError?: Effect.Effect; + /** + * Completes when the app-server process exits. Its stdout can stay open + * after that, while a command it started still holds the pipe. + */ + readonly exited?: Effect.Effect; readonly logIncoming?: boolean; readonly logOutgoing?: boolean; // Test override; production uses MAX_INCOMING_MESSAGE_BYTES. @@ -468,6 +474,27 @@ export const makeCodexAppServerPatchedProtocol = Effect.fn("makeCodexAppServerPa yield* Stream.fromQueue(outgoing).pipe(Stream.run(options.stdio.stdout()), Effect.forkScoped); + if (options.exited) { + yield* options.exited.pipe( + // Output written just before exit still arrives; the pipe normally + // closes with the process and terminates the transport first. + Effect.tap(() => Effect.sleep(EXITED_OUTPUT_GRACE)), + Effect.flatMap((error) => handleTermination(() => Effect.succeed(error))), + Effect.forkScoped, + ); + } + + // Deliberate closure fails pending requests without reporting a runtime failure. + yield* Effect.addFinalizer(() => + Effect.gen(function* () { + const closed = new CodexError.CodexAppServerInputStreamEndedError({}); + yield* Ref.update(terminationFailure, (current) => + Option.isSome(current) ? current : Option.some(closed), + ); + yield* failAllPending(closed); + }), + ); + const request = Effect.fn("CodexAppServerProtocol.request")(function* ( method: string, payload?: unknown,