diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts index f8c63587fec6..0e207f41933b 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.test.ts @@ -1,3 +1,5 @@ +import * as ProviderSessionManager from "../ProviderSessionManager.ts"; +import * as Option from "effect/Option"; import { historyResponseItems } from "@t3tools/provider-core/server/handoffBudget"; import { makeProviderFailure } from "@t3tools/provider-core/server/failure"; import { @@ -1236,7 +1238,9 @@ describe("CodexAdapterV2 post-settle continuation", () => { Effect.void, onRequest: (method: string, params: unknown) => Effect.Effect = () => Effect.void, readChildMetadata?: Parameters[2], - beforeEmitInbound?: CodexReplay.CodexAppServerReplayDriver["beforeEmitInbound"], + replayOptions: + | Parameters[1] + | CodexReplay.CodexAppServerReplayDriver["beforeEmitInbound"] = {}, onUsageLimits?: CodexAdapterV2.CodexAdapterV2Options["onUsageLimits"], promptSuggestions: Effect.Effect = Effect.succeed(false), ) => @@ -1249,13 +1253,13 @@ describe("CodexAdapterV2 post-settle continuation", () => { const changes = yield* makeChangeSignal; const continuationRequests: Array = []; - const driver = yield* CodexReplay.makeReplayDriver( + const replayDriver = yield* CodexReplay.makeReplayDriver( transcript, - beforeEmitInbound === undefined ? {} : { beforeEmitInbound }, + typeof replayOptions === "function" ? { beforeEmitInbound: replayOptions } : replayOptions, ); const clientFactory: CodexAdapterV2.CodexAppServerClientFactoryShape = { open: (openInput) => - Layer.build(CodexReplay.layerReplayWithDriver(driver)).pipe( + Layer.build(CodexReplay.layerReplayWithDriver(replayDriver)).pipe( Effect.mapError( (cause) => new ProviderAdapter.ProviderAdapterOpenSessionError({ @@ -9093,6 +9097,570 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect.each(["no hold", "held active goal", "unrecorded goal"] as const)( + "keeps manual compaction separate from an earlier goal turn with %s", + (ownership) => + Effect.gen(function* () { + const heldGoal = ownership === "held active goal"; + const item = { type: "contextCompaction", id: "compact-item" }; + const transcript = makeCodexReplayTranscript({ + scenario: `goal-${ownership}-during-compaction`, + entries: [ + ...(heldGoal + ? [ + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "goal-turn-1", + prompt: objective, + }), + notification("goal active", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("active", 0), + }), + notification("first turn done", "turn/completed", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "goal-turn-1", status: "completed" }), + }), + ] + : sessionStart), + ...request( + heldGoal ? 4 : 3, + "thread/compact/start", + { threadId: nativeThreadId }, + {}, + ), + // A resumed thread has no goal state until Codex reports it. + ...(ownership === "unrecorded goal" + ? [] + : [ + notification("goal progress", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("active", 10), + }), + ]), + notification("continuation", "turn/started", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "goal-turn-2", status: "inProgress" }), + }), + notification("continuation done", "turn/completed", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ + id: "goal-turn-2", + status: heldGoal ? "completed" : "interrupted", + }), + }), + ...(ownership === "no hold" + ? [ + notification("failed continuation", "turn/started", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "failed-goal-turn", status: "inProgress" }), + }), + notification("continuation error", "error", { + threadId: nativeThreadId, + turnId: "failed-goal-turn", + willRetry: false, + error: { + message: "goal continuation failed", + codexErrorInfo: null, + additionalDetails: null, + }, + }), + notification("failed continuation done", "turn/completed", { + threadId: nativeThreadId, + turn: { + ...makeCodexReplayTurn({ id: "failed-goal-turn", status: "failed" }), + error: { + message: "goal continuation failed", + codexErrorInfo: null, + additionalDetails: null, + }, + }, + }), + ] + : []), + notification("compaction", "turn/started", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "compact-turn", status: "inProgress" }), + }), + ...(["item/started", "item/completed"] as const).map((method) => + notification(method, method, { + threadId: nativeThreadId, + turnId: "compact-turn", + item, + }), + ), + notification("compaction done", "turn/completed", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "compact-turn", status: "completed" }), + }), + notification("accounted", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("active", 20), + }), + ], + }); + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const providerTurnId = (nativeTurnId: string) => + idAllocator.derive.providerTurn({ + driver: CodexAdapterV2.CODEX_DRIVER_KIND, + nativeTurnId, + }); + // This milestone also arrives when ownership is broken and compaction + // never emits a terminal, so regressions fail assertions instead of hanging. + const accounted = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness(transcript, (event) => + event.type === "provider_thread.updated" && event.providerThread.goal?.tokensUsed === 20 + ? Deferred.succeed(accounted, undefined) + : Effect.void, + ); + if (heldGoal) { + yield* harness.runtime.startTurn(yield* goalTurnInput(harness, objective)); + } + yield* harness.runtime.compactThread!( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("compact-attempt"), + text: "/compact", + }), + ).pipe(Effect.forkScoped); + yield* Deferred.await(accounted); + + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "provider_turn.updated" && event.providerTurn.status === "running" + ? [ + [ + event.providerTurn.id, + event.providerTurn.runAttemptId, + event.providerTurn.nodeId, + ], + ] + : [], + ), + [ + ...(heldGoal + ? ["goal-turn-1", "goal-turn-2"].map((turnId) => [ + providerTurnId(turnId), + RunAttemptId.make("goal-attempt"), + NodeId.make("node-goal-attempt"), + ]) + : []), + [ + providerTurnId("compact-turn"), + RunAttemptId.make("compact-attempt"), + NodeId.make("node-compact-attempt"), + ], + ], + ); + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "turn_item.updated" && event.turnItem.type === "compaction" + ? [[event.turnItem.providerTurnId, event.turnItem.runId, event.turnItem.status]] + : [], + ), + [ + [providerTurnId("compact-turn"), RunId.make("run-compact-attempt"), "running"], + [providerTurnId("compact-turn"), RunId.make("run-compact-attempt"), "completed"], + ], + ); + assert.deepEqual( + harness + .terminalEvents() + .filter( + (event) => !heldGoal || event.providerTurnId !== providerTurnId("goal-turn-2"), + ) + .map((event) => [event.providerTurnId, event.status]), + [[providerTurnId("compact-turn"), "completed"]], + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + it.effect.each(["interrupted", "failed"] as const)( + "settles a manual compaction that ends %s before its first item", + (terminalStatus) => + Effect.gen(function* () { + const failure = { + message: "compaction failed", + codexErrorInfo: null, + additionalDetails: null, + }; + const hookRun = (status: "running" | "stopped") => ({ + threadId: nativeThreadId, + turnId: "compact-turn", + run: { + id: "pre-compact-hook", + eventName: "preCompact", + source: "project", + sourcePath: "/workspace/.codex/hooks.json", + scope: "turn", + executionMode: "sync", + handlerType: "command", + displayOrder: 1, + startedAt: 1782622440, + status, + entries: status === "stopped" ? [{ kind: "stop", text: "Not now" }] : [], + }, + }); + const transcript = makeCodexReplayTranscript({ + scenario: `itemless-compaction-${terminalStatus}`, + entries: [ + ...sessionStart, + // An active goal could own an itemless start; only the PreCompact hook names it. + ...(terminalStatus === "interrupted" + ? [ + notification("goal active", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("active", 10), + }), + ] + : []), + ...request(3, "thread/compact/start", { threadId: nativeThreadId }, {}), + notification("compaction", "turn/started", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "compact-turn", status: "inProgress" }), + }), + ...(terminalStatus === "interrupted" + ? [ + notification("hook started", "hook/started", hookRun("running")), + notification("hook stopped", "hook/completed", hookRun("stopped")), + ] + : [ + notification("compaction error", "error", { + threadId: nativeThreadId, + turnId: "compact-turn", + willRetry: false, + error: failure, + }), + ]), + notification("compaction terminal", "turn/completed", { + threadId: nativeThreadId, + turn: { + ...makeCodexReplayTurn({ id: "compact-turn", status: terminalStatus }), + ...(terminalStatus === "failed" ? { error: failure } : {}), + }, + }), + notification("accounted", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("active", 20), + }), + ], + }); + const accounted = yield* Deferred.make(); + const harness = yield* makeCodexReplayHarness(transcript, (event) => + event.type === "provider_thread.updated" && event.providerThread.goal?.tokensUsed === 20 + ? Deferred.succeed(accounted, undefined) + : Effect.void, + ); + yield* harness.runtime.compactThread!( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("compact-attempt"), + text: "/compact", + }), + ).pipe(Effect.forkScoped); + yield* Deferred.await(accounted); + + assert.deepEqual( + harness.terminalEvents().map((event) => [event.providerTurnId, event.status]), + [ + [ + (yield* IdAllocator.IdAllocatorV2).derive.providerTurn({ + driver: CodexAdapterV2.CODEX_DRIVER_KIND, + nativeTurnId: "compact-turn", + }), + terminalStatus, + ], + ], + ); + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + + it.effect("run.interrupt stops compact after RPC success and before its first item", () => + Effect.scoped( + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const cwd = yield* fs.makeTempDirectoryScoped({ prefix: "t3-compact-stop-" }); + const interrupt = request( + 5, + "turn/interrupt", + { threadId: nativeThreadId, turnId: "compact-turn" }, + {}, + ); + const transcript = makeCodexReplayTranscript({ + scenario: "compact-stop-before-first-item", + entries: [ + ...codexReplayPreamble({ + nativeThreadId, + nativeTurnId: "seed-turn", + prompt: "Prepare context", + }), + notification("seed completed", "turn/completed", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "seed-turn", status: "completed" }), + }), + notification("goal active before compact", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("active", 10), + }), + ...request(4, "thread/compact/start", { threadId: nativeThreadId }, {}), + notification("compact started", "turn/started", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "compact-turn", status: "inProgress" }), + }), + notification("start accounted", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("active", 11), + }), + ...interrupt.slice(0, 1), + notification("goal update before interrupt response", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("active", 11), + }), + ...interrupt.slice(1), + ...(["item/started", "item/completed"] as const).map((method) => + notification(method, method, { + threadId: nativeThreadId, + turnId: "compact-turn", + item: { type: "contextCompaction", id: "late-compact-item" }, + }), + ), + notification("compact interrupted", "turn/completed", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "compact-turn", status: "interrupted" }), + }), + notification("late events accounted", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("paused", 12), + }), + ], + }); + const localTranscript = yield* decodeReplayTranscriptJson( + (yield* encodeReplayTranscriptJson(transcript)).replaceAll( + yield* encodeStringJson("/workspace"), + yield* encodeStringJson(cwd), + ), + ); + const goalActiveAllowed = yield* Deferred.make(); + const lateItemsAllowed = yield* Deferred.make(); + const nativeInterrupt = yield* Deferred.make(); + const replayDriver = yield* CodexReplay.makeReplayDriver(localTranscript, { + beforeEmitInbound: (entry) => + entry.label === "goal active before compact" + ? Deferred.await(goalActiveAllowed) + : entry.label === "turn/interrupt" + ? Deferred.succeed(nativeInterrupt, undefined).pipe(Effect.asVoid) + : entry.label === "item/started" + ? Deferred.await(lateItemsAllowed) + : Effect.void, + }); + const spawner = yield* ChildProcessSpawner.ChildProcessSpawner; + for (const args of [ + ["init", "--quiet"], + [ + "-c", + "user.name=Test", + "-c", + "user.email=test@example.com", + "commit", + "--allow-empty", + "--quiet", + "-m", + "Initial commit", + ], + ]) { + assert.equal( + Number(yield* spawner.exitCode(ChildProcess.make("git", args, { cwd }))), + 0, + ); + } + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const threadId = ThreadId.make("thread:compact-stop"); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create-compact-stop"), + threadId, + projectId: ProjectId.make("project:compact-stop"), + title: "Compact Stop", + modelSelection: CODEX_TEST_MODEL_SELECTION, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const seedSettled = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "run.updated" && + (event.payload.status === "waiting" || event.payload.status === "completed"), + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("seed-compact"), + threadId, + messageId: MessageId.make("message:seed-compact"), + text: "Prepare context", + attachments: [], + createdBy: "user", + creationSource: "web", + dispatchMode: { type: "start_immediately" }, + }); + yield* worker.drain(); + assert.isNull((yield* Ref.get(replayDriver.state)).failure); + yield* Fiber.join(seedSettled); + yield* worker.drain(); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runs.at(-1)?.status, + "completed", + ); + const session = Option.getOrThrow( + yield* (yield* ProviderSessionManager.ProviderSessionManagerV2).get( + (yield* orchestrator.getThreadProjection(threadId)).providerThreads.at(-1)! + .providerSessionId!, + ), + ); + const goalReady = yield* session.events.pipe( + Stream.filter( + (event) => + event.type === "provider_thread.updated" && + event.providerThread.goal?.tokensUsed === 10, + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(goalActiveAllowed, undefined); + yield* Fiber.join(goalReady); + const started = yield* orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "provider-thread.updated" && event.payload.goal?.tokensUsed === 11, + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("start-compact"), + threadId, + messageId: MessageId.make("message:compact"), + text: "/compact", + attachments: [], + createdBy: "user", + creationSource: "web", + dispatchMode: { type: "start_immediately" }, + }); + // The start effect may remain unsettled until ownership is classified. + const starting = yield* worker.drain().pipe(Effect.forkScoped); + yield* Fiber.join(started); + const before = yield* orchestrator.getThreadProjection(threadId); + const run = before.runs.at(-1)!; + assert.equal(before.providerThreads.at(-1)?.goal?.status, "active"); + assert.isFalse( + before.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId), + ); + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("stop-compact"), + threadId, + runId: run.id, + }); + yield* Fiber.join(starting); + yield* worker.drain(); + assert.isNull((yield* Ref.get(replayDriver.state)).failure); + yield* Deferred.await(nativeInterrupt); + const stopped = yield* orchestrator.getThreadProjection(threadId); + assert.equal(stopped.runs.at(-1)?.status, "interrupted"); + assert.equal(stopped.attempts.at(-1)?.status, "interrupted"); + const runStatuses: Array = []; + const attemptStatuses: Array = []; + yield* orchestrator.streamDomainEvents.pipe( + Stream.runForEach((event) => + Effect.sync(() => { + if (event.type === "run.updated" && event.payload.id === run.id) { + runStatuses.push(event.payload.status); + } + if ( + event.type === "run-attempt.updated" && + event.payload.id === run.activeAttemptId + ) { + attemptStatuses.push(event.payload.status); + } + }), + ), + Effect.forkChild({ startImmediately: true }), + ); + // Stopped runs reject persisted thread updates; observe the processed adapter event. + const accounted = yield* session.events.pipe( + Stream.filter( + (event) => + event.type === "provider_thread.updated" && + event.providerThread.goal?.tokensUsed === 12, + ), + Stream.runHead, + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(lateItemsAllowed, undefined); + yield* Fiber.join(accounted); + yield* worker.drain(); + const after = yield* orchestrator.getThreadProjection(threadId); + assert.equal(after.runs.at(-1)?.status, "interrupted"); + assert.equal(after.attempts.at(-1)?.status, "interrupted"); + assert.deepEqual( + runStatuses.filter((status) => status !== "interrupted"), + [], + ); + assert.deepEqual( + attemptStatuses.filter((status) => status !== "interrupted"), + [], + ); + assert.isFalse( + after.providerTurns.some((turn) => turn.runAttemptId === run.activeAttemptId), + ); + assert.isFalse(after.turnItems.some((item) => item.type === "compaction")); + assert.isNull((yield* Ref.get(replayDriver.state)).failure); + }).pipe( + Effect.provide( + ProviderReplayHarness.layerWithRegistry( + { name: "codex-compact-stop", runtimePolicyOverride: { cwd } }, + CodexAdapterV2Testkit.layer({ transcript: localTranscript, driver: replayDriver }), + { runEffectWorker: false }, + ), + ), + ); + }).pipe(Effect.provide(NodeServices.layer)), + ), + ); + it.effect("settles a run held for the next goal turn when Stop arrives in between", () => Effect.gen(function* () { const prompt = "Keep going"; @@ -9190,6 +9758,359 @@ describe("CodexAdapterV2 post-settle continuation", () => { ), ); + it.effect.each(["goal first item", "goal starts after Stop", "compact first item"] as const)( + "classifies the deferred turn after goal Stop (%s)", + (scenario) => + Effect.gen(function* () { + const compactTurn = scenario === "compact first item"; + const stopBeforeStart = scenario === "goal starts after Stop"; + const nativeTurnId = compactTurn ? "compact-turn" : "goal-turn-b"; + const prompt = "Keep going"; + const nativeStart = notification("deferred start", "turn/started", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: nativeTurnId, status: "inProgress" }), + }); + const classified = notification("classification accounted", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("paused", 12), + }); + const firstItem = notification("late first item", "item/started", { + threadId: nativeThreadId, + turnId: nativeTurnId, + item: compactTurn + ? { type: "contextCompaction", id: "compact-item" } + : { type: "agentMessage", id: "late-message", text: "Late continuation" }, + }); + const transcript = makeCodexReplayTranscript({ + scenario: `goal-stop-classification-${scenario}`, + entries: [ + ...codexReplayPreamble({ nativeThreadId, nativeTurnId: "goal-turn-a", prompt }), + notification("goal active", "thread/goal/updated", { + threadId: nativeThreadId, + turnId: "goal-turn-a", + goal: codexGoal("active", 10), + }), + notification("first turn done", "turn/completed", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "goal-turn-a", status: "completed" }), + }), + ...request(4, "thread/compact/start", { threadId: nativeThreadId }, {}), + ...(stopBeforeStart ? [] : [nativeStart]), + notification("goal accounted", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("active", 11), + }), + ...request( + 5, + "thread/goal/set", + { threadId: nativeThreadId, status: "paused" }, + { goal: codexGoal("paused", 11) }, + ), + ...(stopBeforeStart ? [nativeStart] : []), + firstItem, + ...(compactTurn ? [] : [classified]), + ...(!compactTurn + ? request( + 6, + "thread/goal/set", + { threadId: nativeThreadId, status: "paused" }, + { goal: codexGoal("paused", 12) }, + ) + : []), + ...(compactTurn + ? [] + : request( + 7, + "turn/interrupt", + { threadId: nativeThreadId, turnId: nativeTurnId }, + {}, + )), + ...(compactTurn + ? [ + notification("compaction item done", "item/completed", { + threadId: nativeThreadId, + turnId: nativeTurnId, + item: { type: "contextCompaction", id: "compact-item" }, + }), + ] + : []), + notification("native terminal", "turn/completed", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ + id: nativeTurnId, + status: compactTurn ? "completed" : "interrupted", + }), + }), + ...(compactTurn + ? [ + ...request(6, "thread/compact/start", { threadId: nativeThreadId }, {}), + notification("late goal start", "turn/started", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "goal-turn-b", status: "inProgress" }), + }), + notification("late goal item", "item/started", { + threadId: nativeThreadId, + turnId: "goal-turn-b", + item: { + type: "agentMessage", + id: "late-goal-message", + text: "Late continuation", + }, + }), + classified, + ...request( + 7, + "thread/goal/set", + { threadId: nativeThreadId, status: "paused" }, + { goal: codexGoal("paused", 12) }, + ), + ...request( + 8, + "turn/interrupt", + { threadId: nativeThreadId, turnId: "goal-turn-b" }, + {}, + ), + notification("late goal interrupted", "turn/completed", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "goal-turn-b", status: "interrupted" }), + }), + notification("second compact start", "turn/started", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "compact-turn-b", status: "inProgress" }), + }), + notification("second compact item", "item/started", { + threadId: nativeThreadId, + turnId: "compact-turn-b", + item: { type: "contextCompaction", id: "compact-item-b" }, + }), + notification("second compact item done", "item/completed", { + threadId: nativeThreadId, + turnId: "compact-turn-b", + item: { type: "contextCompaction", id: "compact-item-b" }, + }), + notification("second compact terminal", "turn/completed", { + threadId: nativeThreadId, + turn: makeCodexReplayTurn({ id: "compact-turn-b", status: "completed" }), + }), + ] + : []), + notification("stop accounted", "thread/goal/updated", { + threadId: nativeThreadId, + goal: codexGoal("paused", 13), + }), + ], + }); + const started = yield* Deferred.make(); + const accounted = yield* Deferred.make(); + const classificationAccounted = yield* Deferred.make(); + const firstItemAllowed = yield* Deferred.make(); + const interruptRequested = yield* Deferred.make(); + const firstCompactCompleted = yield* Deferred.make(); + const lateGoalStartAllowed = yield* Deferred.make(); + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const providerTurnId = (id: string) => + idAllocator.derive.providerTurn({ + driver: CodexAdapterV2.CODEX_DRIVER_KIND, + nativeTurnId: id, + }); + const interrupts: Array = []; + const pauses: Array = []; + const harness = yield* makeCodexReplayHarness( + transcript, + (event) => + event.type === "turn.terminal" && + event.providerTurnId === providerTurnId("compact-turn") && + event.status === "completed" + ? Deferred.succeed(firstCompactCompleted, undefined) + : event.type === "provider_thread.updated" + ? event.providerThread.goal?.tokensUsed === 11 + ? Deferred.succeed(started, undefined) + : event.providerThread.goal?.tokensUsed === 12 + ? Deferred.succeed(classificationAccounted, undefined) + : event.providerThread.goal?.tokensUsed === 13 + ? Deferred.succeed(accounted, undefined) + : Effect.void + : Effect.void, + (method, params) => + method === "turn/interrupt" + ? Effect.sync(() => interrupts.push(params)).pipe( + Effect.andThen(Deferred.succeed(interruptRequested, undefined)), + Effect.asVoid, + ) + : method === "thread/goal/set" + ? Effect.sync(() => { + pauses.push(params); + }) + : Effect.void, + undefined, + { + beforeEmitInbound: (entry) => + entry.label === "late first item" || + (stopBeforeStart && entry.label === "deferred start") + ? Deferred.await(firstItemAllowed) + : compactTurn && entry.label === "late goal start" + ? Deferred.await(lateGoalStartAllowed) + : Effect.void, + }, + ); + yield* harness.runtime.startTurn( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("goal-successor-attempt"), + text: prompt, + }), + ); + yield* harness.runtime.compactThread!( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("compact-attempt"), + text: "/compact", + }), + ).pipe(Effect.forkScoped); + yield* Deferred.await(started); + const stop = yield* harness.runtime + .interruptTurn({ + providerThread: harness.providerThread, + providerTurnId: providerTurnId("goal-turn-a"), + requestRuntimeRestart: true, + }) + .pipe(Effect.forkScoped); + // Either Stop returns without guessing, or the invalid native interrupt + // provides an immediate failure milestone while the item is still gated. + yield* Effect.raceFirst(Fiber.join(stop), Deferred.await(interruptRequested)); + assert.deepEqual(interrupts, [], "Stop must not guess the deferred turn's owner"); + yield* Deferred.succeed(firstItemAllowed, undefined); + yield* Fiber.join(stop); + if (compactTurn) { + yield* Deferred.await(firstCompactCompleted); + assert.equal(providerGoals(harness.events).at(-1), "paused"); + yield* harness.runtime.compactThread!( + makeCodexTestTurnInput({ + threadId: harness.threadId, + providerThread: harness.providerThread, + now: yield* DateTime.now, + attemptId: RunAttemptId.make("compact-attempt-b"), + text: "/compact", + }), + ).pipe(Effect.forkScoped); + yield* Deferred.succeed(lateGoalStartAllowed, undefined); + } + // This notification follows classification even if containment never starts. + yield* Deferred.await(classificationAccounted); + assert.equal(providerGoals(harness.events).at(-1), "paused"); + if (compactTurn) { + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "provider_turn.updated" && + event.providerTurn.id === providerTurnId("goal-turn-b") + ? [[event.providerTurn.runAttemptId, event.providerTurn.status]] + : [], + ), + [], + "The queued ordinary goal turn must not claim the second compact attempt", + ); + } + assert.deepEqual( + pauses, + Array.from({ length: 2 }, () => ({ threadId: nativeThreadId, status: "paused" })), + "Stop must retain intent until the late goal continuation is contained", + ); + yield* Deferred.await(accounted); + assert.deepEqual(interrupts, [{ threadId: nativeThreadId, turnId: "goal-turn-b" }]); + assert.isFalse( + harness.events.some( + (event) => + (event.type === "provider_turn.updated" && + event.providerTurn.id === providerTurnId("goal-turn-b")) || + (event.type === "turn_item.updated" && + event.turnItem.providerTurnId === providerTurnId("goal-turn-b")), + ), + ); + assert.deepEqual( + harness.terminalEvents().map((event) => [event.providerTurnId, event.status]), + [ + [providerTurnId("goal-turn-a"), "interrupted"], + ...(compactTurn + ? [ + [providerTurnId(nativeTurnId), "completed"], + [providerTurnId("compact-turn-b"), "completed"], + ] + : []), + ], + ); + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "provider_turn.updated" && + event.providerTurn.id === providerTurnId(nativeTurnId) + ? [[event.providerTurn.runAttemptId, event.providerTurn.status]] + : [], + ), + compactTurn + ? (["running", "completed"] as const).map((status) => [ + RunAttemptId.make("compact-attempt"), + status, + ]) + : [], + ); + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.providerTurnId === providerTurnId(nativeTurnId) + ? [[event.turnItem.runId, event.turnItem.type, event.turnItem.status]] + : [], + ), + compactTurn + ? (["running", "completed"] as const).map((status) => [ + RunId.make("run-compact-attempt"), + "compaction" as const, + status, + ]) + : [], + ); + if (compactTurn) { + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "provider_turn.updated" && + event.providerTurn.id === providerTurnId("compact-turn-b") + ? [[event.providerTurn.runAttemptId, event.providerTurn.status]] + : [], + ), + (["running", "completed"] as const).map((status) => [ + RunAttemptId.make("compact-attempt-b"), + status, + ]), + ); + assert.deepEqual( + harness.events.flatMap((event) => + event.type === "turn_item.updated" && + event.turnItem.providerTurnId === providerTurnId("compact-turn-b") + ? [[event.turnItem.runId, event.turnItem.type, event.turnItem.status]] + : [], + ), + (["running", "completed"] as const).map((status) => [ + RunId.make("run-compact-attempt-b"), + "compaction" as const, + status, + ]), + ); + } + }).pipe( + Effect.scoped, + Effect.provide( + Layer.mergeAll( + IdAllocator.layer, + McpProviderSessions.layer, + NodeServices.layer, + NodeCrypto.layer, + ), + ), + ), + ); + it.effect("moves a Stop that names an earlier goal turn to the turn Codex continued with", () => Effect.gen(function* () { const prompt = "Keep going"; diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 6953636a8dcf..33312c434d5f 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -1910,6 +1910,21 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( const pendingRootTurns = yield* Ref.make( new Map(), ); + const pendingCompactions = new Map< + string, + { + readonly input: ProviderAdapter.ProviderAdapterV2TurnInput; + readonly classified: Deferred.Deferred; + } + >(); + const stoppedRootThreads = new Map(); + // While a /compact is pending, a native start may still be a goal or resumed turn + // whose goal state is not known yet. Classify it before binding it to a run. + const unclaimedRootTurns = new Map< + string, + { readonly threadId: string; readonly startedAt: DateTime.Utc } + >(); + const manualCompactionTurns = new Set(); const turnWaiters = yield* Ref.make(new Map>()); const subagentThreads = yield* Ref.make(new Map()); const subagentSelections = new Map>(); @@ -4730,6 +4745,141 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }), ); + const continueGoalTurn = Effect.fnUntraced(function* (input: { + readonly threadId: string; + readonly turnId: string; + readonly startedAt: DateTime.Utc; + }) { + const goalHold = goalHolds.get(input.threadId); + if (goalHold !== undefined) { + goalHolds.delete(input.threadId); + yield* emitProviderEvent({ + type: "provider_turn.updated", + driver: CODEX_PROVIDER, + threadId: goalHold.context.projectionThreadId, + providerTurn: goalHold.completedTurn, + }); + const next = yield* registerRootTurn({ + turnInput: { + ...goalHold.context.input, + providerTurnOrdinal: goalHold.context.providerTurnOrdinal + 1, + }, + nativeTurnId: input.turnId, + startedAt: input.startedAt, + }); + const goalRun = continuedRuns.get(goalHold.context.providerTurnId) ?? [ + goalHold.context, + ]; + goalRun.push(next); + continuedRuns.set(goalHold.context.providerTurnId, goalRun); + continuedRuns.set(next.providerTurnId, goalRun); + yield* Deferred.succeed(goalHold.next, next); + return true; + } + return false; + }); + + const stopUnclaimedTurn = (input: { readonly threadId: string; readonly turnId: string }) => + client.request("turn/interrupt", input).pipe( + Effect.timeoutOption("10 seconds"), + Effect.catch((cause) => + Effect.logWarning("orchestration-v2.codex-unclaimed-turn-stop-failed", { cause }), + ), + ); + + const stopOrphanGoalTurn = Effect.fnUntraced(function* (input: { + readonly threadId: string; + readonly turnId: string; + }) { + // A goal turn with no run to own it (Codex continued after the run + // settled, or raced a Stop): stop it and pause the goal rather than + // work out of sight. + if ( + rootProviderThreads.has(input.threadId) && + (goalsByNativeThread.get(input.threadId)?.status === "active" || + goalStops.has(input.threadId) || + stoppedRootThreads.get(input.threadId) === "goal") + ) { + yield* Effect.logWarning("orchestration-v2.codex-goal-turn-without-run", { + nativeThreadId: input.threadId, + nativeTurnId: input.turnId, + }); + yield* client + .request("thread/goal/set", { threadId: input.threadId, status: "paused" }) + .pipe( + Effect.catch((cause) => + Effect.logWarning("orchestration-v2.codex-goal-pause-failed", { cause }), + ), + Effect.andThen( + client.request("turn/interrupt", { + threadId: input.threadId, + turnId: input.turnId, + }), + ), + Effect.tap(() => + Effect.sync(() => { + if (stoppedRootThreads.get(input.threadId) === "goal") { + stoppedRootThreads.delete(input.threadId); + } + }), + ), + Effect.catch((cause) => + Effect.logWarning("orchestration-v2.codex-goal-turn-stop-failed", { cause }), + ), + Effect.forkIn(scope), + ); + return true; + } + return false; + }); + + // Call with the terminalization permit, as goal ownership changes here. + const claimRootTurn = Effect.fnUntraced(function* (input: { + readonly threadId: string; + readonly turnId: string; + readonly compaction: boolean; + readonly status?: CodexSchema.V2TurnCompletedNotification__TurnStatus; + readonly startedAt?: DateTime.Utc; + }) { + if ((yield* Ref.get(activeTurns)).has(input.turnId)) return; + const started = unclaimedRootTurns.get(input.turnId); + unclaimedRootTurns.delete(input.turnId); + const compaction = pendingCompactions.get(input.threadId); + // An itemless start that completes or fails with no goal to own it is the compact + // failing before its first item; an itemless interrupted start can be a resumed turn. + const itemlessCompaction = + started !== undefined && + (input.status === "completed" || input.status === "failed") && + !goalHolds.has(input.threadId) && + goalsByNativeThread.get(input.threadId)?.status !== "active" && + stoppedRootThreads.get(input.threadId) !== "goal"; + if (compaction !== undefined && (input.compaction || itemlessCompaction)) { + pendingCompactions.delete(input.threadId); + manualCompactionTurns.add(input.turnId); + yield* releaseGoalHold(input.threadId); + yield* registerRootTurn({ + turnInput: compaction.input, + nativeTurnId: input.turnId, + startedAt: started?.startedAt ?? input.startedAt ?? (yield* DateTime.now), + }); + yield* Deferred.succeed(compaction.classified, undefined); + } else if (started !== undefined) { + if (yield* continueGoalTurn({ ...input, startedAt: started.startedAt })) return; + if (input.status === undefined) yield* stopOrphanGoalTurn(input); + } + }); + + // A PreCompact hook runs before the contextCompaction item and can stop the turn itemless. + yield* client.handleServerNotification("hook/started", (payload) => + payload.turnId == null || payload.run.eventName !== "preCompact" + ? Effect.void + : claimRootTurn({ + threadId: payload.threadId, + turnId: payload.turnId, + compaction: true, + }).pipe(turnTerminalizationPermit.withPermits(1)), + ); + yield* client.handleServerNotification("turn/started", (payload) => Effect.gen(function* () { const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); @@ -4753,61 +4903,30 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( }); return; } - const goalHold = goalHolds.get(payload.threadId); - if (goalHold !== undefined) { - goalHolds.delete(payload.threadId); - yield* emitProviderEvent({ - type: "provider_turn.updated", - driver: CODEX_PROVIDER, - threadId: goalHold.context.projectionThreadId, - providerTurn: goalHold.completedTurn, - }); - const next = yield* registerRootTurn({ - turnInput: { - ...goalHold.context.input, - providerTurnOrdinal: goalHold.context.providerTurnOrdinal + 1, - }, - nativeTurnId: payload.turn.id, + if (pendingCompactions.has(payload.threadId)) { + unclaimedRootTurns.set(payload.turn.id, { + threadId: payload.threadId, startedAt: codexTimestamp(payload.turn.startedAt), }); - const goalRun = continuedRuns.get(goalHold.context.providerTurnId) ?? [ - goalHold.context, - ]; - goalRun.push(next); - continuedRuns.set(goalHold.context.providerTurnId, goalRun); - continuedRuns.set(next.providerTurnId, goalRun); - yield* Deferred.succeed(goalHold.next, next); return; } - // Pause orphan goal turns so they cannot work without an owning run. - if ( - rootProviderThreads.has(payload.threadId) && - (goalsByNativeThread.get(payload.threadId)?.status === "active" || - goalStops.has(payload.threadId)) - ) { - yield* Effect.logWarning("orchestration-v2.codex-goal-turn-without-run", { - nativeThreadId: payload.threadId, - nativeTurnId: payload.turn.id, - }); - yield* client - .request("thread/goal/set", { threadId: payload.threadId, status: "paused" }) - .pipe( - Effect.catch((cause) => - Effect.logWarning("orchestration-v2.codex-goal-pause-failed", { cause }), - ), - Effect.andThen( - client.request("turn/interrupt", { - threadId: payload.threadId, - turnId: payload.turn.id, - }), - ), - Effect.catch((cause) => - Effect.logWarning("orchestration-v2.codex-goal-turn-stop-failed", { cause }), - ), - Effect.forkIn(scope), - ); + if (stoppedRootThreads.get(payload.threadId) === "compaction") { + const stopped = { threadId: payload.threadId, turnId: payload.turn.id }; + if (!(yield* stopOrphanGoalTurn(stopped))) { + yield* stopUnclaimedTurn(stopped).pipe(Effect.forkIn(scope)); + } return; } + if ( + yield* continueGoalTurn({ + threadId: payload.threadId, + turnId: payload.turn.id, + startedAt: codexTimestamp(payload.turn.startedAt), + }) + ) + return; + if (yield* stopOrphanGoalTurn({ threadId: payload.threadId, turnId: payload.turn.id })) + return; yield* rememberSubagentTurnStarted({ nativeThreadId: payload.threadId, nativeTurnId: payload.turn.id, @@ -4833,6 +4952,14 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("error", (payload) => Effect.gen(function* () { + if (!payload.willRetry) { + yield* claimRootTurn({ + threadId: payload.threadId, + turnId: payload.turnId, + compaction: false, + status: "failed", + }).pipe(turnTerminalizationPermit.withPermits(1)); + } const context = yield* awaitActiveTurn(payload.turnId); if (context === undefined) { return; @@ -4984,6 +5111,12 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("item/started", (payload) => Effect.gen(function* () { + yield* claimRootTurn({ + threadId: payload.threadId, + turnId: payload.turnId, + compaction: payload.item.type === "contextCompaction", + startedAt: DateTime.makeUnsafe(payload.startedAtMs), + }); const context = yield* awaitActiveTurn(payload.turnId); if (context === undefined) { return; @@ -5129,6 +5262,11 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("item/completed", (payload) => Effect.gen(function* () { + yield* claimRootTurn({ + threadId: payload.threadId, + turnId: payload.turnId, + compaction: payload.item.type === "contextCompaction", + }); if (payload.item.type === "contextCompaction") // Fork requests off the notification reader so it can read their responses. yield* restoreAdditionalContext(payload.threadId).pipe( @@ -6234,6 +6372,10 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ) { const hold = goalHolds.get(nativeThreadId); if (hold === undefined || goalStops.has(nativeThreadId)) return; + if ( + Array.from(unclaimedRootTurns.values()).some((turn) => turn.threadId === nativeThreadId) + ) + return; if (expected !== undefined && hold.event !== expected) return; goalHolds.delete(nativeThreadId); yield* settleGoalHold(hold, "completed"); @@ -6389,6 +6531,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( nativeThreadId == null ? undefined : goalActivations.get(nativeThreadId); const holdsForGoal = input.context.subagent === null && + !manualCompactionTurns.has(input.nativeTurnId) && input.status === "completed" && nativeThreadId != null && (goalsByNativeThread.get(nativeThreadId)?.status === "active" || @@ -6594,6 +6737,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( if (!input.continueRun && input.context.capacityRetryPending !== undefined) { yield* Deferred.succeed(input.context.capacityRetryPending, undefined); } + manualCompactionTurns.delete(input.nativeTurnId); yield* flushReadyRootTerminals(); if (!retainSettledContext && !interruptInProgress) { yield* Ref.update(runningCommandItemsByTurn, (current) => { @@ -6773,6 +6917,13 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( yield* client.handleServerNotification("turn/completed", (payload) => Effect.gen(function* () { + yield* claimRootTurn({ + threadId: payload.threadId, + turnId: payload.turn.id, + compaction: payload.turn.items.some((item) => item.type === "contextCompaction"), + status: payload.turn.status, + startedAt: codexTimestamp(payload.turn.startedAt), + }).pipe(turnTerminalizationPermit.withPermits(1)); const context = (yield* Ref.get(activeTurns)).get(payload.turn.id); if (context === undefined) { return; @@ -6937,6 +7088,24 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( } return yield* turnTerminalizationPermit.withPermits(1)( Effect.gen(function* () { + if ( + goalHolds.get(goalThreadId)?.context.providerTurnId === + latestRunTurnId(turnInput.providerTurnId) + ) { + if (pendingCompactions.has(goalThreadId)) { + // A queued goal start can arrive after the pause and compact classification. + stoppedRootThreads.set(goalThreadId, "goal"); + } else { + const deferredStart = Array.from(unclaimedRootTurns).findLast( + ([, started]) => started.threadId === goalThreadId, + ); + if (deferredStart !== undefined) { + const [turnId, started] = deferredStart; + unclaimedRootTurns.delete(turnId); + yield* continueGoalTurn({ ...started, turnId }); + } + } + } const providerTurnId = latestRunTurnId(turnInput.providerTurnId); const hold = goalHolds.get(goalThreadId); const held = hold !== undefined && hold.context.providerTurnId === providerTurnId; @@ -7020,6 +7189,7 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( ...(yield* Ref.get(settledTurns)), ...(yield* Ref.get(activeTurns)), ]); + stoppedRootThreads.delete(threadId); const started = yield* client.request("turn/start", turnStartParams); const previousTurn = previousTurns.get(started.turn.id) ?? @@ -7391,17 +7561,37 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( compactThread: Effect.fn("CodexAdapterV2.compactThread")( function* (turnInput) { const threadId = yield* getNativeThreadId(turnInput.providerThread); - yield* Ref.update(pendingRootTurns, (current) => - new Map(current).set(threadId, turnInput), - ); - // Clear on interruption; a late turn/started must not be adopted after an interrupted start. - yield* client.request("thread/compact/start", { threadId }).pipe( - Effect.onError(() => - Ref.update(pendingRootTurns, (current) => { - const next = new Map(current); - next.delete(threadId); - return next; - }), + const pending = { input: turnInput, classified: yield* Deferred.make() }; + if (stoppedRootThreads.get(threadId) === "compaction") { + stoppedRootThreads.delete(threadId); + } + pendingCompactions.set(threadId, pending); + // Keep the start effect cancellable until Stop can target a published native turn. + yield* Effect.gen(function* () { + yield* client.request("thread/compact/start", { threadId }); + yield* Deferred.await(pending.classified).pipe( + Effect.raceFirst(client.awaitTermination), + ); + }).pipe( + Effect.onError((cause) => + turnTerminalizationPermit.withPermits(1)( + Effect.gen(function* () { + if (pendingCompactions.get(threadId) !== pending) return; + pendingCompactions.delete(threadId); + const cancelled = Cause.hasInterrupts(cause); + // Codex can report its native start after cancellation of the RPC. + if (cancelled) stoppedRootThreads.set(threadId, "compaction"); + for (const [turnId, started] of unclaimedRootTurns) { + if (started.threadId !== threadId) continue; + unclaimedRootTurns.delete(turnId); + if (cancelled) { + yield* stopUnclaimedTurn({ threadId, turnId }).pipe(Effect.forkIn(scope)); + } else if (!(yield* continueGoalTurn({ ...started, turnId }))) { + yield* stopOrphanGoalTurn({ ...started, turnId }); + } + } + }), + ), ), ); },