From 8e3811d2684a84cd204f8b136c5519c0a331fd63 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 16:56:41 +0000 Subject: [PATCH 1/9] test(server): reproduce #30988, Force stop tears down a queued follow-up's session Force stop settles the targeted run, which lets a queued follow-up start its turn on the same session and provider thread before the release runs. The release ignores busy turns of the thread's own provider thread, so it tears the session down under the follow-up. These tests reproduce #30988. Co-Authored-By: Claude Opus 5.5 --- .../src/orchestration-v2/ThreadStop.test.ts | 177 ++++++++++++++++++ 1 file changed, 177 insertions(+) diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts index 6b3ea08a86449..44fa22e59bef7 100644 --- a/apps/server/src/orchestration-v2/ThreadStop.test.ts +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -20,7 +20,9 @@ import { import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as Stream from "effect/Stream"; @@ -38,8 +40,10 @@ import * as EffectWorker from "./EffectWorker.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; +import * as ProviderSessionManager from "./ProviderSessionManager.ts"; import * as ThreadManagementService from "./ThreadManagementService.ts"; import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; const instanceId = ProviderInstanceId.make("codex"); @@ -1742,3 +1746,176 @@ it.effect.each(["Stop", "Force stop"] as const)( ); }).pipe(Effect.provide(layerTest)), ); + +it.effect("Force stop keeps the session a queued follow-up started its turn on", () => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("thread-stop-force-stop-follow-up"); + const driver = adapter.driver; + const events = yield* Queue.unbounded(); + const started: Array = []; + const interrupted: Array = []; + const liveAdapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + ...adapter, + openSession: (input) => + Effect.gen(function* () { + const now = yield* DateTime.now; + return { + instanceId, + driver, + providerSessionId: input.providerSessionId, + providerSession: { + id: input.providerSessionId, + driver, + providerInstanceId: instanceId, + status: "ready", + cwd, + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.fromQueue(events), + ensureThread: ({ threadId }) => + Effect.succeed({ + id: ProviderThreadId.make(`provider-thread:codex:${threadId}`), + driver, + providerInstanceId: instanceId, + providerSessionId: input.providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => + Effect.gen(function* () { + started.push(turn); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: { + driver, + nativeId: `native:${turn.attemptId}`, + strength: "strong", + }, + ordinal: turn.providerTurnOrdinal, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + }), + steerTurn: () => Effect.die("unused"), + interruptTurn: (interrupt) => Effect.sync(() => interrupted.push(interrupt)), + respondToRuntimeRequest: () => Effect.die("unused"), + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }; + // The follow-up the settlement promotes starts its turn just before Force stop releases. + const layerSessionsStartingFollowUpFirst = Layer.effect( + ProviderSessionManager.ProviderSessionManagerV2, + Effect.gen(function* () { + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + return ProviderSessionManager.ProviderSessionManagerV2.of({ + ...sessions, + releaseUnlessBusy: (input) => + worker.drain().pipe(Effect.orDie, Effect.andThen(sessions.releaseUnlessBusy(input))), + }); + }), + ); + const layerLive = ThreadManagementService.layer.pipe( + Layer.provide(layerSessionsStartingFollowUpFirst), + Layer.provideMerge( + Layer.mergeAll( + layerDatabase, + ProjectionStore.layer.pipe(Layer.provide(layerDatabase)), + EventStore.layer.pipe(Layer.provide(layerDatabase)), + EffectOutbox.layer.pipe(Layer.provide(layerDatabase)), + ProviderReplayHarness.layerWithRegistry( + { name: "thread-stop-force-stop-follow-up" }, + ProviderAdapterRegistry.layerFromAdapters([liveAdapter]), + { databaseLayer: layerDatabase, runEffectWorker: false }, + ), + ), + ), + Layer.provideMerge(ServerSettings.layerTest()), + Layer.provideMerge(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))), + ); + + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const threads = yield* ThreadManagementService.ThreadManagementService; + const threadId = ThreadId.make("thread:force-stop-follow-up"); + const runningTurn = orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => event.type === "provider-turn.updated" && event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create:force-stop-follow-up"), + threadId, + projectId: ProjectId.make("project:thread-stop"), + title: "Force stop follow-up", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const firstRunning = yield* runningTurn; + yield* send(threadId, "first", "start_immediately"); + yield* worker.drain(); + yield* Fiber.join(firstRunning); + yield* send(threadId, "follow-up", "queue_after_active"); + + yield* threads.forceStopThread({ threadId }); + yield* worker.drain(); + + assert.deepEqual( + started.map((turn) => turn.runOrdinal), + [1, 2], + "the queued follow-up did not start its turn before the release", + ); + const followUp = started[1]!; + const providerSessionId = (yield* orchestrator.getThreadProjection(threadId)) + .providerThreads[0]!.providerSessionId!; + assert.isFalse( + interrupted.some((interrupt) => + String(interrupt.providerTurnId).includes(followUp.attemptId), + ), + "Force stop of the first run interrupted the follow-up's turn", + ); + assert.isTrue( + yield* sessions.isLive(providerSessionId), + "Force stop of the first run released the session the follow-up's turn is running on", + ); + }).pipe(Effect.provide(layerLive)); + }), + ), +); From f15fe88c79186b673fb1269a9ff4ef2bd616bd3e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 17:05:37 +0000 Subject: [PATCH 2/9] fix(server): Force stop leaves a queued follow-up's session running Force stop settles the run it targets, which can promote a queued follow-up that starts its turn on the same session before the release runs. The release ignored every busy turn of the thread's own provider thread, so it tore the session down under the follow-up, and a declined release fell back to detaching the thread, which would interrupt the follow-up too. The release now ignores only the thread's busy turns from runs up to the targeted run's ordinal, still checked atomically with turn starts. When the release is declined while a later run of the thread is active, Force stop leaves the session and the thread alone and reports `released`. Co-Authored-By: Claude Opus 5.5 --- .../ProviderSessionManager.test.ts | 1 + .../ProviderSessionManager.ts | 31 +++++++++++++------ .../ThreadManagementService.ts | 13 +++++++- packages/contracts/src/orchestrationV2.ts | 8 ++--- 4 files changed, 39 insertions(+), 14 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index d8c046c018651..b3c92cb5dfa2f 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -1572,6 +1572,7 @@ it.effect( manager.releaseUnlessBusy({ providerSessionId, ownProviderThreadIds: [providerThreadOf(own).id], + throughRunOrdinal: 1, reason: "manual_shutdown", }); diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index f06011676c667..3d21053294a66 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -63,6 +63,17 @@ export const PROVIDER_INTERRUPT_TIMEOUT_MS = 15 * 1000; const busyTurnPrefix = (providerThreadId: ProviderThreadId) => `${providerThreadId}#`; const busyTurnKey = (providerThreadId: ProviderThreadId, runOrdinal: number) => `${busyTurnPrefix(providerThreadId)}${runOrdinal}`; +interface OwnBusyTurns { + readonly providerThreadIds: ReadonlyArray; + readonly throughRunOrdinal: number; +} +const isOwnBusyTurn = (turnKey: string, own: OwnBusyTurns) => + own.providerThreadIds.some((id) => { + const prefix = busyTurnPrefix(id); + return ( + turnKey.startsWith(prefix) && Number(turnKey.slice(prefix.length)) <= own.throughRunOrdinal + ); + }); const UNLOAD_THREAD_TIMEOUT_MS = 10 * 1000; const WORKING_RUN_STATUSES: ReadonlySet = new Set([ "preparing", @@ -190,13 +201,14 @@ export interface ProviderSessionManagerV2Shape { readonly revokeMcpCredential?: boolean; }) => Effect.Effect; /** - * Releases the session unless, at the moment of removal, a turn of a provider thread outside - * `ownProviderThreadIds` is busy on it. Checked atomically with turn starts. Succeeds with - * whether it was released. + * Releases the session unless, at the moment of removal, a turn is busy on it other than one + * of `ownProviderThreadIds` from a run up to `throughRunOrdinal`. Checked atomically with turn + * starts. Succeeds with whether it was released. */ readonly releaseUnlessBusy: (input: { readonly providerSessionId: ProviderSessionId; readonly ownProviderThreadIds: ReadonlyArray; + readonly throughRunOrdinal: number; readonly reason: ProviderSessionReleaseReason; readonly detail?: string; }) => Effect.Effect; @@ -922,7 +934,7 @@ export const layerWithOptions = ( function* (input: { readonly providerSessionId: ProviderSessionId; readonly onlyIfIdleGeneration?: number; - readonly keepIfBusyOtherThan?: ReadonlyArray; + readonly keepIfBusyOtherThan?: OwnBusyTurns; }): Effect.fn.Return, DateTime.Utc]> { const key = sessionKey(input.providerSessionId); const candidate = (yield* Ref.get(sessions)).get(key); @@ -945,9 +957,7 @@ export const layerWithOptions = ( const own = input.keepIfBusyOtherThan; if ( own !== undefined && - [...existing.busyTurns].some( - (turnKey) => !own.some((id) => turnKey.startsWith(busyTurnPrefix(id))), - ) + [...existing.busyTurns].some((turnKey) => !isOwnBusyTurn(turnKey, own)) ) { return ["kept", current] as const; } @@ -1009,7 +1019,7 @@ export const layerWithOptions = ( readonly detachedThreadId?: ThreadId; readonly cancelIdleFiber?: boolean; readonly onlyIfIdleGeneration?: number; - readonly keepIfBusyOtherThan?: ReadonlyArray; + readonly keepIfBusyOtherThan?: OwnBusyTurns; readonly gracefulSubscribers?: boolean; }) => Effect.acquireUseRelease( @@ -2366,7 +2376,10 @@ export const layerWithOptions = ( providerSessionId: input.providerSessionId, reason: input.reason, ...(input.detail === undefined ? {} : { detail: input.detail }), - keepIfBusyOtherThan: input.ownProviderThreadIds, + keepIfBusyOtherThan: { + providerThreadIds: input.ownProviderThreadIds, + throughRunOrdinal: input.throughRunOrdinal, + }, }); // Identity, not presence: open can install a replacement under the same id meanwhile. return before !== undefined && (yield* Ref.get(sessions)).get(key) !== before; diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index ed2935fedbf12..8326f2d0d67d2 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1186,6 +1186,16 @@ const make = Effect.gen(function* () { const ownProviderThreadIds = records.providerThreads.flatMap((providerThread) => providerThread.appThreadId === input.threadId ? [providerThread.id] : [], ); + // Settling the target can promote a queued follow-up onto the same session; that run is + // not Force stop's to end. + const throughRunOrdinal = target?.ordinal ?? 0; + const laterRunActive = orchestrator + .getThreadRecords(input.threadId, ["runs"]) + .pipe( + Effect.map(({ runs }) => + runs.some((run) => run.ordinal > throughRunOrdinal && isActiveRun(run)), + ), + ); const detach = (providerSessionId: ProviderSessionId) => sessions .detach({ providerSessionId, threadId: input.threadId, detail: FORCE_STOP_REASON }) @@ -1223,6 +1233,7 @@ const make = Effect.gen(function* () { .releaseUnlessBusy({ providerSessionId, ownProviderThreadIds, + throughRunOrdinal, reason: "manual_shutdown", detail: FORCE_STOP_REASON, }) @@ -1237,7 +1248,7 @@ const make = Effect.gen(function* () { ); if (Option.isNone(released)) { releasing = true; - } else if (!released.value) { + } else if (!released.value && !(yield* laterRunActive)) { shared = true; yield* detach(release.providerSessionId); } diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 785f41e3aa0fb..8bee3bd27d338 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -3983,10 +3983,10 @@ export type OrchestrationV2ForceStopThreadInput = typeof OrchestrationV2ForceSto export const OrchestrationV2ForceStopThreadResult = Schema.Struct({ /** - * `killed`: the thread's own agent processes were killed. `released`: its provider session - * was torn down, which kills the agent's process group. `releasing`: that teardown has not - * finished yet and continues on the server. `shared`: the agent process also works for - * `sharedWithThreadIds`, so only this thread's turn was ended. + * `killed`: its own agent processes were killed. `released`: its provider session was torn + * down (killing the agent's process group) or handed to its newer run. `releasing`: teardown + * continues on the server. `shared`: the agent process also works for `sharedWithThreadIds`, + * so only this thread's turn was ended. */ outcome: Schema.Literals(["killed", "released", "releasing", "shared"]), sharedWithThreadIds: Schema.Array(ThreadId), From f93e9486060d328950ab86b0b5e97ad5b6da0654 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 17:18:03 +0000 Subject: [PATCH 3/9] test(server): Force stop reports released when it leaves the follow-up's session Co-Authored-By: Claude Opus 5.5 --- apps/server/src/orchestration-v2/ThreadStop.test.ts | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts index 44fa22e59bef7..1d9e89cc0967f 100644 --- a/apps/server/src/orchestration-v2/ThreadStop.test.ts +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -1894,7 +1894,7 @@ it.effect("Force stop keeps the session a queued follow-up started its turn on", yield* Fiber.join(firstRunning); yield* send(threadId, "follow-up", "queue_after_active"); - yield* threads.forceStopThread({ threadId }); + const result = yield* threads.forceStopThread({ threadId }); yield* worker.drain(); assert.deepEqual( @@ -1915,6 +1915,11 @@ it.effect("Force stop keeps the session a queued follow-up started its turn on", yield* sessions.isLive(providerSessionId), "Force stop of the first run released the session the follow-up's turn is running on", ); + assert.deepEqual( + result, + { outcome: "released", sharedWithThreadIds: [] }, + "Force stop detached the thread the follow-up runs on as if its session were shared", + ); }).pipe(Effect.provide(layerLive)); }), ), From 6a54259b539a7fa460741208a61838c1c4d0a696 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 4 Nov 2026 17:32:36 +0000 Subject: [PATCH 4/9] fix(server): a kept session drops the stopped run's busy turns so it can go idle Co-Authored-By: Claude Opus 5.5 --- .../orchestration-v2/ProviderSessionManager.ts | 18 +++++++++++++++--- .../src/orchestration-v2/ThreadStop.test.ts | 18 ++++++++++++++++++ 2 files changed, 33 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 3d21053294a66..abd1d067d863c 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -959,7 +959,16 @@ export const layerWithOptions = ( own !== undefined && [...existing.busyTurns].some((turnKey) => !isOwnBusyTurn(turnKey, own)) ) { - return ["kept", current] as const; + // The caller settled its own turns; one whose end never arrives must not keep + // the session from going idle once the turn it was kept for ends. + const kept = new Map(current); + kept.set(key, { + ...existing, + busyTurns: new Set( + [...existing.busyTurns].filter((turnKey) => !isOwnBusyTurn(turnKey, own)), + ), + }); + return ["kept", kept] as const; } const updated = new Map(current); updated.delete(key); @@ -2381,8 +2390,11 @@ export const layerWithOptions = ( throughRunOrdinal: input.throughRunOrdinal, }, }); - // Identity, not presence: open can install a replacement under the same id meanwhile. - return before !== undefined && (yield* Ref.get(sessions)).get(key) !== before; + // Scope, not presence: open can install a replacement under the same id meanwhile, and + // a kept entry is rewritten with the same scope. + return ( + before !== undefined && (yield* Ref.get(sessions)).get(key)?.scope !== before.scope + ); }), otherWorkingThreads: Effect.fn("ProviderSessionManager.otherWorkingThreads")( function* (input) { diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts index 1d9e89cc0967f..4b2c1172ef461 100644 --- a/apps/server/src/orchestration-v2/ThreadStop.test.ts +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -1920,6 +1920,24 @@ it.effect("Force stop keeps the session a queued follow-up started its turn on", { outcome: "released", sharedWithThreadIds: [] }, "Force stop detached the thread the follow-up runs on as if its session were shared", ); + + // The first run's provider never confirms the stop; the follow-up's turn ends normally. + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: followUp.providerThread.id, + providerTurnId: ProviderTurnId.make(`provider-turn:${followUp.attemptId}`), + runOrdinal: followUp.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* worker.drain(); + yield* TestClock.adjust("31 minutes"); + assert.isFalse( + yield* sessions.isLive(providerSessionId), + "the stopped run's unconfirmed turn kept the session from going idle", + ); }).pipe(Effect.provide(layerLive)); }), ), From 4bb412f054115abaab651e767270818ac811ed56 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Tue, 10 Nov 2026 23:09:52 +0000 Subject: [PATCH 5/9] fix(server): Force stop keeps a session its promoted follow-up has not marked busy yet The follow-up run is active as soon as the settlement promotes it, before its turn marks the session busy. Check for it under the session's request permit so the release no longer kills the agent the follow-up is about to use. AI-assisted; verified with the tests and tools listed above. --- .../ProviderSessionManager.ts | 84 ++-- .../ThreadManagementService.ts | 2 + .../src/orchestration-v2/ThreadStop.test.ts | 375 +++++++++--------- 3 files changed, 241 insertions(+), 220 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index 7203ec6b39c28..b7c42222f7cd0 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -213,6 +213,8 @@ export interface ProviderSessionManagerV2Shape { readonly providerSessionId: ProviderSessionId; readonly ownProviderThreadIds: ReadonlyArray; readonly throughRunOrdinal: number; + /** Keeps the session when it succeeds with true, checked atomically with the removal. */ + readonly keepIf?: Effect.Effect; readonly reason: ProviderSessionReleaseReason; readonly detail?: string; }) => Effect.Effect; @@ -994,47 +996,48 @@ export const layerWithOptions = ( if (candidate === undefined) { return [Option.none(), yield* DateTime.now] as const; } - const remove = Effect.zip( - Ref.modify(sessions, (current) => { - const existing = current.get(key); - if (existing !== candidate) { - return [existing === undefined ? "gone" : "changed", current] as const; - } - if ( - input.onlyIfIdleGeneration !== undefined && - (existing.busyTurns.size > 0 || - existing.idleGeneration !== input.onlyIfIdleGeneration) - ) { - return ["kept", current] as const; - } - const own = input.keepIfBusyOtherThan; - if ( - own !== undefined && - [...existing.busyTurns].some((turnKey) => !isOwnBusyTurn(turnKey, own)) - ) { - // The caller settled its own turns; one whose end never arrives must not keep - // the session from going idle once the turn it was kept for ends. - const kept = new Map(current); - kept.set(key, { - ...existing, - busyTurns: new Set( - [...existing.busyTurns].filter((turnKey) => !isOwnBusyTurn(turnKey, own)), - ), - }); - return ["kept", kept] as const; - } - const updated = new Map(current); - updated.delete(key); - return ["removed", updated] as const; - }), - DateTime.now, + const remove = (keepRequested: boolean) => + Effect.zip( + Ref.modify(sessions, (current) => { + const existing = current.get(key); + if (existing !== candidate) { + return [existing === undefined ? "gone" : "changed", current] as const; + } + if ( + input.onlyIfIdleGeneration !== undefined && + (existing.busyTurns.size > 0 || + existing.idleGeneration !== input.onlyIfIdleGeneration) + ) { + return ["kept", current] as const; + } + const own = input.keepIfBusyOtherThan; + if ( + keepRequested || + (own !== undefined && + [...existing.busyTurns].some((turnKey) => !isOwnBusyTurn(turnKey, own))) + ) { + if (own === undefined) return ["kept", current] as const; + // The caller settled its own turns; one whose end never arrives must not keep + // the session from going idle once the turn it was kept for ends. + const kept = new Map(current); + kept.set(key, { + ...existing, + busyTurns: new Set( + [...existing.busyTurns].filter((turnKey) => !isOwnBusyTurn(turnKey, own)), + ), + }); + return ["kept", kept] as const; + } + const updated = new Map(current); + updated.delete(key); + return ["removed", updated] as const; + }), + DateTime.now, + ); + const [outcome, releasedAt] = yield* (input.keepIf ?? Effect.succeed(false)).pipe( + Effect.flatMap(remove), + candidate.requestEventPermit.withPermits(1), ); - const [outcome, releasedAt] = yield* Effect.gen(function* () { - if (input.keepIf !== undefined && (yield* input.keepIf)) { - return ["kept" as const, yield* DateTime.now] as const; - } - return yield* remove; - }).pipe(candidate.requestEventPermit.withPermits(1)); if (outcome === "changed") return yield* removeLiveEntry(input); return [ outcome === "removed" ? Option.some(candidate) : Option.none(), @@ -2669,6 +2672,7 @@ export const layerWithOptions = ( providerThreadIds: input.ownProviderThreadIds, throughRunOrdinal: input.throughRunOrdinal, }, + ...(input.keepIf === undefined ? {} : { keepIf: input.keepIf }), }); // Scope, not presence: open can install a replacement under the same id meanwhile, and // a kept entry is rewritten with the same scope. diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 358ddc0f00fdb..8cc47ec0bef44 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1309,6 +1309,8 @@ const make = Effect.gen(function* () { providerSessionId, ownProviderThreadIds, throughRunOrdinal, + // A promoted follow-up may not have marked its turn busy yet. + keepIf: laterRunActive.pipe(Effect.orElseSucceed(() => true)), reason: "manual_shutdown", detail: FORCE_STOP_REASON, }) diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts index 4b2c1172ef461..01fde69ee09d6 100644 --- a/apps/server/src/orchestration-v2/ThreadStop.test.ts +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -1747,198 +1747,213 @@ it.effect.each(["Stop", "Force stop"] as const)( }).pipe(Effect.provide(layerTest)), ); -it.effect("Force stop keeps the session a queued follow-up started its turn on", () => - Effect.scoped( - Effect.gen(function* () { - const cwd = yield* checkpointWorkspace("thread-stop-force-stop-follow-up"); - const driver = adapter.driver; - const events = yield* Queue.unbounded(); - const started: Array = []; - const interrupted: Array = []; - const liveAdapter: ProviderAdapter.ProviderAdapterV2["Service"] = { - ...adapter, - openSession: (input) => - Effect.gen(function* () { - const now = yield* DateTime.now; - return { - instanceId, - driver, - providerSessionId: input.providerSessionId, - providerSession: { - id: input.providerSessionId, +it.effect.each([ + { followUp: "started its turn", startedBeforeRelease: true }, + { followUp: "was promoted", startedBeforeRelease: false }, +])( + "Force stop keeps the session of a queued follow-up that $followUp", + ({ startedBeforeRelease }) => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("thread-stop-force-stop-follow-up"); + const driver = adapter.driver; + const events = yield* Queue.unbounded(); + const started: Array = []; + const interrupted: Array = []; + let opened = 0; + const liveAdapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + ...adapter, + openSession: (input) => + Effect.gen(function* () { + opened += 1; + const now = yield* DateTime.now; + return { + instanceId, driver, - providerInstanceId: instanceId, - status: "ready", - cwd, - model: modelSelection.model, - capabilities: CodexProviderCapabilitiesV2, - createdAt: now, - updatedAt: now, - lastError: null, - }, - events: Stream.fromQueue(events), - ensureThread: ({ threadId }) => - Effect.succeed({ - id: ProviderThreadId.make(`provider-thread:codex:${threadId}`), + providerSessionId: input.providerSessionId, + providerSession: { + id: input.providerSessionId, driver, providerInstanceId: instanceId, - providerSessionId: input.providerSessionId, - appThreadId: threadId, - ownerNodeId: null, - nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" }, - nativeConversationHeadRef: null, - status: "idle", - firstRunOrdinal: null, - lastRunOrdinal: null, - handoffIds: [], - forkedFrom: null, + status: "ready", + cwd, + model: modelSelection.model, + capabilities: CodexProviderCapabilitiesV2, createdAt: now, updatedAt: now, - }), - resumeThread: ({ providerThread }) => Effect.succeed(providerThread), - startTurn: (turn) => - Effect.gen(function* () { - started.push(turn); - yield* Queue.offer(events, { - type: "provider_turn.updated", + lastError: null, + }, + events: Stream.fromQueue(events), + ensureThread: ({ threadId }) => + Effect.succeed({ + id: ProviderThreadId.make(`provider-thread:codex:${threadId}`), driver, - providerTurn: { - id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), - providerThreadId: turn.providerThread.id, - nodeId: turn.rootNodeId, - runAttemptId: turn.attemptId, - nativeTurnRef: { - driver, - nativeId: `native:${turn.attemptId}`, - strength: "strong", + providerInstanceId: instanceId, + providerSessionId: input.providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => + Effect.gen(function* () { + started.push(turn); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: { + driver, + nativeId: `native:${turn.attemptId}`, + strength: "strong", + }, + ordinal: turn.providerTurnOrdinal, + status: "running", + startedAt: now, + completedAt: null, }, - ordinal: turn.providerTurnOrdinal, - status: "running", - startedAt: now, - completedAt: null, - }, - }); - }), - steerTurn: () => Effect.die("unused"), - interruptTurn: (interrupt) => Effect.sync(() => interrupted.push(interrupt)), - respondToRuntimeRequest: () => Effect.die("unused"), - readThreadSnapshot: () => Effect.die("unused"), - rollbackThread: () => Effect.die("unused"), - forkThread: () => Effect.die("unused"), - }; + }); + }), + steerTurn: () => Effect.die("unused"), + interruptTurn: (interrupt) => Effect.sync(() => interrupted.push(interrupt)), + respondToRuntimeRequest: () => Effect.die("unused"), + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }; + // The settlement promotes the follow-up; optionally its turn starts before Force stop releases. + const layerSessionsStartingFollowUpFirst = Layer.effect( + ProviderSessionManager.ProviderSessionManagerV2, + Effect.gen(function* () { + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + return ProviderSessionManager.ProviderSessionManagerV2.of({ + ...sessions, + releaseUnlessBusy: (input) => + (startedBeforeRelease ? worker.drain().pipe(Effect.orDie) : Effect.void).pipe( + Effect.andThen(sessions.releaseUnlessBusy(input)), + ), + }); }), - }; - // The follow-up the settlement promotes starts its turn just before Force stop releases. - const layerSessionsStartingFollowUpFirst = Layer.effect( - ProviderSessionManager.ProviderSessionManagerV2, - Effect.gen(function* () { - const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - return ProviderSessionManager.ProviderSessionManagerV2.of({ - ...sessions, - releaseUnlessBusy: (input) => - worker.drain().pipe(Effect.orDie, Effect.andThen(sessions.releaseUnlessBusy(input))), - }); - }), - ); - const layerLive = ThreadManagementService.layer.pipe( - Layer.provide(layerSessionsStartingFollowUpFirst), - Layer.provideMerge( - Layer.mergeAll( - layerDatabase, - ProjectionStore.layer.pipe(Layer.provide(layerDatabase)), - EventStore.layer.pipe(Layer.provide(layerDatabase)), - EffectOutbox.layer.pipe(Layer.provide(layerDatabase)), - ProviderReplayHarness.layerWithRegistry( - { name: "thread-stop-force-stop-follow-up" }, - ProviderAdapterRegistry.layerFromAdapters([liveAdapter]), - { databaseLayer: layerDatabase, runEffectWorker: false }, + ); + const layerLive = ThreadManagementService.layer.pipe( + Layer.provide(layerSessionsStartingFollowUpFirst), + Layer.provideMerge( + Layer.mergeAll( + layerDatabase, + ProjectionStore.layer.pipe(Layer.provide(layerDatabase)), + EventStore.layer.pipe(Layer.provide(layerDatabase)), + EffectOutbox.layer.pipe(Layer.provide(layerDatabase)), + ProviderReplayHarness.layerWithRegistry( + { name: "thread-stop-force-stop-follow-up" }, + ProviderAdapterRegistry.layerFromAdapters([liveAdapter]), + { databaseLayer: layerDatabase, runEffectWorker: false }, + ), ), ), - ), - Layer.provideMerge(ServerSettings.layerTest()), - Layer.provideMerge(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))), - ); - - yield* Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; - const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; - const threads = yield* ThreadManagementService.ThreadManagementService; - const threadId = ThreadId.make("thread:force-stop-follow-up"); - const runningTurn = orchestrator.streamDomainEvents.pipe( - Stream.filter( - (event) => event.type === "provider-turn.updated" && event.payload.status === "running", - ), - Stream.take(1), - Stream.runDrain, - Effect.forkScoped, + Layer.provideMerge(ServerSettings.layerTest()), + Layer.provideMerge(CommandReceiptStore.layer.pipe(Layer.provide(layerDatabase))), ); - yield* orchestrator.dispatch({ - type: "thread.create", - commandId: CommandId.make("create:force-stop-follow-up"), - threadId, - projectId: ProjectId.make("project:thread-stop"), - title: "Force stop follow-up", - modelSelection, - runtimeMode: "full-access", - interactionMode: "default", - branch: null, - worktreePath: cwd, - createdBy: "user", - creationSource: "web", - }); - const firstRunning = yield* runningTurn; - yield* send(threadId, "first", "start_immediately"); - yield* worker.drain(); - yield* Fiber.join(firstRunning); - yield* send(threadId, "follow-up", "queue_after_active"); - const result = yield* threads.forceStopThread({ threadId }); - yield* worker.drain(); + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const sessions = yield* ProviderSessionManager.ProviderSessionManagerV2; + const threads = yield* ThreadManagementService.ThreadManagementService; + const threadId = ThreadId.make("thread:force-stop-follow-up"); + const runningTurn = orchestrator.streamDomainEvents.pipe( + Stream.filter( + (event) => + event.type === "provider-turn.updated" && event.payload.status === "running", + ), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create:force-stop-follow-up"), + threadId, + projectId: ProjectId.make("project:thread-stop"), + title: "Force stop follow-up", + modelSelection, + runtimeMode: "full-access", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const firstRunning = yield* runningTurn; + yield* send(threadId, "first", "start_immediately"); + yield* worker.drain(); + yield* Fiber.join(firstRunning); + yield* send(threadId, "follow-up", "queue_after_active"); - assert.deepEqual( - started.map((turn) => turn.runOrdinal), - [1, 2], - "the queued follow-up did not start its turn before the release", - ); - const followUp = started[1]!; - const providerSessionId = (yield* orchestrator.getThreadProjection(threadId)) - .providerThreads[0]!.providerSessionId!; - assert.isFalse( - interrupted.some((interrupt) => - String(interrupt.providerTurnId).includes(followUp.attemptId), - ), - "Force stop of the first run interrupted the follow-up's turn", - ); - assert.isTrue( - yield* sessions.isLive(providerSessionId), - "Force stop of the first run released the session the follow-up's turn is running on", - ); - assert.deepEqual( - result, - { outcome: "released", sharedWithThreadIds: [] }, - "Force stop detached the thread the follow-up runs on as if its session were shared", - ); + const result = yield* threads.forceStopThread({ threadId }); + yield* worker.drain(); - // The first run's provider never confirms the stop; the follow-up's turn ends normally. - yield* Queue.offer(events, { - type: "turn.terminal", - driver, - providerThreadId: followUp.providerThread.id, - providerTurnId: ProviderTurnId.make(`provider-turn:${followUp.attemptId}`), - runOrdinal: followUp.runOrdinal, - status: "completed", - failure: null, - threadDisposition: "reusable", - }); - yield* worker.drain(); - yield* TestClock.adjust("31 minutes"); - assert.isFalse( - yield* sessions.isLive(providerSessionId), - "the stopped run's unconfirmed turn kept the session from going idle", - ); - }).pipe(Effect.provide(layerLive)); - }), - ), + assert.deepEqual( + started.map((turn) => turn.runOrdinal), + [1, 2], + "the queued follow-up did not start its turn", + ); + assert.equal( + opened, + 1, + "Force stop tore down the session the follow-up was promoted onto", + ); + const followUp = started[1]!; + const providerSessionId = (yield* orchestrator.getThreadProjection(threadId)) + .providerThreads[0]!.providerSessionId!; + assert.isFalse( + interrupted.some((interrupt) => + String(interrupt.providerTurnId).includes(followUp.attemptId), + ), + "Force stop of the first run interrupted the follow-up's turn", + ); + assert.isTrue( + yield* sessions.isLive(providerSessionId), + "Force stop of the first run released the session the follow-up's turn is running on", + ); + assert.deepEqual( + result, + { outcome: "released", sharedWithThreadIds: [] }, + "Force stop detached the thread the follow-up runs on as if its session were shared", + ); + + // The first run's provider never confirms the stop; the follow-up's turn ends normally. + yield* Queue.offer(events, { + type: "turn.terminal", + driver, + providerThreadId: followUp.providerThread.id, + providerTurnId: ProviderTurnId.make(`provider-turn:${followUp.attemptId}`), + runOrdinal: followUp.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* worker.drain(); + yield* TestClock.adjust("31 minutes"); + assert.isFalse( + yield* sessions.isLive(providerSessionId), + "the stopped run's unconfirmed turn kept the session from going idle", + ); + }).pipe(Effect.provide(layerLive)); + }), + ), ); From 553bd61dc9b7385b90085b484358d8e5c90de852 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 00:08:47 +0000 Subject: [PATCH 6/9] refactor(server): Force stop guards both reads of a newer active run AI-assisted; verified with the tests and tools listed above. --- .../orchestration-v2/ThreadManagementService.ts | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 8cc47ec0bef44..139e1e01a0b59 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1262,15 +1262,14 @@ const make = Effect.gen(function* () { providerThread.appThreadId === input.threadId ? [providerThread.id] : [], ); // Settling the target can promote a queued follow-up onto the same session; that run is - // not Force stop's to end. + // not Force stop's to end. An unreadable thread counts as one, so its session is kept. const throughRunOrdinal = target?.ordinal ?? 0; - const laterRunActive = orchestrator - .getThreadRecords(input.threadId, ["runs"]) - .pipe( - Effect.map(({ runs }) => - runs.some((run) => run.ordinal > throughRunOrdinal && isActiveRun(run)), - ), - ); + const laterRunActive = orchestrator.getThreadRecords(input.threadId, ["runs"]).pipe( + Effect.map(({ runs }) => + runs.some((run) => run.ordinal > throughRunOrdinal && isActiveRun(run)), + ), + Effect.orElseSucceed(() => true), + ); const detach = (providerSessionId: ProviderSessionId) => sessions .detach({ providerSessionId, threadId: input.threadId, detail: FORCE_STOP_REASON }) @@ -1310,7 +1309,7 @@ const make = Effect.gen(function* () { ownProviderThreadIds, throughRunOrdinal, // A promoted follow-up may not have marked its turn busy yet. - keepIf: laterRunActive.pipe(Effect.orElseSucceed(() => true)), + keepIf: laterRunActive, reason: "manual_shutdown", detail: FORCE_STOP_REASON, }) From dfb8edfb4f0d2d424a5dc026fb4fae57922aa6bd Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 00:21:55 +0000 Subject: [PATCH 7/9] fix(server): a session Force stop keeps still idles out AI-assisted; verified with the tests and tools listed above. --- .../ProviderSessionManager.test.ts | 67 +++++++++++++++++++ .../ProviderSessionManager.ts | 8 ++- 2 files changed, 72 insertions(+), 3 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 50776be626c64..1acc8306cf70e 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -1524,6 +1524,73 @@ it.effect("ProviderSessionManagerV2 gives up on a provider that never confirms a }), ); +it.effect("ProviderSessionManagerV2 still idles out a session releaseUnlessBusy kept", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("thread-provider-session-manager-kept-idles"); + const providerSessionId = idAllocator.derive.providerSession({ + providerInstanceId: modelSelection.instanceId, + }); + yield* eventSink.write({ + events: [yield* makeThreadCreatedEvent({ idAllocator, threadId, now })], + }); + const runtime = yield* manager.open({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + const providerThread = makeProviderThread({ + idAllocator, + threadId, + providerSessionId, + now, + nativeThreadId: `native-${threadId}`, + }); + const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); + yield* runtime.startTurn({ + appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, + threadId, + runId, + runOrdinal: 1, + providerTurnOrdinal: 1, + attemptId: idAllocator.derive.runAttempt({ runId, attemptOrdinal: 1 }), + rootNodeId: idAllocator.derive.rootNode({ runId }), + providerThread, + message: { + createdBy: "user", + creationSource: "web", + messageId: yield* idAllocator.allocate.message({ threadId, ordinal: 1 }), + text: "turn", + attachments: [], + }, + modelSelection, + runtimePolicy, + }); + + assert.isFalse( + yield* manager.releaseUnlessBusy({ + providerSessionId, + ownProviderThreadIds: [providerThread.id], + throughRunOrdinal: 1, + keepIf: Effect.succeed(true), + reason: "manual_shutdown", + }), + ); + assert.isTrue(yield* manager.isLive(providerSessionId)); + // The stopped run 1 never reports its end. + yield* TestClock.adjust("2 seconds"); + assert.isFalse(yield* manager.isLive(providerSessionId)); + }).pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 1_000 }))); + }), +); + it.effect( "ProviderSessionManagerV2 keeps a shared session a sibling is busy on when releasing", () => diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index b7c42222f7cd0..356eba5e80b63 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2676,9 +2676,11 @@ export const layerWithOptions = ( }); // Scope, not presence: open can install a replacement under the same id meanwhile, and // a kept entry is rewritten with the same scope. - return ( - before !== undefined && (yield* Ref.get(sessions)).get(key)?.scope !== before.scope - ); + const released = + before !== undefined && (yield* Ref.get(sessions)).get(key)?.scope !== before.scope; + // The stopped turn's markBusy cancelled the idle timer and its markIdle may never come. + if (!released) yield* scheduleIdleReleaseInternal(input.providerSessionId); + return released; }), otherWorkingThreads: Effect.fn("ProviderSessionManager.otherWorkingThreads")( function* (input) { From 0b5c0f1501cb24ad33252ba024da16e7a7116348 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 03:08:51 +0000 Subject: [PATCH 8/9] fix(server): Force stop keeps only the session a newer run uses A queued follow-up can switch provider instance on promotion. Force stop kept every session of the thread whenever any newer run was active, so the stopped run's session on the old instance stayed alive for the idle timeout while the reply said released. The newer-run check is now per session and only counts a run on that session's provider instance. Co-Authored-By: Claude Opus 5.5 --- .../ThreadManagementService.ts | 34 ++++++++--- .../src/orchestration-v2/ThreadStop.test.ts | 59 ++++++++++++++----- 2 files changed, 69 insertions(+), 24 deletions(-) diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index a41f8c426a452..990a072021f29 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1304,14 +1304,30 @@ const make = Effect.gen(function* () { providerThread.appThreadId === input.threadId ? [providerThread.id] : [], ); // Settling the target can promote a queued follow-up onto the same session; that run is - // not Force stop's to end. An unreadable thread counts as one, so its session is kept. + // not Force stop's to end. Sessions are per provider instance and a follow-up may switch + // instance, so only a session of the newer run's instance is kept. An unreadable thread + // counts as a newer run, so its sessions are kept. const throughRunOrdinal = target?.ordinal ?? 0; - const laterRunActive = orchestrator.getThreadRecords(input.threadId, ["runs"]).pipe( - Effect.map(({ runs }) => - runs.some((run) => run.ordinal > throughRunOrdinal && isActiveRun(run)), - ), - Effect.orElseSucceed(() => true), - ); + const laterRunActive = (providerSessionId: ProviderSessionId) => { + const instanceIds = new Set( + records.providerThreads.flatMap((providerThread) => + providerThread.providerSessionId === providerSessionId + ? [providerThread.providerInstanceId] + : [], + ), + ); + return orchestrator.getThreadRecords(input.threadId, ["runs"]).pipe( + Effect.map(({ runs }) => + runs.some( + (run) => + run.ordinal > throughRunOrdinal && + isActiveRun(run) && + instanceIds.has(run.providerInstanceId), + ), + ), + Effect.orElseSucceed(() => true), + ); + }; const detach = (providerSessionId: ProviderSessionId) => sessions .detach({ providerSessionId, threadId: input.threadId, detail: FORCE_STOP_REASON }) @@ -1351,7 +1367,7 @@ const make = Effect.gen(function* () { ownProviderThreadIds, throughRunOrdinal, // A promoted follow-up may not have marked its turn busy yet. - keepIf: laterRunActive, + keepIf: laterRunActive(providerSessionId), reason: "manual_shutdown", detail: FORCE_STOP_REASON, }) @@ -1366,7 +1382,7 @@ const make = Effect.gen(function* () { ); if (Option.isNone(released)) { releasing = true; - } else if (!released.value && !(yield* laterRunActive)) { + } else if (!released.value && !(yield* laterRunActive(release.providerSessionId))) { shared = true; yield* detach(release.providerSessionId); } diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts index 5bb128aa2b2ec..cdd6c6f317984 100644 --- a/apps/server/src/orchestration-v2/ThreadStop.test.ts +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -1840,25 +1840,34 @@ it.effect.each(["Stop", "Force stop"] as const)( ); it.effect.each([ - { followUp: "started its turn", startedBeforeRelease: true }, - { followUp: "was promoted", startedBeforeRelease: false }, + { followUp: "started its turn", startedBeforeRelease: true, switchesInstance: false }, + { followUp: "was promoted", startedBeforeRelease: false, switchesInstance: false }, + { followUp: "switched provider instance", startedBeforeRelease: true, switchesInstance: true }, ])( - "Force stop keeps the session of a queued follow-up that $followUp", - ({ startedBeforeRelease }) => + "Force stop keeps only the session of a queued follow-up that $followUp", + ({ startedBeforeRelease, switchesInstance }) => Effect.scoped( Effect.gen(function* () { const cwd = yield* checkpointWorkspace("thread-stop-force-stop-follow-up"); const driver = adapter.driver; - const events = yield* Queue.unbounded(); + const otherInstanceId = ProviderInstanceId.make("codex-other"); + const eventQueues = new Map([ + [instanceId, yield* Queue.unbounded()], + [otherInstanceId, yield* Queue.unbounded()], + ]); const started: Array = []; const interrupted: Array = []; let opened = 0; - const liveAdapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + const liveAdapter = ( + instanceId: ProviderInstanceId, + ): ProviderAdapter.ProviderAdapterV2["Service"] => ({ ...adapter, + instanceId, openSession: (input) => Effect.gen(function* () { opened += 1; const now = yield* DateTime.now; + const events = eventQueues.get(instanceId)!; return { instanceId, driver, @@ -1878,7 +1887,7 @@ it.effect.each([ events: Stream.fromQueue(events), ensureThread: ({ threadId }) => Effect.succeed({ - id: ProviderThreadId.make(`provider-thread:codex:${threadId}`), + id: ProviderThreadId.make(`provider-thread:${instanceId}:${threadId}`), driver, providerInstanceId: instanceId, providerSessionId: input.providerSessionId, @@ -1926,7 +1935,7 @@ it.effect.each([ forkThread: () => Effect.die("unused"), }; }), - }; + }); // The settlement promotes the follow-up; optionally its turn starts before Force stop releases. const layerSessionsStartingFollowUpFirst = Layer.effect( ProviderSessionManager.ProviderSessionManagerV2, @@ -1952,7 +1961,10 @@ it.effect.each([ EffectOutbox.layer.pipe(Layer.provide(layerDatabase)), ProviderReplayHarness.layerWithRegistry( { name: "thread-stop-force-stop-follow-up" }, - ProviderAdapterRegistry.layerFromAdapters([liveAdapter]), + ProviderAdapterRegistry.layerFromAdapters([ + liveAdapter(instanceId), + liveAdapter(otherInstanceId), + ]), { databaseLayer: layerDatabase, runEffectWorker: false }, ), ), @@ -1994,7 +2006,20 @@ it.effect.each([ yield* send(threadId, "first", "start_immediately"); yield* worker.drain(); yield* Fiber.join(firstRunning); - yield* send(threadId, "follow-up", "queue_after_active"); + yield* orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make("send:force-stop-follow-up"), + threadId, + messageId: MessageId.make("message:force-stop-follow-up"), + text: "follow-up", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + ...(switchesInstance + ? { modelSelection: { ...modelSelection, instanceId: otherInstanceId } } + : {}), + createdBy: "user", + creationSource: "web", + }); const result = yield* threads.forceStopThread({ threadId }); yield* worker.drain(); @@ -2006,12 +2031,16 @@ it.effect.each([ ); assert.equal( opened, - 1, + switchesInstance ? 2 : 1, "Force stop tore down the session the follow-up was promoted onto", ); - const followUp = started[1]!; - const providerSessionId = (yield* orchestrator.getThreadProjection(threadId)) - .providerThreads[0]!.providerSessionId!; + const [stopped, followUp] = [started[0]!, started[1]!]; + const providerSessionId = followUp.providerThread.providerSessionId!; + assert.equal( + yield* sessions.isLive(stopped.providerThread.providerSessionId!), + !switchesInstance, + "Force stop kept the stopped run's session for a follow-up on another instance", + ); assert.isFalse( interrupted.some((interrupt) => String(interrupt.providerTurnId).includes(followUp.attemptId), @@ -2029,7 +2058,7 @@ it.effect.each([ ); // The first run's provider never confirms the stop; the follow-up's turn ends normally. - yield* Queue.offer(events, { + yield* Queue.offer(eventQueues.get(followUp.providerThread.providerInstanceId)!, { type: "turn.terminal", driver, providerThreadId: followUp.providerThread.id, From 45f491e3876ff116bad52f5132a1af787dbdfecb Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 03:23:41 +0000 Subject: [PATCH 9/9] fix(server): Force stop keeps a session only for a newer run on it Matching the newer run by provider instance is not enough: one-thread-per- session providers run a fresh-context follow-up on its own session of the same instance, so Force stop kept the stopped run's session alive while it reported released. Keep a session only when a newer active run is bound to a provider thread on that session. Co-Authored-By: Claude Opus 5.5 --- .../ThreadManagementService.ts | 33 +++++++------- .../src/orchestration-v2/ThreadStop.test.ts | 44 +++++++++++++------ 2 files changed, 46 insertions(+), 31 deletions(-) diff --git a/apps/server/src/orchestration-v2/ThreadManagementService.ts b/apps/server/src/orchestration-v2/ThreadManagementService.ts index 990a072021f29..a69d308884c72 100644 --- a/apps/server/src/orchestration-v2/ThreadManagementService.ts +++ b/apps/server/src/orchestration-v2/ThreadManagementService.ts @@ -1304,30 +1304,27 @@ const make = Effect.gen(function* () { providerThread.appThreadId === input.threadId ? [providerThread.id] : [], ); // Settling the target can promote a queued follow-up onto the same session; that run is - // not Force stop's to end. Sessions are per provider instance and a follow-up may switch - // instance, so only a session of the newer run's instance is kept. An unreadable thread - // counts as a newer run, so its sessions are kept. + // not Force stop's to end. Only a newer run bound to a provider thread on this session + // counts. An unreadable thread counts as one, so its sessions are kept. const throughRunOrdinal = target?.ordinal ?? 0; - const laterRunActive = (providerSessionId: ProviderSessionId) => { - const instanceIds = new Set( - records.providerThreads.flatMap((providerThread) => - providerThread.providerSessionId === providerSessionId - ? [providerThread.providerInstanceId] - : [], - ), - ); - return orchestrator.getThreadRecords(input.threadId, ["runs"]).pipe( - Effect.map(({ runs }) => - runs.some( + const laterRunActive = (providerSessionId: ProviderSessionId) => + orchestrator.getThreadRecords(input.threadId, ["runs", "providerThreads"]).pipe( + Effect.map(({ runs, providerThreads }) => { + const onSession = new Set( + providerThreads.flatMap((providerThread) => + providerThread.providerSessionId === providerSessionId ? [providerThread.id] : [], + ), + ); + return runs.some( (run) => run.ordinal > throughRunOrdinal && isActiveRun(run) && - instanceIds.has(run.providerInstanceId), - ), - ), + run.providerThreadId !== null && + onSession.has(run.providerThreadId), + ); + }), Effect.orElseSucceed(() => true), ); - }; const detach = (providerSessionId: ProviderSessionId) => sessions .detach({ providerSessionId, threadId: input.threadId, detail: FORCE_STOP_REASON }) diff --git a/apps/server/src/orchestration-v2/ThreadStop.test.ts b/apps/server/src/orchestration-v2/ThreadStop.test.ts index cdd6c6f317984..d07cbc7867718 100644 --- a/apps/server/src/orchestration-v2/ThreadStop.test.ts +++ b/apps/server/src/orchestration-v2/ThreadStop.test.ts @@ -1840,34 +1840,49 @@ it.effect.each(["Stop", "Force stop"] as const)( ); it.effect.each([ - { followUp: "started its turn", startedBeforeRelease: true, switchesInstance: false }, - { followUp: "was promoted", startedBeforeRelease: false, switchesInstance: false }, + { followUp: "started its turn", startedBeforeRelease: true }, + { followUp: "was promoted", startedBeforeRelease: false }, { followUp: "switched provider instance", startedBeforeRelease: true, switchesInstance: true }, + { followUp: "has fresh context", startedBeforeRelease: true, freshContext: true }, ])( "Force stop keeps only the session of a queued follow-up that $followUp", - ({ startedBeforeRelease, switchesInstance }) => + ({ startedBeforeRelease, switchesInstance = false, freshContext = false }) => Effect.scoped( Effect.gen(function* () { const cwd = yield* checkpointWorkspace("thread-stop-force-stop-follow-up"); const driver = adapter.driver; const otherInstanceId = ProviderInstanceId.make("codex-other"); - const eventQueues = new Map([ - [instanceId, yield* Queue.unbounded()], - [otherInstanceId, yield* Queue.unbounded()], - ]); + const eventQueues = new Map< + ProviderSessionId, + Queue.Queue + >(); const started: Array = []; const interrupted: Array = []; let opened = 0; + // Fresh context on a one-thread-per-session provider puts the follow-up on its own session. + // The session itself reports shared threads so fresh-context cleanup leaves the stopped + // run's session to Force stop, as pending background work would. + const adapterCapabilities = freshContext + ? { + ...CodexProviderCapabilitiesV2, + sessions: { + ...CodexProviderCapabilitiesV2.sessions, + supportsMultipleProviderThreadsPerSession: false, + }, + } + : CodexProviderCapabilitiesV2; const liveAdapter = ( instanceId: ProviderInstanceId, ): ProviderAdapter.ProviderAdapterV2["Service"] => ({ ...adapter, instanceId, + getCapabilities: () => Effect.succeed(adapterCapabilities), openSession: (input) => Effect.gen(function* () { opened += 1; const now = yield* DateTime.now; - const events = eventQueues.get(instanceId)!; + const events = yield* Queue.unbounded(); + eventQueues.set(input.providerSessionId, events); return { instanceId, driver, @@ -1887,7 +1902,9 @@ it.effect.each([ events: Stream.fromQueue(events), ensureThread: ({ threadId }) => Effect.succeed({ - id: ProviderThreadId.make(`provider-thread:${instanceId}:${threadId}`), + id: ProviderThreadId.make( + `provider-thread:${input.providerSessionId}:${threadId}`, + ), driver, providerInstanceId: instanceId, providerSessionId: input.providerSessionId, @@ -2017,6 +2034,7 @@ it.effect.each([ ...(switchesInstance ? { modelSelection: { ...modelSelection, instanceId: otherInstanceId } } : {}), + ...(freshContext ? { freshContext } : {}), createdBy: "user", creationSource: "web", }); @@ -2031,15 +2049,15 @@ it.effect.each([ ); assert.equal( opened, - switchesInstance ? 2 : 1, + switchesInstance || freshContext ? 2 : 1, "Force stop tore down the session the follow-up was promoted onto", ); const [stopped, followUp] = [started[0]!, started[1]!]; const providerSessionId = followUp.providerThread.providerSessionId!; assert.equal( yield* sessions.isLive(stopped.providerThread.providerSessionId!), - !switchesInstance, - "Force stop kept the stopped run's session for a follow-up on another instance", + !switchesInstance && !freshContext, + "Force stop kept the stopped run's session for a follow-up on another session", ); assert.isFalse( interrupted.some((interrupt) => @@ -2058,7 +2076,7 @@ it.effect.each([ ); // The first run's provider never confirms the stop; the follow-up's turn ends normally. - yield* Queue.offer(eventQueues.get(followUp.providerThread.providerInstanceId)!, { + yield* Queue.offer(eventQueues.get(providerSessionId)!, { type: "turn.terminal", driver, providerThreadId: followUp.providerThread.id,