From b3ca0b0e3316cebaba4a76a36e35ae96456ef870 Mon Sep 17 00:00:00 2001 From: Zack Jackson <25274700+ScriptedAlchemy@users.noreply.github.com> Date: Wed, 18 Nov 2026 15:33:43 -0800 Subject: [PATCH] fix(server): preserve task and session ownership during cancellation --- .../ProviderSessionManager.test.ts | 117 +++++-- .../ProviderSessionManager.ts | 166 +++++----- apps/server/src/peer/RemoteDelegation.test.ts | 247 +++++++++++---- apps/server/src/peer/RemoteDelegation.ts | 293 ++++++++++-------- apps/server/src/serverSettings.test.ts | 72 ++++- apps/server/src/serverSettings.ts | 45 +-- 6 files changed, 613 insertions(+), 327 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 23d64d4a13c84..0156e951cb7a2 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -581,6 +581,8 @@ const makeProviderAdapter = Effect.fnUntraced(function* ( resumeCount: current.resumeCount + 1, })).pipe(Effect.as(threadInput.providerThread)), startTurn: () => options.startTurn ?? Effect.void, + compactThread: () => options.startTurn ?? Effect.void, + resumeTurn: () => options.startTurn ?? Effect.void, steerTurn: () => Effect.void, interruptTurn: () => Ref.update(state, (current) => ({ @@ -1574,9 +1576,9 @@ it.effect("ProviderSessionManagerV2 cleans up an open interrupted mid-handshake" }), ); -it.effect( - "ProviderSessionManagerV2 keeps a shared session busy when another thread's start is stopped early", - () => +it.effect.each(["startTurn", "compactThread", "resumeTurn"] as const)( + "ProviderSessionManagerV2 keeps a shared session busy during %s when another thread's start is stopped early", + (entrypoint) => Effect.gen(function* () { const state = yield* Ref.make(emptyState); const projectId = ProjectId.make("project-provider-session-manager-shared-stopped-start"); @@ -1621,10 +1623,13 @@ it.effect( modelSelection, runtimePolicy, }); - const startTurn = (threadId: ThreadId) => + const startTurn = ( + threadId: ThreadId, + method: "startTurn" | "compactThread" | "resumeTurn" = "startTurn", + ) => Effect.gen(function* () { const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); - return yield* runtime.startTurn({ + const input = { appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, threadId, runId, @@ -1642,10 +1647,15 @@ it.effect( }, modelSelection, runtimePolicy, - }); + } satisfies ProviderAdapter.ProviderAdapterV2TurnInput; + if (method === "resumeTurn") { + const { message, ...resumeInput } = input; + return yield* runtime.resumeTurn!({ ...resumeInput, messageId: message.messageId }); + } + return yield* runtime[method]!(input); }); - yield* startTurn(threadB); + yield* startTurn(threadB, entrypoint); yield* manager.detach({ providerSessionId, threadId: threadA }); yield* Ref.set(holdProjectRead, true); @@ -1654,9 +1664,39 @@ it.effect( yield* Fiber.interrupt(startingA); yield* TestClock.adjust("2 seconds"); - yield* Effect.yieldNow; assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); assert.equal((yield* Ref.get(state)).closeCount, 0); + + const subscription = yield* runtime.subscribeEvents!; + const ended = yield* subscription.events.pipe( + Stream.filter((event) => event.type === "turn.terminal"), + Stream.runHead, + Effect.forkChild, + ); + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(queue); + yield* Queue.offer(queue!, { + type: "turn.terminal", + driver: CODEX_DRIVER, + providerThreadId: makeProviderThread({ + idAllocator, + threadId: threadB, + providerSessionId, + now, + }).id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-shared-stopped-b", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + yield* Fiber.join(ended); + yield* TestClock.adjust("1 second"); + assert.isTrue(Option.isNone(yield* manager.get(providerSessionId))); + assert.equal((yield* Ref.get(state)).closeCount, 1); }).pipe( Effect.provide( layerTest({ @@ -1673,9 +1713,13 @@ it.effect( ); // regression: #15571 an interrupted session start did not clean up -it.effect( - "ProviderSessionManagerV2 keeps a shared session busy when a stopped start still ends its turn", - () => +it.effect.each([ + "shared runtime", + "replacement runtime", + "replacement runtime after a late terminal event", +] as const)( + "ProviderSessionManagerV2 keeps a %s busy when a stopped start still ends its turn", + (scenario) => Effect.gen(function* () { const state = yield* Ref.make(emptyState); // Like OpenCode2, a stopped start can still emit turn.terminal later. @@ -1718,10 +1762,10 @@ it.effect( now, nativeThreadId: `native-${threadId}`, }); - const startTurn = (threadId: ThreadId) => + const startTurn = (threadId: ThreadId, currentRuntime = runtime) => Effect.gen(function* () { const runId = idAllocator.derive.run({ threadId, ordinal: 1 }); - return yield* runtime.startTurn({ + return yield* currentRuntime.startTurn({ appThread: (yield* projectionStore.getThreadProjection(threadId)).thread, threadId, runId, @@ -1743,32 +1787,47 @@ it.effect( }); yield* startTurn(threadB); + const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); + assert.isDefined(queue); yield* Ref.set(holdStart, true); const startingA = yield* startTurn(threadA).pipe(Effect.forkChild); yield* Deferred.await(startHeld); + if (scenario === "replacement runtime after a late terminal event") { + yield* Fiber.interrupt(startingA); + } + if (scenario !== "shared runtime") { + yield* manager.close(providerSessionId); + yield* Ref.set(holdStart, false); + const replacement = yield* manager.open({ + threadId: threadA, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* startTurn(threadA, replacement); + } yield* Fiber.interrupt(startingA); - const queue = (yield* Ref.get(state)).eventQueues.get(String(providerSessionId)); - assert.isDefined(queue); - yield* Queue.offer(queue!, { - type: "turn.terminal", - driver: CODEX_DRIVER, - providerThreadId: providerThreadOf(threadA).id, - providerTurnId: idAllocator.derive.providerTurn({ + if (scenario !== "replacement runtime") { + yield* Queue.offer(queue!, { + type: "turn.terminal", driver: CODEX_DRIVER, - nativeTurnId: "native-turn-stopped-a", - }), - runOrdinal: 1, - status: "completed", - failure: null, - threadDisposition: "reusable", - }); + providerThreadId: providerThreadOf(threadA).id, + providerTurnId: idAllocator.derive.providerTurn({ + driver: CODEX_DRIVER, + nativeTurnId: "native-turn-stopped-a", + }), + runOrdinal: 1, + status: "completed", + failure: null, + threadDisposition: "reusable", + }); + } yield* TestClock.adjust("2 seconds"); - yield* Effect.yieldNow; assert.isTrue(Option.isSome(yield* manager.get(providerSessionId))); - assert.equal((yield* Ref.get(state)).closeCount, 0); + assert.equal((yield* Ref.get(state)).closeCount, scenario === "shared runtime" ? 0 : 1); }).pipe( Effect.provide( layerTest({ diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index f2a2ee5017842..0962b2b9fc456 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -1881,15 +1881,16 @@ export const layerWithOptions = ( providerSessionId: ProviderSessionId, providerThreadId: ProviderThreadId, runOrdinal: number, + runtime: ProviderAdapter.ProviderAdapterV2SessionRuntime, ) => withActivityError( providerSessionId, Effect.gen(function* () { const key = sessionKey(providerSessionId); - yield* Ref.update(sessions, (current) => { + const marked = yield* Ref.modify(sessions, (current) => { const entry = current.get(key); - if (entry === undefined) { - return current; + if (entry === undefined || entry.runtime !== runtime) { + return [false, current] as const; } const busyTurns = new Set(entry.busyTurns); busyTurns.delete(busyTurnKey(providerThreadId, runOrdinal)); @@ -1898,8 +1899,9 @@ export const layerWithOptions = ( ...entry, busyTurns, }); - return updated; + return [true, updated] as const; }); + if (!marked) return; yield* scheduleIdleReleaseInternal(providerSessionId); yield* scheduleThreadUnload(providerSessionId, providerThreadId); }), @@ -2351,6 +2353,86 @@ export const layerWithOptions = ( modelFamily: normalizeModelMetricLabel(model), }, }); + const trackTurnStart = + < + Input extends + | ProviderAdapter.ProviderAdapterV2TurnInput + | ProviderAdapter.ProviderAdapterV2ResumeTurnInput, + >( + start: (input: Input) => Effect.Effect, + ) => + (input: Input) => + observeActivity( + providerSessionId, + ensureThreadAttached({ + providerSessionId, + threadId: input.threadId, + providerInstanceId: runtime.instanceId, + }), + ).pipe( + // Failed or stopped starts may omit turn.terminal; clear their turns so sessions can idle. + Effect.andThen( + holdThreadLoaded({ + providerSessionId, + threadId: input.threadId, + providerThread: input.providerThread, + }), + ), + Effect.andThen( + Effect.acquireUseRelease( + markBusy( + providerSessionId, + busyTurnKey(input.providerThread.id, input.runOrdinal), + runtime, + ).pipe( + Effect.catchCause((cause) => + Effect.logWarning("orchestration-v2.driver-session.activity-failed", { + providerSessionId, + cause, + }).pipe(Effect.as(true)), + ), + // A release between attach and markBusy leaves nothing to mark: fail the + // start instead of running it on a runtime being torn down. + Effect.flatMap((marked) => + marked + ? Effect.void + : Effect.fail( + new ProviderAdapter.ProviderAdapterTurnStartError({ + driver: runtime.driver, + threadId: input.threadId, + providerThreadId: input.providerThread.id, + runId: input.runId, + cause: "The provider session was released while the turn started.", + }), + ), + ), + ), + () => start(input).pipe(turnMetrics("send", input.modelSelection.model)), + (_, exit) => + Exit.isFailure(exit) + ? observeActivity( + providerSessionId, + markIdle( + providerSessionId, + input.providerThread.id, + input.runOrdinal, + runtime, + ), + ).pipe( + // A failed start can leave its native thread gone (Claude rejecting a + // resume), so the next start must ask the adapter to resume it again. + Effect.andThen( + forgetProviderThreadLoaded({ + providerSessionId, + threadId: input.threadId, + runtime, + }), + ), + ) + : Effect.void, + ), + ), + ); return { ...runtime, // Pi replaces its session snapshot as metadata changes. @@ -2512,74 +2594,13 @@ export const layerWithOptions = ( ), Effect.map(({ loaded }) => bindContinuationIdentity(loaded, continuationKey)), ), - startTurn: (input) => - observeActivity( - providerSessionId, - ensureThreadAttached({ - providerSessionId, - threadId: input.threadId, - providerInstanceId: runtime.instanceId, - }), - ).pipe( - // Failed or stopped starts may omit turn.terminal; clear their turns so sessions can idle. - Effect.andThen( - holdThreadLoaded({ - providerSessionId, - threadId: input.threadId, - providerThread: input.providerThread, - }), - ), - Effect.andThen( - Effect.acquireUseRelease( - markBusy( - providerSessionId, - busyTurnKey(input.providerThread.id, input.runOrdinal), - runtime, - ).pipe( - Effect.catchCause((cause) => - Effect.logWarning("orchestration-v2.driver-session.activity-failed", { - providerSessionId, - cause, - }).pipe(Effect.as(true)), - ), - // A release between attach and markBusy leaves nothing to mark: fail the - // start instead of running it on a runtime being torn down. - Effect.flatMap((marked) => - marked - ? Effect.void - : Effect.fail( - new ProviderAdapter.ProviderAdapterTurnStartError({ - driver: runtime.driver, - threadId: input.threadId, - providerThreadId: input.providerThread.id, - runId: input.runId, - cause: "The provider session was released while the turn started.", - }), - ), - ), - ), - () => - runtime.startTurn(input).pipe(turnMetrics("send", input.modelSelection.model)), - (_, exit) => - Exit.isFailure(exit) - ? observeActivity( - providerSessionId, - markIdle(providerSessionId, input.providerThread.id, input.runOrdinal), - ).pipe( - // A failed start can leave its native thread gone (Claude rejecting a - // resume), so the next start must ask the adapter to resume it again. - Effect.andThen( - forgetProviderThreadLoaded({ - providerSessionId, - threadId: input.threadId, - runtime, - }), - ), - ) - : Effect.void, - ), - ), - ), + startTurn: trackTurnStart(runtime.startTurn), + ...(runtime.compactThread === undefined + ? {} + : { compactThread: trackTurnStart(runtime.compactThread) }), + ...(runtime.resumeTurn === undefined + ? {} + : { resumeTurn: trackTurnStart(runtime.resumeTurn) }), steerTurn: (input) => observeActivity(providerSessionId, touchActivity(providerSessionId)).pipe( Effect.andThen(runtime.steerTurn(input).pipe(turnMetrics("steer"))), @@ -2889,6 +2910,7 @@ export const layerWithOptions = ( entry.runtime.providerSessionId, event.providerThreadId, event.runOrdinal, + entry.runtime, ) : metadataOnly ? Effect.void diff --git a/apps/server/src/peer/RemoteDelegation.test.ts b/apps/server/src/peer/RemoteDelegation.test.ts index b638db854875f..bd5edbb7cc03e 100644 --- a/apps/server/src/peer/RemoteDelegation.test.ts +++ b/apps/server/src/peer/RemoteDelegation.test.ts @@ -683,6 +683,7 @@ const makeLaptop = ( kept?: Context.Context>>, /** Runs as each follow attempt here starts, on its follower's fiber. */ beforeFollow: Effect.Effect = Effect.void, + beforeRecord: Effect.Effect = Effect.void, ) => Effect.gen(function* () { const linking = kept ?? (yield* layerLinkingEnvironment(laptop).pipe(Layer.build)); @@ -718,6 +719,10 @@ const makeLaptop = ( const threads = yield* ThreadManagement.ThreadManagementService; return { ...threads, + dispatch: (command) => + command.type === "delegated_task.remote.request" + ? beforeRecord.pipe(Effect.andThen(threads.dispatch(command))) + : threads.dispatch(command), getThreadEventSequence: (threadId: ThreadId) => beforeFollow.pipe(Effect.andThen(threads.getThreadEventSequence(threadId))), }; @@ -1162,6 +1167,74 @@ it.effect("coalesces concurrent retries before asking the peer to launch", () => ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); +it.effect("keeps an acknowledged remote launch owned when its caller is interrupted", () => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const linking = yield* layerLinkingEnvironment(laptop).pipe(Layer.build); + const client = Context.get(linking, PeerMcpClient.PeerMcpClient); + const links = Context.get(linking, PeerLinks.PeerLinks); + const acknowledged = yield* Ref.make(false); + const recording = yield* Deferred.make(); + const release = yield* Deferred.make(); + const a = yield* makeLaptop( + database, + offers, + linking.pipe( + Context.add(PeerMcpClient.PeerMcpClient, { + call: (input) => + client + .call(input) + .pipe( + Effect.tap(() => + input.tool === "t3_thread_launch" ? Ref.set(acknowledged, true) : Effect.void, + ), + ), + }), + Context.add(PeerLinks.PeerLinks, { + ...links, + get: (environmentId) => + Effect.gen(function* () { + if (yield* Ref.getAndSet(acknowledged, false)) { + yield* Deferred.succeed(recording, undefined); + yield* Deferred.await(release); + } + return yield* links.get(environmentId); + }), + }), + ), + ); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + const pending = yield* delegateToBox(a).pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.gen(function* () { + yield* Deferred.await(recording); + const interrupted = yield* Fiber.interrupt(pending).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(interrupted); + const tasks = (yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents; + expect(tasks).toHaveLength(1); + expect(tasks[0]?.remoteChild?.threadId).toBe(yield* Ref.get(thread.launched)); + expect(yield* Ref.get(thread.launchCount)).toBe(1); + yield* Ref.set(thread.reply, "Finished after the request was cancelled."); + yield* Ref.set(thread.status, "completed"); + yield* Deferred.succeed(thread.finished, undefined); + expect(yield* taskResult(a, tasks[0]!.id)).toMatchObject({ + status: "completed", + result: "Finished after the request was cancelled.", + }); + }).pipe(Effect.ensuring(Deferred.succeed(release, undefined))); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), +); + it.effect.each(["no active parent run", "a newer run owned by another provider"] as const)( "preserves remote observation guards with %s", (observer) => @@ -2349,65 +2422,125 @@ it.effect("a follower that keeps hitting defects leaves a stop it owes owed", () ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); -it.effect("stopping the parent stops follow-up work on a task there that has its result", () => - Effect.scoped( - Effect.gen(function* () { - const thread = yield* makeBoxThread; - const b = yield* serveBox(thread); - const offers = yield* SubscriptionRef.make< - ReadonlyArray - >([]); - const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); - const a = yield* makeLaptop(database, offers); - yield* a.remote.start(); - yield* linkTo(a.links, b, "full-access"); - yield* seedParent(a); - const delegated = yield* delegateToBox(a); - yield* Ref.set(thread.reply, "All green."); - yield* Ref.set(thread.status, "completed"); - yield* Deferred.succeed(thread.finished, undefined); - expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ - status: "completed", - result: "All green.", - }); - - // The parent sends the thread there a follow-up, which starts a run there as a new task. - const sent = yield* a.call("t3_thread_send", { - environmentId: box.environmentId, - threadId: delegated.remoteChild.threadId, - message: "Now run the slow suite too.", - }); - expect(sent.isError, JSON.stringify(sent.content)).toBe(false); - const followUpId = (sent.structuredContent as { taskId: NodeId }).taskId; - expect(yield* Ref.get(thread.sent)).toEqual(["Now run the slow suite too."]); - expect(yield* Ref.get(thread.interrupted)).toBe(0); +it.effect.each(["none", "preflight", "record"] as const)( + "stopping the parent stops recorded follow-up work (request interrupted at %s)", + (interruptedAt) => + Effect.scoped( + Effect.gen(function* () { + const thread = yield* makeBoxThread; + const b = yield* serveBox(thread); + const offers = yield* SubscriptionRef.make< + ReadonlyArray + >([]); + const database = yield* SqlitePersistence.layerMemory.pipe(Layer.build); + const holdRecord = yield* Ref.make(false); + const recording = yield* Deferred.make(); + const release = yield* Deferred.make(); + const a = yield* makeLaptop( + database, + offers, + undefined, + Effect.void, + Ref.get(holdRecord).pipe( + Effect.flatMap((held) => + held + ? Deferred.succeed(recording, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ) + : Effect.void, + ), + ), + ); + yield* a.remote.start(); + yield* linkTo(a.links, b, "full-access"); + yield* seedParent(a); + const delegated = yield* delegateToBox(a); + yield* Ref.set(thread.reply, "All green."); + yield* Ref.set(thread.status, "completed"); + yield* Deferred.succeed(thread.finished, undefined); + expect(yield* taskResult(a, delegated.taskId)).toMatchObject({ + status: "completed", + result: "All green.", + }); - yield* a.threads.stopDelegatedTasks({ - threadId: parentThreadId, - commandId: CommandId.make("command:stop:laptop-parent-after-result"), - }); - // The thread there is stopped as Stop does, for each task, and the first keeps its result. - const task = yield* stoppedThere(a, delegated.taskId); - expect(yield* stoppedThere(a, followUpId, null)).toMatchObject({ status: "interrupted" }); - expect((yield* Ref.get(thread.stops)).toSorted()).toEqual([ - "delegated-tasks", - "delegated-tasks", - "thread.stop", - "thread.stop", - ]); - expect(task).toMatchObject({ status: "completed", result: "All green." }); - expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]); + // The parent sends the thread there a follow-up, which starts a run there as a new task. + const send = a.call("t3_thread_send", { + environmentId: box.environmentId, + threadId: delegated.remoteChild.threadId, + message: "Now run the slow suite too.", + clientRequestId: "slow-suite-followup", + }); + if (interruptedAt === "preflight") { + yield* Ref.set(thread.holdRead, true); + const pending = yield* send.pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.gen(function* () { + yield* Deferred.await(thread.readStarted); + yield* Fiber.interrupt(pending); + expect(yield* Ref.get(thread.sent)).toEqual([]); + expect( + (yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents, + ).toHaveLength(1); + }).pipe( + Effect.ensuring( + Ref.set(thread.holdRead, false).pipe( + Effect.andThen(Deferred.succeed(thread.releaseRead, undefined)), + ), + ), + ); + } + if (interruptedAt === "record") { + yield* Ref.set(holdRecord, true); + const pending = yield* send.pipe(Effect.forkChild({ startImmediately: true })); + yield* Effect.gen(function* () { + yield* Deferred.await(recording); + const stopping = yield* Fiber.interrupt(pending).pipe( + Effect.forkChild({ startImmediately: true }), + ); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(stopping); + }).pipe(Effect.ensuring(Deferred.succeed(release, undefined))); + yield* Ref.set(holdRecord, false); + } else { + const sent = yield* send; + expect(sent.isError, JSON.stringify(sent.content)).toBe(false); + } + const tasks = (yield* a.orchestrator.getThreadProjection(parentThreadId)).subagents; + expect(tasks).toHaveLength(2); + const followUpId = tasks.find((task) => task.id !== delegated.taskId)!.id; + if (interruptedAt === "record") { + const retried = yield* send; + expect(retried.isError, JSON.stringify(retried.content)).toBe(false); + expect(retried.structuredContent).toMatchObject({ taskId: followUpId }); + } + expect(yield* Ref.get(thread.sent)).toEqual(["Now run the slow suite too."]); + expect(yield* Ref.get(thread.interrupted)).toBe(0); - // A second Stop owes a second stop there, not a replay of the first. - yield* a.threads.stopDelegatedTasks({ - threadId: parentThreadId, - commandId: CommandId.make("command:stop:laptop-parent-again"), - }); - expect(yield* stoppedThere(a, delegated.taskId, 2)).toMatchObject({ status: "completed" }); - expect(yield* stoppedThere(a, followUpId, 1)).toMatchObject({ status: "interrupted" }); - expect(yield* Ref.get(thread.interrupted)).toBe(4); - }), - ).pipe(Effect.provide(NodeHttpServer.layerTest)), + yield* a.threads.stopDelegatedTasks({ + threadId: parentThreadId, + commandId: CommandId.make("command:stop:laptop-parent-after-result"), + }); + // The thread there is stopped as Stop does, for each task, and the first keeps its result. + const task = yield* stoppedThere(a, delegated.taskId); + expect(yield* stoppedThere(a, followUpId, null)).toMatchObject({ status: "interrupted" }); + expect((yield* Ref.get(thread.stops)).toSorted()).toEqual([ + "delegated-tasks", + "delegated-tasks", + "thread.stop", + "thread.stop", + ]); + expect(task).toMatchObject({ status: "completed", result: "All green." }); + expect(yield* a.projections.getOpenRemoteDelegatedTasks).toEqual([]); + + // A second Stop owes a second stop there, not a replay of the first. + yield* a.threads.stopDelegatedTasks({ + threadId: parentThreadId, + commandId: CommandId.make("command:stop:laptop-parent-again"), + }); + expect(yield* stoppedThere(a, delegated.taskId, 2)).toMatchObject({ status: "completed" }); + expect(yield* stoppedThere(a, followUpId, 1)).toMatchObject({ status: "interrupted" }); + expect(yield* Ref.get(thread.interrupted)).toBe(4); + }), + ).pipe(Effect.provide(NodeHttpServer.layerTest)), ); it.effect.each([ diff --git a/apps/server/src/peer/RemoteDelegation.ts b/apps/server/src/peer/RemoteDelegation.ts index 245f96df07d71..b12719a8516cf 100644 --- a/apps/server/src/peer/RemoteDelegation.ts +++ b/apps/server/src/peer/RemoteDelegation.ts @@ -369,98 +369,100 @@ const make = Effect.gen(function* () { ); const task = taskPrompt(input); const title = input.title ?? parent.thread.title; - const launched = yield* forwarding.call( - scope, - ProjectToolkit.tools.t3_thread_launch, - target.environmentId, - { - projectId, - title, - modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) }, - ...(runtimeMode === undefined ? {} : { runtimeMode }), - ...(interactionMode === undefined ? {} : { interactionMode }), - message: task, - clientRequestId: `delegate:${identity}`, - // The thread there shows it is this thread's subagent, with a way back. - delegatedFrom: { - environmentId: hereId, - threadId: parent.thread.id, - title: parent.thread.title, - }, - }, - ); - const link = yield* links.get(target.environmentId).pipe( - Effect.map(Option.getOrUndefined), - Effect.orElseSucceed(() => undefined), - ); - const remoteChild: OrchestrationV2RemoteTaskChild = { - environmentId: target.environmentId, - threadId: launched.threadId, - label: link?.label ?? target.environmentId, - title, - ...(launched.runId === null ? {} : { runId: launched.runId }), - }; - const recorded = yield* threads - .dispatch({ - type: "delegated_task.remote.request", - commandId, - parentThreadId: scope.thread.threadId, - parentRunId: parentRun.id, - parentNodeId, - task, - ...(input.title === undefined ? {} : { title: input.title }), - // The driver runs there; this side only labels the task with it. - driver: ProviderDriverKind.make("remote"), - modelSelection: launched.modelSelection, - remoteChild, - completionWake: input.mode === "wait" ? "settled_only" : "always", - }) - .pipe( - // The cause goes to the log; the caller learns only that recording failed. - Effect.tapError((cause) => - Effect.logWarning("remote-delegation.record-failed", { - parentThreadId: scope.thread.threadId, - cause, + // An acknowledged launch must acquire its durable local owner before cancellation lands. + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const launched = yield* restore( + forwarding.call(scope, ProjectToolkit.tools.t3_thread_launch, target.environmentId, { + projectId, + title, + modelSelection: { instanceId, model, ...(options === undefined ? {} : { options }) }, + ...(runtimeMode === undefined ? {} : { runtimeMode }), + ...(interactionMode === undefined ? {} : { interactionMode }), + message: task, + clientRequestId: `delegate:${identity}`, + // The thread there shows it is this thread's subagent, with a way back. + delegatedFrom: { + environmentId: hereId, + threadId: parent.thread.id, + title: parent.thread.title, + }, }), - ), - Effect.mapError( - () => - new OrchestratorMcpFailure({ - code: "orchestration_error", - message: "Unable to record the delegated task.", - }), - ), - // Nothing here follows a thread the task was never recorded for, so stop it there. - // A crash between the launch and this record still leaves it running there - // until a retry with the same clientRequestId records it. - Effect.tapError(() => - stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop").pipe( - Effect.ignoreCause({ log: true }), - ), - ), - ); - const taskEvent = recorded.storedEvents.find( - (stored) => stored.event.type === "subagent.updated", - ); - const taskId = - taskEvent?.event.type === "subagent.updated" - ? taskEvent.event.payload.id - : // A replayed request recorded the task the first time. - yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( - Effect.map( - ({ subagents }) => - subagents.find((task) => task.remoteChild?.threadId === launched.threadId)?.id, + ); + const link = yield* links.get(target.environmentId).pipe( + Effect.map(Option.getOrUndefined), + Effect.orElseSucceed(() => undefined), + ); + const remoteChild: OrchestrationV2RemoteTaskChild = { + environmentId: target.environmentId, + threadId: launched.threadId, + label: link?.label ?? target.environmentId, + title, + ...(launched.runId === null ? {} : { runId: launched.runId }), + }; + const recorded = yield* threads + .dispatch({ + type: "delegated_task.remote.request", + commandId, + parentThreadId: scope.thread.threadId, + parentRunId: parentRun.id, + parentNodeId, + task, + ...(input.title === undefined ? {} : { title: input.title }), + // The driver runs there; this side only labels the task with it. + driver: ProviderDriverKind.make("remote"), + modelSelection: launched.modelSelection, + remoteChild, + completionWake: input.mode === "wait" ? "settled_only" : "always", + }) + .pipe( + // The cause goes to the log; the caller learns only that recording failed. + Effect.tapError((cause) => + Effect.logWarning("remote-delegation.record-failed", { + parentThreadId: scope.thread.threadId, + cause, + }), + ), + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "Unable to record the delegated task.", + }), + ), + // Nothing here follows a thread the task was never recorded for, so stop it there. + // A crash between the launch and this record still leaves it running there + // until a retry with the same clientRequestId records it. + Effect.tapError(() => + restore(stopThere(followScope(scope.thread, remoteChild), remoteChild, "stop")).pipe( + Effect.ignoreCause({ log: true }), + ), ), - Effect.orElseSucceed(() => undefined), ); - if (taskId === undefined) { - return yield* new OrchestratorMcpFailure({ - code: "orchestration_error", - message: "The delegated task was not recorded.", - }); - } - yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers)); - return { taskId }; + const taskEvent = recorded.storedEvents.find( + (stored) => stored.event.type === "subagent.updated", + ); + const taskId = + taskEvent?.event.type === "subagent.updated" + ? taskEvent.event.payload.id + : // A replayed request recorded the task the first time. + yield* threads.getThreadRecords(scope.thread.threadId, ["subagents"]).pipe( + Effect.map( + ({ subagents }) => + subagents.find((task) => task.remoteChild?.threadId === launched.threadId)?.id, + ), + Effect.orElseSucceed(() => undefined), + ); + if (taskId === undefined) { + return yield* new OrchestratorMcpFailure({ + code: "orchestration_error", + message: "The delegated task was not recorded.", + }); + } + yield* followOne(scope.thread.threadId, taskId).pipe(Effect.forkIn(followers)); + return { taskId }; + }), + ); }); const delegate: RemoteDelegation["Service"]["delegate"] = Effect.fn("RemoteDelegation.delegate")( @@ -1027,6 +1029,7 @@ const make = Effect.gen(function* () { "Delegated follow-ups require an active run owned by this MCP provider session.", }); } + const parentNodeId = owner.rootNodeId; const taskId = IdAllocator.derive.delegatedTaskNode({ commandId }); const recorded = yield* readTask(parent.thread.id, taskId).pipe( Effect.mapError(unavailable), @@ -1041,64 +1044,80 @@ const make = Effect.gen(function* () { message: "This clientRequestId already belongs to another delegated follow-up.", }); } - const sent = yield* forwarding.sendToThread(scope, input.environmentId, { + const send = forwarding.sendToThread(scope, input.environmentId, { ...input, clientRequestId: key, }); - const existing = parent.subagents.find( - (task) => - task.result === null && - task.remoteChild?.environmentId === input.environmentId && - task.remoteChild.threadId === input.threadId && - task.remoteChild.runId === sent.runId, - ); - const tracked = recorded.task ?? existing; - if (tracked !== undefined) { - yield* followOne(parent.thread.id, tracked.id).pipe(Effect.forkIn(followers)); - return { ...sent, taskId: tracked.id }; + const recordedTask = recorded.task; + if (recordedTask !== undefined) { + return yield* Effect.uninterruptibleMask((restore) => + restore(send).pipe( + Effect.tap(() => + followOne(parent.thread.id, recordedTask.id).pipe(Effect.forkIn(followers)), + ), + Effect.map((sent) => ({ ...sent, taskId: recordedTask.id })), + ), + ); } - if (sent.delivery === "steered") return sent; + // Finish cancellable metadata reads before sending work that needs a local owner. const target = yield* forwarding.readThread(scope, input.environmentId, { threadId: input.threadId, limit: 1, maxCharsPerItem: 1, runLimit: 1, }); - yield* threads - .dispatch({ - type: "delegated_task.remote.request", - commandId, - parentThreadId: parent.thread.id, - parentRunId: owner.id, - parentNodeId: owner.rootNodeId, - task: input.message, - title: target.thread.title, - driver: ProviderDriverKind.make("remote"), - modelSelection: { - instanceId: target.thread.providerInstanceId, - model: target.thread.model, - }, - remoteChild: { - environmentId: input.environmentId, - threadId: input.threadId, - label: remote.label, - title: target.thread.title, - runId: sent.runId, - }, - completionWake: "always", - }) - .pipe( - Effect.mapError( - () => - new OrchestratorMcpFailure({ - code: "orchestration_error", - message: - "The message was sent, but its delegated follow-up could not be recorded. Retry with the same clientRequestId.", - }), - ), - ); - yield* followOne(parent.thread.id, taskId).pipe(Effect.forkIn(followers)); - return { ...sent, taskId }; + return yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const sent = yield* restore(send); + const existing = parent.subagents.find( + (task) => + task.result === null && + task.remoteChild?.environmentId === input.environmentId && + task.remoteChild.threadId === input.threadId && + task.remoteChild.runId === sent.runId, + ); + if (existing !== undefined) { + yield* followOne(parent.thread.id, existing.id).pipe(Effect.forkIn(followers)); + return { ...sent, taskId: existing.id }; + } + if (sent.delivery === "steered") return sent; + yield* threads + .dispatch({ + type: "delegated_task.remote.request", + commandId, + parentThreadId: parent.thread.id, + parentRunId: owner.id, + parentNodeId, + task: input.message, + title: target.thread.title, + driver: ProviderDriverKind.make("remote"), + modelSelection: { + instanceId: target.thread.providerInstanceId, + model: target.thread.model, + }, + remoteChild: { + environmentId: input.environmentId, + threadId: input.threadId, + label: remote.label, + title: target.thread.title, + runId: sent.runId, + }, + completionWake: "always", + }) + .pipe( + Effect.mapError( + () => + new OrchestratorMcpFailure({ + code: "orchestration_error", + message: + "The message was sent, but its delegated follow-up could not be recorded. Retry with the same clientRequestId.", + }), + ), + ); + yield* followOne(parent.thread.id, taskId).pipe(Effect.forkIn(followers)); + return { ...sent, taskId }; + }), + ); }), ); }); diff --git a/apps/server/src/serverSettings.test.ts b/apps/server/src/serverSettings.test.ts index c6db5e4b4803c..d7c63531cfded 100644 --- a/apps/server/src/serverSettings.test.ts +++ b/apps/server/src/serverSettings.test.ts @@ -786,20 +786,68 @@ it.layer(Layer.mergeAll(NodeServices.layer, NodeCrypto.layer))("server settings" }).pipe(Effect.provide(layerServerSettings())), ); - it.effect("buffers changes after a subscription is acquired but before it is consumed", () => - Effect.scoped( + it.effect.each([false, true])( + "buffers committed changes before subscription consumption (interrupted=%s)", + (interrupted) => Effect.gen(function* () { - const serverSettings = yield* ServerSettingsModule.ServerSettingsService; - const changes = yield* serverSettings.subscribeChanges; - - yield* serverSettings.updateSettings({ - addProjectBaseDirectory: "~/next", - }); - - const firstChange = yield* changes.pipe(Stream.runHead, Effect.timeout("1 second")); - assert.equal(Option.getOrUndefined(firstChange)?.addProjectBaseDirectory, "~/next"); + const fileSystem = yield* FileSystem.FileSystem; + const committed = yield* Deferred.make(); + const releaseCommit = yield* Deferred.make(); + let settingsPath: string | undefined; + const gatedFileSystem = FileSystem.FileSystem.of({ + ...fileSystem, + rename: (from, to) => + fileSystem + .rename(from, to) + .pipe( + Effect.tap(() => + to === settingsPath + ? Deferred.succeed(committed, undefined).pipe( + Effect.andThen(Deferred.await(releaseCommit)), + ) + : Effect.void, + ), + ), + }); + yield* Effect.scoped( + Effect.gen(function* () { + const serverSettings = yield* ServerSettingsModule.ServerSettingsService; + const config = yield* ServerConfig.ServerConfig; + yield* serverSettings.getSettings; + const changes = yield* serverSettings.subscribeChanges; + settingsPath = config.settingsPath; + const updating = yield* serverSettings + .updateSettings({ addProjectBaseDirectory: "~/next" }) + .pipe(Effect.forkChild); + yield* Deferred.await(committed).pipe( + Effect.andThen( + interrupted + ? Fiber.interrupt(updating).pipe(Effect.forkChild({ startImmediately: true })) + : Effect.void, + ), + Effect.ensuring(Deferred.succeed(releaseCommit, undefined)), + ); + const result = yield* Fiber.await(updating); + assert.equal(result._tag, interrupted ? "Failure" : "Success"); + const persisted = yield* fileSystem + .readFileString(config.settingsPath) + .pipe(Effect.flatMap(decodeServerSettingsJson)); + assert.equal(persisted.addProjectBaseDirectory, "~/next"); + assert.equal( + (yield* serverSettings.getSettings).addProjectBaseDirectory, + persisted.addProjectBaseDirectory, + ); + const firstChange = yield* changes.pipe(Stream.runHead, Effect.timeout("1 second")); + assert.equal(Option.getOrUndefined(firstChange)?.addProjectBaseDirectory, "~/next"); + }), + ).pipe( + Effect.provide( + layerServerSettings().pipe( + Layer.provideMerge(Layer.succeed(FileSystem.FileSystem, gatedFileSystem)), + ), + ), + ); }), - ).pipe(Effect.provide(layerServerSettings())), ); it.effect("preserves saved usage settings during cleanup edits and removes custom prices", () => diff --git a/apps/server/src/serverSettings.ts b/apps/server/src/serverSettings.ts index fe9cb1c9f5c05..4a48cda5c9d19 100644 --- a/apps/server/src/serverSettings.ts +++ b/apps/server/src/serverSettings.ts @@ -1395,28 +1395,33 @@ const make = Effect.gen(function* () { withoutRemovedSecretProviderReferences(current, vettedProviders.settings), ); const next = yield* normalizeServerSettings(persisted.settings); - const materializedNext = yield* Effect.uninterruptibleMask(() => + return yield* Effect.uninterruptibleMask(() => Effect.gen(function* () { - yield* invalidateMaterialized; - const rollbackSecretChanges = yield* applySettingsSecretChanges(persisted.changes); - const materializedExit = yield* Effect.exit(materializeSettingsSecrets(next)); - if (Exit.isFailure(materializedExit)) { - yield* rollbackSecretChanges; - return yield* Effect.failCause(materializedExit.cause); - } - const writeExit = yield* Effect.exit(writeSettingsAtomically(next)); - if (Exit.isFailure(writeExit)) { - yield* rollbackSecretChanges; - return yield* Effect.failCause(writeExit.cause); - } - return materializedExit.value; - }).pipe(Effect.ensuring(invalidateMaterialized)), + const materializedNext = yield* Effect.gen(function* () { + yield* invalidateMaterialized; + const rollbackSecretChanges = yield* applySettingsSecretChanges(persisted.changes); + const materializedExit = yield* Effect.exit(materializeSettingsSecrets(next)); + if (Exit.isFailure(materializedExit)) { + yield* rollbackSecretChanges; + return yield* Effect.failCause(materializedExit.cause); + } + const writeExit = yield* Effect.exit(writeSettingsAtomically(next)); + if (Exit.isFailure(writeExit)) { + yield* rollbackSecretChanges; + return yield* Effect.failCause(writeExit.cause); + } + return materializedExit.value; + }).pipe(Effect.ensuring(invalidateMaterialized)); + yield* Cache.set(settingsCache, cacheKey, { + settings: next, + parseFailure: Option.none(), + }); + const resolved = toEffective(materializedNext); + materialized = { source: next, value: resolved }; + yield* emitChange(next); + return { settings: resolved, result }; + }), ); - yield* Cache.set(settingsCache, cacheKey, { settings: next, parseFailure: Option.none() }); - const resolved = toEffective(materializedNext); - materialized = { source: next, value: resolved }; - yield* emitChange(next); - return { settings: resolved, result }; }), );