From 8b6ac19730141dcd8cf7ade47ea13ccd9c47d54f Mon Sep 17 00:00:00 2001 From: Kameron Smith Date: Sat, 21 Nov 2026 21:38:34 -0500 Subject: [PATCH] fix(server): share provider update locks, bound peer calls, mark lost child work interrupted - Build one ProviderMaintenanceRunner for the server and hand it to every WebSocket connection, so two clients share the provider update locks. - Time out a forwarded peer tool call after two minutes, so a half-open linked environment cannot hang task_cancel or task_status. - When a provider process is released with awaited background work still on its roster, record that work on the run as restart recovery does, so a delegated child settles as interrupted instead of completed. Co-Authored-By: Claude Opus 5.5 --- .../ProviderRuntimeRecoveryService.ts | 21 +--- .../ProviderSessionManager.test.ts | 66 +++++++++++++ .../ProviderSessionManager.ts | 42 +++++++- .../orchestration-v2/RestartBackgroundNote.ts | 18 ++++ apps/server/src/peer/PeerMcpClient.test.ts | 14 ++- apps/server/src/peer/PeerMcpClient.ts | 10 +- apps/server/src/server.ts | 3 + apps/server/src/ws.test.ts | 97 +++++++++++++++++++ apps/server/src/ws.ts | 9 +- 9 files changed, 255 insertions(+), 25 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts index 0cd4377a23b3d..35fe4cb27e058 100644 --- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts +++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts @@ -1,4 +1,3 @@ -import { runRanAfter } from "@t3tools/shared/orchestrationV2ThreadError"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { CommandId, @@ -40,6 +39,7 @@ import { awaitingProviderThreadId, cancelledRosterTaskWork, cancelledTurnItemWork, + latestStartedRun, mergeRestartCancelledBackgroundWork, neverStarted, } from "./RestartBackgroundNote.ts"; @@ -173,23 +173,6 @@ const cancelledItem = ( updatedAt: now, }; -function latestStartedRun( - projection: ProjectionStore.ProjectionRuntimeRecoveryState, - providerThreadId: ProviderThreadId, -) { - return projection.runs.reduce( - (latest, run) => - run.providerThreadId === providerThreadId && - run.status !== "queued" && - run.status !== "rolled_back" && - !neverStarted(run) && - (latest === undefined || runRanAfter(run, latest)) - ? run - : latest, - undefined, - ); -} - export const make = Effect.gen(function* () { const settings = yield* ServerSettings.ServerSettingsService; const projections = yield* ProjectionStore.ProjectionStoreV2; @@ -976,7 +959,7 @@ export const make = Effect.gen(function* () { }); } for (const [providerThreadId, work] of cancelledBackgroundWork) { - const noteRun = latestStartedRun(projection, providerThreadId); + const noteRun = latestStartedRun(projection.runs, providerThreadId); if (noteRun === undefined) continue; // Use a separate event; stale run snapshots can overwrite intervening lifecycle changes. events.push({ diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts index 98745a12813ca..5af9ff75fb666 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.test.ts @@ -5,6 +5,8 @@ import { assert, it } from "@effect/vitest"; import { AgentScope } from "@t3tools/shared/AgentScope"; import { EnvironmentId, + EventId, + MessageId, type ModelSelection, type OrchestrationV2AppThread, type OrchestrationV2DomainEvent, @@ -60,6 +62,7 @@ import * as ProviderInstanceRegistry from "../provider/ProviderInstanceRegistry. import * as GitVcsDriver from "../vcs/GitVcsDriver.ts"; import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts"; import * as ProviderSessionManager from "./ProviderSessionManager.ts"; +import { runOutcome } from "./RestartBackgroundNote.ts"; import * as CommandOutputHub from "./CommandOutputHub.ts"; import * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; import { @@ -3669,6 +3672,69 @@ it.effect.each(["a runtime whose tasks can outlive it", "server shutdown"] as co }), ); +// regression: a delegated child whose process ended under awaited background work settled as completed +it.effect("ProviderSessionManagerV2 records the background work a released process lost", () => + Effect.gen(function* () { + const state = yield* Ref.make(emptyState); + const effect = Effect.gen(function* () { + const manager = yield* ProviderSessionManager.ProviderSessionManagerV2; + const eventSink = yield* EventSink.EventSinkV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const { threadId, providerSessionId, providerThreads, writeProviderThread } = + yield* openSessionWithBackgroundCommand("provider-session-manager-lost-awaited-work"); + const now = yield* DateTime.now; + const runId = RunId.make("run:child-awaiting-reviewer"); + yield* eventSink.write({ + events: [ + { + id: EventId.make("event:child-run"), + type: "run.updated", + threadId, + runId, + occurredAt: now, + payload: { + id: runId, + threadId, + ordinal: 1, + providerInstanceId: modelSelection.instanceId, + modelSelection, + providerThreadId: providerThreads[0]!.id, + userMessageId: MessageId.make("message:child-prompt"), + rootNodeId: null, + activeAttemptId: null, + status: "completed", + requestedAt: now, + startedAt: now, + completedAt: now, + checkpointId: null, + contextHandoffId: null, + }, + }, + ], + }); + yield* writeProviderThread({ + ...providerThreads[0]!, + pendingBackgroundTasks: [ + ...providerThreads[0]!.pendingBackgroundTasks!, + { taskId: "reviewer", kind: "subagent", description: "Review the diff" }, + ], + }); + + yield* manager.close(providerSessionId); + + const { runs } = yield* projectionStore.getThreadRecords(threadId, ["runs"]); + const run = runs.find((candidate) => candidate.id === runId)!; + assert.sameMembers( + (run.restartCancelledBackgroundWork ?? []).map((work) => work.id), + ["dev-server", "reviewer"], + ); + assert.equal(runOutcome(run), "interrupted"); + }); + + yield* effect.pipe(Effect.provide(layerTest({ state, idleTimeoutMs: 60_000 }))); + }), +); + // regression: #19355 settled and idle-released threads stayed Waiting it.effect("ProviderSessionManagerV2 keeps provider updates that land while release clears", () => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index abcb5fd88196d..277dd273b26d1 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -65,6 +65,11 @@ import { } from "@t3tools/provider-core/server/subagentProjection"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; +import { + cancelledRosterTaskWork, + latestStartedRun, + mergeRestartCancelledBackgroundWork, +} from "./RestartBackgroundNote.ts"; const DEFAULT_IDLE_TIMEOUT_MS = 30 * 60 * 1000; const DEFAULT_MAX_IDLE_PIN_MS = 4 * 60 * 60 * 1000; @@ -967,7 +972,10 @@ export const layerWithOptions = ( }: ReleasedProviderThread) { if (taskIds.size === 0) return; for (let attempt = 0; attempt < 3; attempt++) { - const projection = yield* projectionStore.getThreadRecords(threadId, ["providerThreads"]); + const projection = yield* projectionStore.getThreadRecords(threadId, [ + "providerThreads", + "runs", + ]); const providerThread = projection.providerThreads.find( (candidate) => candidate.id === providerThreadId, ); @@ -975,6 +983,19 @@ export const layerWithOptions = ( const remaining = tasks.filter((task) => !taskIds.has(task.taskId)); if (providerThread === undefined || remaining.length === tasks.length) return; const now = yield* DateTime.now; + // Record the lost work as restart recovery does, so a child that was waiting on it + // settles as interrupted rather than completed. + const lost = + providerThread.ownerNodeId === null + ? tasks + .filter((task) => taskIds.has(task.taskId)) + .filter( + (task) => task.parentTaskId === undefined || !taskIds.has(task.parentTaskId), + ) + .map(cancelledRosterTaskWork) + : []; + const noteRun = + lost.length === 0 ? undefined : latestStartedRun(projection.runs, providerThreadId); const { committed } = yield* eventSink.writeIfProviderThreadUnchanged({ expected: providerThread, events: [ @@ -991,6 +1012,25 @@ export const layerWithOptions = ( updatedAt: now, }, }, + ...(noteRun === undefined + ? [] + : [ + { + id: yield* idAllocator.allocate.event({ threadId, providerSessionId }), + type: "run.background-work-cancelled" as const, + threadId, + runId: noteRun.id, + providerInstanceId: noteRun.providerInstanceId, + occurredAt: now, + payload: { + runId: noteRun.id, + restartCancelledBackgroundWork: mergeRestartCancelledBackgroundWork( + noteRun.restartCancelledBackgroundWork ?? [], + lost, + ), + }, + }, + ]), ], }); if (committed) return; diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index 23a95bc2bd20f..472861bb7cb72 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -185,6 +185,24 @@ export function neverStarted( return run.startedAt === null && run.restartContinuationOfRunId === undefined; } +/** The run that lost a provider thread's background work when its process ended. */ +export function latestStartedRun( + runs: ReadonlyArray, + providerThreadId: OrchestrationV2ProviderThread["id"], +): Run | undefined { + return runs.reduce( + (latest, run) => + run.providerThreadId === providerThreadId && + run.status !== "queued" && + run.status !== "rolled_back" && + !neverStarted(run) && + (latest === undefined || runRanAfter(run, latest)) + ? run + : latest, + undefined, + ); +} + /** Restart-cancelled unstarted runs supersede nothing; user-stopped unstarted runs do. */ export function cancelledBeforeStart( run: Pick, diff --git a/apps/server/src/peer/PeerMcpClient.test.ts b/apps/server/src/peer/PeerMcpClient.test.ts index c47ed35ca8d46..15746562df3b9 100644 --- a/apps/server/src/peer/PeerMcpClient.test.ts +++ b/apps/server/src/peer/PeerMcpClient.test.ts @@ -398,7 +398,7 @@ it.effect("renews a missing peer session once and reuses its replacement", () => }), ); -it.effect("bounds the session handshake, not a tool call that waits longer", () => +it.effect("bounds the handshake tightly and a tool call past any wait there", () => Effect.gen(function* () { const arrived = yield* Queue.unbounded(); const answer = yield* Deferred.make(); @@ -408,8 +408,8 @@ it.effect("bounds the session handshake, not a tool call that waits longer", () const index = requests++; yield* Queue.offer(arrived, index); // The first initialize, then the next initialized notification, never answer; - // the third session's tool call answers late. - if (index === 0 || index === 2) return yield* Effect.never; + // the third session's tool call answers late, and the one after it never does. + if (index === 0 || index === 2 || index === 6) return yield* Effect.never; if (index === 5) yield* Deferred.await(answer); return HttpClientResponse.fromWeb( request, @@ -437,5 +437,13 @@ it.effect("bounds the session handshake, not a tool call that waits longer", () yield* TestClock.adjust(Duration.minutes(1)); yield* Deferred.succeed(answer, undefined); expect(yield* Fiber.join(waiting)).toEqual({ modes: "approval-required/plan" }); + + const hung = yield* probe.pipe(Effect.flip, Effect.forkScoped); + yield* Queue.takeN(arrived, 1); + yield* TestClock.adjust(Duration.minutes(2)); + expect(yield* Fiber.join(hung)).toMatchObject({ + code: "orchestration_error", + message: "The linked environment stopped answering.", + }); }), ); diff --git a/apps/server/src/peer/PeerMcpClient.ts b/apps/server/src/peer/PeerMcpClient.ts index 07ddf93c9750d..a540732bcb7a3 100644 --- a/apps/server/src/peer/PeerMcpClient.ts +++ b/apps/server/src/peer/PeerMcpClient.ts @@ -7,6 +7,7 @@ import { type RuntimeMode, } from "@t3tools/contracts"; import * as Context from "effect/Context"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; @@ -22,6 +23,12 @@ import * as PeerLinks from "./PeerLinks.ts"; /** The only MCP protocol version a T3 Code `/mcp` registers. */ const PROTOCOL_VERSION = "2025-06-18"; +/** + * A forwarded call's ceiling once the session is open. Waits there stop at 45 s, so only a + * half-open peer runs this long; without it, a cancel or status read on that link never returns. + */ +const FORWARDED_CALL_TIMEOUT = Duration.minutes(2); + /** A title can be any length; the header carrying it stays well under HTTP header limits. */ const SENDER_TITLE_MAX_LENGTH = 200; @@ -162,7 +169,7 @@ const make = Effect.gen(function* () { ); return sessionId; }, - // Only the handshake is bounded: a forwarded tool call may wait for minutes. + // The handshake is bounded tightly; a forwarded tool call may legitimately wait longer. (effect, url) => Effect.timeout(effect, PeerLinks.probeTimeout(url)), ); @@ -300,6 +307,7 @@ const make = Effect.gen(function* () { }, (effect, input) => effect.pipe( + Effect.timeout(FORWARDED_CALL_TIMEOUT), Effect.tapError((error) => links.recordOutcome(input.environmentId, { error: error.message }), ), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index d8cf2f4a8c7dd..ee92d33f943de 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -120,6 +120,7 @@ import * as AntigravityInstallation from "./provider/AntigravityInstallation.ts" import * as ProviderInstallation from "./provider/providerInstallation.ts"; import * as CodexInstallation from "./provider/CodexInstallation.ts"; import * as ProviderInstallationGate from "./provider/ProviderInstallationGate.ts"; +import * as ProviderMaintenanceRunner from "./provider/providerMaintenanceRunner.ts"; import * as ProviderInstanceRegistry from "./provider/ProviderInstanceRegistry.ts"; import * as ProviderAdapterRegistry from "./orchestration-v2/ProviderAdapterRegistry.ts"; import * as ProviderRegistry from "./provider/ProviderRegistry.ts"; @@ -912,6 +913,8 @@ const layerMakeRoutes = Layer.mergeAll( Layer.provide(PeerLinkRequests.layer), // Share the service so HTTP/WS clients single-flight reads and invalidate each other's cached patches. Layer.provide(layerPullRequestService), + // Provider updates hold their locks here, shared by every WebSocket client. + Layer.provide(ProviderMaintenanceRunner.layer), // The same activity instance RunExecutionService reports to, by layer memoization. Layer.provide( ServerBrowser.layer.pipe( diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index 823756ffaaf27..023ff19b10300 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -9,6 +9,7 @@ import { assert, it } from "@effect/vitest"; import { AuthAccessReadScope, AuthOrchestrationOperateScope, + AuthProvidersManageScope, AuthOrchestrationReadScope, AuthSessionId, CommandId, @@ -3021,3 +3022,99 @@ it.live( assert.equal(admitted.resumed, false); }).pipe(Effect.scoped), ); + +it.live("every WebSocket connection updates providers through the server's one runner", () => + Effect.gen(function* () { + const updates: Array = []; + const services = Layer.mergeAll( + layerUnusedConfigRpcServices, + Layer.mock(ProviderMaintenanceRunner.ProviderMaintenanceRunner)({ + updateProvider: (target) => + Effect.sync(() => { + updates.push(typeof target === "string" ? target : target.provider); + return { providers: [] }; + }), + }), + Layer.mock(ServerBrowser.ServerBrowser)({ + profileReporter: Effect.succeed(() => Effect.void), + }), + Layer.mock(EnvironmentAuth.EnvironmentAuth)({ + authenticateWebSocketUpgrade: () => + Effect.succeed({ + sessionId: AuthSessionId.make("shared-runner"), + subject: "browser", + method: "browser-session-cookie", + scopes: [AuthProvidersManageScope], + }), + }), + Layer.mock(SessionStore.SessionStore)({ + cookieName: "shared-runner", + legacyCookieName: undefined, + markConnected: () => Effect.void, + markDisconnected: () => Effect.void, + awaitInvalidation: () => Effect.never, + }), + Layer.mock(AnalyticsService.AnalyticsService)({ + recordOwners: () => Effect.void, + record: () => Effect.void, + }), + Layer.mock(ManagedProjectFolders.ManagedProjectFolders)({ + namedProjectsRoot: process.cwd(), + }), + Layer.mock(ScratchThreadPromotion.ScratchThreadPromotion)({}), + Layer.mock(ProviderRegistry.ProviderRegistry)({}), + Layer.mock(UsageLimitSources.UsageLimitSources)({}), + Layer.mock(ExternalLauncher.ExternalLauncher)({}), + Layer.mock(RemoteOpenTargets.RemoteOpenTargets)({}), + Layer.mock(DirectEndpoints.DirectEndpoints)({}), + Layer.mock(ServerEnvironment.ServerEnvironment)({}), + Layer.mock(VcsProcess.VcsProcess)({ + run: (input) => + Effect.fail( + new VcsProcessSpawnError({ + operation: input.operation, + command: input.command, + cwd: input.cwd, + cause: "No VCS commands in the shared-runner fixture", + }), + ), + }), + ProviderInstallationGate.layer, + FetchHttpClient.layer, + ); + const context = yield* Layer.build( + HttpRouter.serve(Ws.layer, { disableListenLog: true }).pipe( + Layer.provideMerge( + NodeHttpServer.layer(NodeHttp.createServer, { host: "127.0.0.1", port: 0 }), + ), + Layer.provide(services), + Layer.provide(layerConfigSources), + ), + ); + const server = Context.get(context, HttpServer.HttpServer); + const url = `${HttpServer.formatAddress(server.address).replace(/^http/, "ws")}/ws?${ORCHESTRATION_PROTOCOL_QUERY_PARAM}=${ORCHESTRATION_PROTOCOL_VERSION}`; + const updateFromNewConnection = (provider: string) => + Effect.gen(function* () { + const client = yield* RpcClient.make(WsRpcGroup); + return yield* client[WS_METHODS.serverUpdateProvider]({ + provider: ProviderDriverKind.make(provider), + }); + }).pipe( + Effect.scoped, + Effect.provide( + RpcClient.layerProtocolSocket().pipe( + Layer.provide( + Socket.layerWebSocket(url, { openTimeout: "10 seconds" }).pipe( + Layer.provide(NodeSocket.layerWebSocketConstructor), + ), + ), + Layer.provide(RpcSerialization.layerJson), + ), + ), + ); + yield* updateFromNewConnection("codex"); + yield* updateFromNewConnection("claudeAgent"); + // A runner built per connection would hold its own "already running" set and package locks. + assert.deepEqual(updates, ["codex", "claudeAgent"]); + }).pipe(Effect.scoped), +); diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 0ccf7fb913a3e..baf4a4aa61a01 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -3748,6 +3748,7 @@ export const layer = Layer.unwrap( const serverSelfUpdate = yield* ServerSelfUpdate.ServerSelfUpdate; const npmRelease = yield* NpmRelease.NpmRelease; const pullRequests = yield* PullRequestService.PullRequestService; + const providerMaintenanceRunner = yield* ProviderMaintenanceRunner.ProviderMaintenanceRunner; const reviewGuides = yield* PullRequestReviewGuideService.PullRequestReviewGuideService; const peerLinks = yield* PeerLinks.PeerLinks; const peerForwarding = yield* PeerForwarding.PeerForwarding; @@ -3840,7 +3841,13 @@ export const layer = Layer.unwrap( Layer.provide(DefectReporter.layer), Layer.provide(AcpRegistrySessions.layer), Layer.provide(Layer.succeed(SqlClient.SqlClient, sql)), - Layer.provide(ProviderMaintenanceRunner.layer), + // One update lock for the server, so two clients cannot run the same update at once. + Layer.provide( + Layer.succeed( + ProviderMaintenanceRunner.ProviderMaintenanceRunner, + providerMaintenanceRunner, + ), + ), Layer.provide(Layer.succeed(NpmRelease.NpmRelease, npmRelease)), Layer.provide(Layer.succeed(ServerSelfUpdate.ServerSelfUpdate, serverSelfUpdate)), // Share server-lifetime PR caches so WS mutations invalidate every client's HTTP diffs.