From 7e71853d6981150ebb486e08c63614842059866b Mon Sep 17 00:00:00 2001 From: Terry Crapts Date: Wed, 4 Nov 2026 16:30:01 +0100 Subject: [PATCH 1/4] fix(server): withdraw approvals when provider turns fail --- apps/server/src/orchestration-v2/EventSink.ts | 32 ++--- .../src/orchestration-v2/ProjectionStore.ts | 70 ++++----- .../ProviderEventIngestor.test.ts | 39 ++--- .../orchestration-v2/ProviderEventIngestor.ts | 22 ++- .../ProviderTurnControlService.test.ts | 3 +- .../RunExecutionService.test.ts | 133 +++++++++++++++--- .../orchestration-v2/RunExecutionService.ts | 30 +++- 7 files changed, 231 insertions(+), 98 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index b210d3c9840ae..22a0f36964229 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -73,7 +73,7 @@ export interface EventSinkV2Shape { OrchestrationV2ProviderThread, "driver" | "providerInstanceId" | "appThreadId" | "nativeThreadRef" >; - readonly guardPendingUserInputCancellations?: boolean; + readonly guardPendingRuntimeRequestCancellations?: boolean; readonly commandId?: CommandId; readonly events: ReadonlyArray; }) => Effect.Effect, EventSinkV2Error>; @@ -87,13 +87,13 @@ export interface EventSinkV2Shape { OrchestrationV2ProviderThread, "driver" | "providerInstanceId" | "appThreadId" | "nativeThreadRef" >; - readonly guardPendingUserInputCancellations?: boolean; + readonly guardPendingRuntimeRequestCancellations?: boolean; readonly commandId?: CommandId; readonly events: ReadonlyArray; readonly effects: ReadonlyArray; }) => Effect.Effect, EventSinkV2Error>; readonly writeIfRunCurrent: (input: { - readonly guardPendingUserInputCancellations?: boolean; + readonly guardPendingRuntimeRequestCancellations?: boolean; readonly commandId?: CommandId; readonly threadId: ThreadId; readonly runId: RunId; @@ -110,7 +110,7 @@ export interface EventSinkV2Shape { >; /** Commits only if the expected attempt and ordinal still own the provider thread, protecting newer attempts. */ readonly writeIfProviderThreadOwner: (input: { - readonly guardPendingUserInputCancellations?: boolean; + readonly guardPendingRuntimeRequestCancellations?: boolean; readonly commandId?: CommandId; readonly providerThreadId: ProviderThreadId; readonly runId: RunId; @@ -285,17 +285,13 @@ const layerBase: Layer.Layer< // A user can answer after terminal normalization reads the pending request. // Recheck inside the write transaction so stale cleanup cannot erase answers. - const guardUserInputCancellations = + const guardRuntimeRequestCancellations = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (events: ReadonlyArray) { const staleRequests = new Set(); const staleNodes = new Set(); for (const event of events) { - if ( - event.type !== "runtime-request.updated" || - event.payload.kind !== "user_input" || - event.payload.status !== "cancelled" - ) + if (event.type !== "runtime-request.updated" || event.payload.status !== "cancelled") continue; const current = yield* projectionStore.getRuntimeRequest( event.threadId, @@ -303,7 +299,6 @@ const layerBase: Layer.Layer< ); if ( current?.status !== "pending" || - current.kind !== "user_input" || current.providerTurnId !== event.payload.providerTurnId || current.responseCapability.type === "message" ) { @@ -319,7 +314,8 @@ const layerBase: Layer.Layer< return event.payload.status !== "cancelled" || !staleNodes.has(event.payload.id); case "turn-item.updated": return ( - event.payload.type !== "user_input_request" || + (event.payload.type !== "user_input_request" && + event.payload.type !== "approval_request") || event.payload.status !== "cancelled" || !staleRequests.has(event.payload.requestId) ); @@ -438,8 +434,8 @@ const layerBase: Layer.Layer< ? yield* updateDelegatedCompletionOnly(input.events, () => true) : input.events; const normalized = yield* normalizeEvents( - input.guardPendingUserInputCancellations === true - ? yield* guardUserInputCancellations(events) + input.guardPendingRuntimeRequestCancellations === true + ? yield* guardRuntimeRequestCancellations(events) : events, ); const committed = yield* eventStore.append({ @@ -497,8 +493,8 @@ const layerBase: Layer.Layer< } const normalized = yield* normalizeEvents( - input.guardPendingUserInputCancellations === true - ? yield* guardUserInputCancellations(input.events) + input.guardPendingRuntimeRequestCancellations === true + ? yield* guardRuntimeRequestCancellations(input.events) : input.events, ); const storedEvents = yield* eventStore.append({ @@ -562,8 +558,8 @@ const layerBase: Layer.Layer< } const normalized = yield* normalizeEvents( - input.guardPendingUserInputCancellations === true - ? yield* guardUserInputCancellations(input.events) + input.guardPendingRuntimeRequestCancellations === true + ? yield* guardRuntimeRequestCancellations(input.events) : input.events, ); const storedEvents = yield* eventStore.append({ diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index b5a0f2cfb5556..5ea0e0b5d10ed 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -315,7 +315,7 @@ export type ProjectionRuntimeRecoveryState = Pick< | "turnItems" >; -export type ProjectionPendingUserInputs = Pick< +export type ProjectionPendingRuntimeRequests = Pick< OrchestrationV2ThreadProjection, "runtimeRequests" | "nodes" | "turnItems" >; @@ -606,10 +606,11 @@ export interface ProjectionStoreV2Shape { readonly getRuntimeRecoveryProjection: ( threadId: ThreadId, ) => Effect.Effect; - readonly getPendingNativeUserInputs: ( + readonly getPendingNativeRuntimeRequests: ( threadId: ThreadId, providerTurnId: ProviderTurnId, - ) => Effect.Effect; + includeApprovals?: boolean, + ) => Effect.Effect; readonly getPlan: ( threadId: ThreadId, planId: PlanId, @@ -4927,46 +4928,51 @@ export const layer: Layer.Layer = ? cause : new ProjectionStoreReadError({ threadId, cause }); - const getPendingNativeUserInputs: ProjectionStoreV2Shape["getPendingNativeUserInputs"] = ( - threadId, - providerTurnId, - ) => - sql - .withTransaction( - Effect.gen(function* () { - const rows = yield* sql` + const getPendingNativeRuntimeRequests: ProjectionStoreV2Shape["getPendingNativeRuntimeRequests"] = + (threadId, providerTurnId, includeApprovals = false) => + sql + .withTransaction( + Effect.gen(function* () { + const rows = yield* sql` SELECT payload_json FROM orchestration_v2_projection_runtime_requests WHERE thread_id = ${threadId} AND provider_turn_id = ${providerTurnId} - AND kind = 'user_input' AND status = 'pending' + AND status = 'pending' `; - const runtimeRequests = (yield* decodeRows( - decodeRuntimeRequestPayload, - threadId, - )(rows)).filter((request) => request.responseCapability.type !== "message"); - if (runtimeRequests.length === 0) return { runtimeRequests, nodes: [], turnItems: [] }; - const nodeRows = yield* sql` + const runtimeRequests = (yield* decodeRows( + decodeRuntimeRequestPayload, + threadId, + )(rows)).filter( + (request) => + request.responseCapability.type !== "message" && + (includeApprovals || request.kind === "user_input"), + ); + if (runtimeRequests.length === 0) + return { runtimeRequests, nodes: [], turnItems: [] }; + const nodeRows = yield* sql` SELECT payload_json FROM orchestration_v2_projection_nodes WHERE thread_id = ${threadId} AND ${sql.in( "node_id", runtimeRequests.map((request) => request.nodeId), )} `; - const itemRows = yield* sql` + const itemRows = yield* sql` SELECT payload_json FROM orchestration_v2_projection_turn_items WHERE thread_id = ${threadId} AND provider_turn_id = ${providerTurnId} - AND type = 'user_input_request' + AND type IN ('user_input_request', 'approval_request') `; - const requestIds = new Set(runtimeRequests.map((request) => request.id)); - return { - runtimeRequests, - nodes: yield* decodeRows(decodeNodePayload, threadId)(nodeRows), - turnItems: (yield* decodeRows(decodeTurnItemPayload, threadId)(itemRows)).filter( - (item) => item.type === "user_input_request" && requestIds.has(item.requestId), - ), - }; - }), - ) - .pipe(Effect.mapError(controlReadError(threadId))); + const requestIds = new Set(runtimeRequests.map((request) => request.id)); + return { + runtimeRequests, + nodes: yield* decodeRows(decodeNodePayload, threadId)(nodeRows), + turnItems: (yield* decodeRows(decodeTurnItemPayload, threadId)(itemRows)).filter( + (item) => + (item.type === "user_input_request" || item.type === "approval_request") && + requestIds.has(item.requestId), + ), + }; + }), + ) + .pipe(Effect.mapError(controlReadError(threadId))); const getPlan: ProjectionStoreV2Shape["getPlan"] = (threadId, planId) => sql @@ -7401,7 +7407,7 @@ export const layer: Layer.Layer = getRunMessage: readOnReader(getRunMessage), canStartQueuedRun: readOnReader(canStartQueuedRun), getHeldQueuedRuns: readOnReader(getHeldQueuedRuns), - getPendingNativeUserInputs: readOnReader(getPendingNativeUserInputs), + getPendingNativeRuntimeRequests: readOnReader(getPendingNativeRuntimeRequests), hasUnpairedRunInterruptRequest: readOnReader(hasUnpairedRunInterruptRequest), isNodeStored: readOnReader(isNodeStored), getMessageCount: readOnReader(getMessageCount), diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 568129c56cb8f..d21875deab5a2 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -1111,8 +1111,8 @@ layer("ProviderEventIngestorV2", (it) => { }), ); - it.effect.each(["interrupted", "control"] as const)( - "dismisses only native questions when a provider turn ends with %s", + it.effect.each(["interrupted", "failed", "completed", "cancelled", "control"] as const)( + "withdraws approvals only when explicitly abandoned (%s)", (terminal) => Effect.gen(function* () { const now = yield* DateTime.now; @@ -1227,6 +1227,7 @@ layer("ProviderEventIngestorV2", (it) => { providerSessionId, providerInstanceId: modelSelection.instanceId, threadId, + withdrawApprovals: terminal === "control", event: terminal === "control" ? { @@ -1258,7 +1259,10 @@ layer("ProviderEventIngestorV2", (it) => { const stored = yield* ingestor.ingestNormalized(input); const projection = yield* projectionStore.getThreadProjection(threadId); for (const fixture of fixtures) { - const closed = fixture.key === "native" || fixture.key === "unavailable"; + const closed = + fixture.key === "native" || + fixture.key === "unavailable" || + (fixture.key === "approval" && terminal === "control"); const request = projection.runtimeRequests.find( (item) => item.id === fixture.request.id, )!; @@ -1279,10 +1283,10 @@ layer("ProviderEventIngestorV2", (it) => { } assert.equal( stored.filter((entry) => entry.event.type === "runtime-request.updated").length, - 2, + terminal === "control" ? 3 : 2, ); assert.isEmpty( - (yield* projectionStore.getPendingNativeUserInputs(threadId, providerTurnId)) + (yield* projectionStore.getPendingNativeRuntimeRequests(threadId, providerTurnId)) .runtimeRequests, ); const replayed = yield* eventStore.read({ threadId }).pipe(Stream.runCollect); @@ -1292,16 +1296,16 @@ layer("ProviderEventIngestorV2", (it) => { entry.event.type === "runtime-request.updated" && entry.event.payload.status === "cancelled", ).length, - 2, + terminal === "control" ? 3 : 2, ); const repeated = yield* ingestor.ingestNormalized(input); assert.isFalse(repeated.some((entry) => entry.event.type === "runtime-request.updated")); }), ); - it.effect( - "preserves an answer committed after terminal normalization reads a pending question", - () => + it.effect.each(["question", "approval"] as const)( + "preserves an answer committed after terminal normalization reads a pending %s", + (kind) => Effect.scoped( Effect.gen(function* () { const eventSink = yield* EventSink.EventSinkV2; @@ -1329,7 +1333,7 @@ layer("ProviderEventIngestorV2", (it) => { nodeId, providerTurnId, nativeRequestRef: null, - kind: "user_input", + kind: kind === "approval" ? "command" : "user_input", status: "pending", responseCapability: { type: "live", providerSessionId }, createdAt: now, @@ -1341,7 +1345,7 @@ layer("ProviderEventIngestorV2", (it) => { runId: null, parentNodeId: null, rootNodeId: nodeId, - kind: "user_input_request", + kind: kind === "approval" ? "approval_request" : "user_input_request", status: "waiting", countsForRun: false, providerThreadId, @@ -1367,9 +1371,9 @@ layer("ProviderEventIngestorV2", (it) => { startedAt: now, completedAt: null, updatedAt: now, - type: "user_input_request", - requestId: request.id, - questions: [], + ...(kind === "approval" + ? { type: "approval_request", requestId: request.id, requestKind: "command" } + : { type: "user_input_request", requestId: request.id, questions: [] }), }; const seedEvents: Array = [threadEvent]; for (const payload of [ @@ -1400,6 +1404,7 @@ layer("ProviderEventIngestorV2", (it) => { ); const terminal = yield* ingestor .ingestNormalized({ + withdrawApprovals: kind === "approval", providerSessionId, providerInstanceId: modelSelection.instanceId, threadId, @@ -1422,11 +1427,12 @@ layer("ProviderEventIngestorV2", (it) => { .pipe(Effect.forkScoped); yield* Deferred.await(normalized); const answers = { decision: "Use the existing workspace" }; + const response = kind === "approval" ? { decision: "accept" as const } : { answers }; const responseEvents: Array = []; for (const payload of [ { type: "runtime-request.updated" as const, - payload: { ...request, status: "resolved" as const, answers, resolvedAt: now }, + payload: { ...request, status: "resolved" as const, ...response, resolvedAt: now }, }, { type: "node.updated" as const, @@ -1453,7 +1459,8 @@ layer("ProviderEventIngestorV2", (it) => { const projection = yield* projections.getThreadProjection(threadId); const answered = projection.runtimeRequests.find((entry) => entry.id === request.id)!; assert.equal(answered.status, "resolved"); - assert.deepEqual(answered.answers, answers); + if (kind === "approval") assert.equal(answered.decision, "accept"); + else assert.deepEqual(answered.answers, answers); assert.equal(projection.nodes.find((entry) => entry.id === nodeId)!.status, "completed"); assert.equal( projection.turnItems.find((entry) => entry.id === item.id)!.status, diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index f27580d909aff..489189df99f14 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -199,6 +199,7 @@ export interface ProviderEventIngestInput { readonly runId?: RunId; readonly nodeId?: NodeId; readonly rawEventId?: RawEventId; + readonly withdrawApprovals?: boolean; readonly event: ProviderAdapter.ProviderAdapterV2Event; readonly analyticsContext?: ProviderTurnAnalyticsContext; } @@ -293,13 +294,17 @@ export const layer: Layer.Layer< ); }); - const dismissNativeUserInputs = Effect.fn("ProviderEventIngestor.dismissNativeUserInputs")( + const dismissNativeRuntimeRequests = Effect.fn("ProviderEventIngestor.dismissNativeUserInputs")( function* ( input: ProviderEventIngestInput, providerTurnId: ProviderTurnId, threadId = input.threadId, ) { - const pending = yield* projections.getPendingNativeUserInputs(threadId, providerTurnId); + const pending = yield* projections.getPendingNativeRuntimeRequests( + threadId, + providerTurnId, + input.withdrawApprovals, + ); const now = yield* DateTime.now; const events: Array = []; for (const request of pending.runtimeRequests) { @@ -523,7 +528,7 @@ export const layer: Layer.Layer< ...(["completed", "interrupted", "failed", "cancelled"].includes( input.event.providerTurn.status, ) - ? yield* dismissNativeUserInputs( + ? yield* dismissNativeRuntimeRequests( input, input.event.providerTurn.id, input.event.threadId, @@ -622,7 +627,10 @@ export const layer: Layer.Layer< // Live output is routed to the command output hub before ingestion. return []; case "turn.terminal": - const dismissed = yield* dismissNativeUserInputs(input, input.event.providerTurnId); + const dismissed = yield* dismissNativeRuntimeRequests( + input, + input.event.providerTurnId, + ); if (input.event.status !== "failed") { return dismissed; } @@ -686,7 +694,7 @@ export const layer: Layer.Layer< if (input.writeIfProviderThreadOwner !== undefined) { const ownerResult = yield* eventSink .writeIfProviderThreadOwner({ - guardPendingUserInputCancellations: true, + guardPendingRuntimeRequestCancellations: true, ...(input.commandId === undefined ? {} : { commandId: input.commandId }), ...input.writeIfProviderThreadOwner, events, @@ -697,7 +705,7 @@ export const layer: Layer.Layer< if (input.writeIfRunCurrent === undefined) { return yield* eventSink .write({ - guardPendingUserInputCancellations: true, + guardPendingRuntimeRequestCancellations: true, ...(input.commandId === undefined ? {} : { commandId: input.commandId }), events, }) @@ -705,7 +713,7 @@ export const layer: Layer.Layer< } const result = yield* eventSink .writeIfRunCurrent({ - guardPendingUserInputCancellations: true, + guardPendingRuntimeRequestCancellations: true, ...(input.commandId === undefined ? {} : { commandId: input.commandId }), threadId: input.threadId, ...input.writeIfRunCurrent, diff --git a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts index ad5341dd99e49..506bed4516fa1 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnControlService.test.ts @@ -247,7 +247,8 @@ it.effect( getRunningTurnContext: () => Effect.die("unused getRunningTurnContext"), getThreadProviderContext: () => Effect.die("unused getThreadProviderContext"), getRuntimeResponseContext: () => Effect.die("unused getRuntimeResponseContext"), - getPendingNativeUserInputs: () => Effect.die("unused getPendingNativeUserInputs"), + getPendingNativeRuntimeRequests: () => + Effect.die("unused getPendingNativeRuntimeRequests"), getProviderControlContext: (_threadId, target) => Ref.get(projection).pipe( Effect.map((current) => ({ diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index f3f1542146c66..793fd27f0f8de 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -4,6 +4,7 @@ import { ChatAttachmentId, ChatFileAttachment, CommandId, + EventId, MessageId, NodeId, type OrchestrationV2AppThread, @@ -23,6 +24,7 @@ import { ProviderTurnId, RunAttemptId, RunId, + RuntimeRequestId, ServerSettingsError, ThreadId, TurnItemId, @@ -76,6 +78,7 @@ const layerRunExecutionTest = RunExecutionService.layer.pipe( Layer.mock(EventSink.EventSinkV2)({}), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: () => Effect.succeed([]), }), ServerSettings.layerTest(), @@ -903,6 +906,7 @@ it.effect("passes the thread's MCP app context to the provider under a safe key" Layer.mock(EventSink.EventSinkV2)({}), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: () => Effect.succeed([]), }), ServerSettings.layerTest(), @@ -937,6 +941,7 @@ it.effect("fails the run when its ownership check cannot be read before calling }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: () => Effect.succeed([]), }), ServerSettings.layerTest(), @@ -1149,6 +1154,7 @@ it.effect("starts the provider when checkpoint baseline capture fails", () => }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: () => Effect.succeed([]), }), ServerSettings.layerTest(), @@ -1271,6 +1277,7 @@ it.effect.each(["failure", "interruption", "stale-attempt", "start-guard"] as co }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: () => Effect.succeed([]), }), scenario === "start-guard" @@ -1451,6 +1458,7 @@ it.effect("ingests new member threads of an owned child after the root turn term }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: (input) => Effect.gen(function* () { const eventType = input.event.type; @@ -1706,6 +1714,7 @@ it.effect("ends a nested native subagent and its thread when the root run is int }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: () => Effect.succeed([]), }), ServerSettings.layerTest(), @@ -2118,6 +2127,7 @@ it.effect( }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: (input) => Ref.update(ingested, (current) => [...current, input.event]).pipe(Effect.as([])), }), @@ -2663,28 +2673,106 @@ it.effect("does not refresh pull requests for auxiliary or stale provider termin }), ); -it.effect("refreshes pull requests after a provider stream exits with an error", () => - Effect.gen(function* () { - const { written, observed } = yield* captureRootRunTermination({ - key: "pull-request-refresh:stream-error", - shouldFinalizeRun: () => Effect.succeed(true), - events: () => - Stream.fail( - new ProviderAdapter.ProviderAdapterEventStreamError({ - driver, - providerSessionId: ProviderSessionId.make("session:exited"), - cause: "provider process exited", - }), +it.effect.each(["stream", "terminal", "continuing", "completed"] as const)( + "withdraws failed-turn approvals only when children stop (%s)", + (failureMode) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const requestId = RuntimeRequestId.make("request:stream-error"); + const observedTurnId = ProviderTurnId.make("provider-turn:stream-error:observed"); + const guarded = yield* Ref.make(false); + const { written, observed, finalEvents } = yield* captureRootRunTermination({ + key: "pull-request-refresh:stream-error", + shouldFinalizeRun: () => Effect.succeed(true), + initialProviderTurnId: null, + normalize: (input) => + Effect.succeed( + input.withdrawApprovals && + input.event.type === "turn.terminal" && + input.event.providerTurnId === observedTurnId + ? [ + { + id: EventId.make("event:stream-error:withdraw-request"), + type: "runtime-request.updated", + threadId: input.threadId, + occurredAt: now, + payload: { + id: requestId, + nodeId: NodeId.make("node:stream-error:approval"), + providerTurnId: + input.event.type === "turn.terminal" ? input.event.providerTurnId : null, + nativeRequestRef: null, + kind: "file-change", + status: "cancelled", + responseCapability: { + type: "not_resumable", + reason: "Provider stream ended", + }, + createdAt: now, + resolvedAt: now, + }, + }, + ] + : [], + ), + beforeTerminalWrite: (input) => { + return Ref.set(guarded, input.guardPendingRuntimeRequestCancellations === true).pipe( + Effect.as(true), + ); + }, + events: (ids) => + Stream.concat( + Stream.make({ + type: "provider_turn.updated", + driver, + providerTurn: { + id: observedTurnId, + providerThreadId: ids.providerThreadId, + nodeId: ids.rootNodeId, + runAttemptId: ids.attemptId, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }), + failureMode === "stream" + ? Stream.fail( + new ProviderAdapter.ProviderAdapterEventStreamError({ + driver, + providerSessionId: ProviderSessionId.make("session:exited"), + cause: "provider process exited", + }), + ) + : Stream.make({ + ...rootTerminalEvent( + { ...ids, rootProviderTurnId: observedTurnId }, + failureMode === "completed" ? "completed" : "failed", + ), + ...(failureMode === "continuing" ? { childTurnsContinue: true as const } : {}), + }), + ), + }); + assert.isTrue(yield* Ref.get(guarded)); + assert.deepEqual(observed, [ + failureMode === "completed" ? "run:waiting" : "run:failed", + "pull-requests-refreshed", + ]); + assert.deepEqual( + written.map((item) => item.type), + failureMode === "stream" ? ["error"] : [], + ); + const error = written.find((item) => item.type === "error"); + if (failureMode === "stream") + assert.include(error?.failure.message ?? "", "provider event stream closed unexpectedly"); + assert.equal( + finalEvents.some( + (event) => event.type === "runtime-request.updated" && event.payload.id === requestId, ), - }); - assert.deepEqual(observed, ["run:failed", "pull-requests-refreshed"]); - assert.deepEqual( - written.map((item) => item.type), - ["error"], - ); - const error = written.find((item) => item.type === "error"); - assert.include(error?.failure.message ?? "", "provider event stream closed unexpectedly"); - }), + failureMode === "stream" || failureMode === "terminal", + ); + }), ); it.effect("records a provider's input rejection at turn start in the terminal error", () => @@ -3148,6 +3236,7 @@ function captureRootRunTermination(input: { readonly shouldStartProviderTurn?: RunExecutionService.RunExecutionServiceV2StartRootRunInput["shouldStartProviderTurn"]; readonly appContext?: ReturnType; readonly ingestNormalized?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["ingestNormalized"]; + readonly normalize?: ProviderEventIngestor.ProviderEventIngestorV2["Service"]["normalize"]; readonly beforeTerminalWrite?: ( request: Parameters[0], ) => Effect.Effect; @@ -3228,6 +3317,7 @@ function captureRootRunTermination(input: { }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: input.normalize ?? (() => Effect.succeed([])), ingestNormalized: input.ingestNormalized ?? (() => Effect.succeed([])), }), ServerSettings.layerTest(), @@ -3733,6 +3823,7 @@ function runBackgroundItemScenario( }), IdAllocator.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: (input) => Effect.gen(function* () { const event = input.event; diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 6a52fac447926..8c8eb0dfb3ee0 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -621,6 +621,7 @@ export const layer: Layer.Layer< const writeFinalRunEvents = Effect.fn("RunExecutionService.writeFinalRunEvents")( function* (input: { readonly run: OrchestrationV2Run; + readonly providerSessionId: ProviderSessionId; readonly rootNode: OrchestrationV2ExecutionNode; readonly checkpointScope: OrchestrationV2CheckpointScope; readonly providerThread: OrchestrationV2ProviderThread; @@ -738,8 +739,24 @@ export const layer: Layer.Layer< const checkpointCaptureCommandId = CommandId.make( `command:effect:checkpoint.capture:${input.run.id}`, ); - // Stopped runs capture rollback points for the next message; enqueue before later starts on this effect lane. + // Failed turns without continuing children have no consumer for their pending callbacks. + const cancelledRequestEvents = + input.terminal.status === "failed" && !childTurnsContinue(input.terminal) + ? (yield* providerEventIngestor.normalize({ + providerSessionId: input.providerSessionId, + providerInstanceId: input.run.providerInstanceId, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + event: input.terminal, + withdrawApprovals: true, + })).filter( + (event) => event.type !== "turn-item.updated" || event.payload.type !== "error", + ) + : []; const finalization = { + guardPendingRuntimeRequestCancellations: true, + // Stopped runs capture rollback points for the next message; enqueue before later starts on this effect lane. effects: input.terminal.status === "completed" || input.terminal.status === "interrupted" || @@ -758,6 +775,7 @@ export const layer: Layer.Layer< ] : [], events: [ + ...cancelledRequestEvents, ...cascadedSubagentEvents, ...(input.abandonedProviderTurn === undefined ? [] @@ -860,12 +878,11 @@ export const layer: Layer.Layer< } satisfies Parameters[0]; if (input.writeIfRunCurrent !== undefined) { const result = yield* eventSink.writeIfRunCurrent({ + ...finalization, threadId: input.run.threadId, runId: input.run.id, activeAttemptId: input.writeIfRunCurrent.activeAttemptId, expectedStatus: input.writeIfRunCurrent.expectedStatus, - events: finalization.events, - effects: finalization.effects, }); if (!result.committed) { return; @@ -906,11 +923,13 @@ export const layer: Layer.Layer< const makeFailedTerminalEvent = ( failure: OrchestrationV2ProviderFailure, failureItemOrdinal: number, + providerTurnId?: ProviderTurnId | null, ): ProviderTerminalEvent => ({ type: "turn.terminal", driver: input.providerThread.driver, providerThreadId: input.providerThread.id, providerTurnId: + providerTurnId ?? input.attempt.providerTurnId ?? idAllocator.derive.providerTurn({ driver: input.providerThread.driver, @@ -966,6 +985,7 @@ export const layer: Layer.Layer< cause, }); yield* writeFinalRunEvents({ + providerSessionId: input.providerSessionId, run: input.run, rootNode: input.rootNode, checkpointScope: input.checkpointScope, @@ -1082,6 +1102,7 @@ export const layer: Layer.Layer< const providerThread = yield* Ref.get(latestProviderThread); const openSubagents = yield* Ref.get(openRunOwnedSubagents); yield* writeFinalRunEvents({ + providerSessionId: input.providerSessionId, run: input.run, rootNode: input.rootNode, checkpointScope: input.checkpointScope, @@ -1601,6 +1622,7 @@ export const layer: Layer.Layer< const squashed = Cause.squash(cause); let nextRetryWarningAt = completedAt; yield* writeFinalRunEvents({ + providerSessionId: input.providerSessionId, run: input.run, rootNode: input.rootNode, checkpointScope: input.checkpointScope, @@ -1622,6 +1644,7 @@ export const layer: Layer.Layer< class: "unknown", }), latestItemOrdinal + 1, + routing.rootProviderTurnId, ), failureItemPersisted: false, refreshAfterTurn, @@ -1762,6 +1785,7 @@ export const layer: Layer.Layer< Ref.get(openRunOwnedSubagents).pipe( Effect.flatMap((openSubagents) => writeFinalRunEvents({ + providerSessionId: input.providerSessionId, run: input.run, rootNode: input.rootNode, checkpointScope: input.checkpointScope, From 573d81f8230ec8c829025b1277017fdc4645291d Mon Sep 17 00:00:00 2001 From: Terry Crapts Date: Wed, 4 Nov 2026 16:46:20 +0100 Subject: [PATCH 2/4] fix(server): withdraw failed runs' child approvals --- .../RunExecutionService.test.ts | 109 +++++++++++++++--- .../orchestration-v2/RunExecutionService.ts | 64 +++++++--- 2 files changed, 141 insertions(+), 32 deletions(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 793fd27f0f8de..078bd98ee4ee8 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2673,12 +2673,18 @@ it.effect("does not refresh pull requests for auxiliary or stale provider termin }), ); -it.effect.each(["stream", "terminal", "continuing", "completed"] as const)( +it.effect.each(["stream", "terminal", "continuing", "completed", "transferred"] as const)( "withdraws failed-turn approvals only when children stop (%s)", (failureMode) => Effect.gen(function* () { const now = yield* DateTime.now; const requestId = RuntimeRequestId.make("request:stream-error"); + const childRequestId = RuntimeRequestId.make("request:stream-error:child"); + const childTurnId = ProviderTurnId.make("provider-turn:stream-error:child"); + const childProviderThreadId = ProviderThreadId.make("provider-thread:stream-error:child"); + const childThreadId = backgroundScenarioIds( + "pull-request-refresh:stream-error", + ).childThreadId; const observedTurnId = ProviderTurnId.make("provider-turn:stream-error:observed"); const guarded = yield* Ref.make(false); const { written, observed, finalEvents } = yield* captureRootRunTermination({ @@ -2689,15 +2695,16 @@ it.effect.each(["stream", "terminal", "continuing", "completed"] as const)( Effect.succeed( input.withdrawApprovals && input.event.type === "turn.terminal" && - input.event.providerTurnId === observedTurnId + (input.event.providerTurnId === observedTurnId || + (input.event.providerTurnId === childTurnId && input.threadId === childThreadId)) ? [ { - id: EventId.make("event:stream-error:withdraw-request"), + id: EventId.make(`event:stream-error:withdraw-request:${input.threadId}`), type: "runtime-request.updated", threadId: input.threadId, occurredAt: now, payload: { - id: requestId, + id: input.threadId === childThreadId ? childRequestId : requestId, nodeId: NodeId.make("node:stream-error:approval"), providerTurnId: input.event.type === "turn.terminal" ? input.event.providerTurnId : null, @@ -2722,21 +2729,78 @@ it.effect.each(["stream", "terminal", "continuing", "completed"] as const)( }, events: (ids) => Stream.concat( - Stream.make({ - type: "provider_turn.updated", - driver, - providerTurn: { - id: observedTurnId, - providerThreadId: ids.providerThreadId, - nodeId: ids.rootNodeId, - runAttemptId: ids.attemptId, - nativeTurnRef: null, - ordinal: 1, - status: "running", - startedAt: now, - completedAt: null, + Stream.make( + childThreadCreatedEvent(ids), + { + type: "provider_turn.updated", + threadId: childThreadId, + driver, + providerTurn: { + id: childTurnId, + providerThreadId: childProviderThreadId, + nodeId: NodeId.make("node:stream-error:child-root"), + runAttemptId: null, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, }, - }), + { + type: "runtime_request.updated", + driver, + threadId: childThreadId, + runtimeRequest: { + id: childRequestId, + nodeId: NodeId.make("node:stream-error:child-approval"), + providerTurnId: childTurnId, + nativeRequestRef: null, + kind: "command", + status: "pending", + responseCapability: { + type: "live", + providerSessionId: ProviderSessionId.make("session:stream-error"), + }, + createdAt: now, + resolvedAt: null, + }, + }, + { + type: "provider_turn.updated", + driver, + providerTurn: { + id: observedTurnId, + providerThreadId: ids.providerThreadId, + nodeId: ids.rootNodeId, + runAttemptId: ids.attemptId, + nativeTurnRef: null, + ordinal: 1, + status: "running", + startedAt: now, + completedAt: null, + }, + }, + ...(failureMode === "transferred" + ? [ + { + type: "subagent.updated" as const, + driver, + subagent: { + ...makeRunOwnedSubagentFixture({ + ids, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId, + driver, + status: "running", + }), + providerThreadId: childProviderThreadId, + runId: RunId.make("run:replacement"), + }, + }, + ] + : []), + ), failureMode === "stream" ? Stream.fail( new ProviderAdapter.ProviderAdapterEventStreamError({ @@ -2768,10 +2832,17 @@ it.effect.each(["stream", "terminal", "continuing", "completed"] as const)( assert.include(error?.failure.message ?? "", "provider event stream closed unexpectedly"); assert.equal( finalEvents.some( - (event) => event.type === "runtime-request.updated" && event.payload.id === requestId, + (event) => + event.type === "runtime-request.updated" && event.payload.id === childRequestId, ), failureMode === "stream" || failureMode === "terminal", ); + assert.equal( + finalEvents.some( + (event) => event.type === "runtime-request.updated" && event.payload.id === requestId, + ), + failureMode === "stream" || failureMode === "terminal" || failureMode === "transferred", + ); }), ); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 8c8eb0dfb3ee0..162b44c5040cc 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -21,6 +21,7 @@ import { type ProviderThreadId, type ProviderTurnId, type RunAttemptId, + type RuntimeRequestId, type ThreadId, type TurnItemId, } from "@t3tools/contracts"; @@ -152,6 +153,11 @@ export function selectInheritedBackgroundTurnItems(input: { type SubagentTurnItem = Extract; +type PendingRequestTurn = { + readonly threadId: ThreadId; + readonly providerTurnId: ProviderTurnId; +}; + type OpenRunOwnedSubagentProjection = { readonly subagents: ReadonlyMap; readonly turnItems: ReadonlyMap; @@ -635,6 +641,7 @@ export const layer: Layer.Layer< readonly terminal: ProviderTerminalEvent; readonly completedAt?: DateTime.Utc; readonly failureItemPersisted: boolean; + readonly pendingRequestTurns?: ReadonlyArray; readonly refreshAfterTurn: Effect.Effect; readonly writeIfRunCurrent?: { readonly activeAttemptId: RunAttemptId; @@ -740,20 +747,33 @@ export const layer: Layer.Layer< `command:effect:checkpoint.capture:${input.run.id}`, ); // Failed turns without continuing children have no consumer for their pending callbacks. - const cancelledRequestEvents = - input.terminal.status === "failed" && !childTurnsContinue(input.terminal) - ? (yield* providerEventIngestor.normalize({ - providerSessionId: input.providerSessionId, - providerInstanceId: input.run.providerInstanceId, - threadId: input.run.threadId, - runId: input.run.id, - nodeId: input.rootNode.id, - event: input.terminal, - withdrawApprovals: true, - })).filter( + const cancelledRequestEvents: Array = []; + if (input.terminal.status === "failed" && !childTurnsContinue(input.terminal)) { + const turns = new Map>(); + for (const turn of [ + { threadId: input.run.threadId, providerTurnId: input.terminal.providerTurnId }, + ...(input.pendingRequestTurns ?? []), + ]) { + const seen = turns.get(turn.threadId) ?? new Set(); + if (seen.has(turn.providerTurnId)) continue; + seen.add(turn.providerTurnId); + turns.set(turn.threadId, seen); + const events = yield* providerEventIngestor.normalize({ + providerSessionId: input.providerSessionId, + providerInstanceId: input.run.providerInstanceId, + threadId: turn.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + event: { ...input.terminal, providerTurnId: turn.providerTurnId }, + withdrawApprovals: true, + }); + cancelledRequestEvents.push( + ...events.filter( (event) => event.type !== "turn-item.updated" || event.payload.type !== "error", - ) - : []; + ), + ); + } + } const finalization = { guardPendingRuntimeRequestCancellations: true, // Stopped runs capture rollback points for the next message; enqueue before later starts on this effect lane. @@ -1083,6 +1103,13 @@ export const layer: Layer.Layer< : { relatedProviderThreadIds: input.relatedProviderThreadIds }), }), ); + const pendingRequestTurns = new Map(); + const ownedPendingRequestTurns = (routing: ProviderEventRoutingState) => + [...pendingRequestTurns.values()].filter( + (turn) => + routing.ownedProviderTurns.has(turn.providerTurnId) || + (turn.threadId !== input.run.threadId && routing.ownedThreadIds.has(turn.threadId)), + ); const rootTerminalSeen = yield* Ref.make(false); const rootRunFinalized = yield* Ref.make(false); const providerThreadOwnerLost = yield* Ref.make(false); @@ -1128,6 +1155,7 @@ export const layer: Layer.Layer< ? {} : { loadRunInterruptReason: input.loadRunInterruptReason }), openRunOwnedSubagents: openSubagents, + pendingRequestTurns: ownedPendingRequestTurns(yield* Ref.get(eventRouting)), terminal, failureItemPersisted: terminal.status === "failed", refreshAfterTurn, @@ -1145,6 +1173,15 @@ export const layer: Layer.Layer< deliverable: boolean, ) { const routing = yield* Ref.get(eventRouting); + if (event.type === "runtime_request.updated") { + const request = event.runtimeRequest; + if (request.status === "pending" && request.providerTurnId !== null) { + pendingRequestTurns.set(request.id, { + threadId: event.threadId ?? input.run.threadId, + providerTurnId: request.providerTurnId, + }); + } else pendingRequestTurns.delete(request.id); + } if (event.type === "provider_turn.updated") { const isRoot = event.providerTurn.runAttemptId === input.attempt.id || @@ -1633,6 +1670,7 @@ export const layer: Layer.Layer< expectedStatus: "running", }, openRunOwnedSubagents: openSubagents, + pendingRequestTurns: ownedPendingRequestTurns(routing), ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), completedAt, terminal: makeFailedTerminalEvent( From 436aece34e405c80b7b06e337dc48648df4b59b9 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Sun, 15 Nov 2026 00:32:35 +0000 Subject: [PATCH 3/4] test(server): stub provider event normalization in start settlement test Failed-run finalization now normalizes the failed terminal to withdraw pending approvals, so the mocked ingestor needs a normalize stub. Co-Authored-By: Claude Opus 5.5 --- .../server/src/orchestration-v2/ProviderTurnStartService.test.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts index 7b0759ae99b0b..d0643188a5005 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.test.ts @@ -1690,6 +1690,7 @@ effectIt.effect("preserves a replacement committed before exhausted start settle IdAllocator.layer, ThreadCommandExecutor.layer, Layer.mock(ProviderEventIngestor.ProviderEventIngestorV2)({ + normalize: () => Effect.succeed([]), ingestNormalized: () => Effect.sync(() => { harness.setProjection({ ...harness.projection(), providerTurns: [providerTurn] }); From d2dd40e1c66bd142aa0a9e3562eb3e3a31006af3 Mon Sep 17 00:00:00 2001 From: Terry Crapts Date: Tue, 17 Nov 2026 10:28:11 +0100 Subject: [PATCH 4/4] fix(server): withdrawn child approvals stay cancelled when a failed run cascades The failed-run cascade wrote failed snapshots for the same approval node and item that the withdrawal had just cancelled. The later snapshot won, and the answer guard only drops cancellations, so it could overwrite a just-committed answer. Skip cascade snapshots for records the withdrawal already covers. Co-Authored-By: Claude Opus 5.5 --- .../RunExecutionService.test.ts | 105 ++++++++++++++++++ .../orchestration-v2/RunExecutionService.ts | 14 ++- 2 files changed, 118 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 9d8901df0c76e..a31862bcc8524 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -3405,6 +3405,111 @@ it.effect.each(["stream", "terminal", "continuing", "completed", "transferred"] }), ); +it.effect("keeps withdrawn child approvals cancelled when the failed run cascades", () => + Effect.gen(function* () { + const now = yield* DateTime.now; + const key = "withdrawn-child-approval-cascade"; + const scenario = backgroundScenarioIds(key); + const childRequestId = RuntimeRequestId.make(`request:${key}:child`); + const approvalNode: OrchestrationV2ExecutionNode = { + ...makeRunOwnedSubagentChildNodeFixture({ ids: scenario, status: "running" }), + id: NodeId.make(`node:${key}:child-approval`), + kind: "approval_request", + status: "waiting", + runtimeRequestId: childRequestId, + }; + const approvalItem: OrchestrationV2TurnItem = { + id: TurnItemId.make(`turn-item:${key}:child-approval`), + threadId: scenario.childThreadId, + runId: null, + nodeId: approvalNode.id, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + status: "waiting", + title: null, + startedAt: now, + completedAt: null, + updatedAt: now, + type: "approval_request", + requestId: childRequestId, + requestKind: "command", + prompt: "rm -rf build", + }; + const { finalEvents } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + normalize: (input) => + Effect.succeed( + input.withdrawApprovals && input.threadId === scenario.childThreadId + ? [ + { + id: EventId.make(`event:${key}:withdraw-node`), + type: "node.updated", + threadId: input.threadId, + occurredAt: now, + payload: { ...approvalNode, status: "cancelled", completedAt: now }, + }, + { + id: EventId.make(`event:${key}:withdraw-item`), + type: "turn-item.updated", + threadId: input.threadId, + occurredAt: now, + payload: { ...approvalItem, status: "cancelled", completedAt: now }, + }, + ] + : [], + ), + events: (ids) => + Stream.make( + childThreadCreatedEvent(ids), + { + type: "subagent.updated", + driver, + subagent: makeRunOwnedSubagentFixture({ + ids, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId: ids.childThreadId, + driver, + status: "running", + }), + }, + { + type: "runtime_request.updated", + driver, + threadId: ids.childThreadId, + runtimeRequest: { + id: childRequestId, + nodeId: approvalNode.id, + providerTurnId: ProviderTurnId.make(`provider-turn:${key}:child`), + nativeRequestRef: null, + kind: "command", + status: "pending", + responseCapability: { + type: "live", + providerSessionId: ProviderSessionId.make(`session:${key}`), + }, + createdAt: now, + resolvedAt: null, + }, + }, + { type: "node.updated", driver, node: approvalNode }, + { type: "turn_item.updated", driver, turnItem: approvalItem }, + rootTerminalEvent(ids, "failed"), + ), + }); + const approvalStatuses = finalEvents.flatMap((event) => + (event.type === "node.updated" || event.type === "turn-item.updated") && + (event.payload.id === approvalNode.id || event.payload.id === approvalItem.id) + ? [`${event.type}:${event.payload.status}`] + : [], + ); + assert.deepEqual(approvalStatuses, ["node.updated:cancelled", "turn-item.updated:cancelled"]); + }), +); + it.effect("records a provider's input rejection at turn start in the terminal error", () => Effect.gen(function* () { const rejection = "goal objective must be at most 4000 characters"; diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 916c7070ba689..e4f24fc0e53cd 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -879,6 +879,13 @@ export const layer: Layer.Layer< ); } } + const withdrawnRecordIds = new Set( + cancelledRequestEvents.flatMap((event) => + event.type === "node.updated" || event.type === "turn-item.updated" + ? [event.payload.id] + : [], + ), + ); const finalization = { guardPendingRuntimeRequestCancellations: true, // Stopped runs capture rollback points for the next message; enqueue before later starts on this effect lane. @@ -901,7 +908,12 @@ export const layer: Layer.Layer< : [], events: [ ...cancelledRequestEvents, - ...cascadedSubagentEvents, + // A later failed snapshot would replace the withdrawal and slip past the answer guard. + ...cascadedSubagentEvents.filter( + (event) => + (event.type !== "node.updated" && event.type !== "turn-item.updated") || + !withdrawnRecordIds.has(event.payload.id), + ), ...(yield* interruptLostToolCalls({ run: input.run, toolCalls: input.lostToolCalls ?? [],