From 2bcbd16036eaac249d8a52cf9312019f1f027b24 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 21 Oct 2026 15:56:22 +0000 Subject: [PATCH 1/8] fix(server): cancel approval-blocked tools when runs end --- .../src/orchestration-v2/ProjectionStore.ts | 19 +- .../ProviderTurnStartService.ts | 2 + .../orchestration-v2/RunExecutionService.ts | 86 ++++ .../TerminalShellItems.integration.test.ts | 387 ++++++++++++++++++ 4 files changed, 492 insertions(+), 2 deletions(-) create mode 100644 apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 3fb7c15257801..72dc6c8846486 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1,4 +1,5 @@ import { applyThreadPullRequestLinkUpdate } from "@t3tools/shared/threadPullRequests"; +import { isOrchestrationV2WorkActive } from "@t3tools/contracts"; import { projectTurnItemForWire } from "./WireProjection.ts"; import * as Stream from "effect/Stream"; import { makeThreadFind, findProjectedThreadItems } from "./ThreadFind.ts"; @@ -896,11 +897,20 @@ export function applyToProjection( ...base, messages: upsertById(base.messages, event.payload), }; - case "turn-item.updated": + case "turn-item.updated": { + const previous = base.turnItems.find((item) => item.id === event.payload.id); + // A delayed tool_use frame cannot restart work cancelled before execution. + const item = + previous?.toolNonExecutionKind === "cancelled" && + (previous.type === "command_execution" || previous.type === "dynamic_tool") && + isOrchestrationV2WorkActive(event.payload.status) + ? previous + : event.payload; return withLocalVisibleTurnItems({ ...base, - turnItems: upsertById(base.turnItems, event.payload), + turnItems: upsertById(base.turnItems, item), }); + } case "plan.updated": return { ...base, @@ -2578,6 +2588,11 @@ export const layer: Layer.Layer = status = excluded.status, updated_at = excluded.updated_at, payload_json = excluded.payload_json + WHERE NOT ( + orchestration_v2_projection_turn_items.type IN ('command_execution', 'dynamic_tool') + AND json_extract(orchestration_v2_projection_turn_items.payload_json, '$.toolNonExecutionKind') IS 'cancelled' + AND excluded.status IN ('pending', 'running', 'waiting') + ) `; break; } diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 325c4557636fc..547c142786659 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -166,6 +166,7 @@ export const layer: Layer.Layer< ), Effect.catchCause(() => Effect.succeed(input.inheritedBackgroundTurnItems)), ), + loadRunOwnedTools: () => projectionStore.getRuntimeRecoveryProjection(input.threadId), shouldStartProviderTurn: () => isCurrentAttemptInStatus("running"), shouldFinalizeRun: () => projectionStore.getRuntimeRecoveryProjection(input.threadId).pipe( @@ -1233,6 +1234,7 @@ export const layer: Layer.Layer< attempt: runningAttempt, attemptId: attempt.id, loadInheritedBackgroundTurnItems: runControls.loadInheritedBackgroundTurnItems, + loadRunOwnedTools: runControls.loadRunOwnedTools, relatedThreadIds: routableSubagents.flatMap((subagent) => subagent.childThreadId === null ? [] : [subagent.childThreadId], ), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 022b67d9cf48b..117e2d15700a4 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -17,6 +17,7 @@ import { type OrchestrationV2RunAttempt, type OrchestrationV2Subagent, type OrchestrationV2TurnItem, + type OrchestrationV2ThreadProjection, type ProviderSessionId, type ProviderThreadId, type ProviderTurnId, @@ -144,6 +145,11 @@ export function selectInheritedBackgroundTurnItems(input: { ); } +type RunOwnedTools = Pick< + OrchestrationV2ThreadProjection, + "turnItems" | "nodes" | "runtimeRequests" +>; + type SubagentTurnItem = Extract; type OpenRunOwnedSubagentProjection = { @@ -572,6 +578,7 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly relatedProviderThreadIds?: ReadonlyArray; readonly shouldStartProviderTurn?: () => Effect.Effect; readonly shouldFinalizeRun?: () => Effect.Effect; + readonly loadRunOwnedTools?: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly message: ProviderAdapter.ProviderAdapterV2TurnMessage; readonly modelSelection: ModelSelection; @@ -622,6 +629,7 @@ export const layer: Layer.Layer< readonly shouldFinalizeRun?: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection; + readonly loadRunOwnedTools?: () => Effect.Effect; readonly terminal: ProviderTerminalEvent; readonly completedAt?: DateTime.Utc; readonly failureItemPersisted: boolean; @@ -691,6 +699,71 @@ export const layer: Layer.Layer< allocateEventId, }) : []; + const cancelledToolEvents: OrchestrationV2DomainEvent[] = []; + if (input.loadRunOwnedTools !== undefined && !childTurnsContinue(input.terminal)) { + const { turnItems, nodes, runtimeRequests } = yield* input.loadRunOwnedTools(); + const pendingApprovalNodes = new Set( + runtimeRequests + .filter((request) => request.status === "pending") + .map((request) => request.nodeId), + ); + const blockedToolNodes = new Set( + nodes.flatMap((node) => + node.runId === input.run.id && + node.threadId === input.run.threadId && + node.kind === "approval_request" && + node.status === "waiting" && + pendingApprovalNodes.has(node.id) && + node.parentNodeId !== null + ? [node.parentNodeId] + : [], + ), + ); + for (const item of turnItems) { + // Claude marks tool_use as running before asking permission. Only its + // pending approval child proves that a running tool never started. + if ( + item.runId !== input.run.id || + item.threadId !== input.run.threadId || + (item.type !== "command_execution" && item.type !== "dynamic_tool") || + !isOrchestrationV2WorkActive(item.status) || + !( + item.status === "pending" || + (item.nodeId !== null && blockedToolNodes.has(item.nodeId)) + ) + ) + continue; + cancelledToolEvents.push({ + id: yield* allocateEventId(), + type: "turn-item.updated", + threadId: item.threadId, + runId: input.run.id, + ...(item.nodeId === null ? {} : { nodeId: item.nodeId }), + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { + ...item, + status: "cancelled", + toolNonExecutionKind: "cancelled", + completedAt, + updatedAt: completedAt, + }, + }); + const node = nodes.find((candidate) => candidate.id === item.nodeId); + if (node !== undefined && isOrchestrationV2WorkActive(node.status)) { + cancelledToolEvents.push({ + id: yield* allocateEventId(), + type: "node.updated", + threadId: node.threadId, + runId: input.run.id, + nodeId: node.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { ...node, status: "cancelled", completedAt }, + }); + } + } + } const persistedStatus = input.terminal.status === "completed" ? "waiting" : input.terminal.status; // Completion cohorts are advanced by Orchestrator while a provider @@ -743,6 +816,7 @@ export const layer: Layer.Layer< : [], events: [ ...cascadedSubagentEvents, + ...cancelledToolEvents, ...(finalizedAttempt === null ? [] : [ @@ -937,6 +1011,9 @@ export const layer: Layer.Layer< }), input.providerTurnOrdinal * 100 + 1, ), + ...(input.loadRunOwnedTools === undefined + ? {} + : { loadRunOwnedTools: input.loadRunOwnedTools }), failureItemPersisted: false, refreshAfterTurn, writeIfRunCurrent: { @@ -1041,6 +1118,9 @@ export const layer: Layer.Layer< hasUnpairedRunInterruptRequest: input.hasUnpairedRunInterruptRequest, }), openRunOwnedSubagents: openSubagents, + ...(input.loadRunOwnedTools === undefined + ? {} + : { loadRunOwnedTools: input.loadRunOwnedTools }), terminal, failureItemPersisted: terminal.status === "failed", refreshAfterTurn, @@ -1450,6 +1530,9 @@ export const layer: Layer.Layer< expectedStatus: "running", }, openRunOwnedSubagents: openSubagents, + ...(input.loadRunOwnedTools === undefined + ? {} + : { loadRunOwnedTools: input.loadRunOwnedTools }), completedAt, terminal: makeFailedTerminalEvent( makeProviderFailure({ cause: Cause.squash(cause), class: "unknown" }), @@ -1603,6 +1686,9 @@ export const layer: Layer.Layer< expectedStatus: "running", }, openRunOwnedSubagents: openSubagents, + ...(input.loadRunOwnedTools === undefined + ? {} + : { loadRunOwnedTools: input.loadRunOwnedTools }), terminal: makeFailedTerminalEvent( makeProviderFailure({ cause: Cause.squash(cause), diff --git a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts new file mode 100644 index 0000000000000..d1fbf3cc57fb1 --- /dev/null +++ b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts @@ -0,0 +1,387 @@ +import { assert, it } from "@effect/vitest"; +import { + CommandId, + EventId, + NodeId, + MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ProviderThreadId, + ProviderTurnId, + RuntimeRequestId, + ThreadId, + TurnItemId, + type OrchestrationV2DomainEvent, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as Fiber from "effect/Fiber"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import { ClaudeProviderCapabilitiesV2 } from "./Adapters/ClaudeAdapterV2.ts"; +import * as EffectWorker from "./EffectWorker.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; + +import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; + +const driver = ProviderDriverKind.make("claudeAgent"); +const instanceId = ProviderInstanceId.make("claudeAgent"); +const modelSelection = { instanceId, model: "test-model" }; +const databaseLayer = SqlitePersistence.layerMemory; +const capabilities = { + ...ClaudeProviderCapabilitiesV2, + sessions: { + ...ClaudeProviderCapabilitiesV2.sessions, + supportsMultipleProviderThreadsPerSession: false, + supportsRuntimeModeSwitchInSession: false, + }, +}; + +it.effect.each([ + "approval status", + "approval shell selection", + "confirmed background", + "completed", + "dynamic approval", + "accepted approval", + "pending", + "unlinked running", +] as const)("runtime mode change and dispatch preserve terminal shell invariants: %s", (check) => + Effect.scoped( + Effect.gen(function* () { + const cwd = yield* checkpointWorkspace("terminal-shell-items"); + const events = yield* Queue.unbounded(); + const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + instanceId, + driver, + getCapabilities: () => Effect.succeed(capabilities), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: (input) => + Effect.gen(function* () { + const now = yield* DateTime.now; + return { + instanceId, + driver, + providerSessionId: input.providerSessionId, + providerSession: { + id: input.providerSessionId, + driver, + providerInstanceId: instanceId, + status: "ready", + cwd, + model: modelSelection.model, + capabilities, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.fromQueue(events), + ensureThread: ({ threadId }) => + Effect.succeed({ + id: ProviderThreadId.make(`provider-thread:claudeAgent:${threadId}`), + driver, + providerInstanceId: instanceId, + providerSessionId: input.providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => + Effect.gen(function* () { + started.push(turn); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: { + driver, + nativeId: `native:${turn.attemptId}`, + strength: "strong", + }, + ordinal: turn.providerTurnOrdinal, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + }), + steerTurn: () => Effect.die("unused"), + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.void, + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }; + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread:terminal-shell-items"); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create"), + threadId, + projectId: ProjectId.make("project:terminal-shell-items"), + title: "Terminal shell items", + modelSelection, + runtimeMode: "approval-required", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) => + orchestrator.streamDomainEvents.pipe( + Stream.filter(predicate), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const running = yield* watch( + (event) => event.type === "provider-turn.updated" && event.payload.status === "running", + ); + const dispatch = (id: string) => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(id), + threadId, + messageId: MessageId.make(`message:${id}`), + text: "Continue", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* dispatch("first"); + yield* worker.drain(); + yield* Fiber.join(running); + const first = started[0]!; + const projection = yield* orchestrator.getThreadProjection(threadId); + const turn = projection.providerTurns[0]!; + const now = yield* DateTime.now; + const requestId = RuntimeRequestId.make("request:pending-shell"); + const commandItem = { + id: TurnItemId.make("item:pending-shell"), + threadId, + runId: first.runId, + nodeId: NodeId.make("node:command"), + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + nativeItemRef: { driver, nativeId: "shell-native", strength: "strong" as const }, + parentItemId: null, + ordinal: 100, + status: + check === "completed" + ? ("completed" as const) + : check === "pending" + ? ("pending" as const) + : ("running" as const), + title: "Approval-blocked shell", + startedAt: now, + completedAt: check === "completed" ? now : null, + updatedAt: now, + ...(check === "dynamic approval" + ? { type: "dynamic_tool" as const, toolName: "mcp:test", input: {} } + : { type: "command_execution" as const, input: "echo approved" }), + }; + const commandNode = { + id: NodeId.make("node:command"), + threadId, + runId: first.runId, + parentNodeId: first.rootNodeId, + rootNodeId: first.rootNodeId, + kind: "tool_call" as const, + status: commandItem.status, + countsForRun: true, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + nativeItemRef: commandItem.nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: commandItem.completedAt, + }; + const approvalPending = check.includes("approval"); + const ingested = yield* watch((event) => + approvalPending + ? event.type === "runtime-request.updated" && event.payload.id === requestId + : event.type === "turn-item.updated" && event.payload.id === commandItem.id, + ); + if (check === "confirmed background") { + yield* Queue.offer(events, { + type: "provider_thread.updated", + driver, + providerThread: { + ...projection.providerThreads[0]!, + pendingBackgroundTasks: [ + { taskId: "shell-native", kind: "command", description: "Confirmed shell" }, + ], + }, + }); + } + yield* Queue.offer(events, { type: "node.updated", driver, node: commandNode }); + yield* Queue.offer(events, { type: "turn_item.updated", driver, turnItem: commandItem }); + if (approvalPending) { + const approvalNodeId = NodeId.make("node:approval"); + yield* Queue.offer(events, { + type: "node.updated", + driver, + node: { + ...commandNode, + id: approvalNodeId, + parentNodeId: commandNode.id, + kind: "approval_request", + status: "waiting", + runtimeRequestId: requestId, + }, + }); + yield* Queue.offer(events, { + type: "runtime_request.updated", + driver, + threadId, + runtimeRequest: { + id: requestId, + nodeId: approvalNodeId, + providerTurnId: turn.id, + nativeRequestRef: null, + kind: "command", + status: "pending", + responseCapability: { + type: "live", + providerSessionId: projection.providerSessions[0]!.id, + }, + createdAt: now, + resolvedAt: null, + }, + }); + } + yield* Fiber.join(ingested); + const before = (yield* orchestrator.getThreadProjection(threadId)).turnItems.find( + (item) => item.id === commandItem.id, + ); + assert.ok(before); + assert.equal( + before.status, + check === "completed" ? "completed" : check === "pending" ? "pending" : "running", + ); + const ingestedProjection = yield* orchestrator.getThreadProjection(threadId); + if (approvalPending) { + assert.equal(ingestedProjection.runtimeRequests[0]?.status, "pending"); + assert.equal(ingestedProjection.runtimeRequests[0]?.nodeId, NodeId.make("node:approval")); + assert.equal( + ingestedProjection.nodes.find((node) => node.id === NodeId.make("node:approval")) + ?.parentNodeId, + commandNode.id, + ); + } else { + assert.deepEqual(ingestedProjection.runtimeRequests, []); + } + if (check === "accepted approval") { + yield* orchestrator.dispatch({ + type: "runtime-request.respond", + commandId: CommandId.make("approve"), + threadId, + requestId, + decision: "accept", + }); + yield* worker.drain(); + assert.equal( + (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests[0]?.status, + "resolved", + ); + } + const failed = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "failed", + ); + yield* orchestrator.dispatch({ + type: "thread.runtime-mode.set", + commandId: CommandId.make("change-mode"), + threadId, + runtimeMode: "auto", + }); + yield* worker.drain(); + yield* Fiber.join(failed); + // Read before the next run hides background tasks behind its active status. + const snapshot = yield* projections.getShellSnapshot(); + const shell = snapshot.threads.find((thread) => thread.id === threadId)!; + assert.ok(shell); + const pending = shell.pendingBackgroundTasks ?? []; + yield* dispatch("second"); + yield* worker.drain(); + const after = yield* orchestrator.getThreadProjection(threadId); + assert.equal(after.runs.find((run) => run.id === first.runId)?.status, "failed"); + const item = after.turnItems.find((item) => item.id === commandItem.id)!; + if (check === "approval status" || check === "dynamic approval" || check === "pending") { + assert.include(["interrupted", "cancelled"], item.status); + assert.isNotNull(item.completedAt); + assert.equal(item.toolNonExecutionKind, "cancelled"); + const lateEvent: OrchestrationV2DomainEvent = { + id: EventId.make("late-tool-use"), + type: "turn-item.updated", + threadId, + runId: first.runId, + occurredAt: now, + payload: commandItem, + }; + assert.deepEqual( + ProjectionStore.applyToProjection(after, lateEvent).turnItems.find( + (candidate) => candidate.id === item.id, + ), + item, + ); + yield* projections.apply(lateEvent); + assert.deepEqual(yield* projections.getTurnItem({ threadId, itemId: item.id }), item); + } else if (check === "approval shell selection") { + assert.isFalse(pending.some((task) => task.taskId === "shell-native")); + } else if (check === "confirmed background") { + assert.deepEqual(item, before); + assert.isTrue(pending.some((task) => task.taskId === "shell-native")); + } else { + assert.deepEqual(item, before); + assert.equal( + pending.some((task) => task.taskId === "shell-native"), + check !== "completed", + ); + } + }).pipe( + Effect.provide( + Layer.mergeAll( + ProviderReplayHarness.layerWithRegistry( + { name: "terminal-shell-items" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + { runEffectWorker: false, databaseLayer }, + ), + ProjectionStore.layer.pipe(Layer.provide(databaseLayer)), + ), + ), + ); + }), + ), +); From 3d1490255c944813506bb922e5496bce808abec3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 21 Oct 2026 16:01:28 +0000 Subject: [PATCH 2/8] fix(client-runtime): preserve cancelled tools through late active updates --- .../state/orchestrationV2Projection.test.ts | 44 +++++++++++++++++++ .../src/state/orchestrationV2Projection.ts | 32 +++++++++----- 2 files changed, 64 insertions(+), 12 deletions(-) diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index 9d96d8001430d..29a0c09b344a5 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -259,6 +259,50 @@ describe("applyOrchestrationV2ProjectionEvent", () => { expect(next?.visibleTurnItems[1]).toBe(secondRow); }); + for (const type of ["command_execution", "dynamic_tool"] as const) { + it.each(["pending", "running", "waiting"] as const)( + `keeps cancelled ${type} items and visible rows through a late %s update`, + (status) => { + const command = commandItem("item-cancelled", ""); + if (command.type !== "command_execution") throw new Error("Expected a command item"); + const item: OrchestrationV2TurnItem = { + ...command, + ...(type === "dynamic_tool" ? { type, toolName: "test_tool" } : { type }), + status: "cancelled", + toolNonExecutionKind: "cancelled", + }; + const event = { + id: "event-item-cancelled", + type: "turn-item.updated", + threadId, + runId, + occurredAt: now, + payload: item, + } as OrchestrationV2DomainEvent; + const cancelled = applyOrchestrationV2ProjectionEvent( + { ...emptyProjection, runs: [run] }, + event, + )!; + expect(cancelled.turnItems).toEqual([item]); + expect(cancelled.visibleTurnItems.map((row) => row.item)).toEqual([item]); + + const next = applyOrchestrationV2ProjectionEvent(cancelled, { + ...event, + id: "event-item-late-active" as OrchestrationV2DomainEvent["id"], + payload: { + ...item, + status, + toolNonExecutionKind: undefined, + completedAt: null, + updatedAt: DateTime.add(now, { seconds: 1 }), + }, + } as OrchestrationV2DomainEvent); + expect(next?.turnItems).toEqual([item]); + expect(next?.visibleTurnItems).toEqual(cancelled.visibleTurnItems); + }, + ); + } + it("inserts live turn items by authoritative ordinal", () => { const queuedFuture = commandItem("item-queued-future", "queued", 300); const activeAssistant = commandItem("item-active-assistant", "done", 201); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.ts b/packages/client-runtime/src/state/orchestrationV2Projection.ts index d954e8a1eda33..59d379dbf6745 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.ts @@ -1,9 +1,10 @@ import { applyThreadPullRequestLinkUpdate } from "@t3tools/shared/threadPullRequests"; -import type { - OrchestrationV2DomainEvent, - OrchestrationV2Run, - OrchestrationV2ThreadProjection, - OrchestrationV2TurnItem, +import { + isOrchestrationV2WorkActive, + type OrchestrationV2DomainEvent, + type OrchestrationV2Run, + type OrchestrationV2ThreadProjection, + type OrchestrationV2TurnItem, } from "@t3tools/contracts"; import { createOrchestrationV2TurnItemVisibility, @@ -326,22 +327,29 @@ export function applyOrchestrationV2ProjectionEvent( ) { return projection; } + const previous = itemIndex === -1 ? undefined : projection.turnItems[itemIndex]; + // A delayed tool_use frame cannot restart work cancelled before execution. + const item = + previous?.toolNonExecutionKind === "cancelled" && + (previous.type === "command_execution" || previous.type === "dynamic_tool") && + isOrchestrationV2WorkActive(event.payload.status) + ? previous + : event.payload; const turnItems = [...projection.turnItems]; - if (itemIndex === -1) turnItems.push(event.payload); - else turnItems[itemIndex] = event.payload; + if (itemIndex === -1) turnItems.push(item); + else turnItems[itemIndex] = item; const next = { ...base, turnItems }; // Only interrupt requests can change another item's visibility. Streaming // text/tool updates must not recheck every row against every run. - const previous = itemIndex === -1 ? undefined : projection.turnItems[itemIndex]; const visible = - event.payload.type === "run_interrupt_request" || previous?.type === "run_interrupt_request" + item.type === "run_interrupt_request" || previous?.type === "run_interrupt_request" ? { ...next, visibleTurnItems: activeVisibleTurnItems(next) } : next; return { ...next, - visibleTurnItems: shouldShowLocalTurnItem(next, event.payload) - ? upsertVisibleTurnItem(visible, event.payload, partialTimeline, latestLocalTurnOrdinal) - : removeVisibleItem(visible.visibleTurnItems, event.payload.id), + visibleTurnItems: shouldShowLocalTurnItem(next, item) + ? upsertVisibleTurnItem(visible, item, partialTimeline, latestLocalTurnOrdinal) + : removeVisibleItem(visible.visibleTurnItems, item.id), }; } case "checkpoint-scope.created": From 147b22734b84fdb8cd6aa5d1103010566e99fc0f Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 21 Oct 2026 16:04:18 +0000 Subject: [PATCH 3/8] fix(server): cancel approvals linked to finalized tools --- .../src/orchestration-v2/ProjectionStore.ts | 4 +- .../orchestration-v2/RunExecutionService.ts | 65 +++++++++++ .../TerminalShellItems.integration.test.ts | 103 +++++++++++++++++- 3 files changed, 167 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 72dc6c8846486..44891160f392d 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -2590,7 +2590,9 @@ export const layer: Layer.Layer = payload_json = excluded.payload_json WHERE NOT ( orchestration_v2_projection_turn_items.type IN ('command_execution', 'dynamic_tool') - AND json_extract(orchestration_v2_projection_turn_items.payload_json, '$.toolNonExecutionKind') IS 'cancelled' + AND CASE WHEN orchestration_v2_projection_turn_items.status = 'cancelled' + THEN json_extract(orchestration_v2_projection_turn_items.payload_json, '$.toolNonExecutionKind') IS 'cancelled' + ELSE FALSE END AND excluded.status IN ('pending', 'running', 'waiting') ) `; diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 117e2d15700a4..df98fe8e2edbc 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -719,6 +719,7 @@ export const layer: Layer.Layer< : [], ), ); + const cancelledToolNodeIds = new Set(); for (const item of turnItems) { // Claude marks tool_use as running before asking permission. Only its // pending approval child proves that a running tool never started. @@ -749,6 +750,7 @@ export const layer: Layer.Layer< updatedAt: completedAt, }, }); + if (item.nodeId !== null) cancelledToolNodeIds.add(item.nodeId); const node = nodes.find((candidate) => candidate.id === item.nodeId); if (node !== undefined && isOrchestrationV2WorkActive(node.status)) { cancelledToolEvents.push({ @@ -763,6 +765,69 @@ export const layer: Layer.Layer< }); } } + for (const approvalNode of nodes) { + if ( + approvalNode.kind !== "approval_request" || + approvalNode.runId !== input.run.id || + approvalNode.threadId !== input.run.threadId || + approvalNode.parentNodeId === null || + !cancelledToolNodeIds.has(approvalNode.parentNodeId) + ) + continue; + const request = runtimeRequests.find( + (candidate) => candidate.nodeId === approvalNode.id && candidate.status === "pending", + ); + if (request === undefined) continue; + cancelledToolEvents.push( + { + id: yield* allocateEventId(), + type: "runtime-request.updated", + threadId: input.run.threadId, + runId: input.run.id, + nodeId: approvalNode.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { + ...request, + status: "cancelled", + responseCapability: { + type: "not_resumable", + reason: "The tool was cancelled before this approval was resolved.", + }, + resolvedAt: completedAt, + }, + }, + { + id: yield* allocateEventId(), + type: "node.updated", + threadId: input.run.threadId, + runId: input.run.id, + nodeId: approvalNode.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { ...approvalNode, status: "cancelled", completedAt }, + }, + ); + for (const approvalItem of turnItems) { + if (approvalItem.type !== "approval_request" || approvalItem.requestId !== request.id) + continue; + cancelledToolEvents.push({ + id: yield* allocateEventId(), + type: "turn-item.updated", + threadId: input.run.threadId, + ...(approvalItem.runId === null ? {} : { runId: approvalItem.runId }), + ...(approvalItem.nodeId === null ? {} : { nodeId: approvalItem.nodeId }), + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { + ...approvalItem, + status: "cancelled", + completedAt, + updatedAt: completedAt, + }, + }); + } + } } const persistedStatus = input.terminal.status === "completed" ? "waiting" : input.terminal.status; diff --git a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts index d1fbf3cc57fb1..6fb26448dfce7 100644 --- a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts +++ b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts @@ -259,6 +259,50 @@ it.effect.each([ runtimeRequestId: requestId, }, }); + yield* Queue.offer(events, { + type: "turn_item.updated", + driver, + turnItem: { + ...commandItem, + id: TurnItemId.make("item:approval"), + nodeId: approvalNodeId, + type: "approval_request", + requestId, + requestKind: "command", + status: "waiting", + }, + }); + yield* Queue.offer(events, { + type: "node.updated", + driver, + node: { + ...commandNode, + id: NodeId.make("node:unrelated-approval"), + parentNodeId: first.rootNodeId, + kind: "approval_request", + status: "waiting", + runtimeRequestId: RuntimeRequestId.make("request:unrelated"), + }, + }); + yield* Queue.offer(events, { + type: "runtime_request.updated", + driver, + threadId, + runtimeRequest: { + id: RuntimeRequestId.make("request:unrelated"), + nodeId: NodeId.make("node:unrelated-approval"), + providerTurnId: turn.id, + nativeRequestRef: null, + kind: "command", + status: "pending", + responseCapability: { + type: "live", + providerSessionId: projection.providerSessions[0]!.id, + }, + createdAt: now, + resolvedAt: null, + }, + }); yield* Queue.offer(events, { type: "runtime_request.updated", driver, @@ -290,8 +334,14 @@ it.effect.each([ ); const ingestedProjection = yield* orchestrator.getThreadProjection(threadId); if (approvalPending) { - assert.equal(ingestedProjection.runtimeRequests[0]?.status, "pending"); - assert.equal(ingestedProjection.runtimeRequests[0]?.nodeId, NodeId.make("node:approval")); + assert.equal( + ingestedProjection.runtimeRequests.find((request) => request.id === requestId)?.status, + "pending", + ); + assert.equal( + ingestedProjection.runtimeRequests.find((request) => request.id === requestId)?.nodeId, + NodeId.make("node:approval"), + ); assert.equal( ingestedProjection.nodes.find((node) => node.id === NodeId.make("node:approval")) ?.parentNodeId, @@ -310,7 +360,9 @@ it.effect.each([ }); yield* worker.drain(); assert.equal( - (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests[0]?.status, + (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests.find( + (request) => request.id === requestId, + )?.status, "resolved", ); } @@ -339,7 +391,8 @@ it.effect.each([ assert.equal(after.runs.find((run) => run.id === first.runId)?.status, "failed"); const item = after.turnItems.find((item) => item.id === commandItem.id)!; if (check === "approval status" || check === "dynamic approval" || check === "pending") { - assert.include(["interrupted", "cancelled"], item.status); + assert.equal(item.status, "cancelled"); + assert.equal(after.nodes.find((node) => node.id === commandNode.id)?.status, "cancelled"); assert.isNotNull(item.completedAt); assert.equal(item.toolNonExecutionKind, "cancelled"); const lateEvent: OrchestrationV2DomainEvent = { @@ -370,6 +423,48 @@ it.effect.each([ check !== "completed", ); } + if (approvalPending) { + const unrelated = after.runtimeRequests.find( + (request) => request.id === RuntimeRequestId.make("request:unrelated"), + )!; + assert.equal(unrelated.status, "pending"); + assert.equal(unrelated.responseCapability.type, "live"); + assert.equal(after.nodes.find((node) => node.id === unrelated.nodeId)?.status, "waiting"); + } + if (approvalPending && check !== "accepted approval") { + assert.equal(item.status, "cancelled"); + const request = after.runtimeRequests.find((request) => request.id === requestId)!; + assert.equal(request.status, "cancelled"); + assert.deepEqual(request.responseCapability, { + type: "not_resumable", + reason: "The tool was cancelled before this approval was resolved.", + }); + assert.isNotNull(request.resolvedAt); + const approvalNode = after.nodes.find((node) => node.id === request.nodeId)!; + assert.equal(approvalNode.status, "cancelled"); + assert.isNotNull(approvalNode.completedAt); + const approvalItem = after.turnItems.find( + (item) => item.id === TurnItemId.make("item:approval"), + )!; + assert.equal(approvalItem.status, "cancelled"); + assert.isNotNull(approvalItem.completedAt); + const response = yield* orchestrator + .dispatch({ + type: "runtime-request.respond", + commandId: CommandId.make("late-approve"), + threadId, + requestId, + decision: "accept", + }) + .pipe(Effect.flip); + assert.equal(response._tag, "OrchestratorDispatchError"); + assert.equal(response.cause, `Runtime request ${requestId} is cancelled.`); + } else if (check === "accepted approval") { + assert.equal( + after.runtimeRequests.find((request) => request.id === requestId)?.status, + "resolved", + ); + } }).pipe( Effect.provide( Layer.mergeAll( From 661329512e65c3a08adabdad89cf0ddacdec9625 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 21 Oct 2026 16:18:37 +0000 Subject: [PATCH 4/8] test(server): cover blocked approvals through successor completion --- .../TerminalShellItems.integration.test.ts | 308 +++++++++++++++++- 1 file changed, 306 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts index 6fb26448dfce7..3ce5bf9da3dca 100644 --- a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts +++ b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts @@ -16,6 +16,7 @@ import { } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; +import * as Deferred from "effect/Deferred"; import * as Layer from "effect/Layer"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; @@ -45,6 +46,7 @@ const capabilities = { }; it.effect.each([ + "approval query failure after successor dispatch", "approval status", "approval shell selection", "confirmed background", @@ -56,13 +58,20 @@ it.effect.each([ ] as const)("runtime mode change and dispatch preserve terminal shell invariants: %s", (check) => Effect.scoped( Effect.gen(function* () { + const testCapabilities = + check === "approval query failure after successor dispatch" + ? { + ...capabilities, + sessions: { ...capabilities.sessions, supportsRuntimeModeSwitchInSession: true }, + } + : capabilities; const cwd = yield* checkpointWorkspace("terminal-shell-items"); const events = yield* Queue.unbounded(); const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { instanceId, driver, - getCapabilities: () => Effect.succeed(capabilities), + getCapabilities: () => Effect.succeed(testCapabilities), planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), openSession: (input) => Effect.gen(function* () { @@ -78,7 +87,7 @@ it.effect.each([ status: "ready", cwd, model: modelSelection.model, - capabilities, + capabilities: testCapabilities, createdAt: now, updatedAt: now, lastError: null, @@ -366,6 +375,301 @@ it.effect.each([ "resolved", ); } + if (check === "approval query failure after successor dispatch") { + const secondCommand = { + ...commandItem, + id: TurnItemId.make("item:pending-shell-2"), + nodeId: NodeId.make("node:command-2"), + nativeItemRef: { ...commandItem.nativeItemRef, nativeId: "shell-native-2" }, + ordinal: 101, + }; + const secondRequestId = RuntimeRequestId.make("request:pending-shell-2"); + const secondApprovalNodeId = NodeId.make("node:approval-2"); + const secondIngested = yield* watch( + (event) => + event.type === "runtime-request.updated" && event.payload.id === secondRequestId, + ); + yield* Queue.offerAll(events, [ + { + type: "node.updated", + driver, + node: { + ...commandNode, + id: secondCommand.nodeId, + nativeItemRef: secondCommand.nativeItemRef, + }, + }, + { type: "turn_item.updated", driver, turnItem: secondCommand }, + { + type: "node.updated", + driver, + node: { + ...commandNode, + id: secondApprovalNodeId, + parentNodeId: secondCommand.nodeId, + kind: "approval_request", + status: "waiting", + runtimeRequestId: secondRequestId, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...secondCommand, + id: TurnItemId.make("item:approval-2"), + nodeId: secondApprovalNodeId, + type: "approval_request", + requestId: secondRequestId, + requestKind: "command", + status: "waiting", + }, + }, + { + type: "runtime_request.updated", + driver, + threadId, + runtimeRequest: { + id: secondRequestId, + nodeId: secondApprovalNodeId, + providerTurnId: turn.id, + nativeRequestRef: null, + kind: "command", + status: "pending", + responseCapability: { + type: "live", + providerSessionId: projection.providerSessions[0]!.id, + }, + createdAt: now, + resolvedAt: null, + }, + }, + ]); + yield* Fiber.join(secondIngested); + const originals = [commandItem, secondCommand]; + const requestIds = [requestId, secondRequestId]; + const assertBlocked = (state: typeof projection) => { + assert.equal(state.runs.find((run) => run.id === first.runId)?.status, "running"); + for (const original of originals) { + assert.equal( + state.turnItems.find((item) => item.id === original.id)?.status, + "running", + ); + } + for (const id of requestIds) { + assert.equal( + state.runtimeRequests.find((request) => request.id === id)?.status, + "pending", + ); + } + }; + assertBlocked(yield* orchestrator.getThreadProjection(threadId)); + yield* orchestrator.dispatch({ + type: "thread.runtime-mode.set", + commandId: CommandId.make("incident-change-mode"), + threadId, + runtimeMode: "auto", + }); + yield* worker.drain(); + assertBlocked(yield* orchestrator.getThreadProjection(threadId)); + const closeQuery = yield* Deferred.make(); + const failed = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "failed", + ); + const successorRunning = yield* watch( + (event) => + event.type === "provider-turn.updated" && + event.payload.runAttemptId !== first.attemptId && + event.payload.status === "running", + ); + // Simulate the provider closing its query in response to the successor + // dispatch, with a latch so both approvals are observed pending first. + const queryClosed = yield* Effect.gen(function* () { + yield* Deferred.await(closeQuery); + yield* Queue.offerAll(events, [ + { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "failed", completedAt: now }, + }, + { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status: "failed", + failureItemOrdinal: 200, + failure: { + class: "provider_error", + message: "Query closed after successor dispatch", + code: null, + retryable: null, + }, + threadDisposition: "reusable", + }, + ]); + }).pipe(Effect.forkScoped); + yield* dispatch("incident-successor"); + const dispatched = yield* orchestrator.getThreadProjection(threadId); + assertBlocked(dispatched); + assert.equal( + dispatched.runs.find( + (run) => run.userMessageId === MessageId.make("message:incident-successor"), + )?.status, + "queued", + ); + yield* Deferred.succeed(closeQuery, undefined); + yield* Fiber.join(queryClosed); + yield* Fiber.join(failed); + yield* worker.drain(); + // A provider error holds queued work; resume the already-dispatched + // successor rather than sending another message after the failure. + yield* orchestrator.dispatch({ + type: "queue.resume", + commandId: CommandId.make("resume-incident-successor"), + threadId, + }); + yield* worker.drain(); + assert.equal(started.length, 2); + yield* Fiber.join(successorRunning); + const successor = started[1]!; + assert.equal(successor.runtimePolicy.runtimeMode, "auto"); + assert.equal(successor.message.messageId, MessageId.make("message:incident-successor")); + const successorProjection = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + successorProjection.runs.find((run) => run.id === successor.runId)?.status, + "running", + ); + const successorTurn = successorProjection.providerTurns.find( + (candidate) => candidate.runAttemptId === successor.attemptId, + )!; + for (const [index, original] of originals.entries()) { + const replacement = { + ...original, + id: TurnItemId.make(`item:successor-shell-${index}`), + runId: successor.runId, + nodeId: NodeId.make(`node:successor-command-${index}`), + providerThreadId: successorTurn.providerThreadId, + providerTurnId: successorTurn.id, + nativeItemRef: { ...original.nativeItemRef, nativeId: `successor-shell-${index}` }, + status: "running" as const, + }; + const replacementNode = { + ...commandNode, + id: replacement.nodeId, + runId: successor.runId, + parentNodeId: successor.rootNodeId, + rootNodeId: successor.rootNodeId, + providerThreadId: successorTurn.providerThreadId, + providerTurnId: successorTurn.id, + nativeItemRef: replacement.nativeItemRef, + }; + const completed = yield* watch( + (event) => + event.type === "turn-item.updated" && + event.payload.id === replacement.id && + event.payload.status === "completed", + ); + yield* Queue.offerAll(events, [ + { type: "node.updated", driver, node: replacementNode }, + { type: "turn_item.updated", driver, turnItem: replacement }, + { + type: "node.updated", + driver, + node: { ...replacementNode, status: "completed", completedAt: now }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { ...replacement, status: "completed", completedAt: now }, + }, + ]); + yield* Fiber.join(completed); + } + const settled = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === successor.runId && + event.payload.status === "waiting", + ); + yield* Queue.offerAll(events, [ + { + type: "provider_turn.updated", + driver, + providerTurn: { ...successorTurn, status: "completed", completedAt: now }, + }, + { + type: "turn.terminal", + driver, + providerThreadId: successorTurn.providerThreadId, + providerTurnId: successorTurn.id, + runOrdinal: successor.runOrdinal, + status: "completed", + failure: null, + threadDisposition: "reusable", + }, + ]); + yield* Fiber.join(settled); + yield* worker.drain(); + const final = yield* orchestrator.getThreadProjection(threadId); + assert.equal(final.runs.find((run) => run.id === first.runId)?.status, "failed"); + const finished = final.runs.find((run) => run.id === successor.runId)!; + assert.equal(finished.status, "completed"); + assert.isNotNull(finished.completedAt); + assert.isNotNull(finished.checkpointId); + assert.equal( + final.checkpoints.find((checkpoint) => checkpoint.id === finished.checkpointId)?.status, + "ready", + ); + for (const [index, original] of originals.entries()) { + const cancelled = final.turnItems.find((item) => item.id === original.id)!; + assert.equal(cancelled.status, "cancelled"); + assert.equal(cancelled.toolNonExecutionKind, "cancelled"); + assert.isNotNull(cancelled.completedAt); + assert.equal( + final.nodes.find((node) => node.id === original.nodeId)?.status, + "cancelled", + ); + assert.equal( + final.turnItems.find( + (item) => item.id === TurnItemId.make(`item:successor-shell-${index}`), + )?.status, + "completed", + ); + } + for (const id of requestIds) { + const request = final.runtimeRequests.find((candidate) => candidate.id === id)!; + assert.equal(request.status, "cancelled"); + assert.isNotNull(request.resolvedAt); + assert.equal(request.responseCapability.type, "not_resumable"); + assert.equal( + final.nodes.find((node) => node.id === request.nodeId)?.status, + "cancelled", + ); + assert.equal( + final.turnItems.find( + (item) => item.type === "approval_request" && item.requestId === id, + )?.status, + "cancelled", + ); + } + const freshShell = (yield* projections.getShellSnapshot()).threads.find( + (thread) => thread.id === threadId, + )!; + assert.ok(freshShell); + for (const original of originals) { + assert.isFalse( + (freshShell.pendingBackgroundTasks ?? []).some( + (task) => task.taskId === original.nativeItemRef.nativeId, + ), + ); + } + return; + } const failed = yield* watch( (event) => event.type === "run.updated" && From 17a4a3851efa9b98b5b01f7c25a478564d260df3 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 05:58:09 +0000 Subject: [PATCH 5/8] fix(server): cancel tools whose approval a session release already expired Main now expires a detached session's pending approvals before the run finalizes, so the cancel pass treats any unresolved approval child as proof the tool never started. It reads only the run's tool and approval rows instead of the whole recovery projection. Trim the integration test to the cases that fail without the fix plus the two that must keep running. AI-assisted; verified with the tests and tools listed above. --- .../ProviderTurnStartService.ts | 19 +- .../orchestration-v2/RunExecutionService.ts | 12 +- .../TerminalShellItems.integration.test.ts | 505 +++--------------- 3 files changed, 110 insertions(+), 426 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 10e3ef80f6b4a..890fb0ab3c7a3 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -259,7 +259,24 @@ export const layer: Layer.Layer< ), Effect.catchCause(() => Effect.succeed(input.inheritedBackgroundTurnItems)), ), - loadRunOwnedTools: () => projectionStore.getRuntimeRecoveryProjection(input.threadId), + loadRunOwnedTools: Effect.fn("ProviderTurnStartService.loadRunOwnedTools")(function* () { + const { turnItems, runtimeRequests } = yield* projectionStore.getThreadRecords( + input.threadId, + ["turnItems", "runtimeRequests"], + { + turnItemRunIds: [input.runId], + turnItemTypes: ["command_execution", "dynamic_tool", "approval_request"], + turnItemStatuses: ["pending", "running", "waiting", "cancelled", "failed"], + runtimeRequestStatuses: ["pending", "expired", "cancelled"], + }, + ); + const nodeIds = turnItems.flatMap((item) => (item.nodeId === null ? [] : [item.nodeId])); + if (nodeIds.length === 0) return { turnItems, nodes: [], runtimeRequests }; + const { nodes } = yield* projectionStore.getThreadRecords(input.threadId, ["nodes"], { + nodeIds, + }); + return { turnItems, nodes, runtimeRequests }; + }), shouldStartProviderTurn: () => isCurrentAttemptInStatus("running"), shouldFinalizeRun: () => projectionStore.getRuntimeRecoveryProjection(input.threadId).pipe( diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 52c9c9d420390..51151efde0ff4 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -732,9 +732,10 @@ export const layer: Layer.Layer< const cancelledToolEvents: OrchestrationV2DomainEvent[] = []; if (input.loadRunOwnedTools !== undefined && !childTurnsContinue(input.terminal)) { const { turnItems, nodes, runtimeRequests } = yield* input.loadRunOwnedTools(); - const pendingApprovalNodes = new Set( + // A session release may already have expired the approval before the run ends. + const unresolvedApprovalNodes = new Set( runtimeRequests - .filter((request) => request.status === "pending") + .filter((request) => request.status !== "resolved") .map((request) => request.nodeId), ); const blockedToolNodes = new Set( @@ -742,8 +743,7 @@ export const layer: Layer.Layer< node.runId === input.run.id && node.threadId === input.run.threadId && node.kind === "approval_request" && - node.status === "waiting" && - pendingApprovalNodes.has(node.id) && + unresolvedApprovalNodes.has(node.id) && node.parentNodeId !== null ? [node.parentNodeId] : [], @@ -751,8 +751,8 @@ export const layer: Layer.Layer< ); const cancelledToolNodeIds = new Set(); for (const item of turnItems) { - // Claude marks tool_use as running before asking permission. Only its - // pending approval child proves that a running tool never started. + // Claude marks tool_use as running before asking permission. Only an + // unresolved approval child proves that a running tool never started. if ( item.runId !== input.run.id || item.threadId !== input.run.threadId || diff --git a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts index 3ce5bf9da3dca..59ab526cac49e 100644 --- a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts +++ b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts @@ -16,7 +16,6 @@ import { } from "@t3tools/contracts"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; -import * as Deferred from "effect/Deferred"; import * as Layer from "effect/Layer"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; import * as ProjectionStore from "./ProjectionStore.ts"; @@ -46,20 +45,17 @@ const capabilities = { }; it.effect.each([ - "approval query failure after successor dispatch", - "approval status", - "approval shell selection", - "confirmed background", - "completed", + "approval", "dynamic approval", "accepted approval", "pending", "unlinked running", -] as const)("runtime mode change and dispatch preserve terminal shell invariants: %s", (check) => + "approval on provider failure", +] as const)("ending a run cancels only tools that never started: %s", (check) => Effect.scoped( Effect.gen(function* () { const testCapabilities = - check === "approval query failure after successor dispatch" + check === "approval on provider failure" ? { ...capabilities, sessions: { ...capabilities.sessions, supportsRuntimeModeSwitchInSession: true }, @@ -203,15 +199,10 @@ it.effect.each([ nativeItemRef: { driver, nativeId: "shell-native", strength: "strong" as const }, parentItemId: null, ordinal: 100, - status: - check === "completed" - ? ("completed" as const) - : check === "pending" - ? ("pending" as const) - : ("running" as const), + status: check === "pending" ? ("pending" as const) : ("running" as const), title: "Approval-blocked shell", startedAt: now, - completedAt: check === "completed" ? now : null, + completedAt: null, updatedAt: now, ...(check === "dynamic approval" ? { type: "dynamic_tool" as const, toolName: "mcp:test", input: {} } @@ -240,18 +231,6 @@ it.effect.each([ ? event.type === "runtime-request.updated" && event.payload.id === requestId : event.type === "turn-item.updated" && event.payload.id === commandItem.id, ); - if (check === "confirmed background") { - yield* Queue.offer(events, { - type: "provider_thread.updated", - driver, - providerThread: { - ...projection.providerThreads[0]!, - pendingBackgroundTasks: [ - { taskId: "shell-native", kind: "command", description: "Confirmed shell" }, - ], - }, - }); - } yield* Queue.offer(events, { type: "node.updated", driver, node: commandNode }); yield* Queue.offer(events, { type: "turn_item.updated", driver, turnItem: commandItem }); if (approvalPending) { @@ -337,28 +316,6 @@ it.effect.each([ (item) => item.id === commandItem.id, ); assert.ok(before); - assert.equal( - before.status, - check === "completed" ? "completed" : check === "pending" ? "pending" : "running", - ); - const ingestedProjection = yield* orchestrator.getThreadProjection(threadId); - if (approvalPending) { - assert.equal( - ingestedProjection.runtimeRequests.find((request) => request.id === requestId)?.status, - "pending", - ); - assert.equal( - ingestedProjection.runtimeRequests.find((request) => request.id === requestId)?.nodeId, - NodeId.make("node:approval"), - ); - assert.equal( - ingestedProjection.nodes.find((node) => node.id === NodeId.make("node:approval")) - ?.parentNodeId, - commandNode.id, - ); - } else { - assert.deepEqual(ingestedProjection.runtimeRequests, []); - } if (check === "accepted approval") { yield* orchestrator.dispatch({ type: "runtime-request.respond", @@ -368,406 +325,116 @@ it.effect.each([ decision: "accept", }); yield* worker.drain(); - assert.equal( - (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests.find( - (request) => request.id === requestId, - )?.status, - "resolved", - ); } - if (check === "approval query failure after successor dispatch") { - const secondCommand = { - ...commandItem, - id: TurnItemId.make("item:pending-shell-2"), - nodeId: NodeId.make("node:command-2"), - nativeItemRef: { ...commandItem.nativeItemRef, nativeId: "shell-native-2" }, - ordinal: 101, - }; - const secondRequestId = RuntimeRequestId.make("request:pending-shell-2"); - const secondApprovalNodeId = NodeId.make("node:approval-2"); - const secondIngested = yield* watch( - (event) => - event.type === "runtime-request.updated" && event.payload.id === secondRequestId, - ); + const failed = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "failed", + ); + if (check === "approval on provider failure") { + // The session stays live, so only the run's own finalization can close the approval. yield* Queue.offerAll(events, [ { - type: "node.updated", - driver, - node: { - ...commandNode, - id: secondCommand.nodeId, - nativeItemRef: secondCommand.nativeItemRef, - }, - }, - { type: "turn_item.updated", driver, turnItem: secondCommand }, - { - type: "node.updated", - driver, - node: { - ...commandNode, - id: secondApprovalNodeId, - parentNodeId: secondCommand.nodeId, - kind: "approval_request", - status: "waiting", - runtimeRequestId: secondRequestId, - }, - }, - { - type: "turn_item.updated", + type: "provider_turn.updated", driver, - turnItem: { - ...secondCommand, - id: TurnItemId.make("item:approval-2"), - nodeId: secondApprovalNodeId, - type: "approval_request", - requestId: secondRequestId, - requestKind: "command", - status: "waiting", - }, + providerTurn: { ...turn, status: "failed", completedAt: now }, }, { - type: "runtime_request.updated", + type: "turn.terminal", driver, - threadId, - runtimeRequest: { - id: secondRequestId, - nodeId: secondApprovalNodeId, - providerTurnId: turn.id, - nativeRequestRef: null, - kind: "command", - status: "pending", - responseCapability: { - type: "live", - providerSessionId: projection.providerSessions[0]!.id, - }, - createdAt: now, - resolvedAt: null, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status: "failed", + failureItemOrdinal: 200, + failure: { + class: "provider_error", + message: "Query closed", + code: null, + retryable: null, }, + threadDisposition: "reusable", }, ]); - yield* Fiber.join(secondIngested); - const originals = [commandItem, secondCommand]; - const requestIds = [requestId, secondRequestId]; - const assertBlocked = (state: typeof projection) => { - assert.equal(state.runs.find((run) => run.id === first.runId)?.status, "running"); - for (const original of originals) { - assert.equal( - state.turnItems.find((item) => item.id === original.id)?.status, - "running", - ); - } - for (const id of requestIds) { - assert.equal( - state.runtimeRequests.find((request) => request.id === id)?.status, - "pending", - ); - } - }; - assertBlocked(yield* orchestrator.getThreadProjection(threadId)); + } else { + // Claude cannot switch modes in session, so this detaches it and fails the run. yield* orchestrator.dispatch({ type: "thread.runtime-mode.set", - commandId: CommandId.make("incident-change-mode"), + commandId: CommandId.make("change-mode"), threadId, runtimeMode: "auto", }); - yield* worker.drain(); - assertBlocked(yield* orchestrator.getThreadProjection(threadId)); - const closeQuery = yield* Deferred.make(); - const failed = yield* watch( - (event) => - event.type === "run.updated" && - event.payload.id === first.runId && - event.payload.status === "failed", - ); - const successorRunning = yield* watch( - (event) => - event.type === "provider-turn.updated" && - event.payload.runAttemptId !== first.attemptId && - event.payload.status === "running", - ); - // Simulate the provider closing its query in response to the successor - // dispatch, with a latch so both approvals are observed pending first. - const queryClosed = yield* Effect.gen(function* () { - yield* Deferred.await(closeQuery); - yield* Queue.offerAll(events, [ - { - type: "provider_turn.updated", - driver, - providerTurn: { ...turn, status: "failed", completedAt: now }, - }, - { - type: "turn.terminal", - driver, - providerThreadId: turn.providerThreadId, - providerTurnId: turn.id, - runOrdinal: first.runOrdinal, - status: "failed", - failureItemOrdinal: 200, - failure: { - class: "provider_error", - message: "Query closed after successor dispatch", - code: null, - retryable: null, - }, - threadDisposition: "reusable", - }, - ]); - }).pipe(Effect.forkScoped); - yield* dispatch("incident-successor"); - const dispatched = yield* orchestrator.getThreadProjection(threadId); - assertBlocked(dispatched); - assert.equal( - dispatched.runs.find( - (run) => run.userMessageId === MessageId.make("message:incident-successor"), - )?.status, - "queued", - ); - yield* Deferred.succeed(closeQuery, undefined); - yield* Fiber.join(queryClosed); - yield* Fiber.join(failed); - yield* worker.drain(); - // A provider error holds queued work; resume the already-dispatched - // successor rather than sending another message after the failure. - yield* orchestrator.dispatch({ - type: "queue.resume", - commandId: CommandId.make("resume-incident-successor"), - threadId, - }); - yield* worker.drain(); - assert.equal(started.length, 2); - yield* Fiber.join(successorRunning); - const successor = started[1]!; - assert.equal(successor.runtimePolicy.runtimeMode, "auto"); - assert.equal(successor.message.messageId, MessageId.make("message:incident-successor")); - const successorProjection = yield* orchestrator.getThreadProjection(threadId); - assert.equal( - successorProjection.runs.find((run) => run.id === successor.runId)?.status, - "running", - ); - const successorTurn = successorProjection.providerTurns.find( - (candidate) => candidate.runAttemptId === successor.attemptId, - )!; - for (const [index, original] of originals.entries()) { - const replacement = { - ...original, - id: TurnItemId.make(`item:successor-shell-${index}`), - runId: successor.runId, - nodeId: NodeId.make(`node:successor-command-${index}`), - providerThreadId: successorTurn.providerThreadId, - providerTurnId: successorTurn.id, - nativeItemRef: { ...original.nativeItemRef, nativeId: `successor-shell-${index}` }, - status: "running" as const, - }; - const replacementNode = { - ...commandNode, - id: replacement.nodeId, - runId: successor.runId, - parentNodeId: successor.rootNodeId, - rootNodeId: successor.rootNodeId, - providerThreadId: successorTurn.providerThreadId, - providerTurnId: successorTurn.id, - nativeItemRef: replacement.nativeItemRef, - }; - const completed = yield* watch( - (event) => - event.type === "turn-item.updated" && - event.payload.id === replacement.id && - event.payload.status === "completed", - ); - yield* Queue.offerAll(events, [ - { type: "node.updated", driver, node: replacementNode }, - { type: "turn_item.updated", driver, turnItem: replacement }, - { - type: "node.updated", - driver, - node: { ...replacementNode, status: "completed", completedAt: now }, - }, - { - type: "turn_item.updated", - driver, - turnItem: { ...replacement, status: "completed", completedAt: now }, - }, - ]); - yield* Fiber.join(completed); - } - const settled = yield* watch( - (event) => - event.type === "run.updated" && - event.payload.id === successor.runId && - event.payload.status === "waiting", - ); - yield* Queue.offerAll(events, [ - { - type: "provider_turn.updated", - driver, - providerTurn: { ...successorTurn, status: "completed", completedAt: now }, - }, - { - type: "turn.terminal", - driver, - providerThreadId: successorTurn.providerThreadId, - providerTurnId: successorTurn.id, - runOrdinal: successor.runOrdinal, - status: "completed", - failure: null, - threadDisposition: "reusable", - }, - ]); - yield* Fiber.join(settled); - yield* worker.drain(); - const final = yield* orchestrator.getThreadProjection(threadId); - assert.equal(final.runs.find((run) => run.id === first.runId)?.status, "failed"); - const finished = final.runs.find((run) => run.id === successor.runId)!; - assert.equal(finished.status, "completed"); - assert.isNotNull(finished.completedAt); - assert.isNotNull(finished.checkpointId); - assert.equal( - final.checkpoints.find((checkpoint) => checkpoint.id === finished.checkpointId)?.status, - "ready", - ); - for (const [index, original] of originals.entries()) { - const cancelled = final.turnItems.find((item) => item.id === original.id)!; - assert.equal(cancelled.status, "cancelled"); - assert.equal(cancelled.toolNonExecutionKind, "cancelled"); - assert.isNotNull(cancelled.completedAt); - assert.equal( - final.nodes.find((node) => node.id === original.nodeId)?.status, - "cancelled", - ); - assert.equal( - final.turnItems.find( - (item) => item.id === TurnItemId.make(`item:successor-shell-${index}`), - )?.status, - "completed", - ); - } - for (const id of requestIds) { - const request = final.runtimeRequests.find((candidate) => candidate.id === id)!; - assert.equal(request.status, "cancelled"); - assert.isNotNull(request.resolvedAt); - assert.equal(request.responseCapability.type, "not_resumable"); - assert.equal( - final.nodes.find((node) => node.id === request.nodeId)?.status, - "cancelled", - ); - assert.equal( - final.turnItems.find( - (item) => item.type === "approval_request" && item.requestId === id, - )?.status, - "cancelled", - ); - } - const freshShell = (yield* projections.getShellSnapshot()).threads.find( - (thread) => thread.id === threadId, - )!; - assert.ok(freshShell); - for (const original of originals) { - assert.isFalse( - (freshShell.pendingBackgroundTasks ?? []).some( - (task) => task.taskId === original.nativeItemRef.nativeId, - ), - ); - } - return; } - const failed = yield* watch( - (event) => - event.type === "run.updated" && - event.payload.id === first.runId && - event.payload.status === "failed", - ); - yield* orchestrator.dispatch({ - type: "thread.runtime-mode.set", - commandId: CommandId.make("change-mode"), - threadId, - runtimeMode: "auto", - }); yield* worker.drain(); yield* Fiber.join(failed); // Read before the next run hides background tasks behind its active status. - const snapshot = yield* projections.getShellSnapshot(); - const shell = snapshot.threads.find((thread) => thread.id === threadId)!; - assert.ok(shell); + const shell = (yield* projections.getShellSnapshot()).threads.find( + (thread) => thread.id === threadId, + )!; const pending = shell.pendingBackgroundTasks ?? []; yield* dispatch("second"); yield* worker.drain(); const after = yield* orchestrator.getThreadProjection(threadId); - assert.equal(after.runs.find((run) => run.id === first.runId)?.status, "failed"); - const item = after.turnItems.find((item) => item.id === commandItem.id)!; - if (check === "approval status" || check === "dynamic approval" || check === "pending") { - assert.equal(item.status, "cancelled"); - assert.equal(after.nodes.find((node) => node.id === commandNode.id)?.status, "cancelled"); - assert.isNotNull(item.completedAt); - assert.equal(item.toolNonExecutionKind, "cancelled"); - const lateEvent: OrchestrationV2DomainEvent = { - id: EventId.make("late-tool-use"), - type: "turn-item.updated", - threadId, - runId: first.runId, - occurredAt: now, - payload: commandItem, - }; - assert.deepEqual( - ProjectionStore.applyToProjection(after, lateEvent).turnItems.find( - (candidate) => candidate.id === item.id, - ), - item, - ); - yield* projections.apply(lateEvent); - assert.deepEqual(yield* projections.getTurnItem({ threadId, itemId: item.id }), item); - } else if (check === "approval shell selection") { - assert.isFalse(pending.some((task) => task.taskId === "shell-native")); - } else if (check === "confirmed background") { + const item = after.turnItems.find((candidate) => candidate.id === commandItem.id)!; + if (check === "accepted approval" || check === "unlinked running") { assert.deepEqual(item, before); assert.isTrue(pending.some((task) => task.taskId === "shell-native")); - } else { - assert.deepEqual(item, before); - assert.equal( - pending.some((task) => task.taskId === "shell-native"), - check !== "completed", - ); + if (check === "accepted approval") { + assert.equal( + after.runtimeRequests.find((request) => request.id === requestId)?.status, + "resolved", + ); + } + return; } - if (approvalPending) { + assert.equal(item.status, "cancelled"); + assert.equal(item.toolNonExecutionKind, "cancelled"); + assert.isNotNull(item.completedAt); + assert.equal(after.nodes.find((node) => node.id === commandNode.id)?.status, "cancelled"); + assert.isFalse(pending.some((task) => task.taskId === "shell-native")); + const lateEvent: OrchestrationV2DomainEvent = { + id: EventId.make("late-tool-use"), + type: "turn-item.updated", + threadId, + runId: first.runId, + occurredAt: now, + payload: commandItem, + }; + assert.deepEqual( + ProjectionStore.applyToProjection(after, lateEvent).turnItems.find( + (candidate) => candidate.id === item.id, + ), + item, + ); + yield* projections.apply(lateEvent); + assert.deepEqual(yield* projections.getTurnItem({ threadId, itemId: item.id }), item); + if (!approvalPending) return; + const request = after.runtimeRequests.find((candidate) => candidate.id === requestId)!; + assert.notEqual(request.status, "pending"); + assert.equal(request.responseCapability.type, "not_resumable"); + assert.equal(after.nodes.find((node) => node.id === request.nodeId)?.status, "cancelled"); + assert.equal( + after.turnItems.find((candidate) => candidate.id === TurnItemId.make("item:approval")) + ?.status, + "cancelled", + ); + const response = yield* orchestrator + .dispatch({ + type: "runtime-request.respond", + commandId: CommandId.make("late-approve"), + threadId, + requestId, + decision: "accept", + }) + .pipe(Effect.flip); + assert.equal(response._tag, "OrchestratorDispatchError"); + if (check === "approval on provider failure") { const unrelated = after.runtimeRequests.find( - (request) => request.id === RuntimeRequestId.make("request:unrelated"), + (candidate) => candidate.id === RuntimeRequestId.make("request:unrelated"), )!; assert.equal(unrelated.status, "pending"); assert.equal(unrelated.responseCapability.type, "live"); - assert.equal(after.nodes.find((node) => node.id === unrelated.nodeId)?.status, "waiting"); - } - if (approvalPending && check !== "accepted approval") { - assert.equal(item.status, "cancelled"); - const request = after.runtimeRequests.find((request) => request.id === requestId)!; - assert.equal(request.status, "cancelled"); - assert.deepEqual(request.responseCapability, { - type: "not_resumable", - reason: "The tool was cancelled before this approval was resolved.", - }); - assert.isNotNull(request.resolvedAt); - const approvalNode = after.nodes.find((node) => node.id === request.nodeId)!; - assert.equal(approvalNode.status, "cancelled"); - assert.isNotNull(approvalNode.completedAt); - const approvalItem = after.turnItems.find( - (item) => item.id === TurnItemId.make("item:approval"), - )!; - assert.equal(approvalItem.status, "cancelled"); - assert.isNotNull(approvalItem.completedAt); - const response = yield* orchestrator - .dispatch({ - type: "runtime-request.respond", - commandId: CommandId.make("late-approve"), - threadId, - requestId, - decision: "accept", - }) - .pipe(Effect.flip); - assert.equal(response._tag, "OrchestratorDispatchError"); - assert.equal(response.cause, `Runtime request ${requestId} is cancelled.`); - } else if (check === "accepted approval") { - assert.equal( - after.runtimeRequests.find((request) => request.id === requestId)?.status, - "resolved", - ); } }).pipe( Effect.provide( From f2c82830a8d32c7248e08acffc8aaa5fc44c0231 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 06:14:04 +0000 Subject: [PATCH 6/8] fix(server): run finalization keeps approvals answered after its read Finalization builds approval cancellations from a read taken before its write, so an approval accepted in between was overwritten as cancelled and its tool marked never started. The write now rechecks each approval inside the transaction and drops the cancellation, and the blocked tool's, when the request is no longer pending. Only server-closed approvals (pending, expired, or cancelled as not_resumable) prove a tool never started. Claude also cancels a request a PermissionRequest hook allowed, and that tool runs. AI-assisted; verified with the tests and tools listed above. --- apps/server/src/orchestration-v2/EventSink.ts | 70 +++++++++++++++- .../orchestration-v2/RunExecutionService.ts | 17 +++- .../TerminalShellItems.integration.test.ts | 82 ++++++++++++++++++- 3 files changed, 161 insertions(+), 8 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 0c42ac73ff9cb..ec0e1aa113d5b 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -116,12 +116,14 @@ export interface EventSinkV2Shape { "driver" | "providerInstanceId" | "appThreadId" | "nativeThreadRef" >; readonly guardPendingUserInputCancellations?: boolean; + readonly guardPendingApprovalCancellations?: boolean; readonly commandId?: CommandId; readonly events: ReadonlyArray; readonly effects: ReadonlyArray; }) => Effect.Effect, EventSinkV2Error>; readonly writeIfRunCurrent: (input: { readonly guardPendingUserInputCancellations?: boolean; + readonly guardPendingApprovalCancellations?: boolean; readonly commandId?: CommandId; readonly threadId: ThreadId; readonly runId: RunId; @@ -357,6 +359,58 @@ const layerBase: Layer.Layer< }); }); + // Run finalization cancels approvals it read as pending. One answered since then keeps + // its decision, and the tool it unblocked is not marked as never started. + const guardApprovalCancellations = + /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ + Effect.fnUntraced(function* (events: ReadonlyArray) { + const staleRequests = new Set(); + const staleNodes = new Set(); + for (const event of events) { + if ( + event.type !== "runtime-request.updated" || + event.payload.kind === "user_input" || + event.payload.status !== "cancelled" + ) + continue; + const current = yield* projectionStore.getRuntimeRequest( + event.threadId, + event.payload.id, + ); + if (current?.status !== "pending") { + staleRequests.add(event.payload.id); + staleNodes.add(event.payload.nodeId); + } + } + if (staleRequests.size === 0) return events; + for (const event of events) { + if ( + event.type === "node.updated" && + staleNodes.has(event.payload.id) && + event.payload.parentNodeId !== null + ) { + staleNodes.add(event.payload.parentNodeId); + } + } + return events.filter((event) => { + switch (event.type) { + case "runtime-request.updated": + return !staleRequests.has(event.payload.id); + case "node.updated": + return event.payload.status !== "cancelled" || !staleNodes.has(event.payload.id); + case "turn-item.updated": + return ( + event.payload.status !== "cancelled" || + !(event.payload.type === "approval_request" + ? staleRequests.has(event.payload.requestId) + : event.payload.nodeId !== null && staleNodes.has(event.payload.nodeId)) + ); + default: + return true; + } + }); + }); + const normalizeEvents = (events: ReadonlyArray) => { const runOrdinals = new Map( events.flatMap((event) => @@ -467,10 +521,14 @@ const layerBase: Layer.Layer< const events = input.updateDelegatedCompletionOnly ? yield* updateDelegatedCompletionOnly(input.events, () => true) : input.events; - const normalized = yield* normalizeEvents( + const guarded = input.guardPendingUserInputCancellations === true ? yield* guardUserInputCancellations(events) - : events, + : events; + const normalized = yield* normalizeEvents( + input.guardPendingApprovalCancellations === true + ? yield* guardApprovalCancellations(guarded) + : guarded, ); const committed = yield* eventStore.append({ ...(input.commandId === undefined ? {} : { commandId: input.commandId }), @@ -526,10 +584,14 @@ const layerBase: Layer.Layer< }; } - const normalized = yield* normalizeEvents( + const guarded = input.guardPendingUserInputCancellations === true ? yield* guardUserInputCancellations(input.events) - : input.events, + : input.events; + const normalized = yield* normalizeEvents( + input.guardPendingApprovalCancellations === true + ? yield* guardApprovalCancellations(guarded) + : guarded, ); const storedEvents = yield* eventStore.append({ ...(input.commandId === undefined ? {} : { commandId: input.commandId }), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 51151efde0ff4..cad4975b5d972 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -732,10 +732,17 @@ export const layer: Layer.Layer< const cancelledToolEvents: OrchestrationV2DomainEvent[] = []; if (input.loadRunOwnedTools !== undefined && !childTurnsContinue(input.terminal)) { const { turnItems, nodes, runtimeRequests } = yield* input.loadRunOwnedTools(); - // A session release may already have expired the approval before the run ends. + // A session release may already have closed the approval. Claude also cancels + // requests a hook allowed, and those tools run, so their capability stays live. const unresolvedApprovalNodes = new Set( runtimeRequests - .filter((request) => request.status !== "resolved") + .filter( + (request) => + request.status === "pending" || + request.status === "expired" || + (request.status === "cancelled" && + request.responseCapability.type === "not_resumable"), + ) .map((request) => request.nodeId), ); const blockedToolNodes = new Set( @@ -1009,6 +1016,7 @@ export const layer: Layer.Layer< } satisfies Parameters[0]; if (input.writeIfRunCurrent !== undefined) { const result = yield* eventSink.writeIfRunCurrent({ + guardPendingApprovalCancellations: true, threadId: input.run.threadId, runId: input.run.id, activeAttemptId: input.writeIfRunCurrent.activeAttemptId, @@ -1020,7 +1028,10 @@ export const layer: Layer.Layer< return; } } else { - yield* eventSink.writeWithEffects(finalization); + yield* eventSink.writeWithEffects({ + ...finalization, + guardPendingApprovalCancellations: true, + }); } yield* turnAnalytics.runSettled({ runId: input.run.id, diff --git a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts index 59ab526cac49e..b804512da8709 100644 --- a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts +++ b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts @@ -24,6 +24,7 @@ import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import { ClaudeProviderCapabilitiesV2 } from "./Adapters/ClaudeAdapterV2.ts"; import * as EffectWorker from "./EffectWorker.ts"; +import * as EventSink from "./EventSink.ts"; import * as Orchestrator from "./Orchestrator.ts"; import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; @@ -48,6 +49,7 @@ it.effect.each([ "approval", "dynamic approval", "accepted approval", + "withdrawn approval", "pending", "unlinked running", "approval on provider failure", @@ -317,6 +319,7 @@ it.effect.each([ ); assert.ok(before); if (check === "accepted approval") { + const stale = yield* orchestrator.getThreadProjection(threadId); yield* orchestrator.dispatch({ type: "runtime-request.respond", commandId: CommandId.make("approve"), @@ -325,6 +328,74 @@ it.effect.each([ decision: "accept", }); yield* worker.drain(); + // Finalization that read the approval before it was answered must not erase it. + const request = stale.runtimeRequests.find((candidate) => candidate.id === requestId)!; + const cancelledAt = { status: "cancelled" as const, completedAt: now }; + const staleEvent = (id: string) => ({ + id: EventId.make(`stale-${id}`), + threadId, + runId: first.runId, + occurredAt: now, + }); + yield* (yield* EventSink.EventSinkV2).writeWithEffects({ + guardPendingApprovalCancellations: true, + effects: [], + events: [ + { + ...staleEvent("tool"), + type: "turn-item.updated", + payload: { ...before, ...cancelledAt, toolNonExecutionKind: "cancelled" }, + }, + { + ...staleEvent("tool-node"), + type: "node.updated", + payload: { + ...stale.nodes.find((node) => node.id === commandNode.id)!, + ...cancelledAt, + }, + }, + { + ...staleEvent("request"), + type: "runtime-request.updated", + payload: { ...request, status: "cancelled", resolvedAt: now }, + }, + { + ...staleEvent("approval-node"), + type: "node.updated", + payload: { + ...stale.nodes.find((node) => node.id === request.nodeId)!, + ...cancelledAt, + }, + }, + { + ...staleEvent("approval-item"), + type: "turn-item.updated", + payload: { + ...stale.turnItems.find((item) => item.id === TurnItemId.make("item:approval"))!, + ...cancelledAt, + }, + }, + ], + }); + } + if (check === "withdrawn approval") { + // Claude withdraws the prompt when a PermissionRequest hook allows the tool, which then runs. + const withdrawn = yield* watch( + (event) => + event.type === "runtime-request.updated" && + event.payload.id === requestId && + event.payload.status === "cancelled", + ); + const request = (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests.find( + (candidate) => candidate.id === requestId, + )!; + yield* Queue.offer(events, { + type: "runtime_request.updated", + driver, + threadId, + runtimeRequest: { ...request, status: "cancelled", resolvedAt: now }, + }); + yield* Fiber.join(withdrawn); } const failed = yield* watch( (event) => @@ -377,7 +448,11 @@ it.effect.each([ yield* worker.drain(); const after = yield* orchestrator.getThreadProjection(threadId); const item = after.turnItems.find((candidate) => candidate.id === commandItem.id)!; - if (check === "accepted approval" || check === "unlinked running") { + if ( + check === "accepted approval" || + check === "withdrawn approval" || + check === "unlinked running" + ) { assert.deepEqual(item, before); assert.isTrue(pending.some((task) => task.taskId === "shell-native")); if (check === "accepted approval") { @@ -385,6 +460,11 @@ it.effect.each([ after.runtimeRequests.find((request) => request.id === requestId)?.status, "resolved", ); + assert.equal( + after.turnItems.find((candidate) => candidate.id === TurnItemId.make("item:approval")) + ?.status, + "completed", + ); } return; } From 190c7de4d6f3b9b9b67700e52a2caf576cde4b58 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Wed, 11 Nov 2026 06:31:00 +0000 Subject: [PATCH 7/8] fix(server): a session release during finalization still cancels the blocked tool The finalization guard kept the tool running whenever its approval was no longer pending. A server-closed (not_resumable) approval proves the tool never started, so only an accepted approval, or one Claude withdrew with a live capability after a hook allowed the tool, now preserves it. The released approval's own outcome is still kept. AI-assisted; verified with the tests and tools listed above. --- apps/server/src/orchestration-v2/EventSink.ts | 20 +++-- .../TerminalShellItems.integration.test.ts | 77 ++++++++++++++----- 2 files changed, 72 insertions(+), 25 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index ec0e1aa113d5b..012067c7cd6f1 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -359,13 +359,14 @@ const layerBase: Layer.Layer< }); }); - // Run finalization cancels approvals it read as pending. One answered since then keeps - // its decision, and the tool it unblocked is not marked as never started. + // Run finalization cancels approvals it read as pending. One closed since then keeps + // its outcome; if it let the tool run, the tool is not marked as never started. const guardApprovalCancellations = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (events: ReadonlyArray) { const staleRequests = new Set(); const staleNodes = new Set(); + const startedToolParents = new Set(); for (const event of events) { if ( event.type !== "runtime-request.updated" || @@ -377,16 +378,23 @@ const layerBase: Layer.Layer< event.threadId, event.payload.id, ); - if (current?.status !== "pending") { - staleRequests.add(event.payload.id); - staleNodes.add(event.payload.nodeId); + if (current?.status === "pending") continue; + staleRequests.add(event.payload.id); + staleNodes.add(event.payload.nodeId); + // Accepted, or withdrawn by Claude after a hook allowed it. A server-closed + // (not_resumable) approval still proves the tool never started. + if ( + current?.status === "resolved" || + (current?.status === "cancelled" && current.responseCapability.type === "live") + ) { + startedToolParents.add(event.payload.nodeId); } } if (staleRequests.size === 0) return events; for (const event of events) { if ( event.type === "node.updated" && - staleNodes.has(event.payload.id) && + startedToolParents.has(event.payload.id) && event.payload.parentNodeId !== null ) { staleNodes.add(event.payload.parentNodeId); diff --git a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts index b804512da8709..f4ce2039dcb8e 100644 --- a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts +++ b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts @@ -49,6 +49,7 @@ it.effect.each([ "approval", "dynamic approval", "accepted approval", + "released approval", "withdrawn approval", "pending", "unlinked running", @@ -318,18 +319,14 @@ it.effect.each([ (item) => item.id === commandItem.id, ); assert.ok(before); - if (check === "accepted approval") { + if (check === "accepted approval" || check === "released approval") { + const eventSink = yield* EventSink.EventSinkV2; const stale = yield* orchestrator.getThreadProjection(threadId); - yield* orchestrator.dispatch({ - type: "runtime-request.respond", - commandId: CommandId.make("approve"), - threadId, - requestId, - decision: "accept", - }); - yield* worker.drain(); - // Finalization that read the approval before it was answered must not erase it. const request = stale.runtimeRequests.find((candidate) => candidate.id === requestId)!; + const approvalNode = stale.nodes.find((node) => node.id === request.nodeId)!; + const approvalItem = stale.turnItems.find( + (item) => item.id === TurnItemId.make("item:approval"), + )!; const cancelledAt = { status: "cancelled" as const, completedAt: now }; const staleEvent = (id: string) => ({ id: EventId.make(`stale-${id}`), @@ -337,7 +334,43 @@ it.effect.each([ runId: first.runId, occurredAt: now, }); - yield* (yield* EventSink.EventSinkV2).writeWithEffects({ + if (check === "accepted approval") { + yield* orchestrator.dispatch({ + type: "runtime-request.respond", + commandId: CommandId.make("approve"), + threadId, + requestId, + decision: "accept", + }); + yield* worker.drain(); + } else { + yield* eventSink.write({ + events: [ + { + ...staleEvent("release-request"), + type: "runtime-request.updated", + payload: { + ...request, + status: "cancelled", + responseCapability: { type: "not_resumable", reason: "Session closed." }, + resolvedAt: now, + }, + }, + { + ...staleEvent("release-node"), + type: "node.updated", + payload: { ...approvalNode, ...cancelledAt }, + }, + { + ...staleEvent("release-item"), + type: "turn-item.updated", + payload: { ...approvalItem, ...cancelledAt }, + }, + ], + }); + } + // Finalization read the approval as pending before it was answered or released. + yield* eventSink.writeWithEffects({ guardPendingApprovalCancellations: true, effects: [], events: [ @@ -362,21 +395,27 @@ it.effect.each([ { ...staleEvent("approval-node"), type: "node.updated", - payload: { - ...stale.nodes.find((node) => node.id === request.nodeId)!, - ...cancelledAt, - }, + payload: { ...approvalNode, ...cancelledAt }, }, { ...staleEvent("approval-item"), type: "turn-item.updated", - payload: { - ...stale.turnItems.find((item) => item.id === TurnItemId.make("item:approval"))!, - ...cancelledAt, - }, + payload: { ...approvalItem, ...cancelledAt }, }, ], }); + if (check === "released approval") { + const released = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + released.turnItems.find((item) => item.id === commandItem.id)?.status, + "cancelled", + ); + assert.deepEqual( + released.runtimeRequests.find((candidate) => candidate.id === requestId) + ?.responseCapability, + { type: "not_resumable", reason: "Session closed." }, + ); + } } if (check === "withdrawn approval") { // Claude withdraws the prompt when a PermissionRequest hook allows the tool, which then runs. From 4dc9446f30c3c397014601bd1cc3944367654fda Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Fri, 13 Nov 2026 16:32:43 +0000 Subject: [PATCH 8/8] test(client-runtime): one late-update case per cancelled tool type Co-Authored-By: Claude Opus 5.5 --- .../state/orchestrationV2Projection.test.ts | 84 +++++++++---------- 1 file changed, 41 insertions(+), 43 deletions(-) diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index 6c0e291d6c100..6cda9874c8bbb 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -261,49 +261,47 @@ describe("applyOrchestrationV2ProjectionEvent", () => { expect(next?.visibleTurnItems[1]).toBe(secondRow); }); - for (const type of ["command_execution", "dynamic_tool"] as const) { - it.each(["pending", "running", "waiting"] as const)( - `keeps cancelled ${type} items and visible rows through a late %s update`, - (status) => { - const command = commandItem("item-cancelled", ""); - if (command.type !== "command_execution") throw new Error("Expected a command item"); - const item: OrchestrationV2TurnItem = { - ...command, - ...(type === "dynamic_tool" ? { type, toolName: "test_tool" } : { type }), - status: "cancelled", - toolNonExecutionKind: "cancelled", - }; - const event = { - id: "event-item-cancelled", - type: "turn-item.updated", - threadId, - runId, - occurredAt: now, - payload: item, - } as OrchestrationV2DomainEvent; - const cancelled = applyOrchestrationV2ProjectionEvent( - { ...emptyProjection, runs: [run] }, - event, - )!; - expect(cancelled.turnItems).toEqual([item]); - expect(cancelled.visibleTurnItems.map((row) => row.item)).toEqual([item]); - - const next = applyOrchestrationV2ProjectionEvent(cancelled, { - ...event, - id: "event-item-late-active" as OrchestrationV2DomainEvent["id"], - payload: { - ...item, - status, - toolNonExecutionKind: undefined, - completedAt: null, - updatedAt: DateTime.add(now, { seconds: 1 }), - }, - } as OrchestrationV2DomainEvent); - expect(next?.turnItems).toEqual([item]); - expect(next?.visibleTurnItems).toEqual(cancelled.visibleTurnItems); - }, - ); - } + it.each(["command_execution", "dynamic_tool"] as const)( + "keeps a cancelled %s item and its visible row through a late running update", + (type) => { + const command = commandItem("item-cancelled", ""); + if (command.type !== "command_execution") throw new Error("Expected a command item"); + const item: OrchestrationV2TurnItem = { + ...command, + ...(type === "dynamic_tool" ? { type, toolName: "test_tool" } : { type }), + status: "cancelled", + toolNonExecutionKind: "cancelled", + }; + const event = { + id: "event-item-cancelled", + type: "turn-item.updated", + threadId, + runId, + occurredAt: now, + payload: item, + } as OrchestrationV2DomainEvent; + const cancelled = applyOrchestrationV2ProjectionEvent( + { ...emptyProjection, runs: [run] }, + event, + )!; + expect(cancelled.turnItems).toEqual([item]); + expect(cancelled.visibleTurnItems.map((row) => row.item)).toEqual([item]); + + const next = applyOrchestrationV2ProjectionEvent(cancelled, { + ...event, + id: "event-item-late-active" as OrchestrationV2DomainEvent["id"], + payload: { + ...item, + status: "running", + toolNonExecutionKind: undefined, + completedAt: null, + updatedAt: DateTime.add(now, { seconds: 1 }), + }, + } as OrchestrationV2DomainEvent); + expect(next?.turnItems).toEqual([item]); + expect(next?.visibleTurnItems).toEqual(cancelled.visibleTurnItems); + }, + ); it("inserts live turn items by authoritative ordinal", () => { const queuedFuture = commandItem("item-queued-future", "queued", 300);