diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 3f6744e576ed..ee543a867903 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -130,12 +130,14 @@ export interface EventSinkV2Shape { "driver" | "providerInstanceId" | "appThreadId" | "nativeThreadRef" >; readonly guardPendingUserInputCancellations?: boolean; + readonly guardPendingApprovalCancellations?: boolean; readonly commandId?: CommandId; readonly events: ReadonlyArray; readonly effects: ReadonlyArray; }) => Effect.Effect, EventSinkV2Error>; readonly writeIfRunCurrent: (input: { readonly guardPendingUserInputCancellations?: boolean; + readonly guardPendingApprovalCancellations?: boolean; readonly commandId?: CommandId; readonly threadId: ThreadId; readonly runId: RunId; @@ -400,6 +402,66 @@ const layerBase: Layer.Layer< }); }); + // Run finalization cancels approvals it read as pending. One closed since then keeps + // its outcome; if it let the tool run, the tool is not marked as never started. + const guardApprovalCancellations = + /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ + Effect.fnUntraced(function* (events: ReadonlyArray) { + const staleRequests = new Set(); + const staleNodes = new Set(); + const startedToolParents = new Set(); + for (const event of events) { + if ( + event.type !== "runtime-request.updated" || + event.payload.kind === "user_input" || + event.payload.status !== "cancelled" + ) + continue; + const current = yield* projectionStore.getRuntimeRequest( + event.threadId, + event.payload.id, + ); + if (current?.status === "pending") continue; + staleRequests.add(event.payload.id); + staleNodes.add(event.payload.nodeId); + // Accepted, or withdrawn by Claude after a hook allowed it. A server-closed + // (not_resumable) approval still proves the tool never started. + if ( + current?.status === "resolved" || + (current?.status === "cancelled" && current.responseCapability.type === "live") + ) { + startedToolParents.add(event.payload.nodeId); + } + } + if (staleRequests.size === 0) return events; + for (const event of events) { + if ( + event.type === "node.updated" && + startedToolParents.has(event.payload.id) && + event.payload.parentNodeId !== null + ) { + staleNodes.add(event.payload.parentNodeId); + } + } + return events.filter((event) => { + switch (event.type) { + case "runtime-request.updated": + return !staleRequests.has(event.payload.id); + case "node.updated": + return event.payload.status !== "cancelled" || !staleNodes.has(event.payload.id); + case "turn-item.updated": + return ( + event.payload.status !== "cancelled" || + !(event.payload.type === "approval_request" + ? staleRequests.has(event.payload.requestId) + : event.payload.nodeId !== null && staleNodes.has(event.payload.nodeId)) + ); + default: + return true; + } + }); + }); + const normalizeEvents = (events: ReadonlyArray) => { const runOrdinals = new Map( events.flatMap((event) => @@ -518,10 +580,14 @@ const layerBase: Layer.Layer< const events = input.updateDelegatedCompletionOnly ? yield* updateDelegatedCompletionOnly(input.events, () => true) : input.events; - const normalized = yield* normalizeEvents( + const guarded = input.guardPendingUserInputCancellations === true ? yield* guardUserInputCancellations(events) - : events, + : events; + const normalized = yield* normalizeEvents( + input.guardPendingApprovalCancellations === true + ? yield* guardApprovalCancellations(guarded) + : guarded, ); const committed = yield* eventStore.append({ ...(input.commandId === undefined ? {} : { commandId: input.commandId }), @@ -596,10 +662,14 @@ const layerBase: Layer.Layer< } } - const normalized = yield* normalizeEvents( + const guarded = input.guardPendingUserInputCancellations === true ? yield* guardUserInputCancellations(input.events) - : input.events, + : input.events; + const normalized = yield* normalizeEvents( + input.guardPendingApprovalCancellations === true + ? yield* guardApprovalCancellations(guarded) + : guarded, ); const storedEvents = yield* eventStore.append({ ...(input.commandId === undefined ? {} : { commandId: input.commandId }), diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index e930fa67e383..a5a0e9932cf9 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -1,5 +1,9 @@ import type { ProjectThreadAwarenessV2Input } from "@t3tools/shared/agentAwareness"; -import { applyOrchestrationV2ThreadVisit, pendingGuestMessageCount } from "@t3tools/contracts"; +import { + applyOrchestrationV2ThreadVisit, + isOrchestrationV2WorkActive, + pendingGuestMessageCount, +} from "@t3tools/contracts"; import { applyThreadPullRequestLinkUpdate } from "../sourceControl/threadPullRequests.ts"; import { projectTurnItemForWire } from "./WireProjection.ts"; import * as Stream from "effect/Stream"; @@ -1195,11 +1199,20 @@ export function applyToProjection( ...base, messages: upsertById(base.messages, event.payload), }; - case "turn-item.updated": + case "turn-item.updated": { + const previous = base.turnItems.find((item) => item.id === event.payload.id); + // A delayed tool_use frame cannot restart work cancelled before execution. + const item = + previous?.toolNonExecutionKind === "cancelled" && + (previous.type === "command_execution" || previous.type === "dynamic_tool") && + isOrchestrationV2WorkActive(event.payload.status) + ? previous + : event.payload; return withLocalVisibleTurnItems({ ...base, - turnItems: upsertById(base.turnItems, event.payload), + turnItems: upsertById(base.turnItems, item), }); + } case "plan.updated": return { ...base, @@ -3190,6 +3203,13 @@ export const layer: Layer.Layer = updated_at = excluded.updated_at, payload_json = excluded.payload_json, inline_work_output = excluded.inline_work_output + WHERE NOT ( + orchestration_v2_projection_turn_items.type IN ('command_execution', 'dynamic_tool') + AND CASE WHEN orchestration_v2_projection_turn_items.status = 'cancelled' + THEN json_extract(orchestration_v2_projection_turn_items.payload_json, '$.toolNonExecutionKind') IS 'cancelled' + ELSE FALSE END + AND excluded.status IN ('pending', 'running', 'waiting') + ) `; break; } diff --git a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts index 317dff4b395b..1f006bdd7c63 100644 --- a/apps/server/src/orchestration-v2/ProviderTurnStartService.ts +++ b/apps/server/src/orchestration-v2/ProviderTurnStartService.ts @@ -336,6 +336,24 @@ export const layer: Layer.Layer< ), Effect.catchCause(() => Effect.succeed(input.inheritedBackgroundTurnItems)), ), + loadRunOwnedTools: Effect.fn("ProviderTurnStartService.loadRunOwnedTools")(function* () { + const { turnItems, runtimeRequests } = yield* projectionStore.getThreadRecords( + input.threadId, + ["turnItems", "runtimeRequests"], + { + turnItemRunIds: [input.runId], + turnItemTypes: ["command_execution", "dynamic_tool", "approval_request"], + turnItemStatuses: ["pending", "running", "waiting", "cancelled", "failed"], + runtimeRequestStatuses: ["pending", "expired", "cancelled"], + }, + ); + const nodeIds = turnItems.flatMap((item) => (item.nodeId === null ? [] : [item.nodeId])); + if (nodeIds.length === 0) return { turnItems, nodes: [], runtimeRequests }; + const { nodes } = yield* projectionStore.getThreadRecords(input.threadId, ["nodes"], { + nodeIds, + }); + return { turnItems, nodes, runtimeRequests }; + }), shouldStartProviderTurn: () => isCurrentAttemptInStatus("running"), shouldFinalizeRun: () => isCurrentAttemptIn(["starting", "running"]), hasUnpairedRunInterruptRequest: () => @@ -2244,6 +2262,7 @@ export const layer: Layer.Layer< attemptId: attempt.id, baseline, loadInheritedBackgroundTurnItems: runControls.loadInheritedBackgroundTurnItems, + loadRunOwnedTools: runControls.loadRunOwnedTools, relatedThreadIds: routableSubagents.flatMap((subagent) => subagent.childThreadId === null ? [] : [subagent.childThreadId], ), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index a5eeaa9cb8b9..9ff819b7915b 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -19,6 +19,7 @@ import { type OrchestrationV2RunAttempt, type OrchestrationV2Subagent, type OrchestrationV2TurnItem, + type OrchestrationV2ThreadProjection, type ProviderSessionId, type ProviderThreadId, type ProviderTurnId, @@ -155,6 +156,11 @@ export function selectInheritedBackgroundTurnItems(input: { ); } +type RunOwnedTools = Pick< + OrchestrationV2ThreadProjection, + "turnItems" | "nodes" | "runtimeRequests" +>; + type SubagentTurnItem = Extract; // Tool calls that only a live provider process can end. @@ -659,6 +665,7 @@ export interface RunExecutionServiceV2StartRootRunInput { readonly relatedProviderThreadIds?: ReadonlyArray; readonly shouldStartProviderTurn?: () => Effect.Effect; readonly shouldFinalizeRun?: () => Effect.Effect; + readonly loadRunOwnedTools?: () => Effect.Effect; readonly hasUnpairedRunInterruptRequest?: () => Effect.Effect; readonly loadRunInterruptReason?: () => Effect.Effect; readonly hasWorkspaceMoved?: () => Effect.Effect; @@ -741,6 +748,7 @@ export const layer: Layer.Layer< readonly loadRunInterruptReason?: () => Effect.Effect; readonly openRunOwnedSubagents?: OpenRunOwnedSubagentProjection; readonly openRunOwnedSubagentsTerminal?: ProviderTerminalEvent; + readonly loadRunOwnedTools?: () => Effect.Effect; // Root turn snapshot written with the run when T3 ends the run itself. readonly abandonedProviderTurn?: OrchestrationV2ProviderTurn; // Stored root tool calls still open when the provider runtime was lost; nothing else ends them. @@ -846,6 +854,145 @@ export const layer: Layer.Layer< allocateEventId, }) : []; + const cancelledToolEvents: OrchestrationV2DomainEvent[] = []; + const cancelledToolItemIds = new Set(); + if (input.loadRunOwnedTools !== undefined && !childTurnsContinue(input.terminal)) { + const { turnItems, nodes, runtimeRequests } = yield* input.loadRunOwnedTools(); + // A session release may already have closed the approval. Claude also cancels + // requests a hook allowed, and those tools run, so their capability stays live. + const unresolvedApprovalNodes = new Set( + runtimeRequests + .filter( + (request) => + request.status === "pending" || + request.status === "expired" || + (request.status === "cancelled" && + request.responseCapability.type === "not_resumable"), + ) + .map((request) => request.nodeId), + ); + const blockedToolNodes = new Set( + nodes.flatMap((node) => + node.runId === input.run.id && + node.threadId === input.run.threadId && + node.kind === "approval_request" && + unresolvedApprovalNodes.has(node.id) && + node.parentNodeId !== null + ? [node.parentNodeId] + : [], + ), + ); + const cancelledToolNodeIds = new Set(); + for (const item of turnItems) { + // Claude marks tool_use as running before asking permission. Only an + // unresolved approval child proves that a running tool never started. + if ( + item.runId !== input.run.id || + item.threadId !== input.run.threadId || + (item.type !== "command_execution" && item.type !== "dynamic_tool") || + !isOrchestrationV2WorkActive(item.status) || + !( + item.status === "pending" || + (item.nodeId !== null && blockedToolNodes.has(item.nodeId)) + ) + ) + continue; + cancelledToolEvents.push({ + id: yield* allocateEventId(), + type: "turn-item.updated", + threadId: item.threadId, + runId: input.run.id, + ...(item.nodeId === null ? {} : { nodeId: item.nodeId }), + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { + ...item, + status: "cancelled", + toolNonExecutionKind: "cancelled", + completedAt, + updatedAt: completedAt, + }, + }); + cancelledToolItemIds.add(item.id); + if (item.nodeId !== null) cancelledToolNodeIds.add(item.nodeId); + const node = nodes.find((candidate) => candidate.id === item.nodeId); + if (node !== undefined && isOrchestrationV2WorkActive(node.status)) { + cancelledToolEvents.push({ + id: yield* allocateEventId(), + type: "node.updated", + threadId: node.threadId, + runId: input.run.id, + nodeId: node.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { ...node, status: "cancelled", completedAt }, + }); + } + } + for (const approvalNode of nodes) { + if ( + approvalNode.kind !== "approval_request" || + approvalNode.runId !== input.run.id || + approvalNode.threadId !== input.run.threadId || + approvalNode.parentNodeId === null || + !cancelledToolNodeIds.has(approvalNode.parentNodeId) + ) + continue; + const request = runtimeRequests.find( + (candidate) => candidate.nodeId === approvalNode.id && candidate.status === "pending", + ); + if (request === undefined) continue; + cancelledToolEvents.push( + { + id: yield* allocateEventId(), + type: "runtime-request.updated", + threadId: input.run.threadId, + runId: input.run.id, + nodeId: approvalNode.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { + ...request, + status: "cancelled", + responseCapability: { + type: "not_resumable", + reason: "The tool was cancelled before this approval was resolved.", + }, + resolvedAt: completedAt, + }, + }, + { + id: yield* allocateEventId(), + type: "node.updated", + threadId: input.run.threadId, + runId: input.run.id, + nodeId: approvalNode.id, + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { ...approvalNode, status: "cancelled", completedAt }, + }, + ); + for (const approvalItem of turnItems) { + if (approvalItem.type !== "approval_request" || approvalItem.requestId !== request.id) + continue; + cancelledToolEvents.push({ + id: yield* allocateEventId(), + type: "turn-item.updated", + threadId: input.run.threadId, + ...(approvalItem.runId === null ? {} : { runId: approvalItem.runId }), + ...(approvalItem.nodeId === null ? {} : { nodeId: approvalItem.nodeId }), + providerInstanceId: input.run.providerInstanceId, + occurredAt: completedAt, + payload: { + ...approvalItem, + status: "cancelled", + completedAt, + updatedAt: completedAt, + }, + }); + } + } + } const persistedStatus = input.terminal.status === "completed" ? "waiting" : input.terminal.status; // Omit delegatedCompletion; the startup snapshot must not overwrite newer acknowledgements, successors, or Stop barriers. @@ -895,9 +1042,12 @@ export const layer: Layer.Layer< : [], events: [ ...cascadedSubagentEvents, + ...cancelledToolEvents, ...(yield* interruptLostToolCalls({ run: input.run, - toolCalls: input.lostToolCalls ?? [], + toolCalls: (input.lostToolCalls ?? []).filter( + (toolCall) => !cancelledToolItemIds.has(toolCall.id), + ), completedAt, allocateEventId, })), @@ -1039,6 +1189,7 @@ export const layer: Layer.Layer< } satisfies Parameters[0]; if (input.writeIfRunCurrent !== undefined) { const result = yield* eventSink.writeIfRunCurrent({ + guardPendingApprovalCancellations: true, threadId: input.run.threadId, runId: input.run.id, activeAttemptId: input.writeIfRunCurrent.activeAttemptId, @@ -1071,7 +1222,10 @@ export const layer: Layer.Layer< return false; } } else { - yield* eventSink.writeWithEffects(finalization); + yield* eventSink.writeWithEffects({ + ...finalization, + guardPendingApprovalCancellations: true, + }); } yield* turnAnalytics.runSettled({ runId: input.run.id, @@ -1192,6 +1346,9 @@ export const layer: Layer.Layer< }), input.providerTurnOrdinal * 100 + 1, ), + ...(input.loadRunOwnedTools === undefined + ? {} + : { loadRunOwnedTools: input.loadRunOwnedTools }), failureItemPersisted: false, refreshAfterTurn, writeIfRunCurrent: { @@ -1347,6 +1504,9 @@ export const layer: Layer.Layer< : { loadRunInterruptReason: input.loadRunInterruptReason }), openRunOwnedSubagents: openSubagents, openRunOwnedSubagentsTerminal: terminal, + ...(input.loadRunOwnedTools === undefined + ? {} + : { loadRunOwnedTools: input.loadRunOwnedTools }), ...(abandonedProviderTurn === undefined ? {} : { abandonedProviderTurn }), ...(restoredHandoffs === undefined ? {} : { restoredHandoffs }), terminal: settledTerminal, @@ -2196,6 +2356,9 @@ export const layer: Layer.Layer< }, openRunOwnedSubagents: openSubagents, ...(abandonedProviderTurn === null ? {} : { abandonedProviderTurn }), + ...(input.loadRunOwnedTools === undefined + ? {} + : { loadRunOwnedTools: input.loadRunOwnedTools }), ...(recoveredHandoffs === undefined ? {} : { restoredHandoffs: recoveredHandoffs }), @@ -2371,6 +2534,9 @@ export const layer: Layer.Layer< expectedStatus: "running", }, openRunOwnedSubagents: openSubagents, + ...(input.loadRunOwnedTools === undefined + ? {} + : { loadRunOwnedTools: input.loadRunOwnedTools }), terminal: makeFailedTerminalEvent( makeProviderFailure({ cause: Cause.squash(cause), diff --git a/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts new file mode 100644 index 000000000000..f4ce2039dcb8 --- /dev/null +++ b/apps/server/src/orchestration-v2/TerminalShellItems.integration.test.ts @@ -0,0 +1,572 @@ +import { assert, it } from "@effect/vitest"; +import { + CommandId, + EventId, + NodeId, + MessageId, + ProjectId, + ProviderDriverKind, + ProviderInstanceId, + ProviderThreadId, + ProviderTurnId, + RuntimeRequestId, + ThreadId, + TurnItemId, + type OrchestrationV2DomainEvent, +} from "@t3tools/contracts"; +import * as DateTime from "effect/DateTime"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import * as SqlitePersistence from "../persistence/Sqlite.ts"; +import * as ProjectionStore from "./ProjectionStore.ts"; +import * as Fiber from "effect/Fiber"; +import * as Queue from "effect/Queue"; +import * as Stream from "effect/Stream"; +import { ClaudeProviderCapabilitiesV2 } from "./Adapters/ClaudeAdapterV2.ts"; +import * as EffectWorker from "./EffectWorker.ts"; +import * as EventSink from "./EventSink.ts"; +import * as Orchestrator from "./Orchestrator.ts"; +import * as ProviderAdapterRegistry from "./ProviderAdapterRegistry.ts"; + +import * as ProviderReplayHarness from "./testkit/ProviderReplayHarness.ts"; +import { checkpointWorkspace } from "@t3tools/provider-testing/replayWorkspace"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; + +const driver = ProviderDriverKind.make("claudeAgent"); +const instanceId = ProviderInstanceId.make("claudeAgent"); +const modelSelection = { instanceId, model: "test-model" }; +const databaseLayer = SqlitePersistence.layerMemory; +const capabilities = { + ...ClaudeProviderCapabilitiesV2, + sessions: { + ...ClaudeProviderCapabilitiesV2.sessions, + supportsMultipleProviderThreadsPerSession: false, + supportsRuntimeModeSwitchInSession: false, + }, +}; + +it.effect.each([ + "approval", + "dynamic approval", + "accepted approval", + "released approval", + "withdrawn approval", + "pending", + "unlinked running", + "approval on provider failure", +] as const)("ending a run cancels only tools that never started: %s", (check) => + Effect.scoped( + Effect.gen(function* () { + const testCapabilities = + check === "approval on provider failure" + ? { + ...capabilities, + sessions: { ...capabilities.sessions, supportsRuntimeModeSwitchInSession: true }, + } + : capabilities; + const cwd = yield* checkpointWorkspace("terminal-shell-items"); + const events = yield* Queue.unbounded(); + const started: ProviderAdapter.ProviderAdapterV2TurnInput[] = []; + const adapter: ProviderAdapter.ProviderAdapterV2["Service"] = { + instanceId, + driver, + getCapabilities: () => Effect.succeed(testCapabilities), + planSelectionTransition: () => Effect.succeed({ type: "apply_on_next_turn" }), + openSession: (input) => + Effect.gen(function* () { + const now = yield* DateTime.now; + return { + instanceId, + driver, + providerSessionId: input.providerSessionId, + providerSession: { + id: input.providerSessionId, + driver, + providerInstanceId: instanceId, + status: "ready", + cwd, + model: modelSelection.model, + capabilities: testCapabilities, + createdAt: now, + updatedAt: now, + lastError: null, + }, + events: Stream.fromQueue(events), + ensureThread: ({ threadId }) => + Effect.succeed({ + id: ProviderThreadId.make(`provider-thread:claudeAgent:${threadId}`), + driver, + providerInstanceId: instanceId, + providerSessionId: input.providerSessionId, + appThreadId: threadId, + ownerNodeId: null, + nativeThreadRef: { driver, nativeId: "native-thread", strength: "strong" }, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: null, + lastRunOrdinal: null, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }), + resumeThread: ({ providerThread }) => Effect.succeed(providerThread), + startTurn: (turn) => + Effect.gen(function* () { + started.push(turn); + yield* Queue.offer(events, { + type: "provider_turn.updated", + driver, + providerTurn: { + id: ProviderTurnId.make(`provider-turn:${turn.attemptId}`), + providerThreadId: turn.providerThread.id, + nodeId: turn.rootNodeId, + runAttemptId: turn.attemptId, + nativeTurnRef: { + driver, + nativeId: `native:${turn.attemptId}`, + strength: "strong", + }, + ordinal: turn.providerTurnOrdinal, + status: "running", + startedAt: now, + completedAt: null, + }, + }); + }), + steerTurn: () => Effect.die("unused"), + interruptTurn: () => Effect.void, + respondToRuntimeRequest: () => Effect.void, + readThreadSnapshot: () => Effect.die("unused"), + rollbackThread: () => Effect.die("unused"), + forkThread: () => Effect.die("unused"), + }; + }), + }; + yield* Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const worker = yield* EffectWorker.OrchestrationEffectWorkerV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const threadId = ThreadId.make("thread:terminal-shell-items"); + yield* orchestrator.dispatch({ + type: "thread.create", + commandId: CommandId.make("create"), + threadId, + projectId: ProjectId.make("project:terminal-shell-items"), + title: "Terminal shell items", + modelSelection, + runtimeMode: "approval-required", + interactionMode: "default", + branch: null, + worktreePath: cwd, + createdBy: "user", + creationSource: "web", + }); + const watch = (predicate: (event: OrchestrationV2DomainEvent) => boolean) => + orchestrator.streamDomainEvents.pipe( + Stream.filter(predicate), + Stream.take(1), + Stream.runDrain, + Effect.forkScoped, + ); + const running = yield* watch( + (event) => event.type === "provider-turn.updated" && event.payload.status === "running", + ); + const dispatch = (id: string) => + orchestrator.dispatch({ + type: "message.dispatch", + commandId: CommandId.make(id), + threadId, + messageId: MessageId.make(`message:${id}`), + text: "Continue", + attachments: [], + dispatchMode: { type: "start_immediately" }, + createdBy: "user", + creationSource: "web", + }); + yield* dispatch("first"); + yield* worker.drain(); + yield* Fiber.join(running); + const first = started[0]!; + const projection = yield* orchestrator.getThreadProjection(threadId); + const turn = projection.providerTurns[0]!; + const now = yield* DateTime.now; + const requestId = RuntimeRequestId.make("request:pending-shell"); + const commandItem = { + id: TurnItemId.make("item:pending-shell"), + threadId, + runId: first.runId, + nodeId: NodeId.make("node:command"), + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + nativeItemRef: { driver, nativeId: "shell-native", strength: "strong" as const }, + parentItemId: null, + ordinal: 100, + status: check === "pending" ? ("pending" as const) : ("running" as const), + title: "Approval-blocked shell", + startedAt: now, + completedAt: null, + updatedAt: now, + ...(check === "dynamic approval" + ? { type: "dynamic_tool" as const, toolName: "mcp:test", input: {} } + : { type: "command_execution" as const, input: "echo approved" }), + }; + const commandNode = { + id: NodeId.make("node:command"), + threadId, + runId: first.runId, + parentNodeId: first.rootNodeId, + rootNodeId: first.rootNodeId, + kind: "tool_call" as const, + status: commandItem.status, + countsForRun: true, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + nativeItemRef: commandItem.nativeItemRef, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: commandItem.completedAt, + }; + const approvalPending = check.includes("approval"); + const ingested = yield* watch((event) => + approvalPending + ? event.type === "runtime-request.updated" && event.payload.id === requestId + : event.type === "turn-item.updated" && event.payload.id === commandItem.id, + ); + yield* Queue.offer(events, { type: "node.updated", driver, node: commandNode }); + yield* Queue.offer(events, { type: "turn_item.updated", driver, turnItem: commandItem }); + if (approvalPending) { + const approvalNodeId = NodeId.make("node:approval"); + yield* Queue.offer(events, { + type: "node.updated", + driver, + node: { + ...commandNode, + id: approvalNodeId, + parentNodeId: commandNode.id, + kind: "approval_request", + status: "waiting", + runtimeRequestId: requestId, + }, + }); + yield* Queue.offer(events, { + type: "turn_item.updated", + driver, + turnItem: { + ...commandItem, + id: TurnItemId.make("item:approval"), + nodeId: approvalNodeId, + type: "approval_request", + requestId, + requestKind: "command", + status: "waiting", + }, + }); + yield* Queue.offer(events, { + type: "node.updated", + driver, + node: { + ...commandNode, + id: NodeId.make("node:unrelated-approval"), + parentNodeId: first.rootNodeId, + kind: "approval_request", + status: "waiting", + runtimeRequestId: RuntimeRequestId.make("request:unrelated"), + }, + }); + yield* Queue.offer(events, { + type: "runtime_request.updated", + driver, + threadId, + runtimeRequest: { + id: RuntimeRequestId.make("request:unrelated"), + nodeId: NodeId.make("node:unrelated-approval"), + providerTurnId: turn.id, + nativeRequestRef: null, + kind: "command", + status: "pending", + responseCapability: { + type: "live", + providerSessionId: projection.providerSessions[0]!.id, + }, + createdAt: now, + resolvedAt: null, + }, + }); + yield* Queue.offer(events, { + type: "runtime_request.updated", + driver, + threadId, + runtimeRequest: { + id: requestId, + nodeId: approvalNodeId, + providerTurnId: turn.id, + nativeRequestRef: null, + kind: "command", + status: "pending", + responseCapability: { + type: "live", + providerSessionId: projection.providerSessions[0]!.id, + }, + createdAt: now, + resolvedAt: null, + }, + }); + } + yield* Fiber.join(ingested); + const before = (yield* orchestrator.getThreadProjection(threadId)).turnItems.find( + (item) => item.id === commandItem.id, + ); + assert.ok(before); + if (check === "accepted approval" || check === "released approval") { + const eventSink = yield* EventSink.EventSinkV2; + const stale = yield* orchestrator.getThreadProjection(threadId); + const request = stale.runtimeRequests.find((candidate) => candidate.id === requestId)!; + const approvalNode = stale.nodes.find((node) => node.id === request.nodeId)!; + const approvalItem = stale.turnItems.find( + (item) => item.id === TurnItemId.make("item:approval"), + )!; + const cancelledAt = { status: "cancelled" as const, completedAt: now }; + const staleEvent = (id: string) => ({ + id: EventId.make(`stale-${id}`), + threadId, + runId: first.runId, + occurredAt: now, + }); + if (check === "accepted approval") { + yield* orchestrator.dispatch({ + type: "runtime-request.respond", + commandId: CommandId.make("approve"), + threadId, + requestId, + decision: "accept", + }); + yield* worker.drain(); + } else { + yield* eventSink.write({ + events: [ + { + ...staleEvent("release-request"), + type: "runtime-request.updated", + payload: { + ...request, + status: "cancelled", + responseCapability: { type: "not_resumable", reason: "Session closed." }, + resolvedAt: now, + }, + }, + { + ...staleEvent("release-node"), + type: "node.updated", + payload: { ...approvalNode, ...cancelledAt }, + }, + { + ...staleEvent("release-item"), + type: "turn-item.updated", + payload: { ...approvalItem, ...cancelledAt }, + }, + ], + }); + } + // Finalization read the approval as pending before it was answered or released. + yield* eventSink.writeWithEffects({ + guardPendingApprovalCancellations: true, + effects: [], + events: [ + { + ...staleEvent("tool"), + type: "turn-item.updated", + payload: { ...before, ...cancelledAt, toolNonExecutionKind: "cancelled" }, + }, + { + ...staleEvent("tool-node"), + type: "node.updated", + payload: { + ...stale.nodes.find((node) => node.id === commandNode.id)!, + ...cancelledAt, + }, + }, + { + ...staleEvent("request"), + type: "runtime-request.updated", + payload: { ...request, status: "cancelled", resolvedAt: now }, + }, + { + ...staleEvent("approval-node"), + type: "node.updated", + payload: { ...approvalNode, ...cancelledAt }, + }, + { + ...staleEvent("approval-item"), + type: "turn-item.updated", + payload: { ...approvalItem, ...cancelledAt }, + }, + ], + }); + if (check === "released approval") { + const released = yield* orchestrator.getThreadProjection(threadId); + assert.equal( + released.turnItems.find((item) => item.id === commandItem.id)?.status, + "cancelled", + ); + assert.deepEqual( + released.runtimeRequests.find((candidate) => candidate.id === requestId) + ?.responseCapability, + { type: "not_resumable", reason: "Session closed." }, + ); + } + } + if (check === "withdrawn approval") { + // Claude withdraws the prompt when a PermissionRequest hook allows the tool, which then runs. + const withdrawn = yield* watch( + (event) => + event.type === "runtime-request.updated" && + event.payload.id === requestId && + event.payload.status === "cancelled", + ); + const request = (yield* orchestrator.getThreadProjection(threadId)).runtimeRequests.find( + (candidate) => candidate.id === requestId, + )!; + yield* Queue.offer(events, { + type: "runtime_request.updated", + driver, + threadId, + runtimeRequest: { ...request, status: "cancelled", resolvedAt: now }, + }); + yield* Fiber.join(withdrawn); + } + const failed = yield* watch( + (event) => + event.type === "run.updated" && + event.payload.id === first.runId && + event.payload.status === "failed", + ); + if (check === "approval on provider failure") { + // The session stays live, so only the run's own finalization can close the approval. + yield* Queue.offerAll(events, [ + { + type: "provider_turn.updated", + driver, + providerTurn: { ...turn, status: "failed", completedAt: now }, + }, + { + type: "turn.terminal", + driver, + providerThreadId: turn.providerThreadId, + providerTurnId: turn.id, + runOrdinal: first.runOrdinal, + status: "failed", + failureItemOrdinal: 200, + failure: { + class: "provider_error", + message: "Query closed", + code: null, + retryable: null, + }, + threadDisposition: "reusable", + }, + ]); + } else { + // Claude cannot switch modes in session, so this detaches it and fails the run. + yield* orchestrator.dispatch({ + type: "thread.runtime-mode.set", + commandId: CommandId.make("change-mode"), + threadId, + runtimeMode: "auto", + }); + } + yield* worker.drain(); + yield* Fiber.join(failed); + // Read before the next run hides background tasks behind its active status. + const shell = (yield* projections.getShellSnapshot()).threads.find( + (thread) => thread.id === threadId, + )!; + const pending = shell.pendingBackgroundTasks ?? []; + yield* dispatch("second"); + yield* worker.drain(); + const after = yield* orchestrator.getThreadProjection(threadId); + const item = after.turnItems.find((candidate) => candidate.id === commandItem.id)!; + if ( + check === "accepted approval" || + check === "withdrawn approval" || + check === "unlinked running" + ) { + assert.deepEqual(item, before); + assert.isTrue(pending.some((task) => task.taskId === "shell-native")); + if (check === "accepted approval") { + assert.equal( + after.runtimeRequests.find((request) => request.id === requestId)?.status, + "resolved", + ); + assert.equal( + after.turnItems.find((candidate) => candidate.id === TurnItemId.make("item:approval")) + ?.status, + "completed", + ); + } + return; + } + assert.equal(item.status, "cancelled"); + assert.equal(item.toolNonExecutionKind, "cancelled"); + assert.isNotNull(item.completedAt); + assert.equal(after.nodes.find((node) => node.id === commandNode.id)?.status, "cancelled"); + assert.isFalse(pending.some((task) => task.taskId === "shell-native")); + const lateEvent: OrchestrationV2DomainEvent = { + id: EventId.make("late-tool-use"), + type: "turn-item.updated", + threadId, + runId: first.runId, + occurredAt: now, + payload: commandItem, + }; + assert.deepEqual( + ProjectionStore.applyToProjection(after, lateEvent).turnItems.find( + (candidate) => candidate.id === item.id, + ), + item, + ); + yield* projections.apply(lateEvent); + assert.deepEqual(yield* projections.getTurnItem({ threadId, itemId: item.id }), item); + if (!approvalPending) return; + const request = after.runtimeRequests.find((candidate) => candidate.id === requestId)!; + assert.notEqual(request.status, "pending"); + assert.equal(request.responseCapability.type, "not_resumable"); + assert.equal(after.nodes.find((node) => node.id === request.nodeId)?.status, "cancelled"); + assert.equal( + after.turnItems.find((candidate) => candidate.id === TurnItemId.make("item:approval")) + ?.status, + "cancelled", + ); + const response = yield* orchestrator + .dispatch({ + type: "runtime-request.respond", + commandId: CommandId.make("late-approve"), + threadId, + requestId, + decision: "accept", + }) + .pipe(Effect.flip); + assert.equal(response._tag, "OrchestratorDispatchError"); + if (check === "approval on provider failure") { + const unrelated = after.runtimeRequests.find( + (candidate) => candidate.id === RuntimeRequestId.make("request:unrelated"), + )!; + assert.equal(unrelated.status, "pending"); + assert.equal(unrelated.responseCapability.type, "live"); + } + }).pipe( + Effect.provide( + Layer.mergeAll( + ProviderReplayHarness.layerWithRegistry( + { name: "terminal-shell-items" }, + ProviderAdapterRegistry.layerFromAdapters([adapter]), + { runEffectWorker: false, databaseLayer }, + ), + ProjectionStore.layer.pipe(Layer.provide(databaseLayer)), + ), + ), + ); + }), + ), +); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts index 16301157a6a4..3220dceef303 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.test.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.test.ts @@ -262,6 +262,48 @@ describe("applyOrchestrationV2ProjectionEvent", () => { expect(next?.visibleTurnItems[1]).toBe(secondRow); }); + it.each(["command_execution", "dynamic_tool"] as const)( + "keeps a cancelled %s item and its visible row through a late running update", + (type) => { + const command = commandItem("item-cancelled", ""); + if (command.type !== "command_execution") throw new Error("Expected a command item"); + const item: OrchestrationV2TurnItem = { + ...command, + ...(type === "dynamic_tool" ? { type, toolName: "test_tool" } : { type }), + status: "cancelled", + toolNonExecutionKind: "cancelled", + }; + const event = { + id: "event-item-cancelled", + type: "turn-item.updated", + threadId, + runId, + occurredAt: now, + payload: item, + } as OrchestrationV2DomainEvent; + const cancelled = applyOrchestrationV2ProjectionEvent( + { ...emptyProjection, runs: [run] }, + event, + )!; + expect(cancelled.turnItems).toEqual([item]); + expect(cancelled.visibleTurnItems.map((row) => row.item)).toEqual([item]); + + const next = applyOrchestrationV2ProjectionEvent(cancelled, { + ...event, + id: "event-item-late-active" as OrchestrationV2DomainEvent["id"], + payload: { + ...item, + status: "running", + toolNonExecutionKind: undefined, + completedAt: null, + updatedAt: DateTime.add(now, { seconds: 1 }), + }, + } as OrchestrationV2DomainEvent); + expect(next?.turnItems).toEqual([item]); + expect(next?.visibleTurnItems).toEqual(cancelled.visibleTurnItems); + }, + ); + it("inserts live turn items by authoritative ordinal", () => { const queuedFuture = commandItem("item-queued-future", "queued", 300); const activeAssistant = commandItem("item-active-assistant", "done", 201); diff --git a/packages/client-runtime/src/state/orchestrationV2Projection.ts b/packages/client-runtime/src/state/orchestrationV2Projection.ts index d936fa7726df..5f913a4a6abb 100644 --- a/packages/client-runtime/src/state/orchestrationV2Projection.ts +++ b/packages/client-runtime/src/state/orchestrationV2Projection.ts @@ -1,6 +1,7 @@ import { applyThreadPullRequestLinkUpdate } from "../threadPullRequests.ts"; import { applyOrchestrationV2ThreadVisit, + isOrchestrationV2WorkActive, isQueuedProviderThreadPlaceholder, } from "@t3tools/contracts"; import type { @@ -446,6 +447,15 @@ export function applyOrchestrationV2ProjectionEvent( ) { return projection; } + const previous = itemIndex === -1 ? undefined : projection.turnItems[itemIndex]; + // A delayed tool_use frame cannot restart work cancelled before execution. + if ( + previous?.toolNonExecutionKind === "cancelled" && + (previous.type === "command_execution" || previous.type === "dynamic_tool") && + isOrchestrationV2WorkActive(payload.status) + ) { + payload = previous; + } const turnItems = itemIndex === -1 ? [...projection.turnItems, payload] @@ -453,7 +463,6 @@ export function applyOrchestrationV2ProjectionEvent( const next = { ...base, turnItems }; // Only interrupt requests can change another item's visibility. Streaming // text/tool updates must not recheck every row against every run. - const previous = itemIndex === -1 ? undefined : projection.turnItems[itemIndex]; const visible = payload.type === "run_interrupt_request" || previous?.type === "run_interrupt_request" ? { ...next, visibleTurnItems: activeVisibleTurnItems(next) }