diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index e6e29a1b34a7..72ea19a442db 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -3252,6 +3252,7 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( id: EventId.make(`event:followup-command-awaited-${ending}`), type: "provider-thread.updated", threadId: child.childThreadId, + runId: followupRun.id, driver, providerInstanceId: modelSelection.instanceId, occurredAt: now, @@ -3432,6 +3433,329 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( }), ); + it.effect("settles a follow-up whose own run started no background work", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const parentThreadId = ThreadId.make("thread:followup-earlier-work-parent"); + const projectId = ProjectId.make("project:followup-earlier-work"); + const parentRunId = RunId.make("run:followup-earlier-work-parent"); + const rootNodeId = NodeId.make("node:followup-earlier-work-parent"); + yield* seedParentWithTerminalTask({ + threadId: parentThreadId, + projectId, + runId: parentRunId, + rootNodeId, + taskId: NodeId.make("node:followup-earlier-work-seed"), + deliveryState: "acknowledged", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId, + projectId, + parentRunId, + rootNodeId, + name: "followup-earlier-work-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + yield* orchestrator.recoverDelegatedTasks; + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + const childProviderThread = { + ...parent.providerThreads[0]!, + id: ProviderThreadId.make("provider-thread:followup-earlier-work-child"), + appThreadId: child.childThreadId, + }; + + // An earlier run of the child left an awaited build running. + yield* eventSink.write({ + commandId: reconcileCommandId("followup-earlier-work-build"), + events: [ + { + id: EventId.make("event:followup-earlier-work-build"), + type: "provider-thread.updated", + threadId: child.childThreadId, + runId: child.childRunId, + driver, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + ...childProviderThread, + pendingBackgroundTasks: [ + { taskId: "build", kind: "command" as const, awaited: true as const }, + ], + }, + }, + ], + }); + + const followupMessageId = MessageId.make("message:followup-earlier-work-send"); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make("command:followup-earlier-work-send"), + threadId: child.childThreadId, + senderThreadId: parentThreadId, + messageId: followupMessageId, + text: "Reply with exactly PONG.", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + }); + const followupRun = (yield* orchestrator.getThreadProjection( + child.childThreadId, + )).runs.find((run) => run.userMessageId === followupMessageId)!; + const followupTaskId = followupRun.delegatedTaskId!; + + // The follow-up's run answers and ends without starting any background work. + yield* eventSink.write({ + commandId: reconcileCommandId("followup-earlier-work-answered"), + events: [ + // The provider resends its whole roster during the follow-up's run. + { + id: EventId.make("event:followup-earlier-work-roster"), + type: "provider-thread.updated", + threadId: child.childThreadId, + runId: followupRun.id, + driver, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + ...childProviderThread, + pendingBackgroundTasks: [ + { taskId: "build", kind: "command" as const, awaited: true as const }, + ], + }, + }, + { + id: EventId.make("event:followup-earlier-work-answer"), + type: "message.updated", + threadId: child.childThreadId, + runId: followupRun.id, + occurredAt: now, + payload: { + id: MessageId.make("message:followup-earlier-work-answer"), + threadId: child.childThreadId, + runId: followupRun.id, + nodeId: null, + role: "assistant", + text: "PONG", + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "server", + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("event:followup-earlier-work-run-completed"), + type: "run.updated", + threadId: child.childThreadId, + runId: followupRun.id, + occurredAt: now, + payload: { ...followupRun, status: "completed", startedAt: now, completedAt: now }, + }, + ], + }); + yield* orchestrator.recoverDelegatedTasks; + + const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( + (task) => task.id === followupTaskId, + ); + assert.equal(followup?.status, "completed"); + assert.equal(followup?.result, "PONG"); + }), + ); + + it.effect("settles a follow-up when its own work ends while another's still runs", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const parentThreadId = ThreadId.make("thread:followup-own-work-parent"); + const projectId = ProjectId.make("project:followup-own-work"); + const parentRunId = RunId.make("run:followup-own-work-parent"); + const rootNodeId = NodeId.make("node:followup-own-work-parent"); + yield* seedParentWithTerminalTask({ + threadId: parentThreadId, + projectId, + runId: parentRunId, + rootNodeId, + taskId: NodeId.make("node:followup-own-work-seed"), + deliveryState: "acknowledged", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId, + projectId, + parentRunId, + rootNodeId, + name: "followup-own-work-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + yield* orchestrator.recoverDelegatedTasks; + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + const childProviderThread = { + ...parent.providerThreads[0]!, + id: ProviderThreadId.make("provider-thread:followup-own-work-child"), + appThreadId: child.childThreadId, + }; + const awaited = (taskId: string) => ({ + taskId, + kind: "command" as const, + awaited: true as const, + }); + const roster = (name: string, runId: RunId, tasks: ReadonlyArray) => ({ + id: EventId.make(`event:followup-own-work-roster-${name}`), + type: "provider-thread.updated" as const, + threadId: child.childThreadId, + runId, + driver, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { ...childProviderThread, pendingBackgroundTasks: tasks.map(awaited) }, + }); + const settledTask = (taskId: NodeId, afterSequence: number) => + eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( + Stream.filter( + (stored) => + stored.event.type === "subagent.updated" && + stored.event.payload.id === taskId && + stored.event.payload.status === "completed", + ), + Stream.take(1), + Stream.runDrain, + ); + const send = (name: string) => + Effect.gen(function* () { + const messageId = MessageId.make(`message:followup-own-work-send-${name}`); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`command:followup-own-work-send-${name}`), + threadId: child.childThreadId, + senderThreadId: parentThreadId, + messageId, + text: `Start build ${name} in the background and wait on it.`, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + }); + return (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( + (run) => run.userMessageId === messageId, + )!; + }); + + // Follow-up A starts awaited build a, answers and ends its turn. + const runA = yield* send("a"); + yield* eventSink.write({ + commandId: reconcileCommandId("followup-own-work-a-ended"), + events: [ + roster("a", runA.id, ["a"]), + { + id: EventId.make("event:followup-own-work-a-answer"), + type: "message.updated", + threadId: child.childThreadId, + runId: runA.id, + occurredAt: now, + payload: { + id: MessageId.make("message:followup-own-work-a-answer"), + threadId: child.childThreadId, + runId: runA.id, + nodeId: null, + role: "assistant", + text: "A-DONE", + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "server", + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("event:followup-own-work-a-completed"), + type: "run.updated", + threadId: child.childThreadId, + runId: runA.id, + occurredAt: now, + payload: { ...runA, status: "completed", startedAt: now, completedAt: now }, + }, + ], + }); + yield* orchestrator.recoverDelegatedTasks; + + // Follow-up B starts awaited build b and is still running. + const runB = yield* send("b"); + yield* eventSink.write({ + commandId: reconcileCommandId("followup-own-work-b-started"), + events: [ + { + id: EventId.make("event:followup-own-work-b-running"), + type: "run.updated", + threadId: child.childThreadId, + runId: runB.id, + occurredAt: now, + payload: { ...runB, status: "running", startedAt: now }, + }, + roster("ab", runB.id, ["a", "b"]), + ], + }); + // Run updates react in order, so the barrier settling means B's start was handled too. + const barrier = yield* seedRestartCancelledChild({ + parentThreadId, + projectId, + parentRunId, + rootNodeId, + name: "followup-own-work-barrier", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + const beforeBarrier = yield* eventSink.latestSequence(); + yield* eventSink.write({ + events: [ + { + ...runEvent({ + threadId: barrier.childThreadId, + runId: barrier.childRunId, + ordinal: 1, + status: "completed", + now, + }), + id: EventId.make("event:followup-own-work-barrier-ended"), + }, + ], + }); + yield* settledTask(barrier.taskId, beforeBarrier); + assert.equal( + (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( + (task) => task.id === runA.delegatedTaskId, + )?.status, + "running", + ); + + // Build a ends; the roster still holds b, and no run update follows. + const beforeSettle = yield* eventSink.latestSequence(); + yield* eventSink.write({ + commandId: reconcileCommandId("followup-own-work-a-settled"), + events: [roster("b", runB.id, ["b"])], + }); + yield* settledTask(runA.delegatedTaskId!, beforeSettle); + const tasks = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents; + assert.equal(tasks.find((task) => task.id === runA.delegatedTaskId)?.result, "A-DONE"); + assert.equal(tasks.find((task) => task.id === runB.delegatedTaskId)?.status, "running"); + }), + ); + it.effect("waits for a run the follow-up's child queued on its own thread", () => Effect.gen(function* () { const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index f047fe771f1d..27b7a2e12878 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -15872,18 +15872,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // A child whose run ends while its background tasks or its own subagents still run has no // result yet. When that work settles there is no run update to finalize it, so these events - // do it. A background roster counts once it stops holding after it was seen holding. - const providerThreadsWithBackgroundWork = new Set(); + // do it. A roster counts when a task seen holding leaves it, since that may be one follow-up's. + const holdingTaskIdsByProviderThread = new Map>(); const settlesChildWork = (stored: OrchestrationV2StoredEvent) => { const { event } = stored; if (event.type === "subagent.updated") return !isOrchestrationV2WorkActive(event.payload.status); if (event.type !== "provider-thread.updated") return false; - if (backgroundWorkHoldsCompletion(event.payload.pendingBackgroundTasks ?? [])) { - providerThreadsWithBackgroundWork.add(event.payload.id); - return false; - } - return providerThreadsWithBackgroundWork.delete(event.payload.id); + const holding = new Set( + (event.payload.pendingBackgroundTasks ?? []) + .filter((task) => backgroundWorkHoldsCompletion([task])) + .map((task) => task.taskId), + ); + const previous = holdingTaskIdsByProviderThread.get(event.payload.id); + if (holding.size > 0) holdingTaskIdsByProviderThread.set(event.payload.id, holding); + else holdingTaskIdsByProviderThread.delete(event.payload.id); + return previous !== undefined && [...previous].some((taskId) => !holding.has(taskId)); }; yield* Stream.merge( eventSink.stream({ diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 25a86d351548..289bdce2276c 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -848,6 +848,26 @@ function upsertById(items: ReadonlyArray, return updated; } +/** Adapters resend whole rosters without runs, so a task keeps the run it first appeared under. */ +function attributePendingBackgroundTasks( + next: OrchestrationV2ProviderThread, + previous: Pick | undefined, + runId: OrchestrationV2DomainEvent["runId"], +): OrchestrationV2ProviderThread { + const tasks = next.pendingBackgroundTasks ?? []; + if (tasks.every((task) => task.runId !== undefined)) return next; + const known = new Map( + (previous?.pendingBackgroundTasks ?? []).map((task) => [task.taskId, task.runId]), + ); + return { + ...next, + pendingBackgroundTasks: tasks.map((task) => { + const owner = task.runId ?? (known.has(task.taskId) ? known.get(task.taskId) : runId); + return owner === undefined ? task : { ...task, runId: owner }; + }), + }; +} + function upsertProviderTurn( turns: ReadonlyArray, next: OrchestrationV2ProviderTurn, @@ -1044,7 +1064,14 @@ export function applyToProjection( activeProviderThreadId: event.payload.id, } : base.thread, - providerThreads: upsertById(base.providerThreads, event.payload), + providerThreads: upsertById( + base.providerThreads, + attributePendingBackgroundTasks( + event.payload, + base.providerThreads.find((thread) => thread.id === event.payload.id), + event.runId, + ), + ), }; case "provider-turn.updated": return { @@ -2691,7 +2718,26 @@ export const layer: Layer.Layer = break; } case "provider-thread.updated": { - const payloadJson = yield* encodeProviderThreadPayload(event.payload); + const existingRows = (event.payload.pendingBackgroundTasks ?? []).some( + (task) => task.runId === undefined, + ) + ? yield* sql` + SELECT payload_json + FROM orchestration_v2_projection_provider_threads + WHERE provider_thread_id = ${event.payload.id} + LIMIT 1 + ` + : []; + const existing = existingRows[0]; + const payloadJson = yield* encodeProviderThreadPayload( + attributePendingBackgroundTasks( + event.payload, + existing === undefined + ? undefined + : yield* decodeProviderThreadPayload(existing.payload_json), + event.runId, + ), + ); const payload = parseEncodedPayload(payloadJson); yield* sql` INSERT INTO orchestration_v2_projection_provider_threads ( diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index 23a95bc2bd20..9dae3e539c09 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -227,8 +227,8 @@ export function delegatedTaskRuns( } /** - * A follow-up task's runs still owe a subagent they started its result, or the child still - * awaits background work, as the original task waits on both. + * A follow-up task's runs still owe a subagent they started its result, or still await + * background work they started, as the original task waits on both. */ export function delegatedTaskAwaitsWork( runs: ReadonlyArray, @@ -245,7 +245,11 @@ export function delegatedTaskAwaitsWork( return ( child.subagents.some((task) => runIds.has(task.runId) && subagentHoldsCompletion(task)) || child.providerThreads.some((thread) => - backgroundWorkHoldsCompletion(thread.pendingBackgroundTasks ?? []), + backgroundWorkHoldsCompletion( + (thread.pendingBackgroundTasks ?? []).filter( + (task) => task.runId !== undefined && runIds.has(task.runId), + ), + ), ) ); } diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_monitor_wake/output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_monitor_wake/output.ts index 6c11f179dd4e..9de3d50a316f 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_monitor_wake/output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_monitor_wake/output.ts @@ -39,7 +39,9 @@ export function assertClaudeBackgroundMonitorWakeOutput( .filter((task) => task.taskId === MONITOR_TASK_ID); // The first roster precedes task_started linking the Monitor call, so that entry reads command. assert.isAbove(rosterTasks.length, 1); - for (const task of rosterTasks.slice(1)) { + for (const { runId, ...task } of rosterTasks.slice(1)) { + // Adapter rosters carry no run; server rewrites of the stored thread carry the projection's stamp. + if (runId !== undefined) assert.equal(runId, monitorItem?.runId); assert.deepEqual(task, { taskId: MONITOR_TASK_ID, description: "Background monitor test", diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 38d2e98ef72c..11987980cedf 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1060,6 +1060,8 @@ const PendingBackgroundTaskFields = { parentTaskId: Schema.optional(TrimmedNonEmptyString), /** Native id of the turn item that started this work, when it is not `taskId`. */ nativeItemId: Schema.optional(TrimmedNonEmptyString), + /** The run that was executing when the work first appeared, stamped by the projection. */ + runId: Schema.optional(RunId), /** * The agent declared it resumes when this work ends, so the thread holds like it does for a * subagent. Only the agent knows whether a command is a wait or a dev server it left running.