From c9961feffde048e7424a86294cc6f3000f223850 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 16 Nov 2026 21:38:51 +0000 Subject: [PATCH 1/4] test(server): a follow-up held by the child's earlier background work (#37698) Fails on main: a thread_send follow-up whose own run answers and starts no background work stays running while awaited work from an earlier run is pending. Co-Authored-By: Claude Opus 5.5 --- .../DelegatedCompletionDelivery.test.ts | 121 ++++++++++++++++++ 1 file changed, 121 insertions(+) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index e6e29a1b34a76..4433120278da6 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -3432,6 +3432,127 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( }), ); + it.effect("settles a follow-up whose own run started no background work", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const parentThreadId = ThreadId.make("thread:followup-earlier-work-parent"); + const projectId = ProjectId.make("project:followup-earlier-work"); + const parentRunId = RunId.make("run:followup-earlier-work-parent"); + const rootNodeId = NodeId.make("node:followup-earlier-work-parent"); + yield* seedParentWithTerminalTask({ + threadId: parentThreadId, + projectId, + runId: parentRunId, + rootNodeId, + taskId: NodeId.make("node:followup-earlier-work-seed"), + deliveryState: "acknowledged", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId, + projectId, + parentRunId, + rootNodeId, + name: "followup-earlier-work-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + yield* orchestrator.recoverDelegatedTasks; + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + const childProviderThread = { + ...parent.providerThreads[0]!, + id: ProviderThreadId.make("provider-thread:followup-earlier-work-child"), + appThreadId: child.childThreadId, + }; + + // An earlier run of the child left an awaited build running. + yield* eventSink.write({ + commandId: reconcileCommandId("followup-earlier-work-build"), + events: [ + { + id: EventId.make("event:followup-earlier-work-build"), + type: "provider-thread.updated", + threadId: child.childThreadId, + driver, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + ...childProviderThread, + pendingBackgroundTasks: [ + { taskId: "build", kind: "command" as const, awaited: true as const }, + ], + }, + }, + ], + }); + + const followupMessageId = MessageId.make("message:followup-earlier-work-send"); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make("command:followup-earlier-work-send"), + threadId: child.childThreadId, + senderThreadId: parentThreadId, + messageId: followupMessageId, + text: "Reply with exactly PONG.", + attachments: [], + dispatchMode: { type: "queue_after_active" }, + }); + const followupRun = (yield* orchestrator.getThreadProjection( + child.childThreadId, + )).runs.find((run) => run.userMessageId === followupMessageId)!; + const followupTaskId = followupRun.delegatedTaskId!; + + // The follow-up's run answers and ends without starting any background work. + yield* eventSink.write({ + commandId: reconcileCommandId("followup-earlier-work-answered"), + events: [ + { + id: EventId.make("event:followup-earlier-work-answer"), + type: "message.updated", + threadId: child.childThreadId, + runId: followupRun.id, + occurredAt: now, + payload: { + id: MessageId.make("message:followup-earlier-work-answer"), + threadId: child.childThreadId, + runId: followupRun.id, + nodeId: null, + role: "assistant", + text: "PONG", + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "server", + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("event:followup-earlier-work-run-completed"), + type: "run.updated", + threadId: child.childThreadId, + runId: followupRun.id, + occurredAt: now, + payload: { ...followupRun, status: "completed", startedAt: now, completedAt: now }, + }, + ], + }); + yield* orchestrator.recoverDelegatedTasks; + + const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( + (task) => task.id === followupTaskId, + ); + assert.equal(followup?.status, "completed"); + assert.equal(followup?.result, "PONG"); + }), + ); + it.effect("waits for a run the follow-up's child queued on its own thread", () => Effect.gen(function* () { const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; From 5e8eb7686b5938e51547b666a7994dceb8660bb8 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 16 Nov 2026 21:52:57 +0000 Subject: [PATCH 2/4] fix(server): a thread_send follow-up waits only on background work its own runs started A follow-up task held on any awaited background work on the delegated child, because roster entries did not record the run that started them. A follow-up the child answered without starting work stayed open until an earlier run's work ended, then completed with another follow-up's result. The projection now stamps each pending background task with the run whose event first listed it and keeps that run when the provider resends its roster. delegatedTaskAwaitsWork holds only on awaited work stamped with one of the follow-up's runs. Fixes pingdotgg/t3code#37698 Co-Authored-By: Claude Opus 5.5 --- .../DelegatedCompletionDelivery.test.ts | 18 +++++++ .../src/orchestration-v2/ProjectionStore.ts | 50 ++++++++++++++++++- .../orchestration-v2/RestartBackgroundNote.ts | 10 ++-- packages/contracts/src/orchestrationV2.ts | 2 + 4 files changed, 75 insertions(+), 5 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 4433120278da6..935b5419151f3 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -3252,6 +3252,7 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( id: EventId.make(`event:followup-command-awaited-${ending}`), type: "provider-thread.updated", threadId: child.childThreadId, + runId: followupRun.id, driver, providerInstanceId: modelSelection.instanceId, occurredAt: now, @@ -3477,6 +3478,7 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( id: EventId.make("event:followup-earlier-work-build"), type: "provider-thread.updated", threadId: child.childThreadId, + runId: child.childRunId, driver, providerInstanceId: modelSelection.instanceId, occurredAt: now, @@ -3512,6 +3514,22 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( yield* eventSink.write({ commandId: reconcileCommandId("followup-earlier-work-answered"), events: [ + // The provider resends its whole roster during the follow-up's run. + { + id: EventId.make("event:followup-earlier-work-roster"), + type: "provider-thread.updated", + threadId: child.childThreadId, + runId: followupRun.id, + driver, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { + ...childProviderThread, + pendingBackgroundTasks: [ + { taskId: "build", kind: "command" as const, awaited: true as const }, + ], + }, + }, { id: EventId.make("event:followup-earlier-work-answer"), type: "message.updated", diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 25a86d351548a..289bdce2276c0 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -848,6 +848,26 @@ function upsertById(items: ReadonlyArray, return updated; } +/** Adapters resend whole rosters without runs, so a task keeps the run it first appeared under. */ +function attributePendingBackgroundTasks( + next: OrchestrationV2ProviderThread, + previous: Pick | undefined, + runId: OrchestrationV2DomainEvent["runId"], +): OrchestrationV2ProviderThread { + const tasks = next.pendingBackgroundTasks ?? []; + if (tasks.every((task) => task.runId !== undefined)) return next; + const known = new Map( + (previous?.pendingBackgroundTasks ?? []).map((task) => [task.taskId, task.runId]), + ); + return { + ...next, + pendingBackgroundTasks: tasks.map((task) => { + const owner = task.runId ?? (known.has(task.taskId) ? known.get(task.taskId) : runId); + return owner === undefined ? task : { ...task, runId: owner }; + }), + }; +} + function upsertProviderTurn( turns: ReadonlyArray, next: OrchestrationV2ProviderTurn, @@ -1044,7 +1064,14 @@ export function applyToProjection( activeProviderThreadId: event.payload.id, } : base.thread, - providerThreads: upsertById(base.providerThreads, event.payload), + providerThreads: upsertById( + base.providerThreads, + attributePendingBackgroundTasks( + event.payload, + base.providerThreads.find((thread) => thread.id === event.payload.id), + event.runId, + ), + ), }; case "provider-turn.updated": return { @@ -2691,7 +2718,26 @@ export const layer: Layer.Layer = break; } case "provider-thread.updated": { - const payloadJson = yield* encodeProviderThreadPayload(event.payload); + const existingRows = (event.payload.pendingBackgroundTasks ?? []).some( + (task) => task.runId === undefined, + ) + ? yield* sql` + SELECT payload_json + FROM orchestration_v2_projection_provider_threads + WHERE provider_thread_id = ${event.payload.id} + LIMIT 1 + ` + : []; + const existing = existingRows[0]; + const payloadJson = yield* encodeProviderThreadPayload( + attributePendingBackgroundTasks( + event.payload, + existing === undefined + ? undefined + : yield* decodeProviderThreadPayload(existing.payload_json), + event.runId, + ), + ); const payload = parseEncodedPayload(payloadJson); yield* sql` INSERT INTO orchestration_v2_projection_provider_threads ( diff --git a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts index 23a95bc2bd20f..9dae3e539c09e 100644 --- a/apps/server/src/orchestration-v2/RestartBackgroundNote.ts +++ b/apps/server/src/orchestration-v2/RestartBackgroundNote.ts @@ -227,8 +227,8 @@ export function delegatedTaskRuns( } /** - * A follow-up task's runs still owe a subagent they started its result, or the child still - * awaits background work, as the original task waits on both. + * A follow-up task's runs still owe a subagent they started its result, or still await + * background work they started, as the original task waits on both. */ export function delegatedTaskAwaitsWork( runs: ReadonlyArray, @@ -245,7 +245,11 @@ export function delegatedTaskAwaitsWork( return ( child.subagents.some((task) => runIds.has(task.runId) && subagentHoldsCompletion(task)) || child.providerThreads.some((thread) => - backgroundWorkHoldsCompletion(thread.pendingBackgroundTasks ?? []), + backgroundWorkHoldsCompletion( + (thread.pendingBackgroundTasks ?? []).filter( + (task) => task.runId !== undefined && runIds.has(task.runId), + ), + ), ) ); } diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 38d2e98ef72c2..11987980cedfa 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1060,6 +1060,8 @@ const PendingBackgroundTaskFields = { parentTaskId: Schema.optional(TrimmedNonEmptyString), /** Native id of the turn item that started this work, when it is not `taskId`. */ nativeItemId: Schema.optional(TrimmedNonEmptyString), + /** The run that was executing when the work first appeared, stamped by the projection. */ + runId: Schema.optional(RunId), /** * The agent declared it resumes when this work ends, so the thread holds like it does for a * subagent. Only the agent knows whether a command is a wait or a dev server it left running. From 2b5223d0331df46fb20bdef816febc162ae49a67 Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 16 Nov 2026 22:14:32 +0000 Subject: [PATCH 3/4] fix(server): settle a follow-up when its own awaited work leaves the roster The roster reaction finalized a delegated child only once the whole roster stopped holding, so follow-up A stayed open after its build ended while follow-up B's build still ran. It now reacts whenever a task that was holding leaves the roster, and the per-follow-up check decides which tasks settle. Co-Authored-By: Claude Opus 5.5 --- .../DelegatedCompletionDelivery.test.ts | 185 ++++++++++++++++++ .../src/orchestration-v2/Orchestrator.ts | 18 +- 2 files changed, 196 insertions(+), 7 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index 935b5419151f3..72ea19a442dba 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -3571,6 +3571,191 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))( }), ); + it.effect("settles a follow-up when its own work ends while another's still runs", () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const eventSink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const parentThreadId = ThreadId.make("thread:followup-own-work-parent"); + const projectId = ProjectId.make("project:followup-own-work"); + const parentRunId = RunId.make("run:followup-own-work-parent"); + const rootNodeId = NodeId.make("node:followup-own-work-parent"); + yield* seedParentWithTerminalTask({ + threadId: parentThreadId, + projectId, + runId: parentRunId, + rootNodeId, + taskId: NodeId.make("node:followup-own-work-seed"), + deliveryState: "acknowledged", + now, + }); + const child = yield* seedRestartCancelledChild({ + parentThreadId, + projectId, + parentRunId, + rootNodeId, + name: "followup-own-work-child", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + yield* orchestrator.recoverDelegatedTasks; + const parent = yield* orchestrator.getThreadProjection(parentThreadId); + const childProviderThread = { + ...parent.providerThreads[0]!, + id: ProviderThreadId.make("provider-thread:followup-own-work-child"), + appThreadId: child.childThreadId, + }; + const awaited = (taskId: string) => ({ + taskId, + kind: "command" as const, + awaited: true as const, + }); + const roster = (name: string, runId: RunId, tasks: ReadonlyArray) => ({ + id: EventId.make(`event:followup-own-work-roster-${name}`), + type: "provider-thread.updated" as const, + threadId: child.childThreadId, + runId, + driver, + providerInstanceId: modelSelection.instanceId, + occurredAt: now, + payload: { ...childProviderThread, pendingBackgroundTasks: tasks.map(awaited) }, + }); + const settledTask = (taskId: NodeId, afterSequence: number) => + eventSink.stream({ afterSequence, eventType: "subagent.updated" }).pipe( + Stream.filter( + (stored) => + stored.event.type === "subagent.updated" && + stored.event.payload.id === taskId && + stored.event.payload.status === "completed", + ), + Stream.take(1), + Stream.runDrain, + ); + const send = (name: string) => + Effect.gen(function* () { + const messageId = MessageId.make(`message:followup-own-work-send-${name}`); + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "mcp", + commandId: CommandId.make(`command:followup-own-work-send-${name}`), + threadId: child.childThreadId, + senderThreadId: parentThreadId, + messageId, + text: `Start build ${name} in the background and wait on it.`, + attachments: [], + dispatchMode: { type: "queue_after_active" }, + }); + return (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find( + (run) => run.userMessageId === messageId, + )!; + }); + + // Follow-up A starts awaited build a, answers and ends its turn. + const runA = yield* send("a"); + yield* eventSink.write({ + commandId: reconcileCommandId("followup-own-work-a-ended"), + events: [ + roster("a", runA.id, ["a"]), + { + id: EventId.make("event:followup-own-work-a-answer"), + type: "message.updated", + threadId: child.childThreadId, + runId: runA.id, + occurredAt: now, + payload: { + id: MessageId.make("message:followup-own-work-a-answer"), + threadId: child.childThreadId, + runId: runA.id, + nodeId: null, + role: "assistant", + text: "A-DONE", + attachments: [], + streaming: false, + createdBy: "agent", + creationSource: "server", + createdAt: now, + updatedAt: now, + }, + }, + { + id: EventId.make("event:followup-own-work-a-completed"), + type: "run.updated", + threadId: child.childThreadId, + runId: runA.id, + occurredAt: now, + payload: { ...runA, status: "completed", startedAt: now, completedAt: now }, + }, + ], + }); + yield* orchestrator.recoverDelegatedTasks; + + // Follow-up B starts awaited build b and is still running. + const runB = yield* send("b"); + yield* eventSink.write({ + commandId: reconcileCommandId("followup-own-work-b-started"), + events: [ + { + id: EventId.make("event:followup-own-work-b-running"), + type: "run.updated", + threadId: child.childThreadId, + runId: runB.id, + occurredAt: now, + payload: { ...runB, status: "running", startedAt: now }, + }, + roster("ab", runB.id, ["a", "b"]), + ], + }); + // Run updates react in order, so the barrier settling means B's start was handled too. + const barrier = yield* seedRestartCancelledChild({ + parentThreadId, + projectId, + parentRunId, + rootNodeId, + name: "followup-own-work-barrier", + completionWake: "always", + continuationPending: false, + runStatus: "completed", + now, + }); + const beforeBarrier = yield* eventSink.latestSequence(); + yield* eventSink.write({ + events: [ + { + ...runEvent({ + threadId: barrier.childThreadId, + runId: barrier.childRunId, + ordinal: 1, + status: "completed", + now, + }), + id: EventId.make("event:followup-own-work-barrier-ended"), + }, + ], + }); + yield* settledTask(barrier.taskId, beforeBarrier); + assert.equal( + (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find( + (task) => task.id === runA.delegatedTaskId, + )?.status, + "running", + ); + + // Build a ends; the roster still holds b, and no run update follows. + const beforeSettle = yield* eventSink.latestSequence(); + yield* eventSink.write({ + commandId: reconcileCommandId("followup-own-work-a-settled"), + events: [roster("b", runB.id, ["b"])], + }); + yield* settledTask(runA.delegatedTaskId!, beforeSettle); + const tasks = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents; + assert.equal(tasks.find((task) => task.id === runA.delegatedTaskId)?.result, "A-DONE"); + assert.equal(tasks.find((task) => task.id === runB.delegatedTaskId)?.status, "running"); + }), + ); + it.effect("waits for a run the follow-up's child queued on its own thread", () => Effect.gen(function* () { const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index f047fe771f1d2..27b7a2e12878d 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -15872,18 +15872,22 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio // A child whose run ends while its background tasks or its own subagents still run has no // result yet. When that work settles there is no run update to finalize it, so these events - // do it. A background roster counts once it stops holding after it was seen holding. - const providerThreadsWithBackgroundWork = new Set(); + // do it. A roster counts when a task seen holding leaves it, since that may be one follow-up's. + const holdingTaskIdsByProviderThread = new Map>(); const settlesChildWork = (stored: OrchestrationV2StoredEvent) => { const { event } = stored; if (event.type === "subagent.updated") return !isOrchestrationV2WorkActive(event.payload.status); if (event.type !== "provider-thread.updated") return false; - if (backgroundWorkHoldsCompletion(event.payload.pendingBackgroundTasks ?? [])) { - providerThreadsWithBackgroundWork.add(event.payload.id); - return false; - } - return providerThreadsWithBackgroundWork.delete(event.payload.id); + const holding = new Set( + (event.payload.pendingBackgroundTasks ?? []) + .filter((task) => backgroundWorkHoldsCompletion([task])) + .map((task) => task.taskId), + ); + const previous = holdingTaskIdsByProviderThread.get(event.payload.id); + if (holding.size > 0) holdingTaskIdsByProviderThread.set(event.payload.id, holding); + else holdingTaskIdsByProviderThread.delete(event.payload.id); + return previous !== undefined && [...previous].some((taskId) => !holding.has(taskId)); }; yield* Stream.merge( eventSink.stream({ From 57e5be2bc02f97867774a3d77b65d916c1055f9e Mon Sep 17 00:00:00 2001 From: Julius Marminge Date: Mon, 16 Nov 2026 22:36:51 +0000 Subject: [PATCH 4/4] test(server): accept the run stamp on Claude monitor roster entries Server rewrites of a stored provider thread now carry the run the projection stamped on each pending task, while adapter rosters carry none. The monitor fixture compares entries without the stamp and checks any stamp names the run that launched the monitor. Co-Authored-By: Claude Opus 5.5 --- .../testkit/fixtures/claude_background_monitor_wake/output.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_monitor_wake/output.ts b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_monitor_wake/output.ts index 6c11f179dd4e0..9de3d50a316f1 100644 --- a/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_monitor_wake/output.ts +++ b/apps/server/src/orchestration-v2/testkit/fixtures/claude_background_monitor_wake/output.ts @@ -39,7 +39,9 @@ export function assertClaudeBackgroundMonitorWakeOutput( .filter((task) => task.taskId === MONITOR_TASK_ID); // The first roster precedes task_started linking the Monitor call, so that entry reads command. assert.isAbove(rosterTasks.length, 1); - for (const task of rosterTasks.slice(1)) { + for (const { runId, ...task } of rosterTasks.slice(1)) { + // Adapter rosters carry no run; server rewrites of the stored thread carry the projection's stamp. + if (runId !== undefined) assert.equal(runId, monitorItem?.runId); assert.deepEqual(task, { taskId: MONITOR_TASK_ID, description: "Background monitor test",