From 32e45fcd6b1e1dc020c232ae62930adb21d83861 Mon Sep 17 00:00:00 2001 From: vedmalex Date: Tue, 3 Nov 2026 07:17:11 +0300 Subject: [PATCH 1/8] fix(server): enable Antigravity subagent continuation and idle paragraph streaming --- .../Adapters/AntigravityAdapterV2.ts | 2 + .../src/orchestration-v2/Orchestrator.ts | 41 +- .../RunExecutionService.test.ts | 541 +++++++++++++++++- .../orchestration-v2/RunExecutionService.ts | 219 +++++-- .../assistantStreaming.test.ts | 58 +- .../orchestration-v2/assistantStreaming.ts | 172 ++++-- 6 files changed, 919 insertions(+), 114 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts index 4d2f7bac175e8..a54d911a81cb5 100644 --- a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts @@ -276,6 +276,8 @@ export function makeAntigravityAcpAdapterFlavor( // 1.4.5 returns the root trajectory's Gemini usageMetadata delta; 1.4.1 returns none. reportsPerTurnPromptUsage: true, geminiPromptUsage: true, + deferFinalizeForBackgroundWork: true, + enablePostSettleContinuation: true, ...(profileDirectory === undefined ? {} : { diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 600764704e57f..bf42f52f85e64 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -2624,6 +2624,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly disposition: "stopped" | "disposed"; readonly now: DateTime.Utc; readonly cancelQueuedDelivery?: boolean; + // An interrupted wake disposes only the results it carried. The cohort stays open, and its + // delivery stays the wake's, so the wake's terminal reserves any sibling's result. + readonly interruptedWakeTaskIds?: ReadonlyArray; }) { const parentRun = input.projection.runs.find((candidate) => candidate.id === input.parentRunId); if (parentRun === undefined) { @@ -2637,12 +2640,28 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return; } const emitEvent = emit(input.events, input.command); - const nextDisposition = cohort?.disposition === "disposed" ? "disposed" : input.disposition; - const nextCohort = { - disposition: nextDisposition, - nextGeneration: cohort?.nextGeneration ?? 1, - delivery: null, - } as const; + const wakeTaskIds = input.interruptedWakeTaskIds; + const keptTaskIds = new Set( + wakeTaskIds !== undefined && cohort?.disposition === "open" + ? tasks + .filter( + (task) => + !wakeTaskIds.includes(task.id) && + task.completionDelivery?.state !== "acknowledged" && + task.completionDelivery?.state !== "delivered" && + task.completionDelivery?.state !== "disposed", + ) + .map((task) => task.id) + : [], + ); + const nextCohort = + keptTaskIds.size > 0 + ? cohort! + : ({ + disposition: cohort?.disposition === "disposed" ? "disposed" : input.disposition, + nextGeneration: cohort?.nextGeneration ?? 1, + delivery: null, + } as const); yield* emitEvent({ type: "run.updated", threadId: parentRun.threadId, @@ -2657,6 +2676,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); for (const task of tasks) { if ( + keptTaskIds.has(task.id) || task.completionDelivery?.state === "acknowledged" || task.completionDelivery?.state === "delivered" || task.completionDelivery?.state === "disposed" @@ -2682,7 +2702,11 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); } const deliveryRun = completionDeliveryRun(input.projection, cohort?.delivery ?? null); - if (input.cancelQueuedDelivery !== false && deliveryRun?.status === "queued") { + if ( + keptTaskIds.size === 0 && + input.cancelQueuedDelivery !== false && + deliveryRun?.status === "queued" + ) { yield* emitQueuedRunCancellation({ command: input.command, events: input.events, @@ -11324,6 +11348,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio parentRunId, disposition: "stopped", now, + ...(parentRunId !== run.id && completionMessage?.delegatedCompletion !== undefined + ? { interruptedWakeTaskIds: completionMessage.delegatedCompletion.taskIds } + : {}), }); } }); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index f3f1542146c66..87adb7005d64f 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -29,13 +29,16 @@ import { } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Exit from "effect/Exit"; +import * as Fiber from "effect/Fiber"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; +import * as TestClock from "effect/testing/TestClock"; import { CodexAppServerRequestError } from "effect-codex-app-server/errors"; import { SqlError, classifySqliteError } from "effect/sql/SqlError"; @@ -3681,8 +3684,13 @@ function rootTerminalEvent( function runBackgroundItemScenario( key: string, - makeEvents: (ids: BackgroundScenarioIds) => ReadonlyArray, + makeEvents: ( + ids: BackgroundScenarioIds, + ) => + | ReadonlyArray + | Stream.Stream, options?: { + readonly onIngest?: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect; readonly keepEventStreamOpen?: boolean; readonly hasPendingBackgroundWorkForThread?: ( providerThread: OrchestrationV2ProviderThread, @@ -3698,6 +3706,9 @@ function runBackgroundItemScenario( readonly onFinalized?: ( events: ReadonlyArray, ) => Effect.Effect; + readonly onInterruptTurn?: (input: { + readonly requestRuntimeRestart?: boolean; + }) => Effect.Effect; }, ) { return Effect.gen(function* () { @@ -3736,6 +3747,7 @@ function runBackgroundItemScenario( ingestNormalized: (input) => Effect.gen(function* () { const event = input.event; + yield* options?.onIngest?.(event) ?? Effect.void; if (event.type === "turn_item.updated") { yield* Ref.update(observed, (current) => [ ...current, @@ -3769,7 +3781,8 @@ function runBackgroundItemScenario( : { hasPendingBackgroundWorkForThread: options.hasPendingBackgroundWorkForThread }), subscribeEvents: Effect.gen(function* () { yield* options?.onSubscribe ?? Effect.void; - const events = Stream.fromIterable(makeEvents(ids)).pipe( + const source = makeEvents(ids); + const events = (Stream.isStream(source) ? source : Stream.fromIterable(source)).pipe( Stream.concat( Stream.fromEffect( Effect.andThen( @@ -3788,6 +3801,8 @@ function runBackgroundItemScenario( }; }), startTurn: () => Effect.void, + interruptTurn: (interrupt: { readonly requestRuntimeRestart?: boolean }) => + options?.onInterruptTurn?.(interrupt) ?? Effect.void, } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime, run: { id: ids.runId, @@ -3836,8 +3851,9 @@ function runBackgroundItemScenario( }); }).pipe(Effect.provide(layerTest)); - const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); - assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish"); + // No timeout: tests advance the TestClock, which would fire it early. + yield* Deferred.await(ingestionDone); + return yield* Ref.get(observed); }); } @@ -3889,3 +3905,520 @@ it.effect("keeps a failed root's still-running native children live while the ru ); }), ); + +it.effect("delivers a held paragraph after the provider goes idle with its prompt still open", () => + Effect.gen(function* () { + const key = "idle-paragraph"; + const ids = backgroundScenarioIds(key); + const now = yield* DateTime.now; + const providerEvents = yield* Queue.unbounded(); + const delivered = yield* Ref.make>([]); + const paragraphDelivered = yield* Deferred.make(); + const idleDelivered = yield* Deferred.make(); + const assistantText = (text: string, streaming: boolean) => + [ + { + type: "message.updated", + driver, + message: { + createdBy: "agent", + creationSource: "provider", + id: MessageId.make(`message:${key}`), + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + role: "assistant", + text, + attachments: [], + streaming, + createdAt: now, + updatedAt: now, + }, + }, + { + type: "turn_item.updated", + driver, + turnItem: { + ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }), + id: ids.itemId, + threadId: ids.threadId, + runId: ids.runId, + nodeId: ids.rootNodeId, + providerTurnId: ids.rootProviderTurnId, + text, + streaming, + }, + }, + ] as ReadonlyArray; + + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(providerEvents), { + onIngest: (event) => + event.type === "message.updated" || event.type === "turn_item.updated" + ? Effect.gen(function* () { + const message = event.type === "message.updated" ? event.message : event.turnItem; + const text = "text" in message ? message.text : ""; + const streaming = "streaming" in message && message.streaming; + yield* Ref.update(delivered, (current) => [ + ...current, + `${event.type}:${streaming ? "streaming" : "final"}:${JSON.stringify(text)}`, + ]); + if (event.type === "turn_item.updated") { + yield* Deferred.succeed(paragraphDelivered, undefined); + if (streaming && text === "First.\n\nWaiting on the build") + yield* Deferred.succeed(idleDelivered, undefined); + } + }) + : Effect.void, + }).pipe(Effect.forkChild); + + yield* Queue.offerAll(providerEvents, assistantText("First.\n\nWaiting on the build", true)); + yield* Deferred.await(paragraphDelivered); + // The provider keeps its prompt open while a backgrounded command runs. + yield* TestClock.adjust("1 second"); + yield* Deferred.await(idleDelivered); + yield* Queue.offerAll(providerEvents, [ + ...assistantText("First.\n\nWaiting on the build to", true), + ...assistantText("First.\n\nWaiting on the build to finish.", false), + rootTerminalEvent(ids, "completed"), + ]); + yield* Fiber.join(scenario); + + assert.deepEqual(yield* Ref.get(delivered), [ + 'message.updated:streaming:"First.\\n\\n"', + 'turn_item.updated:streaming:"First.\\n\\n"', + 'message.updated:streaming:"First.\\n\\nWaiting on the build"', + 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"', + 'message.updated:final:"First.\\n\\nWaiting on the build to finish."', + 'turn_item.updated:final:"First.\\n\\nWaiting on the build to finish."', + ]); + }), +); + +const heldChildText = "partial response"; + +function childAssistantSnapshot( + ids: BackgroundScenarioIds, +): ProviderAdapter.ProviderAdapterV2Event { + return { + type: "turn_item.updated", + driver, + turnItem: { + ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }), + text: heldChildText, + }, + } as ProviderAdapter.ProviderAdapterV2Event; +} + +function linkedSubagentEvent( + ids: BackgroundScenarioIds, + status: "running" | "completed", +): ProviderAdapter.ProviderAdapterV2Event { + return { + type: "subagent.updated", + driver, + subagent: { + ...makeRunOwnedSubagentFixture({ + ids, + providerInstanceId: ProviderInstanceId.make("codex"), + childThreadId: ids.childThreadId, + driver, + status: "running", + }), + status, + }, + }; +} + +const isHeldChildSnapshot = (event: ProviderAdapter.ProviderAdapterV2Event) => + event.type === "turn_item.updated" && + "text" in event.turnItem && + event.turnItem.text === heldChildText; + +// Feeds a held child snapshot, then waits until ingestion has read past it. +const holdChildSnapshot = Effect.fn(function* ( + ids: BackgroundScenarioIds, + events: Queue.Enqueue, + subagentIngested: Deferred.Deferred, +) { + yield* Queue.offerAll(events, [ + childThreadCreatedEvent(ids), + childAssistantSnapshot(ids), + linkedSubagentEvent(ids, "running"), + ]); + yield* Deferred.await(subagentIngested); +}); + +it.effect("settles a child message only an idle flush wrote when the root is interrupted", () => + Effect.gen(function* () { + const ids = backgroundScenarioIds("idle-child-interrupt"); + const events = yield* Queue.unbounded(); + const subagentIngested = yield* Deferred.make(); + const flushed = yield* Deferred.make(); + const finalized = yield* Ref.make>([]); + const scenario = yield* runBackgroundItemScenario( + "idle-child-interrupt", + () => Stream.fromQueue(events), + { + onIngest: (event) => + event.type === "subagent.updated" + ? Deferred.succeed(subagentIngested, undefined) + : isHeldChildSnapshot(event) + ? Deferred.succeed(flushed, undefined) + : Effect.void, + onFinalized: (written) => Ref.set(finalized, written), + }, + ).pipe(Effect.forkChild); + + yield* holdChildSnapshot(ids, events, subagentIngested); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(flushed); + yield* Queue.offer(events, rootTerminalEvent(ids, "interrupted")); + yield* Fiber.join(scenario); + + const settled = (yield* Ref.get(finalized)).flatMap((event) => + event.type === "turn-item.updated" && event.payload.id === ids.childItemId + ? [event.payload] + : [], + ); + assert.equal(settled.length, 1); + assert.equal(settled[0]?.status, "interrupted"); + assert.equal("text" in settled[0]! ? settled[0].text : null, heldChildText); + }), +); + +it.effect.each([ + { name: "completed", terminalStatus: "completed" as const }, + { name: "failed with its children continuing", terminalStatus: "failed" as const }, +])("idle-flushes a continuing child's text after the root $name", ({ terminalStatus }) => + Effect.gen(function* () { + const key = `idle-child-after-${terminalStatus}`; + const ids = backgroundScenarioIds(key); + const events = yield* Queue.unbounded(); + const subagentIngested = yield* Deferred.make(); + const rootFinalized = yield* Deferred.make(); + const flushed = yield* Deferred.make(); + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { + onIngest: (event) => + event.type === "subagent.updated" + ? Deferred.succeed(subagentIngested, undefined) + : isHeldChildSnapshot(event) + ? Deferred.succeed(flushed, undefined) + : Effect.void, + onFinalized: () => Deferred.succeed(rootFinalized, undefined), + }).pipe(Effect.forkChild); + + yield* holdChildSnapshot(ids, events, subagentIngested); + const terminal = rootTerminalEvent(ids, terminalStatus); + yield* Queue.offer( + events, + terminalStatus === "failed" + ? ({ ...terminal, childTurnsContinue: true } as ProviderAdapter.ProviderAdapterV2Event) + : terminal, + ); + yield* Deferred.await(rootFinalized); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(flushed); + yield* Queue.offer(events, linkedSubagentEvent(ids, "completed")); + const observed = yield* Fiber.join(scenario); + + assert.deepEqual(observed.slice(-3), [ + "root-finalized", + "turn_item:running", + "subagent:completed", + ]); + }), +); + +it.effect.each([ + { name: "before", afterRoot: false }, + { name: "after", afterRoot: true }, +])("retries a failed idle flush $name the root completes and keeps the run live", ({ afterRoot }) => + Effect.gen(function* () { + const key = `idle-flush-retry-${afterRoot ? "after" : "before"}-root`; + const ids = backgroundScenarioIds(key); + const events = yield* Queue.unbounded(); + const subagentIngested = yield* Deferred.make(); + const rootFinalized = yield* Deferred.make(); + const firstWriteFailed = yield* Deferred.make(); + const failureHandled = yield* Deferred.make(); + const retried = yield* Deferred.make(); + const attempts = yield* Ref.make(0); + const childWrites = yield* Ref.make>([]); + const interrupts = yield* Ref.make(0); + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { + onIngest: (event) => + Effect.gen(function* () { + if (event.type === "subagent.updated") { + yield* Deferred.succeed(subagentIngested, undefined); + if ((yield* Ref.get(attempts)) > 0) yield* Deferred.succeed(failureHandled, undefined); + return; + } + if (event.type !== "turn_item.updated" || event.turnItem.type !== "assistant_message") + return; + if ( + isHeldChildSnapshot(event) && + (yield* Ref.getAndUpdate(attempts, (n) => n + 1)) === 0 + ) { + yield* Deferred.succeed(firstWriteFailed, undefined); + yield* Effect.sleep("2 seconds"); + return yield* Effect.die(new Error("disk I/O error")); + } + const { text, streaming } = event.turnItem; + yield* Ref.update(childWrites, (current) => [ + ...current, + `${streaming ? "streaming" : "final"}:${text}`, + ]); + if (isHeldChildSnapshot(event)) yield* Deferred.succeed(retried, undefined); + }), + onFinalized: () => Deferred.succeed(rootFinalized, undefined), + onInterruptTurn: () => Ref.update(interrupts, (n) => n + 1), + }).pipe(Effect.forkChild); + + yield* holdChildSnapshot(ids, events, subagentIngested); + if (afterRoot) { + yield* Queue.offer(events, rootTerminalEvent(ids, "completed")); + yield* Deferred.await(rootFinalized); + } + yield* TestClock.adjust("1 second"); + yield* Deferred.await(firstWriteFailed); + yield* TestClock.adjust("2 seconds"); + yield* Queue.offer(events, linkedSubagentEvent(ids, "running")); + yield* Deferred.await(failureHandled); + assert.equal(yield* Ref.get(attempts), 1); + assert.deepEqual(yield* Ref.get(childWrites), []); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(retried); + const childText = (text: string, streaming: boolean) => { + const snapshot = childAssistantSnapshot(ids); + return snapshot.type === "turn_item.updated" + ? ({ + ...snapshot, + turnItem: { ...snapshot.turnItem, text, streaming }, + } as ProviderAdapter.ProviderAdapterV2Event) + : snapshot; + }; + yield* Queue.offerAll(events, [ + childText(`${heldChildText} and more`, true), + childText(`${heldChildText} and more.`, false), + ...(afterRoot ? [] : [rootTerminalEvent(ids, "completed")]), + linkedSubagentEvent(ids, "completed"), + ]); + const observed = yield* Fiber.join(scenario); + + assert.deepEqual(yield* Ref.get(childWrites), [ + `streaming:${heldChildText}`, + `final:${heldChildText} and more.`, + ]); + assert.equal(yield* Ref.get(interrupts), 0); + assert.include(observed, "root-finalized"); + assert.equal(observed.at(-1), "subagent:completed"); + }), +); + +it.effect("reschedules idle text when a newer snapshot precedes a failed write's retry", () => + Effect.gen(function* () { + const key = "idle-flush-earlier-deadline"; + const ids = backgroundScenarioIds(key); + const events = yield* Queue.unbounded(); + const subagentIngested = yield* Deferred.make(); + const markers = yield* Queue.unbounded(); + const newerWrite = yield* Deferred.make(); + const attempts = yield* Ref.make(0); + const childWrites = yield* Ref.make>([]); + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { + onIngest: (event) => + Effect.gen(function* () { + if (event.type === "subagent.updated") + yield* Deferred.succeed(subagentIngested, undefined); + if (event.type === "message.updated" && event.message.role === "user") + yield* Queue.offer(markers, undefined); + if (event.type !== "turn_item.updated" || event.turnItem.type !== "assistant_message") + return; + if ((yield* Ref.getAndUpdate(attempts, (n) => n + 1)) < 2) + return yield* Effect.die(new Error("disk I/O error")); + const { text } = event.turnItem; + yield* Ref.update(childWrites, (current) => [...current, text]); + if (text === "New partial response") { + yield* Deferred.succeed(newerWrite, undefined); + } + }), + }).pipe(Effect.forkChild); + const marker = streamedMessageEvent({ + id: `message:${key}:marker`, + threadId: ids.threadId, + runId: ids.runId, + role: "user", + text: "marker", + streaming: false, + }); + + yield* holdChildSnapshot(ids, events, subagentIngested); + yield* TestClock.adjust("1 second"); + yield* Queue.offer(events, marker); + yield* Queue.take(markers); + assert.equal(yield* Ref.get(attempts), 1); + yield* TestClock.adjust("1 second"); + yield* Queue.offer(events, marker); + yield* Queue.take(markers); + assert.equal(yield* Ref.get(attempts), 2); + + const newer = childAssistantSnapshot(ids); + assert(newer.type === "turn_item.updated" && newer.turnItem.type === "assistant_message"); + yield* Queue.offerAll(events, [ + { ...newer, turnItem: { ...newer.turnItem, text: "New partial response" } }, + marker, + ]); + yield* Queue.take(markers); + yield* TestClock.adjust("800 millis"); + yield* Deferred.await(newerWrite); + assert.deepEqual(yield* Ref.get(childWrites), ["New partial response"]); + + yield* Queue.offerAll(events, [ + rootTerminalEvent(ids, "completed"), + linkedSubagentEvent(ids, "completed"), + ]); + yield* Fiber.join(scenario); + }), +); + +function streamedMessageEvent(input: { + readonly id: string; + readonly threadId: ThreadId; + readonly runId: RunId | null; + readonly role: "assistant" | "user"; + readonly text: string; + readonly streaming: boolean; +}): ProviderAdapter.ProviderAdapterV2Event { + const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z"); + return { + type: "message.updated", + driver, + message: { + createdBy: input.role === "user" ? "user" : "agent", + creationSource: "provider", + id: MessageId.make(input.id), + threadId: input.threadId, + runId: input.runId, + nodeId: null, + role: input.role, + text: input.text, + attachments: [], + streaming: input.streaming, + createdAt: now, + updatedAt: now, + }, + } as ProviderAdapter.ProviderAdapterV2Event; +} + +it.effect.each([ + { name: "", failFirst: false }, + { name: " after its first idle write failed", failFirst: true }, +])("drops a child's held text once the child is handed to another run$name", ({ failFirst }) => + Effect.gen(function* () { + const key = `idle-child-handoff${failFirst ? "-after-failure" : ""}`; + const ids = backgroundScenarioIds(key); + const events = yield* Queue.unbounded(); + const subagentIngested = yield* Deferred.make(); + const firstWriteFailed = yield* Deferred.make(); + const handoffRead = yield* Deferred.make(); + const rootFlushed = yield* Deferred.make(); + const staleWrites = yield* Ref.make>([]); + const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { + onIngest: (event) => { + if (event.type === "subagent.updated") return Deferred.succeed(subagentIngested, undefined); + if (event.type !== "message.updated" && event.type !== "turn_item.updated") + return Effect.void; + const message = event.type === "message.updated" ? event.message : event.turnItem; + if (message.threadId === ids.childThreadId) { + return Deferred.succeed(firstWriteFailed, undefined).pipe( + Effect.flatMap((first) => + failFirst && first + ? Effect.die(new Error("disk I/O error")) + : Ref.update(staleWrites, (current) => [...current, event.type]), + ), + ); + } + if (event.type === "message.updated" && event.message.role === "user") + return Deferred.succeed(handoffRead, undefined); + return Deferred.succeed(rootFlushed, undefined); + }, + }).pipe(Effect.forkChild); + + yield* holdChildSnapshot(ids, events, subagentIngested); + if (failFirst) { + // The held snapshot stays held for a retry that must not outlive the handoff. + yield* TestClock.adjust("1 second"); + yield* Deferred.await(firstWriteFailed); + } + const handoff = linkedSubagentEvent(ids, "running"); + yield* Queue.offerAll(events, [ + streamedMessageEvent({ + id: `message:${key}:child`, + threadId: ids.childThreadId, + runId: null, + role: "assistant", + text: heldChildText, + streaming: true, + }), + handoff.type === "subagent.updated" + ? { ...handoff, subagent: { ...handoff.subagent, runId: RunId.make(`run:${key}:next`) } } + : handoff, + // Due in the same flush as the child's text, after it. + streamedMessageEvent({ + id: `message:${key}:root`, + threadId: ids.threadId, + runId: ids.runId, + role: "assistant", + text: "Root partial", + streaming: true, + }), + streamedMessageEvent({ + id: `message:${key}:marker`, + threadId: ids.threadId, + runId: ids.runId, + role: "user", + text: "marker", + streaming: false, + }), + ]); + yield* Deferred.await(handoffRead); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(rootFlushed); + yield* Queue.end(events); + yield* Fiber.join(scenario); + + assert.deepEqual(yield* Ref.get(staleWrites), []); + }), +); + +it.effect("ends ingestion while an idle flush write is still blocked", () => + Effect.gen(function* () { + const ids = backgroundScenarioIds("idle-flush-blocked-close"); + const events = yield* Queue.unbounded(); + const subagentIngested = yield* Deferred.make(); + const writeStarted = yield* Deferred.make(); + const writeInterrupted = yield* Deferred.make(); + const scenario = yield* runBackgroundItemScenario( + "idle-flush-blocked-close", + () => Stream.fromQueue(events), + { + onIngest: (event) => + event.type === "subagent.updated" + ? Deferred.succeed(subagentIngested, undefined) + : isHeldChildSnapshot(event) + ? Deferred.succeed(writeStarted, undefined).pipe( + Effect.andThen(Effect.never), + Effect.onInterrupt(() => Deferred.succeed(writeInterrupted, undefined)), + ) + : Effect.void, + }, + ).pipe(Effect.forkChild); + + yield* holdChildSnapshot(ids, events, subagentIngested); + yield* TestClock.adjust("1 second"); + yield* Deferred.await(writeStarted); + yield* Queue.end(events); + yield* Fiber.join(scenario); + yield* Deferred.await(writeInterrupted); + }), +); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 6a52fac447926..c51d2dfb9d005 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -35,6 +35,7 @@ import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; @@ -1073,6 +1074,10 @@ export const layer: Layer.Layer< ReadonlySet >(new Set(inheritedBackgroundTurnItemsById.keys())); const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection()); + const assistantStreaming = makeAssistantStreamingFilter(responseStreamingMode); + // Set when ingestion ends or finalization settles the run's open items, which a later + // streaming snapshot must not reopen. + let idleFlushClosed = false; const finalizeRootRun = Effect.fn("RunExecutionService.finalizeRootRun")(function* ( terminal: ProviderTerminalEvent, ) { @@ -1115,6 +1120,7 @@ export const layer: Layer.Layer< ); if (isRunOwnedSubagentTerminalStatus(terminal.status) && !childTurnsContinue(terminal)) { yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); + idleFlushClosed = true; } yield* Ref.set(rootRunFinalized, true); }); @@ -1315,7 +1321,151 @@ export const layer: Layer.Layer< } return true; }); - const filterAssistantEvent = makeAssistantStreamingFilter(responseStreamingMode); + // Serializes routing and ingestion with idle flushes, so a flushed snapshot can never land + // after a newer or final one, or after its child moved to another run. + const ingestLock = yield* Semaphore.make(1); + const ingestDelivered = Effect.fnUntraced(function* ( + event: ProviderAdapter.ProviderAdapterV2Event, + deliveredEvent: ProviderAdapter.ProviderAdapterV2Event, + ) { + const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); + const isRootProviderThreadUpdate = + event.type === "provider_thread.updated" && + event.providerThread.id === input.providerThread.id; + const nativeChildTurn = + event.type === "provider_turn.updated" && + event.providerTurn.providerThreadId !== input.providerThread.id; + if (event.type === "provider_turn.updated" && !nativeChildTurn) { + firstRootProviderTurnId ??= event.providerTurn.id; + } + const storedEvents = yield* providerEventIngestor.ingestNormalized({ + analyticsContext: nativeChildTurn + ? { + ...analyticsContext, + origin: NATIVE_SUBAGENT_TURN_ORIGIN, + segment: "native_child", + } + : { + ...analyticsContext, + segment: + event.type === "provider_turn.updated" && + event.providerTurn.id !== firstRootProviderTurnId + ? "continuation" + : input.attempt.reason, + }, + providerSessionId: input.providerSessionId, + providerInstanceId: input.run.providerInstanceId, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + event: deliveredEvent, + ...(isRootProviderThreadUpdate + ? rootTerminalAlreadySeen + ? { + writeIfProviderThreadOwner: { + providerThreadId: input.providerThread.id, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedLastRunOrdinal: input.run.ordinal, + }, + } + : { + writeIfRunCurrent: { + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + }, + } + : {}), + }); + if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEvents.length === 0) { + yield* Ref.set(providerThreadOwnerLost, true); + } + return storedEvents.length; + }); + // Held by the timer until it exits, so closing can always interrupt a blocked write. + let idleFlushTimer: Fiber.Fiber | undefined; + let idleFlushTimerAt: number | null = null; + // A failed idle write keeps its text held and retries with backoff. The provider's next + // update still writes, and fails ingestion, the normal way. + let idleFlushFailures = 0; + let nextIdleFlushWarningAt = Number.NEGATIVE_INFINITY; + const flushDueAssistantText = ingestLock.withPermits(1)( + Effect.gen(function* () { + if (!idleFlushClosed) { + const now = DateTime.toEpochMillis(yield* DateTime.now); + const due = assistantStreaming.dueIdleFlushes(now); + for (const [index, flush] of due.entries()) { + // The held text's child may have been handed to another run since it arrived. + if ( + !routeProviderEvent(flush.event, routeIdentity, yield* Ref.get(eventRouting))[0] + ) { + flush.discard(); + continue; + } + const written = yield* Effect.exit(ingestDelivered(flush.event, flush.event)); + if (Exit.isFailure(written)) { + if (Cause.hasInterruptsOnly(written.cause)) return yield* written; + idleFlushFailures += 1; + const failedAt = DateTime.toEpochMillis(yield* DateTime.now); + if (failedAt >= nextIdleFlushWarningAt) { + yield* Effect.logWarning("orchestration V2 idle assistant text flush failed", { + runId: input.run.id, + failures: idleFlushFailures, + cause: written.cause, + }); + nextIdleFlushWarningAt = failedAt + 10 * 60_000; + } + const retryAt = + DateTime.toEpochMillis(yield* DateTime.now) + + Math.min(1_000 * 2 ** (idleFlushFailures - 1), 30_000); + for (const pending of due.slice(index)) pending.retryAt(retryAt); + break; + } + idleFlushFailures = 0; + nextIdleFlushWarningAt = Number.NEGATIVE_INFINITY; + flush.delivered(now); + yield* trackChildLifecycle(flush.event, true); + } + } + if (idleFlushClosed || assistantStreaming.nextIdleFlushAt() === null) { + idleFlushTimer = undefined; + return true; + } + return false; + }), + ); + const idleFlushLoop: Effect.Effect = Effect.gen(function* () { + while (true) { + const at = assistantStreaming.nextIdleFlushAt(); + idleFlushTimerAt = at; + const now = DateTime.toEpochMillis(yield* DateTime.now); + if (at !== null && at > now) yield* Effect.sleep(Duration.millis(at - now)); + if (yield* flushDueAssistantText) return; + } + }).pipe(Effect.ignoreCause); + // Runs under ingestLock, so the timer cannot exit between this check and the fork. + const scheduleIdleFlush = Effect.suspend(() => { + const at = assistantStreaming.nextIdleFlushAt(); + if ( + idleFlushClosed || + at === null || + (idleFlushTimer !== undefined && (idleFlushTimerAt === null || at >= idleFlushTimerAt)) + ) + return Effect.void; + const previous = idleFlushTimer; + return Effect.gen(function* () { + if (previous !== undefined) yield* Fiber.interrupt(previous); + idleFlushTimerAt = at; + idleFlushTimer = yield* Effect.forkDetach(idleFlushLoop); + }); + }); + const closeIdleFlush = Effect.suspend(() => { + idleFlushClosed = true; + const timer = idleFlushTimer; + idleFlushTimer = undefined; + return timer === undefined ? Effect.void : Fiber.interrupt(timer); + }); // Held text writes nothing, so its item would take a position only // when finally written, behind every item that began after it. const reservedHeldTurnItemIds = new Set(); @@ -1401,7 +1551,7 @@ export const layer: Layer.Layer< ); } return { event, accepted }; - }), + }).pipe(ingestLock.withPermits(1)), ), Stream.tap(({ event, accepted }) => Effect.gen(function* () { @@ -1414,69 +1564,12 @@ export const layer: Layer.Layer< yield* Ref.set(latestRootProviderTurn, event.providerTurn); } let storedEventCount = 0; - const deliveredEvent = filterAssistantEvent( + const deliveredEvent = assistantStreaming.filter( event, DateTime.toEpochMillis(yield* DateTime.now), ); if (deliveredEvent) { - const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); - const isRootProviderThreadUpdate = - event.type === "provider_thread.updated" && - event.providerThread.id === input.providerThread.id; - const nativeChildTurn = - event.type === "provider_turn.updated" && - event.providerTurn.providerThreadId !== input.providerThread.id; - if (event.type === "provider_turn.updated" && !nativeChildTurn) { - firstRootProviderTurnId ??= event.providerTurn.id; - } - const storedEvents = yield* providerEventIngestor.ingestNormalized({ - analyticsContext: nativeChildTurn - ? { - ...analyticsContext, - origin: NATIVE_SUBAGENT_TURN_ORIGIN, - segment: "native_child", - } - : { - ...analyticsContext, - segment: - event.type === "provider_turn.updated" && - event.providerTurn.id !== firstRootProviderTurnId - ? "continuation" - : input.attempt.reason, - }, - providerSessionId: input.providerSessionId, - providerInstanceId: input.run.providerInstanceId, - threadId: input.run.threadId, - runId: input.run.id, - nodeId: input.rootNode.id, - event: deliveredEvent, - ...(isRootProviderThreadUpdate - ? rootTerminalAlreadySeen - ? { - writeIfProviderThreadOwner: { - providerThreadId: input.providerThread.id, - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedLastRunOrdinal: input.run.ordinal, - }, - } - : { - writeIfRunCurrent: { - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - }, - } - : {}), - }); - storedEventCount = storedEvents.length; - if ( - isRootProviderThreadUpdate && - rootTerminalAlreadySeen && - storedEventCount === 0 - ) { - yield* Ref.set(providerThreadOwnerLost, true); - } + storedEventCount = yield* ingestDelivered(event, deliveredEvent); } else if ( event.type === "turn_item.updated" && (event.turnItem.type === "assistant_message" || @@ -1517,7 +1610,8 @@ export const layer: Layer.Layer< } } yield* trackChildLifecycle(event, deliveredEvent !== null); - }), + yield* scheduleIdleFlush; + }).pipe(ingestLock.withPermits(1)), ), // Also after events this run does not accept, such as a child handed to another run. Stream.tap(() => @@ -1527,6 +1621,7 @@ export const layer: Layer.Layer< ), Stream.takeUntilEffect(() => shouldStopProviderEventIngestion), Stream.runDrain, + Effect.ensuring(closeIdleFlush), Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })), Effect.flatMap(() => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/assistantStreaming.test.ts b/apps/server/src/orchestration-v2/assistantStreaming.test.ts index d71042312bfb1..08da5bf79eb38 100644 --- a/apps/server/src/orchestration-v2/assistantStreaming.test.ts +++ b/apps/server/src/orchestration-v2/assistantStreaming.test.ts @@ -1,5 +1,6 @@ -import { describe, expect, it } from "vite-plus/test"; -import { splitBufferedAssistantText } from "./assistantStreaming.ts"; +import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; +import { assert, describe, expect, it } from "vite-plus/test"; +import { makeAssistantStreamingFilter, splitBufferedAssistantText } from "./assistantStreaming.ts"; describe("V2 assistant streaming", () => { it("keeps code fences intact", () => { @@ -24,3 +25,56 @@ it("holds a streamed section heading until its content has a boundary", () => { rest: "Next", }); }); + +it("delivers held text once idle, never after its final, and never in turn mode", () => { + const snapshot = (text: string, streaming = true) => + ({ + type: "turn_item.updated", + turnItem: { id: "item", type: "reasoning", text, streaming }, + }) as unknown as ProviderAdapter.ProviderAdapterV2Event; + const text = (event: ProviderAdapter.ProviderAdapterV2Event | undefined) => + event?.type === "turn_item.updated" && "text" in event.turnItem ? event.turnItem.text : null; + // Flushes everything due and reports what it delivered. + const flush = (filter: ReturnType, now: number) => + filter.dueIdleFlushes(now).map((due) => { + due.delivered(now); + return text(due.event); + }); + + const paragraph = makeAssistantStreamingFilter("paragraph", 800); + assert.equal(text(paragraph.filter(snapshot("A\n\n```ts\nx()"), 0) ?? undefined), "A\n\n"); + // Inside the 400ms throttle even though a boundary is ready. + assert.isNull(paragraph.filter(snapshot("A\n\n```ts\nx()\n```\n\nB"), 100)); + assert.equal(paragraph.nextIdleFlushAt(), 900); + assert.deepEqual(flush(paragraph, 899), []); + // A failed write keeps the snapshot held until its retry time. + paragraph.dueIdleFlushes(900)[0]?.retryAt(1_900); + assert.equal(paragraph.nextIdleFlushAt(), 1_900); + assert.deepEqual(flush(paragraph, 1_899), []); + assert.deepEqual(flush(paragraph, 1_900), ["A\n\n```ts\nx()\n```\n\nB"]); + assert.isNull(paragraph.nextIdleFlushAt()); + + paragraph.filter(snapshot("A\n\n```ts\nx()\n```\n\nB and C"), 2_500); + const superseded = paragraph.dueIdleFlushes(3_300); + assert.isNotNull(paragraph.filter(snapshot("A\n\n```ts\nx()\n```\n\nB and C.", false), 3_400)); + // An outcome for a snapshot the final replaced changes nothing. + superseded[0]?.retryAt(3_500); + assert.isNull(paragraph.nextIdleFlushAt()); + assert.deepEqual(flush(paragraph, 5_000), []); + + // Provider rewrites replace delivered text: shorter at once, same-length after the throttle. + const rewrite = makeAssistantStreamingFilter("paragraph", 800); + assert.equal( + text(rewrite.filter(snapshot("Original long paragraph.\n\n"), 0) ?? undefined), + "Original long paragraph.\n\n", + ); + assert.equal(text(rewrite.filter(snapshot("New.\n\n"), 1_000) ?? undefined), "New.\n\n"); + assert.isNull(rewrite.filter(snapshot("Old.\n\n"), 1_100)); + assert.deepEqual(flush(rewrite, 1_900), ["Old.\n\n"]); + assert.isNull(rewrite.filter(snapshot("Old"), 2_500)); + assert.deepEqual(flush(rewrite, 3_300), ["Old"]); + + const turn = makeAssistantStreamingFilter("turn", 800); + assert.isNull(turn.filter(snapshot("A\n\nB"), 0)); + assert.isNull(turn.nextIdleFlushAt()); +}); diff --git a/apps/server/src/orchestration-v2/assistantStreaming.ts b/apps/server/src/orchestration-v2/assistantStreaming.ts index 62a173d7318c7..59f6aaa47dabf 100644 --- a/apps/server/src/orchestration-v2/assistantStreaming.ts +++ b/apps/server/src/orchestration-v2/assistantStreaming.ts @@ -68,44 +68,138 @@ export function splitBufferedAssistantText(text: string): { ready: string; rest: return { ready: text.slice(0, boundary), rest: text.slice(boundary) }; } -export function makeAssistantStreamingFilter(mode: ResponseStreamingMode) { - const delivered = new Map(); - return ( - event: ProviderAdapter.ProviderAdapterV2Event, - now: number, - ): ProviderAdapter.ProviderAdapterV2Event | null => { - if ( - event.type === "node.updated" && - event.node.kind === "assistant_message" && - event.node.status === "running" - ) - return null; - const message = - event.type === "message.updated" && event.message.role === "assistant" - ? event.message - : event.type === "turn_item.updated" && - (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") - ? event.turnItem - : null; - if (!message) return event; - const key = `${event.type}:${message.id}`; - if (!message.streaming) { - delivered.delete(key); - return event; - } - if (mode === "turn") return null; - const previous = delivered.get(key); - if (previous && now - previous.at < 400) return null; - const { ready } = splitBufferedAssistantText(message.text); - if (!ready || ready === previous?.text) return null; - delivered.set(key, { text: ready, at: now }); - if (event.type === "message.updated") - return { ...event, message: { ...event.message, text: ready } }; - if ( - event.type === "turn_item.updated" && - (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") - ) - return { ...event, turnItem: { ...event.turnItem, text: ready } }; - return event; +// Long enough to ride out normal gaps between chunks; short enough that text written before a +// long tool call or background wait does not look lost. +const ASSISTANT_IDLE_FLUSH_MS = 800; + +type StreamedMessage = { + readonly id: string; + readonly text: string; + readonly streaming: boolean; +}; + +interface StreamTrack { + readonly text: string; + readonly at: number; + // Latest snapshot whose text has not all been delivered yet. + readonly held: ProviderAdapter.ProviderAdapterV2Event | null; + readonly dueAt: number; +} + +/** A held snapshot due for delivery. Each outcome applies only while nothing newer replaced it. */ +interface IdleFlush { + readonly event: ProviderAdapter.ProviderAdapterV2Event; + readonly delivered: (now: number) => void; + readonly retryAt: (at: number) => void; + readonly discard: () => void; +} + +function streamedMessage(event: ProviderAdapter.ProviderAdapterV2Event): StreamedMessage | null { + if (event.type === "message.updated" && event.message.role === "assistant") return event.message; + if ( + event.type === "turn_item.updated" && + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") + ) + return event.turnItem; + return null; +} + +function withText( + event: ProviderAdapter.ProviderAdapterV2Event, + text: string, +): ProviderAdapter.ProviderAdapterV2Event { + if (event.type === "message.updated") return { ...event, message: { ...event.message, text } }; + if ( + event.type === "turn_item.updated" && + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") + ) + return { ...event, turnItem: { ...event.turnItem, text } }; + return event; +} + +/** + * Paragraph mode delivers streamed text at Markdown boundaries, at most every 400ms. A snapshot + * held back that way is delivered whole once its message has been idle for `idleFlushMs`, so a + * provider that pauses mid-turn still shows everything it wrote. + */ +export function makeAssistantStreamingFilter( + mode: ResponseStreamingMode, + idleFlushMs = ASSISTANT_IDLE_FLUSH_MS, +) { + const tracks = new Map(); + return { + filter( + event: ProviderAdapter.ProviderAdapterV2Event, + now: number, + ): ProviderAdapter.ProviderAdapterV2Event | null { + if ( + event.type === "node.updated" && + event.node.kind === "assistant_message" && + event.node.status === "running" + ) + return null; + const message = streamedMessage(event); + if (!message) return event; + const key = `${event.type}:${message.id}`; + if (!message.streaming) { + tracks.delete(key); + return event; + } + if (mode === "turn") return null; + const previous = tracks.get(key); + const delivered = previous?.text ?? ""; + const hold = () => { + tracks.set(key, { + text: delivered, + at: previous?.at ?? Number.NEGATIVE_INFINITY, + held: message.text !== delivered ? event : null, + dueAt: now + idleFlushMs, + }); + return null; + }; + if (previous && now - previous.at < 400) return hold(); + const { ready } = splitBufferedAssistantText(message.text); + // An idle delivery may already show more than the next boundary; never shrink it. A + // provider rewrite or truncation is not a prefix and replaces it. + if (delivered.startsWith(ready)) return hold(); + tracks.set(key, { + text: ready, + at: now, + held: ready !== message.text ? event : null, + dueAt: now + idleFlushMs, + }); + return withText(event, ready); + }, + nextIdleFlushAt(): number | null { + let next: number | null = null; + for (const track of tracks.values()) { + if (track.held !== null && (next === null || track.dueAt < next)) next = track.dueAt; + } + return next; + }, + dueIdleFlushes(now: number): ReadonlyArray { + const due: Array = []; + for (const [key, track] of tracks) { + const event = track.held; + if (event === null || track.dueAt > now) continue; + const settle = (next: (current: StreamTrack) => StreamTrack) => { + const current = tracks.get(key); + if (current?.held === event) tracks.set(key, next(current)); + }; + due.push({ + event, + delivered: (at) => + settle((current) => ({ + text: streamedMessage(event)?.text ?? current.text, + at, + held: null, + dueAt: at, + })), + retryAt: (at) => settle((current) => ({ ...current, dueAt: at })), + discard: () => settle((current) => ({ ...current, held: null })), + }); + } + return due; + }, }; } From 331ffff28534590b5177ac44e12bd80a139c39c6 Mon Sep 17 00:00:00 2001 From: vedmalex Date: Tue, 3 Nov 2026 14:01:34 +0300 Subject: [PATCH 2/8] test(orchestration): verify open-cohort sibling preservation on interrupted wake --- .../DelegatedCompletionDelivery.test.ts | 91 ++++++++++++++++++- .../orchestration-v2/RunExecutionService.ts | 1 + 2 files changed, 90 insertions(+), 2 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index cabc322d31331..e992fe917b988 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -158,13 +158,13 @@ const seedParentWithTerminalTask = (input: { readonly runId: RunId; readonly rootNodeId: NodeId; readonly taskId: NodeId; - readonly deliveryState: "delivered" | "claimed" | "acknowledged" | "disposed"; + readonly deliveryState: "delivered" | "claimed" | "acknowledged" | "disposed" | "pending"; readonly completionWake?: "always" | "settled_only"; readonly deliveryTaskIds?: ReadonlyArray; readonly parentRunStatus?: "running" | "completed"; readonly siblingTasks?: ReadonlyArray<{ readonly id: NodeId; - readonly deliveryState: "delivered" | "claimed"; + readonly deliveryState: "delivered" | "claimed" | "pending"; }>; readonly now: DateTime.Utc; }) => @@ -458,6 +458,93 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { }), ); + it.effect( + "an interrupted wake with an open cohort preserves undelivered sibling tasks for successor delivery", + () => + Effect.gen(function* () { + const orchestrator = yield* Orchestrator.OrchestratorV2; + const sink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const threadId = ThreadId.make("open-cohort-wake-parent"); + const originRunId = RunId.make("open-cohort-wake-origin"); + const carriedTaskId = NodeId.make("open-cohort-wake-carried"); + const siblingTaskId = NodeId.make("open-cohort-wake-sibling"); + + yield* seedParentWithTerminalTask({ + threadId, + projectId: ProjectId.make("open-cohort-wake-project"), + runId: originRunId, + rootNodeId: NodeId.make("open-cohort-wake-root"), + taskId: carriedTaskId, + deliveryState: "claimed", + deliveryTaskIds: [carriedTaskId], + completionWake: "always", + siblingTasks: [ + { + id: siblingTaskId, + deliveryState: "pending", + }, + ], + now, + }); + + const origin = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; + yield* sink.write({ + events: [ + { + ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }), + payload: { ...origin, status: "completed", completedAt: now }, + }, + ], + }); + + const delivery = origin.delegatedCompletion!.delivery!; + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "server", + commandId: CommandId.make("open-cohort-wake-start"), + threadId, + messageId: delivery.messageId, + text: "Delegated task completed", + attachments: [], + modelSelection, + dispatchMode: { type: "queue_after_active" }, + delegatedCompletion: { + parentRunId: originRunId, + generation: delivery.generation, + taskIds: delivery.taskIds, + }, + }); + yield* orchestrator.resumeQueuedRuns; + + const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === delivery.messageId, + )!; + + yield* orchestrator.dispatch({ + type: "run.interrupt", + commandId: CommandId.make("open-cohort-wake-interrupt"), + threadId, + runId: wake.id, + }); + + const projection = yield* orchestrator.getThreadProjection(threadId); + const interruptedWake = projection.runs.find((run) => run.id === wake.id)!; + assert.equal(interruptedWake.status, "interrupted"); + + const parentRun = projection.runs.find((run) => run.id === originRunId)!; + assert.equal(parentRun.delegatedCompletion?.disposition, "open"); + + const carried = projection.subagents.find((task) => task.id === carriedTaskId)!; + assert.equal(carried.completionDelivery?.state, "disposed"); + + const sibling = projection.subagents.find((task) => task.id === siblingTaskId)!; + assert.equal(sibling.completionDelivery?.state, "claimed"); + assert.deepEqual(parentRun.delegatedCompletion?.delivery?.taskIds, [siblingTaskId]); + }), + ); + it.effect("acceptance batches pending siblings without acknowledging their results", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index c51d2dfb9d005..b0d733a7598f2 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1324,6 +1324,7 @@ export const layer: Layer.Layer< // Serializes routing and ingestion with idle flushes, so a flushed snapshot can never land // after a newer or final one, or after its child moved to another run. const ingestLock = yield* Semaphore.make(1); + // Untraced: runs per provider event and per idle flush, so a span per call would add noise. const ingestDelivered = Effect.fnUntraced(function* ( event: ProviderAdapter.ProviderAdapterV2Event, deliveredEvent: ProviderAdapter.ProviderAdapterV2Event, From a8ebd7240a8b4786a5bce93c2817aecbdb03d293 Mon Sep 17 00:00:00 2001 From: vedmalex Date: Tue, 3 Nov 2026 14:15:56 +0300 Subject: [PATCH 3/8] test(orchestration): assert successor wake is dispatched on open cohort sibling preservation --- .../DelegatedCompletionDelivery.test.ts | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index e992fe917b988..8a81458842434 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -542,6 +542,32 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { const sibling = projection.subagents.find((task) => task.id === siblingTaskId)!; assert.equal(sibling.completionDelivery?.state, "claimed"); assert.deepEqual(parentRun.delegatedCompletion?.delivery?.taskIds, [siblingTaskId]); + + const successorDelivery = parentRun.delegatedCompletion!.delivery!; + yield* orchestrator.dispatch({ + type: "message.dispatch", + createdBy: "agent", + creationSource: "server", + commandId: CommandId.make("open-cohort-successor-wake-start"), + threadId, + messageId: successorDelivery.messageId, + text: "Delegated task completed", + attachments: [], + modelSelection, + dispatchMode: { type: "queue_after_active" }, + delegatedCompletion: { + parentRunId: originRunId, + generation: successorDelivery.generation, + taskIds: successorDelivery.taskIds, + }, + }); + yield* orchestrator.resumeQueuedRuns; + + const successorRun = (yield* orchestrator.getThreadProjection(threadId)).runs.find( + (run) => run.userMessageId === successorDelivery.messageId, + )!; + assert.isDefined(successorRun); + assert.equal(successorRun.status, "starting"); }), ); From 8bc3d21b0740c691c2d08e4fbb7929c8e49abb04 Mon Sep 17 00:00:00 2001 From: vedmalex Date: Tue, 3 Nov 2026 15:27:04 +0300 Subject: [PATCH 4/8] fix(antigravity): quarantine corrupted session checkpoints and auto-recover --- .../acp/AntigravityTurnOutcome.test.ts | 52 +++++++++++++++++++ .../provider/acp/AntigravityTurnOutcome.ts | 43 ++++++++++++++- 2 files changed, 94 insertions(+), 1 deletion(-) diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts index d3ca431c50794..f4a69f0e9bf0f 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts @@ -192,4 +192,56 @@ describe("watchAntigravityPromptOutcome", () => { }); }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), ); + + it.effect("detects corrupted checkpoint errors and quarantines the database", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const profileDirectory = yield* fs.makeTempDirectoryScoped(); + const sessionId = "a1b2c3d4-e5f6-4a1b-8c2d-3e4f5a6b7c8d"; + const conversations = path.join(profileDirectory, "antigravity-acp", "conversations"); + yield* fs.makeDirectory(conversations, { recursive: true }); + const dbPath = path.join(conversations, `${sessionId}.db`); + + const database = new NodeSqlite.DatabaseSync(dbPath); + try { + database.exec( + "CREATE TABLE IF NOT EXISTS steps (idx INTEGER PRIMARY KEY, step_type INTEGER, step_payload BLOB)", + ); + } finally { + database.close(); + } + + const check = yield* watchAntigravityPromptOutcome(profileDirectory, sessionId); + + const checkpointCorruptPayload = errorPayload( + "agent executor error: could not find doneCh for checkpoint", + ); + const corruptDb = new NodeSqlite.DatabaseSync(dbPath); + try { + corruptDb + .prepare("INSERT INTO steps (idx, step_type, step_payload) VALUES (?, ?, ?)") + .run(0, ERROR_MESSAGE, checkpointCorruptPayload); + } finally { + corruptDb.close(); + } + + const failure = yield* check({ stopReason: "end_turn" }); + assert.deepEqual(failure, { + class: "provider_error", + message: + "Antigravity session checkpoint was corrupted and has been quarantined. The session will restart.", + code: "SESSION_CORRUPTED", + retryable: true, + }); + + // The original .db file should be quarantined (moved) + const originalExists = yield* fs.exists(dbPath); + assert.isFalse(originalExists); + + const files = yield* fs.readDirectory(conversations); + const quarantined = files.find((f) => f.startsWith(`${sessionId}.db.corrupt-`)); + assert.isDefined(quarantined); + }).pipe(Effect.provide(NodeServices.layer), Effect.scoped), + ); }); diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index 874c28eeb1537..f5e191f17ec4e 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -19,6 +19,12 @@ const MAX_ERROR_PAYLOAD_BYTES = 16_384; // The Go harness's rendering of a Google API error, as the agent's api_errors.py reads it. const API_ERROR = /Error (\d{3}), Message: (.+?), Status: ([A-Z_]+)/s; const QUOTA_EXHAUSTED = /\breason:QUOTA_EXHAUSTED\b/; +const CHECKPOINT_CORRUPTED = /could not find doneCh for checkpoint/i; + +function isAntigravityCheckpointCorrupted(text: string): boolean { + return CHECKPOINT_CORRUPTED.test(text); +} + const RESET_TIMESTAMP = /\bquotaResetTimeStamp:([^\s\]]+)/; const RESET_DELAY = /\bquotaResetDelay:((?:\d+(?:\.\d+)?(?:ns|us|µs|μs|ms|h|m|s))+)(?=[\s\]])/; const GO_DURATION_PART = /(\d+(?:\.\d+)?)(ns|us|µs|μs|ms|h|m|s)/g; @@ -107,6 +113,15 @@ function turnFailure( resetAt: resetAtFrom(text, now) ?? null, }); } + if (isAntigravityCheckpointCorrupted(text)) { + return makeProviderFailure({ + class: "provider_error", + message: + "Antigravity session checkpoint was corrupted and has been quarantined. The session will restart.", + code: "SESSION_CORRUPTED", + retryable: true, + }); + } if (apiError === null) { return makeProviderFailure({ class: "provider_error", @@ -123,6 +138,17 @@ function turnFailure( }); } +async function quarantineAntigravitySessionDatabase( + databasePath: string, + timestamp: number, +): Promise { + const quarantinePath = `${databasePath}.corrupt-${timestamp}`; + await NodeFSP.rename(databasePath, quarantinePath).catch(() => {}); + await NodeFSP.rename(`${databasePath}-wal`, `${quarantinePath}-wal`).catch(() => {}); + await NodeFSP.rename(`${databasePath}-shm`, `${quarantinePath}-shm`).catch(() => {}); + return quarantinePath; +} + const conversationSegments = ["antigravity-acp", "conversations"]; /** @@ -189,6 +215,21 @@ export const watchAntigravityPromptOutcome = Effect.fn("AntigravityTurnOutcome.w Effect.orElseSucceed(() => undefined), ) : undefined; - return turnFailure(result.stopReason, step, yield* DateTime.now); + const now = yield* DateTime.now; + const failure = turnFailure(result.stopReason, step, now); + if (failure?.code === "SESSION_CORRUPTED" && after !== undefined) { + const timestamp = DateTime.toEpochMillis(now); + yield* Effect.tryPromise(() => + quarantineAntigravitySessionDatabase(after.path, timestamp), + ).pipe( + Effect.tap(() => + Effect.logWarning("Quarantined corrupted Antigravity session database", { + path: after.path, + }), + ), + Effect.orElseSucceed(() => undefined), + ); + } + return failure; }); }); From a3d8dc05374b5f18fe595d5ef267568c76115667 Mon Sep 17 00:00:00 2001 From: vedmalex Date: Tue, 3 Nov 2026 21:53:08 +0300 Subject: [PATCH 5/8] fix(antigravity): preserve session db atomicity on quarantine and remove deferFinalize --- .../Adapters/AntigravityAdapterV2.ts | 1 - .../acp/AntigravityTurnOutcome.test.ts | 11 ++++++ .../provider/acp/AntigravityTurnOutcome.ts | 38 ++++++++++++++++--- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts index a54d911a81cb5..0e4221ccfdbe5 100644 --- a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts @@ -276,7 +276,6 @@ export function makeAntigravityAcpAdapterFlavor( // 1.4.5 returns the root trajectory's Gemini usageMetadata delta; 1.4.1 returns none. reportsPerTurnPromptUsage: true, geminiPromptUsage: true, - deferFinalizeForBackgroundWork: true, enablePostSettleContinuation: true, ...(profileDirectory === undefined ? {} diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts index f4a69f0e9bf0f..493fc83980036 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.test.ts @@ -226,6 +226,17 @@ describe("watchAntigravityPromptOutcome", () => { corruptDb.close(); } + const blocked = `${dbPath}.corrupt-0`; + yield* fs.makeDirectory(blocked); + const walPath = `${dbPath}-wal`; + yield* fs.writeFileString(walPath, "fixture wal"); + const failed = yield* check({ stopReason: "end_turn" }); + assert.equal(failed?.retryable, false); + assert.isTrue(yield* fs.exists(dbPath)); + assert.isTrue(yield* fs.exists(walPath)); + yield* fs.remove(blocked, { recursive: true }); + yield* fs.remove(walPath); + const failure = yield* check({ stopReason: "end_turn" }); assert.deepEqual(failure, { class: "provider_error", diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index f5e191f17ec4e..70c184bd50155 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -143,9 +143,25 @@ async function quarantineAntigravitySessionDatabase( timestamp: number, ): Promise { const quarantinePath = `${databasePath}.corrupt-${timestamp}`; - await NodeFSP.rename(databasePath, quarantinePath).catch(() => {}); - await NodeFSP.rename(`${databasePath}-wal`, `${quarantinePath}-wal`).catch(() => {}); - await NodeFSP.rename(`${databasePath}-shm`, `${quarantinePath}-shm`).catch(() => {}); + const moved: Array = []; + try { + for (const suffix of ["-wal", "-shm", ""]) { + const source = `${databasePath}${suffix}`; + const destination = `${quarantinePath}${suffix}`; + try { + await NodeFSP.rename(source, destination); + moved.push([source, destination]); + } catch (error) { + if (suffix !== "" && (error as NodeJS.ErrnoException).code === "ENOENT") continue; + throw error; + } + } + } catch (error) { + for (const [source, destination] of moved.reverse()) { + await NodeFSP.rename(destination, source); + } + throw error; + } return quarantinePath; } @@ -219,7 +235,7 @@ export const watchAntigravityPromptOutcome = Effect.fn("AntigravityTurnOutcome.w const failure = turnFailure(result.stopReason, step, now); if (failure?.code === "SESSION_CORRUPTED" && after !== undefined) { const timestamp = DateTime.toEpochMillis(now); - yield* Effect.tryPromise(() => + const quarantined = yield* Effect.tryPromise(() => quarantineAntigravitySessionDatabase(after.path, timestamp), ).pipe( Effect.tap(() => @@ -227,8 +243,20 @@ export const watchAntigravityPromptOutcome = Effect.fn("AntigravityTurnOutcome.w path: after.path, }), ), - Effect.orElseSucceed(() => undefined), + Effect.as(true), + Effect.tapError((error) => + Effect.logWarning("Could not quarantine Antigravity session database", { error }), + ), + Effect.orElseSucceed(() => false), ); + if (!quarantined) { + return makeProviderFailure({ + class: "provider_error", + message: "Antigravity session checkpoint is corrupted and could not be quarantined.", + code: "SESSION_QUARANTINE_FAILED", + retryable: false, + }); + } } return failure; }); From bbb923783441f5040f9d256bc848792d6f9c15ea Mon Sep 17 00:00:00 2001 From: vedmalex Date: Tue, 10 Nov 2026 08:05:54 +0300 Subject: [PATCH 6/8] fix(server): narrow Antigravity continuation PR scope --- .../DelegatedCompletionDelivery.test.ts | 115 +--- .../src/orchestration-v2/Orchestrator.ts | 41 +- .../RunExecutionService.test.ts | 541 +----------------- .../orchestration-v2/RunExecutionService.ts | 220 ++----- .../assistantStreaming.test.ts | 58 +- .../orchestration-v2/assistantStreaming.ts | 172 ++---- 6 files changed, 115 insertions(+), 1032 deletions(-) diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts index f6314b6a8fac0..3f0f9cf0342ee 100644 --- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts +++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts @@ -164,7 +164,7 @@ const seedParentWithTerminalTask = (input: { readonly parentRunStatus?: "running" | "completed"; readonly siblingTasks?: ReadonlyArray<{ readonly id: NodeId; - readonly deliveryState: "delivered" | "claimed" | "pending"; + readonly deliveryState: "delivered" | "claimed"; }>; readonly now: DateTime.Utc; }) => @@ -458,119 +458,6 @@ it.layer(layerTest)("delegated completion delivery repairs", (it) => { }), ); - it.effect( - "an interrupted wake with an open cohort preserves undelivered sibling tasks for successor delivery", - () => - Effect.gen(function* () { - const orchestrator = yield* Orchestrator.OrchestratorV2; - const sink = yield* EventSink.EventSinkV2; - const now = yield* DateTime.now; - const threadId = ThreadId.make("open-cohort-wake-parent"); - const originRunId = RunId.make("open-cohort-wake-origin"); - const carriedTaskId = NodeId.make("open-cohort-wake-carried"); - const siblingTaskId = NodeId.make("open-cohort-wake-sibling"); - - yield* seedParentWithTerminalTask({ - threadId, - projectId: ProjectId.make("open-cohort-wake-project"), - runId: originRunId, - rootNodeId: NodeId.make("open-cohort-wake-root"), - taskId: carriedTaskId, - deliveryState: "claimed", - deliveryTaskIds: [carriedTaskId], - completionWake: "always", - siblingTasks: [ - { - id: siblingTaskId, - deliveryState: "pending", - }, - ], - now, - }); - - const origin = (yield* orchestrator.getThreadProjection(threadId)).runs[0]!; - yield* sink.write({ - events: [ - { - ...runEvent({ threadId, runId: originRunId, ordinal: 1, status: "completed", now }), - payload: { ...origin, status: "completed", completedAt: now }, - }, - ], - }); - - const delivery = origin.delegatedCompletion!.delivery!; - yield* orchestrator.dispatch({ - type: "message.dispatch", - createdBy: "agent", - creationSource: "server", - commandId: CommandId.make("open-cohort-wake-start"), - threadId, - messageId: delivery.messageId, - text: "Delegated task completed", - attachments: [], - modelSelection, - dispatchMode: { type: "queue_after_active" }, - delegatedCompletion: { - parentRunId: originRunId, - generation: delivery.generation, - taskIds: delivery.taskIds, - }, - }); - yield* orchestrator.resumeQueuedRuns; - - const wake = (yield* orchestrator.getThreadProjection(threadId)).runs.find( - (run) => run.userMessageId === delivery.messageId, - )!; - - yield* orchestrator.dispatch({ - type: "run.interrupt", - commandId: CommandId.make("open-cohort-wake-interrupt"), - threadId, - runId: wake.id, - }); - - const projection = yield* orchestrator.getThreadProjection(threadId); - const interruptedWake = projection.runs.find((run) => run.id === wake.id)!; - assert.equal(interruptedWake.status, "interrupted"); - - const parentRun = projection.runs.find((run) => run.id === originRunId)!; - assert.equal(parentRun.delegatedCompletion?.disposition, "open"); - - const carried = projection.subagents.find((task) => task.id === carriedTaskId)!; - assert.equal(carried.completionDelivery?.state, "disposed"); - - const sibling = projection.subagents.find((task) => task.id === siblingTaskId)!; - assert.equal(sibling.completionDelivery?.state, "claimed"); - assert.deepEqual(parentRun.delegatedCompletion?.delivery?.taskIds, [siblingTaskId]); - - const successorDelivery = parentRun.delegatedCompletion!.delivery!; - yield* orchestrator.dispatch({ - type: "message.dispatch", - createdBy: "agent", - creationSource: "server", - commandId: CommandId.make("open-cohort-successor-wake-start"), - threadId, - messageId: successorDelivery.messageId, - text: "Delegated task completed", - attachments: [], - modelSelection, - dispatchMode: { type: "queue_after_active" }, - delegatedCompletion: { - parentRunId: originRunId, - generation: successorDelivery.generation, - taskIds: successorDelivery.taskIds, - }, - }); - yield* orchestrator.resumeQueuedRuns; - - const successorRun = (yield* orchestrator.getThreadProjection(threadId)).runs.find( - (run) => run.userMessageId === successorDelivery.messageId, - )!; - assert.isDefined(successorRun); - assert.equal(successorRun.status, "starting"); - }), - ); - it.effect("acceptance batches pending siblings without acknowledging their results", () => Effect.gen(function* () { const orchestrator = yield* Orchestrator.OrchestratorV2; diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts index 6a8b8507757fe..1059c88230788 100644 --- a/apps/server/src/orchestration-v2/Orchestrator.ts +++ b/apps/server/src/orchestration-v2/Orchestrator.ts @@ -2624,9 +2624,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio readonly disposition: "stopped" | "disposed"; readonly now: DateTime.Utc; readonly cancelQueuedDelivery?: boolean; - // An interrupted wake disposes only the results it carried. The cohort stays open, and its - // delivery stays the wake's, so the wake's terminal reserves any sibling's result. - readonly interruptedWakeTaskIds?: ReadonlyArray; }) { const parentRun = input.projection.runs.find((candidate) => candidate.id === input.parentRunId); if (parentRun === undefined) { @@ -2640,28 +2637,12 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio return; } const emitEvent = emit(input.events, input.command); - const wakeTaskIds = input.interruptedWakeTaskIds; - const keptTaskIds = new Set( - wakeTaskIds !== undefined && cohort?.disposition === "open" - ? tasks - .filter( - (task) => - !wakeTaskIds.includes(task.id) && - task.completionDelivery?.state !== "acknowledged" && - task.completionDelivery?.state !== "delivered" && - task.completionDelivery?.state !== "disposed", - ) - .map((task) => task.id) - : [], - ); - const nextCohort = - keptTaskIds.size > 0 - ? cohort! - : ({ - disposition: cohort?.disposition === "disposed" ? "disposed" : input.disposition, - nextGeneration: cohort?.nextGeneration ?? 1, - delivery: null, - } as const); + const nextDisposition = cohort?.disposition === "disposed" ? "disposed" : input.disposition; + const nextCohort = { + disposition: nextDisposition, + nextGeneration: cohort?.nextGeneration ?? 1, + delivery: null, + } as const; yield* emitEvent({ type: "run.updated", threadId: parentRun.threadId, @@ -2676,7 +2657,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); for (const task of tasks) { if ( - keptTaskIds.has(task.id) || task.completionDelivery?.state === "acknowledged" || task.completionDelivery?.state === "delivered" || task.completionDelivery?.state === "disposed" @@ -2702,11 +2682,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio }); } const deliveryRun = completionDeliveryRun(input.projection, cohort?.delivery ?? null); - if ( - keptTaskIds.size === 0 && - input.cancelQueuedDelivery !== false && - deliveryRun?.status === "queued" - ) { + if (input.cancelQueuedDelivery !== false && deliveryRun?.status === "queued") { yield* emitQueuedRunCancellation({ command: input.command, events: input.events, @@ -11354,9 +11330,6 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio parentRunId, disposition: "stopped", now, - ...(parentRunId !== run.id && completionMessage?.delegatedCompletion !== undefined - ? { interruptedWakeTaskIds: completionMessage.delegatedCompletion.taskIds } - : {}), }); } }); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 87adb7005d64f..f3f1542146c66 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -29,16 +29,13 @@ import { } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; import * as Exit from "effect/Exit"; -import * as Fiber from "effect/Fiber"; import * as DateTime from "effect/DateTime"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; -import * as Queue from "effect/Queue"; import * as Ref from "effect/Ref"; import * as Stream from "effect/Stream"; -import * as TestClock from "effect/testing/TestClock"; import { CodexAppServerRequestError } from "effect-codex-app-server/errors"; import { SqlError, classifySqliteError } from "effect/sql/SqlError"; @@ -3684,13 +3681,8 @@ function rootTerminalEvent( function runBackgroundItemScenario( key: string, - makeEvents: ( - ids: BackgroundScenarioIds, - ) => - | ReadonlyArray - | Stream.Stream, + makeEvents: (ids: BackgroundScenarioIds) => ReadonlyArray, options?: { - readonly onIngest?: (event: ProviderAdapter.ProviderAdapterV2Event) => Effect.Effect; readonly keepEventStreamOpen?: boolean; readonly hasPendingBackgroundWorkForThread?: ( providerThread: OrchestrationV2ProviderThread, @@ -3706,9 +3698,6 @@ function runBackgroundItemScenario( readonly onFinalized?: ( events: ReadonlyArray, ) => Effect.Effect; - readonly onInterruptTurn?: (input: { - readonly requestRuntimeRestart?: boolean; - }) => Effect.Effect; }, ) { return Effect.gen(function* () { @@ -3747,7 +3736,6 @@ function runBackgroundItemScenario( ingestNormalized: (input) => Effect.gen(function* () { const event = input.event; - yield* options?.onIngest?.(event) ?? Effect.void; if (event.type === "turn_item.updated") { yield* Ref.update(observed, (current) => [ ...current, @@ -3781,8 +3769,7 @@ function runBackgroundItemScenario( : { hasPendingBackgroundWorkForThread: options.hasPendingBackgroundWorkForThread }), subscribeEvents: Effect.gen(function* () { yield* options?.onSubscribe ?? Effect.void; - const source = makeEvents(ids); - const events = (Stream.isStream(source) ? source : Stream.fromIterable(source)).pipe( + const events = Stream.fromIterable(makeEvents(ids)).pipe( Stream.concat( Stream.fromEffect( Effect.andThen( @@ -3801,8 +3788,6 @@ function runBackgroundItemScenario( }; }), startTurn: () => Effect.void, - interruptTurn: (interrupt: { readonly requestRuntimeRestart?: boolean }) => - options?.onInterruptTurn?.(interrupt) ?? Effect.void, } as unknown as ProviderAdapter.ProviderAdapterV2SessionRuntime, run: { id: ids.runId, @@ -3851,9 +3836,8 @@ function runBackgroundItemScenario( }); }).pipe(Effect.provide(layerTest)); - // No timeout: tests advance the TestClock, which would fire it early. - yield* Deferred.await(ingestionDone); - + const closed = yield* Deferred.await(ingestionDone).pipe(Effect.timeoutOption("2 seconds")); + assert.isTrue(Option.isSome(closed), "event ingestion fiber did not finish"); return yield* Ref.get(observed); }); } @@ -3905,520 +3889,3 @@ it.effect("keeps a failed root's still-running native children live while the ru ); }), ); - -it.effect("delivers a held paragraph after the provider goes idle with its prompt still open", () => - Effect.gen(function* () { - const key = "idle-paragraph"; - const ids = backgroundScenarioIds(key); - const now = yield* DateTime.now; - const providerEvents = yield* Queue.unbounded(); - const delivered = yield* Ref.make>([]); - const paragraphDelivered = yield* Deferred.make(); - const idleDelivered = yield* Deferred.make(); - const assistantText = (text: string, streaming: boolean) => - [ - { - type: "message.updated", - driver, - message: { - createdBy: "agent", - creationSource: "provider", - id: MessageId.make(`message:${key}`), - threadId: ids.threadId, - runId: ids.runId, - nodeId: ids.rootNodeId, - role: "assistant", - text, - attachments: [], - streaming, - createdAt: now, - updatedAt: now, - }, - }, - { - type: "turn_item.updated", - driver, - turnItem: { - ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }), - id: ids.itemId, - threadId: ids.threadId, - runId: ids.runId, - nodeId: ids.rootNodeId, - providerTurnId: ids.rootProviderTurnId, - text, - streaming, - }, - }, - ] as ReadonlyArray; - - const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(providerEvents), { - onIngest: (event) => - event.type === "message.updated" || event.type === "turn_item.updated" - ? Effect.gen(function* () { - const message = event.type === "message.updated" ? event.message : event.turnItem; - const text = "text" in message ? message.text : ""; - const streaming = "streaming" in message && message.streaming; - yield* Ref.update(delivered, (current) => [ - ...current, - `${event.type}:${streaming ? "streaming" : "final"}:${JSON.stringify(text)}`, - ]); - if (event.type === "turn_item.updated") { - yield* Deferred.succeed(paragraphDelivered, undefined); - if (streaming && text === "First.\n\nWaiting on the build") - yield* Deferred.succeed(idleDelivered, undefined); - } - }) - : Effect.void, - }).pipe(Effect.forkChild); - - yield* Queue.offerAll(providerEvents, assistantText("First.\n\nWaiting on the build", true)); - yield* Deferred.await(paragraphDelivered); - // The provider keeps its prompt open while a backgrounded command runs. - yield* TestClock.adjust("1 second"); - yield* Deferred.await(idleDelivered); - yield* Queue.offerAll(providerEvents, [ - ...assistantText("First.\n\nWaiting on the build to", true), - ...assistantText("First.\n\nWaiting on the build to finish.", false), - rootTerminalEvent(ids, "completed"), - ]); - yield* Fiber.join(scenario); - - assert.deepEqual(yield* Ref.get(delivered), [ - 'message.updated:streaming:"First.\\n\\n"', - 'turn_item.updated:streaming:"First.\\n\\n"', - 'message.updated:streaming:"First.\\n\\nWaiting on the build"', - 'turn_item.updated:streaming:"First.\\n\\nWaiting on the build"', - 'message.updated:final:"First.\\n\\nWaiting on the build to finish."', - 'turn_item.updated:final:"First.\\n\\nWaiting on the build to finish."', - ]); - }), -); - -const heldChildText = "partial response"; - -function childAssistantSnapshot( - ids: BackgroundScenarioIds, -): ProviderAdapter.ProviderAdapterV2Event { - return { - type: "turn_item.updated", - driver, - turnItem: { - ...makeLinkedChildTurnItemFixture({ ids, driver, type: "assistant_message" }), - text: heldChildText, - }, - } as ProviderAdapter.ProviderAdapterV2Event; -} - -function linkedSubagentEvent( - ids: BackgroundScenarioIds, - status: "running" | "completed", -): ProviderAdapter.ProviderAdapterV2Event { - return { - type: "subagent.updated", - driver, - subagent: { - ...makeRunOwnedSubagentFixture({ - ids, - providerInstanceId: ProviderInstanceId.make("codex"), - childThreadId: ids.childThreadId, - driver, - status: "running", - }), - status, - }, - }; -} - -const isHeldChildSnapshot = (event: ProviderAdapter.ProviderAdapterV2Event) => - event.type === "turn_item.updated" && - "text" in event.turnItem && - event.turnItem.text === heldChildText; - -// Feeds a held child snapshot, then waits until ingestion has read past it. -const holdChildSnapshot = Effect.fn(function* ( - ids: BackgroundScenarioIds, - events: Queue.Enqueue, - subagentIngested: Deferred.Deferred, -) { - yield* Queue.offerAll(events, [ - childThreadCreatedEvent(ids), - childAssistantSnapshot(ids), - linkedSubagentEvent(ids, "running"), - ]); - yield* Deferred.await(subagentIngested); -}); - -it.effect("settles a child message only an idle flush wrote when the root is interrupted", () => - Effect.gen(function* () { - const ids = backgroundScenarioIds("idle-child-interrupt"); - const events = yield* Queue.unbounded(); - const subagentIngested = yield* Deferred.make(); - const flushed = yield* Deferred.make(); - const finalized = yield* Ref.make>([]); - const scenario = yield* runBackgroundItemScenario( - "idle-child-interrupt", - () => Stream.fromQueue(events), - { - onIngest: (event) => - event.type === "subagent.updated" - ? Deferred.succeed(subagentIngested, undefined) - : isHeldChildSnapshot(event) - ? Deferred.succeed(flushed, undefined) - : Effect.void, - onFinalized: (written) => Ref.set(finalized, written), - }, - ).pipe(Effect.forkChild); - - yield* holdChildSnapshot(ids, events, subagentIngested); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(flushed); - yield* Queue.offer(events, rootTerminalEvent(ids, "interrupted")); - yield* Fiber.join(scenario); - - const settled = (yield* Ref.get(finalized)).flatMap((event) => - event.type === "turn-item.updated" && event.payload.id === ids.childItemId - ? [event.payload] - : [], - ); - assert.equal(settled.length, 1); - assert.equal(settled[0]?.status, "interrupted"); - assert.equal("text" in settled[0]! ? settled[0].text : null, heldChildText); - }), -); - -it.effect.each([ - { name: "completed", terminalStatus: "completed" as const }, - { name: "failed with its children continuing", terminalStatus: "failed" as const }, -])("idle-flushes a continuing child's text after the root $name", ({ terminalStatus }) => - Effect.gen(function* () { - const key = `idle-child-after-${terminalStatus}`; - const ids = backgroundScenarioIds(key); - const events = yield* Queue.unbounded(); - const subagentIngested = yield* Deferred.make(); - const rootFinalized = yield* Deferred.make(); - const flushed = yield* Deferred.make(); - const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { - onIngest: (event) => - event.type === "subagent.updated" - ? Deferred.succeed(subagentIngested, undefined) - : isHeldChildSnapshot(event) - ? Deferred.succeed(flushed, undefined) - : Effect.void, - onFinalized: () => Deferred.succeed(rootFinalized, undefined), - }).pipe(Effect.forkChild); - - yield* holdChildSnapshot(ids, events, subagentIngested); - const terminal = rootTerminalEvent(ids, terminalStatus); - yield* Queue.offer( - events, - terminalStatus === "failed" - ? ({ ...terminal, childTurnsContinue: true } as ProviderAdapter.ProviderAdapterV2Event) - : terminal, - ); - yield* Deferred.await(rootFinalized); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(flushed); - yield* Queue.offer(events, linkedSubagentEvent(ids, "completed")); - const observed = yield* Fiber.join(scenario); - - assert.deepEqual(observed.slice(-3), [ - "root-finalized", - "turn_item:running", - "subagent:completed", - ]); - }), -); - -it.effect.each([ - { name: "before", afterRoot: false }, - { name: "after", afterRoot: true }, -])("retries a failed idle flush $name the root completes and keeps the run live", ({ afterRoot }) => - Effect.gen(function* () { - const key = `idle-flush-retry-${afterRoot ? "after" : "before"}-root`; - const ids = backgroundScenarioIds(key); - const events = yield* Queue.unbounded(); - const subagentIngested = yield* Deferred.make(); - const rootFinalized = yield* Deferred.make(); - const firstWriteFailed = yield* Deferred.make(); - const failureHandled = yield* Deferred.make(); - const retried = yield* Deferred.make(); - const attempts = yield* Ref.make(0); - const childWrites = yield* Ref.make>([]); - const interrupts = yield* Ref.make(0); - const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { - onIngest: (event) => - Effect.gen(function* () { - if (event.type === "subagent.updated") { - yield* Deferred.succeed(subagentIngested, undefined); - if ((yield* Ref.get(attempts)) > 0) yield* Deferred.succeed(failureHandled, undefined); - return; - } - if (event.type !== "turn_item.updated" || event.turnItem.type !== "assistant_message") - return; - if ( - isHeldChildSnapshot(event) && - (yield* Ref.getAndUpdate(attempts, (n) => n + 1)) === 0 - ) { - yield* Deferred.succeed(firstWriteFailed, undefined); - yield* Effect.sleep("2 seconds"); - return yield* Effect.die(new Error("disk I/O error")); - } - const { text, streaming } = event.turnItem; - yield* Ref.update(childWrites, (current) => [ - ...current, - `${streaming ? "streaming" : "final"}:${text}`, - ]); - if (isHeldChildSnapshot(event)) yield* Deferred.succeed(retried, undefined); - }), - onFinalized: () => Deferred.succeed(rootFinalized, undefined), - onInterruptTurn: () => Ref.update(interrupts, (n) => n + 1), - }).pipe(Effect.forkChild); - - yield* holdChildSnapshot(ids, events, subagentIngested); - if (afterRoot) { - yield* Queue.offer(events, rootTerminalEvent(ids, "completed")); - yield* Deferred.await(rootFinalized); - } - yield* TestClock.adjust("1 second"); - yield* Deferred.await(firstWriteFailed); - yield* TestClock.adjust("2 seconds"); - yield* Queue.offer(events, linkedSubagentEvent(ids, "running")); - yield* Deferred.await(failureHandled); - assert.equal(yield* Ref.get(attempts), 1); - assert.deepEqual(yield* Ref.get(childWrites), []); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(retried); - const childText = (text: string, streaming: boolean) => { - const snapshot = childAssistantSnapshot(ids); - return snapshot.type === "turn_item.updated" - ? ({ - ...snapshot, - turnItem: { ...snapshot.turnItem, text, streaming }, - } as ProviderAdapter.ProviderAdapterV2Event) - : snapshot; - }; - yield* Queue.offerAll(events, [ - childText(`${heldChildText} and more`, true), - childText(`${heldChildText} and more.`, false), - ...(afterRoot ? [] : [rootTerminalEvent(ids, "completed")]), - linkedSubagentEvent(ids, "completed"), - ]); - const observed = yield* Fiber.join(scenario); - - assert.deepEqual(yield* Ref.get(childWrites), [ - `streaming:${heldChildText}`, - `final:${heldChildText} and more.`, - ]); - assert.equal(yield* Ref.get(interrupts), 0); - assert.include(observed, "root-finalized"); - assert.equal(observed.at(-1), "subagent:completed"); - }), -); - -it.effect("reschedules idle text when a newer snapshot precedes a failed write's retry", () => - Effect.gen(function* () { - const key = "idle-flush-earlier-deadline"; - const ids = backgroundScenarioIds(key); - const events = yield* Queue.unbounded(); - const subagentIngested = yield* Deferred.make(); - const markers = yield* Queue.unbounded(); - const newerWrite = yield* Deferred.make(); - const attempts = yield* Ref.make(0); - const childWrites = yield* Ref.make>([]); - const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { - onIngest: (event) => - Effect.gen(function* () { - if (event.type === "subagent.updated") - yield* Deferred.succeed(subagentIngested, undefined); - if (event.type === "message.updated" && event.message.role === "user") - yield* Queue.offer(markers, undefined); - if (event.type !== "turn_item.updated" || event.turnItem.type !== "assistant_message") - return; - if ((yield* Ref.getAndUpdate(attempts, (n) => n + 1)) < 2) - return yield* Effect.die(new Error("disk I/O error")); - const { text } = event.turnItem; - yield* Ref.update(childWrites, (current) => [...current, text]); - if (text === "New partial response") { - yield* Deferred.succeed(newerWrite, undefined); - } - }), - }).pipe(Effect.forkChild); - const marker = streamedMessageEvent({ - id: `message:${key}:marker`, - threadId: ids.threadId, - runId: ids.runId, - role: "user", - text: "marker", - streaming: false, - }); - - yield* holdChildSnapshot(ids, events, subagentIngested); - yield* TestClock.adjust("1 second"); - yield* Queue.offer(events, marker); - yield* Queue.take(markers); - assert.equal(yield* Ref.get(attempts), 1); - yield* TestClock.adjust("1 second"); - yield* Queue.offer(events, marker); - yield* Queue.take(markers); - assert.equal(yield* Ref.get(attempts), 2); - - const newer = childAssistantSnapshot(ids); - assert(newer.type === "turn_item.updated" && newer.turnItem.type === "assistant_message"); - yield* Queue.offerAll(events, [ - { ...newer, turnItem: { ...newer.turnItem, text: "New partial response" } }, - marker, - ]); - yield* Queue.take(markers); - yield* TestClock.adjust("800 millis"); - yield* Deferred.await(newerWrite); - assert.deepEqual(yield* Ref.get(childWrites), ["New partial response"]); - - yield* Queue.offerAll(events, [ - rootTerminalEvent(ids, "completed"), - linkedSubagentEvent(ids, "completed"), - ]); - yield* Fiber.join(scenario); - }), -); - -function streamedMessageEvent(input: { - readonly id: string; - readonly threadId: ThreadId; - readonly runId: RunId | null; - readonly role: "assistant" | "user"; - readonly text: string; - readonly streaming: boolean; -}): ProviderAdapter.ProviderAdapterV2Event { - const now = DateTime.makeUnsafe("2026-07-21T12:00:00.000Z"); - return { - type: "message.updated", - driver, - message: { - createdBy: input.role === "user" ? "user" : "agent", - creationSource: "provider", - id: MessageId.make(input.id), - threadId: input.threadId, - runId: input.runId, - nodeId: null, - role: input.role, - text: input.text, - attachments: [], - streaming: input.streaming, - createdAt: now, - updatedAt: now, - }, - } as ProviderAdapter.ProviderAdapterV2Event; -} - -it.effect.each([ - { name: "", failFirst: false }, - { name: " after its first idle write failed", failFirst: true }, -])("drops a child's held text once the child is handed to another run$name", ({ failFirst }) => - Effect.gen(function* () { - const key = `idle-child-handoff${failFirst ? "-after-failure" : ""}`; - const ids = backgroundScenarioIds(key); - const events = yield* Queue.unbounded(); - const subagentIngested = yield* Deferred.make(); - const firstWriteFailed = yield* Deferred.make(); - const handoffRead = yield* Deferred.make(); - const rootFlushed = yield* Deferred.make(); - const staleWrites = yield* Ref.make>([]); - const scenario = yield* runBackgroundItemScenario(key, () => Stream.fromQueue(events), { - onIngest: (event) => { - if (event.type === "subagent.updated") return Deferred.succeed(subagentIngested, undefined); - if (event.type !== "message.updated" && event.type !== "turn_item.updated") - return Effect.void; - const message = event.type === "message.updated" ? event.message : event.turnItem; - if (message.threadId === ids.childThreadId) { - return Deferred.succeed(firstWriteFailed, undefined).pipe( - Effect.flatMap((first) => - failFirst && first - ? Effect.die(new Error("disk I/O error")) - : Ref.update(staleWrites, (current) => [...current, event.type]), - ), - ); - } - if (event.type === "message.updated" && event.message.role === "user") - return Deferred.succeed(handoffRead, undefined); - return Deferred.succeed(rootFlushed, undefined); - }, - }).pipe(Effect.forkChild); - - yield* holdChildSnapshot(ids, events, subagentIngested); - if (failFirst) { - // The held snapshot stays held for a retry that must not outlive the handoff. - yield* TestClock.adjust("1 second"); - yield* Deferred.await(firstWriteFailed); - } - const handoff = linkedSubagentEvent(ids, "running"); - yield* Queue.offerAll(events, [ - streamedMessageEvent({ - id: `message:${key}:child`, - threadId: ids.childThreadId, - runId: null, - role: "assistant", - text: heldChildText, - streaming: true, - }), - handoff.type === "subagent.updated" - ? { ...handoff, subagent: { ...handoff.subagent, runId: RunId.make(`run:${key}:next`) } } - : handoff, - // Due in the same flush as the child's text, after it. - streamedMessageEvent({ - id: `message:${key}:root`, - threadId: ids.threadId, - runId: ids.runId, - role: "assistant", - text: "Root partial", - streaming: true, - }), - streamedMessageEvent({ - id: `message:${key}:marker`, - threadId: ids.threadId, - runId: ids.runId, - role: "user", - text: "marker", - streaming: false, - }), - ]); - yield* Deferred.await(handoffRead); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(rootFlushed); - yield* Queue.end(events); - yield* Fiber.join(scenario); - - assert.deepEqual(yield* Ref.get(staleWrites), []); - }), -); - -it.effect("ends ingestion while an idle flush write is still blocked", () => - Effect.gen(function* () { - const ids = backgroundScenarioIds("idle-flush-blocked-close"); - const events = yield* Queue.unbounded(); - const subagentIngested = yield* Deferred.make(); - const writeStarted = yield* Deferred.make(); - const writeInterrupted = yield* Deferred.make(); - const scenario = yield* runBackgroundItemScenario( - "idle-flush-blocked-close", - () => Stream.fromQueue(events), - { - onIngest: (event) => - event.type === "subagent.updated" - ? Deferred.succeed(subagentIngested, undefined) - : isHeldChildSnapshot(event) - ? Deferred.succeed(writeStarted, undefined).pipe( - Effect.andThen(Effect.never), - Effect.onInterrupt(() => Deferred.succeed(writeInterrupted, undefined)), - ) - : Effect.void, - }, - ).pipe(Effect.forkChild); - - yield* holdChildSnapshot(ids, events, subagentIngested); - yield* TestClock.adjust("1 second"); - yield* Deferred.await(writeStarted); - yield* Queue.end(events); - yield* Fiber.join(scenario); - yield* Deferred.await(writeInterrupted); - }), -); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index b0d733a7598f2..6a52fac447926 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -35,7 +35,6 @@ import * as Layer from "effect/Layer"; import * as Ref from "effect/Ref"; import * as Schedule from "effect/Schedule"; import * as Schema from "effect/Schema"; -import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import * as McpSessionRegistry from "../mcp/McpSessionRegistry.ts"; @@ -1074,10 +1073,6 @@ export const layer: Layer.Layer< ReadonlySet >(new Set(inheritedBackgroundTurnItemsById.keys())); const openRunOwnedSubagents = yield* Ref.make(emptyOpenRunOwnedSubagentProjection()); - const assistantStreaming = makeAssistantStreamingFilter(responseStreamingMode); - // Set when ingestion ends or finalization settles the run's open items, which a later - // streaming snapshot must not reopen. - let idleFlushClosed = false; const finalizeRootRun = Effect.fn("RunExecutionService.finalizeRootRun")(function* ( terminal: ProviderTerminalEvent, ) { @@ -1120,7 +1115,6 @@ export const layer: Layer.Layer< ); if (isRunOwnedSubagentTerminalStatus(terminal.status) && !childTurnsContinue(terminal)) { yield* Ref.set(openRunOwnedSubagents, emptyOpenRunOwnedSubagentProjection()); - idleFlushClosed = true; } yield* Ref.set(rootRunFinalized, true); }); @@ -1321,152 +1315,7 @@ export const layer: Layer.Layer< } return true; }); - // Serializes routing and ingestion with idle flushes, so a flushed snapshot can never land - // after a newer or final one, or after its child moved to another run. - const ingestLock = yield* Semaphore.make(1); - // Untraced: runs per provider event and per idle flush, so a span per call would add noise. - const ingestDelivered = Effect.fnUntraced(function* ( - event: ProviderAdapter.ProviderAdapterV2Event, - deliveredEvent: ProviderAdapter.ProviderAdapterV2Event, - ) { - const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); - const isRootProviderThreadUpdate = - event.type === "provider_thread.updated" && - event.providerThread.id === input.providerThread.id; - const nativeChildTurn = - event.type === "provider_turn.updated" && - event.providerTurn.providerThreadId !== input.providerThread.id; - if (event.type === "provider_turn.updated" && !nativeChildTurn) { - firstRootProviderTurnId ??= event.providerTurn.id; - } - const storedEvents = yield* providerEventIngestor.ingestNormalized({ - analyticsContext: nativeChildTurn - ? { - ...analyticsContext, - origin: NATIVE_SUBAGENT_TURN_ORIGIN, - segment: "native_child", - } - : { - ...analyticsContext, - segment: - event.type === "provider_turn.updated" && - event.providerTurn.id !== firstRootProviderTurnId - ? "continuation" - : input.attempt.reason, - }, - providerSessionId: input.providerSessionId, - providerInstanceId: input.run.providerInstanceId, - threadId: input.run.threadId, - runId: input.run.id, - nodeId: input.rootNode.id, - event: deliveredEvent, - ...(isRootProviderThreadUpdate - ? rootTerminalAlreadySeen - ? { - writeIfProviderThreadOwner: { - providerThreadId: input.providerThread.id, - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedLastRunOrdinal: input.run.ordinal, - }, - } - : { - writeIfRunCurrent: { - runId: input.run.id, - activeAttemptId: input.attempt.id, - expectedStatus: "running" as const, - }, - } - : {}), - }); - if (isRootProviderThreadUpdate && rootTerminalAlreadySeen && storedEvents.length === 0) { - yield* Ref.set(providerThreadOwnerLost, true); - } - return storedEvents.length; - }); - // Held by the timer until it exits, so closing can always interrupt a blocked write. - let idleFlushTimer: Fiber.Fiber | undefined; - let idleFlushTimerAt: number | null = null; - // A failed idle write keeps its text held and retries with backoff. The provider's next - // update still writes, and fails ingestion, the normal way. - let idleFlushFailures = 0; - let nextIdleFlushWarningAt = Number.NEGATIVE_INFINITY; - const flushDueAssistantText = ingestLock.withPermits(1)( - Effect.gen(function* () { - if (!idleFlushClosed) { - const now = DateTime.toEpochMillis(yield* DateTime.now); - const due = assistantStreaming.dueIdleFlushes(now); - for (const [index, flush] of due.entries()) { - // The held text's child may have been handed to another run since it arrived. - if ( - !routeProviderEvent(flush.event, routeIdentity, yield* Ref.get(eventRouting))[0] - ) { - flush.discard(); - continue; - } - const written = yield* Effect.exit(ingestDelivered(flush.event, flush.event)); - if (Exit.isFailure(written)) { - if (Cause.hasInterruptsOnly(written.cause)) return yield* written; - idleFlushFailures += 1; - const failedAt = DateTime.toEpochMillis(yield* DateTime.now); - if (failedAt >= nextIdleFlushWarningAt) { - yield* Effect.logWarning("orchestration V2 idle assistant text flush failed", { - runId: input.run.id, - failures: idleFlushFailures, - cause: written.cause, - }); - nextIdleFlushWarningAt = failedAt + 10 * 60_000; - } - const retryAt = - DateTime.toEpochMillis(yield* DateTime.now) + - Math.min(1_000 * 2 ** (idleFlushFailures - 1), 30_000); - for (const pending of due.slice(index)) pending.retryAt(retryAt); - break; - } - idleFlushFailures = 0; - nextIdleFlushWarningAt = Number.NEGATIVE_INFINITY; - flush.delivered(now); - yield* trackChildLifecycle(flush.event, true); - } - } - if (idleFlushClosed || assistantStreaming.nextIdleFlushAt() === null) { - idleFlushTimer = undefined; - return true; - } - return false; - }), - ); - const idleFlushLoop: Effect.Effect = Effect.gen(function* () { - while (true) { - const at = assistantStreaming.nextIdleFlushAt(); - idleFlushTimerAt = at; - const now = DateTime.toEpochMillis(yield* DateTime.now); - if (at !== null && at > now) yield* Effect.sleep(Duration.millis(at - now)); - if (yield* flushDueAssistantText) return; - } - }).pipe(Effect.ignoreCause); - // Runs under ingestLock, so the timer cannot exit between this check and the fork. - const scheduleIdleFlush = Effect.suspend(() => { - const at = assistantStreaming.nextIdleFlushAt(); - if ( - idleFlushClosed || - at === null || - (idleFlushTimer !== undefined && (idleFlushTimerAt === null || at >= idleFlushTimerAt)) - ) - return Effect.void; - const previous = idleFlushTimer; - return Effect.gen(function* () { - if (previous !== undefined) yield* Fiber.interrupt(previous); - idleFlushTimerAt = at; - idleFlushTimer = yield* Effect.forkDetach(idleFlushLoop); - }); - }); - const closeIdleFlush = Effect.suspend(() => { - idleFlushClosed = true; - const timer = idleFlushTimer; - idleFlushTimer = undefined; - return timer === undefined ? Effect.void : Fiber.interrupt(timer); - }); + const filterAssistantEvent = makeAssistantStreamingFilter(responseStreamingMode); // Held text writes nothing, so its item would take a position only // when finally written, behind every item that began after it. const reservedHeldTurnItemIds = new Set(); @@ -1552,7 +1401,7 @@ export const layer: Layer.Layer< ); } return { event, accepted }; - }).pipe(ingestLock.withPermits(1)), + }), ), Stream.tap(({ event, accepted }) => Effect.gen(function* () { @@ -1565,12 +1414,69 @@ export const layer: Layer.Layer< yield* Ref.set(latestRootProviderTurn, event.providerTurn); } let storedEventCount = 0; - const deliveredEvent = assistantStreaming.filter( + const deliveredEvent = filterAssistantEvent( event, DateTime.toEpochMillis(yield* DateTime.now), ); if (deliveredEvent) { - storedEventCount = yield* ingestDelivered(event, deliveredEvent); + const rootTerminalAlreadySeen = yield* Ref.get(rootTerminalSeen); + const isRootProviderThreadUpdate = + event.type === "provider_thread.updated" && + event.providerThread.id === input.providerThread.id; + const nativeChildTurn = + event.type === "provider_turn.updated" && + event.providerTurn.providerThreadId !== input.providerThread.id; + if (event.type === "provider_turn.updated" && !nativeChildTurn) { + firstRootProviderTurnId ??= event.providerTurn.id; + } + const storedEvents = yield* providerEventIngestor.ingestNormalized({ + analyticsContext: nativeChildTurn + ? { + ...analyticsContext, + origin: NATIVE_SUBAGENT_TURN_ORIGIN, + segment: "native_child", + } + : { + ...analyticsContext, + segment: + event.type === "provider_turn.updated" && + event.providerTurn.id !== firstRootProviderTurnId + ? "continuation" + : input.attempt.reason, + }, + providerSessionId: input.providerSessionId, + providerInstanceId: input.run.providerInstanceId, + threadId: input.run.threadId, + runId: input.run.id, + nodeId: input.rootNode.id, + event: deliveredEvent, + ...(isRootProviderThreadUpdate + ? rootTerminalAlreadySeen + ? { + writeIfProviderThreadOwner: { + providerThreadId: input.providerThread.id, + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedLastRunOrdinal: input.run.ordinal, + }, + } + : { + writeIfRunCurrent: { + runId: input.run.id, + activeAttemptId: input.attempt.id, + expectedStatus: "running" as const, + }, + } + : {}), + }); + storedEventCount = storedEvents.length; + if ( + isRootProviderThreadUpdate && + rootTerminalAlreadySeen && + storedEventCount === 0 + ) { + yield* Ref.set(providerThreadOwnerLost, true); + } } else if ( event.type === "turn_item.updated" && (event.turnItem.type === "assistant_message" || @@ -1611,8 +1517,7 @@ export const layer: Layer.Layer< } } yield* trackChildLifecycle(event, deliveredEvent !== null); - yield* scheduleIdleFlush; - }).pipe(ingestLock.withPermits(1)), + }), ), // Also after events this run does not accept, such as a child handed to another run. Stream.tap(() => @@ -1622,7 +1527,6 @@ export const layer: Layer.Layer< ), Stream.takeUntilEffect(() => shouldStopProviderEventIngestion), Stream.runDrain, - Effect.ensuring(closeIdleFlush), Effect.mapError((cause) => new RunExecutionIngestError({ runId: input.run.id, cause })), Effect.flatMap(() => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/assistantStreaming.test.ts b/apps/server/src/orchestration-v2/assistantStreaming.test.ts index 08da5bf79eb38..d71042312bfb1 100644 --- a/apps/server/src/orchestration-v2/assistantStreaming.test.ts +++ b/apps/server/src/orchestration-v2/assistantStreaming.test.ts @@ -1,6 +1,5 @@ -import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter"; -import { assert, describe, expect, it } from "vite-plus/test"; -import { makeAssistantStreamingFilter, splitBufferedAssistantText } from "./assistantStreaming.ts"; +import { describe, expect, it } from "vite-plus/test"; +import { splitBufferedAssistantText } from "./assistantStreaming.ts"; describe("V2 assistant streaming", () => { it("keeps code fences intact", () => { @@ -25,56 +24,3 @@ it("holds a streamed section heading until its content has a boundary", () => { rest: "Next", }); }); - -it("delivers held text once idle, never after its final, and never in turn mode", () => { - const snapshot = (text: string, streaming = true) => - ({ - type: "turn_item.updated", - turnItem: { id: "item", type: "reasoning", text, streaming }, - }) as unknown as ProviderAdapter.ProviderAdapterV2Event; - const text = (event: ProviderAdapter.ProviderAdapterV2Event | undefined) => - event?.type === "turn_item.updated" && "text" in event.turnItem ? event.turnItem.text : null; - // Flushes everything due and reports what it delivered. - const flush = (filter: ReturnType, now: number) => - filter.dueIdleFlushes(now).map((due) => { - due.delivered(now); - return text(due.event); - }); - - const paragraph = makeAssistantStreamingFilter("paragraph", 800); - assert.equal(text(paragraph.filter(snapshot("A\n\n```ts\nx()"), 0) ?? undefined), "A\n\n"); - // Inside the 400ms throttle even though a boundary is ready. - assert.isNull(paragraph.filter(snapshot("A\n\n```ts\nx()\n```\n\nB"), 100)); - assert.equal(paragraph.nextIdleFlushAt(), 900); - assert.deepEqual(flush(paragraph, 899), []); - // A failed write keeps the snapshot held until its retry time. - paragraph.dueIdleFlushes(900)[0]?.retryAt(1_900); - assert.equal(paragraph.nextIdleFlushAt(), 1_900); - assert.deepEqual(flush(paragraph, 1_899), []); - assert.deepEqual(flush(paragraph, 1_900), ["A\n\n```ts\nx()\n```\n\nB"]); - assert.isNull(paragraph.nextIdleFlushAt()); - - paragraph.filter(snapshot("A\n\n```ts\nx()\n```\n\nB and C"), 2_500); - const superseded = paragraph.dueIdleFlushes(3_300); - assert.isNotNull(paragraph.filter(snapshot("A\n\n```ts\nx()\n```\n\nB and C.", false), 3_400)); - // An outcome for a snapshot the final replaced changes nothing. - superseded[0]?.retryAt(3_500); - assert.isNull(paragraph.nextIdleFlushAt()); - assert.deepEqual(flush(paragraph, 5_000), []); - - // Provider rewrites replace delivered text: shorter at once, same-length after the throttle. - const rewrite = makeAssistantStreamingFilter("paragraph", 800); - assert.equal( - text(rewrite.filter(snapshot("Original long paragraph.\n\n"), 0) ?? undefined), - "Original long paragraph.\n\n", - ); - assert.equal(text(rewrite.filter(snapshot("New.\n\n"), 1_000) ?? undefined), "New.\n\n"); - assert.isNull(rewrite.filter(snapshot("Old.\n\n"), 1_100)); - assert.deepEqual(flush(rewrite, 1_900), ["Old.\n\n"]); - assert.isNull(rewrite.filter(snapshot("Old"), 2_500)); - assert.deepEqual(flush(rewrite, 3_300), ["Old"]); - - const turn = makeAssistantStreamingFilter("turn", 800); - assert.isNull(turn.filter(snapshot("A\n\nB"), 0)); - assert.isNull(turn.nextIdleFlushAt()); -}); diff --git a/apps/server/src/orchestration-v2/assistantStreaming.ts b/apps/server/src/orchestration-v2/assistantStreaming.ts index 59f6aaa47dabf..62a173d7318c7 100644 --- a/apps/server/src/orchestration-v2/assistantStreaming.ts +++ b/apps/server/src/orchestration-v2/assistantStreaming.ts @@ -68,138 +68,44 @@ export function splitBufferedAssistantText(text: string): { ready: string; rest: return { ready: text.slice(0, boundary), rest: text.slice(boundary) }; } -// Long enough to ride out normal gaps between chunks; short enough that text written before a -// long tool call or background wait does not look lost. -const ASSISTANT_IDLE_FLUSH_MS = 800; - -type StreamedMessage = { - readonly id: string; - readonly text: string; - readonly streaming: boolean; -}; - -interface StreamTrack { - readonly text: string; - readonly at: number; - // Latest snapshot whose text has not all been delivered yet. - readonly held: ProviderAdapter.ProviderAdapterV2Event | null; - readonly dueAt: number; -} - -/** A held snapshot due for delivery. Each outcome applies only while nothing newer replaced it. */ -interface IdleFlush { - readonly event: ProviderAdapter.ProviderAdapterV2Event; - readonly delivered: (now: number) => void; - readonly retryAt: (at: number) => void; - readonly discard: () => void; -} - -function streamedMessage(event: ProviderAdapter.ProviderAdapterV2Event): StreamedMessage | null { - if (event.type === "message.updated" && event.message.role === "assistant") return event.message; - if ( - event.type === "turn_item.updated" && - (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") - ) - return event.turnItem; - return null; -} - -function withText( - event: ProviderAdapter.ProviderAdapterV2Event, - text: string, -): ProviderAdapter.ProviderAdapterV2Event { - if (event.type === "message.updated") return { ...event, message: { ...event.message, text } }; - if ( - event.type === "turn_item.updated" && - (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") - ) - return { ...event, turnItem: { ...event.turnItem, text } }; - return event; -} - -/** - * Paragraph mode delivers streamed text at Markdown boundaries, at most every 400ms. A snapshot - * held back that way is delivered whole once its message has been idle for `idleFlushMs`, so a - * provider that pauses mid-turn still shows everything it wrote. - */ -export function makeAssistantStreamingFilter( - mode: ResponseStreamingMode, - idleFlushMs = ASSISTANT_IDLE_FLUSH_MS, -) { - const tracks = new Map(); - return { - filter( - event: ProviderAdapter.ProviderAdapterV2Event, - now: number, - ): ProviderAdapter.ProviderAdapterV2Event | null { - if ( - event.type === "node.updated" && - event.node.kind === "assistant_message" && - event.node.status === "running" - ) - return null; - const message = streamedMessage(event); - if (!message) return event; - const key = `${event.type}:${message.id}`; - if (!message.streaming) { - tracks.delete(key); - return event; - } - if (mode === "turn") return null; - const previous = tracks.get(key); - const delivered = previous?.text ?? ""; - const hold = () => { - tracks.set(key, { - text: delivered, - at: previous?.at ?? Number.NEGATIVE_INFINITY, - held: message.text !== delivered ? event : null, - dueAt: now + idleFlushMs, - }); - return null; - }; - if (previous && now - previous.at < 400) return hold(); - const { ready } = splitBufferedAssistantText(message.text); - // An idle delivery may already show more than the next boundary; never shrink it. A - // provider rewrite or truncation is not a prefix and replaces it. - if (delivered.startsWith(ready)) return hold(); - tracks.set(key, { - text: ready, - at: now, - held: ready !== message.text ? event : null, - dueAt: now + idleFlushMs, - }); - return withText(event, ready); - }, - nextIdleFlushAt(): number | null { - let next: number | null = null; - for (const track of tracks.values()) { - if (track.held !== null && (next === null || track.dueAt < next)) next = track.dueAt; - } - return next; - }, - dueIdleFlushes(now: number): ReadonlyArray { - const due: Array = []; - for (const [key, track] of tracks) { - const event = track.held; - if (event === null || track.dueAt > now) continue; - const settle = (next: (current: StreamTrack) => StreamTrack) => { - const current = tracks.get(key); - if (current?.held === event) tracks.set(key, next(current)); - }; - due.push({ - event, - delivered: (at) => - settle((current) => ({ - text: streamedMessage(event)?.text ?? current.text, - at, - held: null, - dueAt: at, - })), - retryAt: (at) => settle((current) => ({ ...current, dueAt: at })), - discard: () => settle((current) => ({ ...current, held: null })), - }); - } - return due; - }, +export function makeAssistantStreamingFilter(mode: ResponseStreamingMode) { + const delivered = new Map(); + return ( + event: ProviderAdapter.ProviderAdapterV2Event, + now: number, + ): ProviderAdapter.ProviderAdapterV2Event | null => { + if ( + event.type === "node.updated" && + event.node.kind === "assistant_message" && + event.node.status === "running" + ) + return null; + const message = + event.type === "message.updated" && event.message.role === "assistant" + ? event.message + : event.type === "turn_item.updated" && + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") + ? event.turnItem + : null; + if (!message) return event; + const key = `${event.type}:${message.id}`; + if (!message.streaming) { + delivered.delete(key); + return event; + } + if (mode === "turn") return null; + const previous = delivered.get(key); + if (previous && now - previous.at < 400) return null; + const { ready } = splitBufferedAssistantText(message.text); + if (!ready || ready === previous?.text) return null; + delivered.set(key, { text: ready, at: now }); + if (event.type === "message.updated") + return { ...event, message: { ...event.message, text: ready } }; + if ( + event.type === "turn_item.updated" && + (event.turnItem.type === "assistant_message" || event.turnItem.type === "reasoning") + ) + return { ...event, turnItem: { ...event.turnItem, text: ready } }; + return event; }; } From f2c967e744c7845973f605cdd6f84cf090e35880 Mon Sep 17 00:00:00 2001 From: vedmalex Date: Tue, 10 Nov 2026 08:50:05 +0300 Subject: [PATCH 7/8] fix(server): type Antigravity quarantine failures --- .../src/provider/acp/AntigravityTurnOutcome.ts | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts index 25706da5abb5f..9ae58ead715ab 100644 --- a/apps/server/src/provider/acp/AntigravityTurnOutcome.ts +++ b/apps/server/src/provider/acp/AntigravityTurnOutcome.ts @@ -59,6 +59,15 @@ class AntigravitySessionLogError extends Schema.TaggedError new AntigravitySessionLogError({ cause }); +class AntigravitySessionQuarantineError extends Schema.TaggedError()( + "AntigravitySessionQuarantineError", + { cause: Schema.Defect() }, +) { + override get message(): string { + return "Could not quarantine the Antigravity session database."; + } +} + const readSteps = (file: string, read: (database: NodeSqlite.DatabaseSync) => A) => Effect.try({ try: () => { @@ -257,9 +266,10 @@ export const watchAntigravityPromptOutcome = Effect.fn("AntigravityTurnOutcome.w const failure = turnFailure(result.stopReason, step, now); if (failure?.code === "SESSION_CORRUPTED" && after !== undefined) { const timestamp = DateTime.toEpochMillis(now); - const quarantined = yield* Effect.tryPromise(() => - quarantineAntigravitySessionDatabase(after.path, timestamp), - ).pipe( + const quarantined = yield* Effect.tryPromise({ + try: () => quarantineAntigravitySessionDatabase(after.path, timestamp), + catch: (cause) => new AntigravitySessionQuarantineError({ cause }), + }).pipe( Effect.tap(() => Effect.logWarning("Quarantined corrupted Antigravity session database", { path: after.path, From b76e7793c97b41deaf52f289b82309bf6adc49e9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=90=D0=BB=D0=B5=D0=BA=D1=81=D0=B5=D0=B9=20=D0=92=D0=B5?= =?UTF-8?q?=D0=B4=D0=BC=D0=B5=D0=B4=D0=B5=D0=BD=D0=BA=D0=BE?= Date: Sat, 14 Nov 2026 09:01:24 +0300 Subject: [PATCH 8/8] fix(antigravity): forward thread MCP env and defer MCP tools AntigravityDriver dropped input.processEnvironment when building the ACP spawn env, hiding T3CODE_THREAD_ID and T3_ACP_MCP_* from Antigravity's built-in shell tool. AntigravityAdapterV2 also omitted deferMcpTools, causing Antigravity ACP's ~40 KB eager schema cap to drop delegate_task, t3_thread_launch, and schedule_task. Merge input.processEnvironment into baseEnv in AntigravityDriver and enable deferMcpTools on AntigravityAdapterV2 so initial tools/list stays under the cap via tool_search (refs #36088). Authored with Gemini 4 Hydrogen via Antigravity. --- .../Adapters/AntigravityAdapterV2.ts | 2 + .../Drivers/AntigravityDriver.test.ts | 69 +++++++++++++++++++ .../src/provider/Drivers/AntigravityDriver.ts | 7 +- .../src/provider/acp/AntigravityAcpSupport.ts | 1 + 4 files changed, 78 insertions(+), 1 deletion(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts index c2a9bc12406ec..9b6672981edcc 100644 --- a/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/AntigravityAdapterV2.ts @@ -212,6 +212,8 @@ export function makeAntigravityAcpAdapterFlavor( supportsNativeGoals: options.supportsNativeGoals ?? Effect.succeed(false), // Agents that advertise HTTP MCP skip the stdio bridge process; others keep stdio. mcpOverHttp: true, + // Keep initial tools/list under Antigravity ACP's ~40 KB eager schema cap via tool_search. + deferMcpTools: true, sessionCarriesPullRequestLinking: antigravityAppendsSystemPrompt, makeRuntime, buildAttachmentPrompt: ({ diff --git a/apps/server/src/provider/Drivers/AntigravityDriver.test.ts b/apps/server/src/provider/Drivers/AntigravityDriver.test.ts index 447745fc673a0..e131b4d62a6a1 100644 --- a/apps/server/src/provider/Drivers/AntigravityDriver.test.ts +++ b/apps/server/src/provider/Drivers/AntigravityDriver.test.ts @@ -3,6 +3,7 @@ import * as NodeServices from "@effect/platform-node/NodeServices"; import { expect, it } from "@effect/vitest"; import { ANTIGRAVITY_DEFAULT_MODEL, + EnvironmentId, ProviderInstanceId, ProviderSessionId, ThreadId, @@ -82,6 +83,7 @@ const makeHarness = Effect.fn("makeAntigravityDriverHarness")(function* ( /** `data` of the agent's "Internal error" response to `session/new`. */ readonly newSessionErrorData?: unknown; readonly initialMode?: string; + readonly httpMcp?: boolean; } = {}, ) { const directory = path.join(root, name); @@ -105,6 +107,7 @@ const makeHarness = Effect.fn("makeAntigravityDriverHarness")(function* ( ...(options.initialMode === undefined ? [] : [`export T3_ACP_INITIAL_MODE=${shellQuote(options.initialMode)}`]), + ...(options.httpMcp ? ["export T3_ACP_MCP_HTTP=1"] : []), `exec ${shellQuote(nodePath)} ${shellQuote(mockAgentPath)} "$@"`, "", ].join("\n"), @@ -142,6 +145,10 @@ const makeHarness = Effect.fn("makeAntigravityDriverHarness")(function* ( credentialKeys: ReadonlyArray; geminiApiKey: string | undefined; tempDirectory: string | undefined; + threadId: string | undefined; + mcpEndpoint: string | undefined; + mcpAuthorization: string | undefined; + mcpNode: string | undefined; handle: ChildProcessSpawner.ChildProcessHandle; }> = []; @@ -200,6 +207,10 @@ const makeHarness = Effect.fn("makeAntigravityDriverHarness")(function* ( geminiApiKey: env("GEMINI_API_KEY"), tempDirectory: environment.ANTIGRAVITY_HARNESS_PATH === undefined ? undefined : env("TMPDIR"), + threadId: env("T3CODE_THREAD_ID"), + mcpEndpoint: env("T3_ACP_MCP_ENDPOINT"), + mcpAuthorization: env("T3_ACP_MCP_AUTHORIZATION"), + mcpNode: env("T3_ACP_MCP_NODE"), handle, }); return handle; @@ -579,4 +590,62 @@ it.layer(layerTest)("AntigravityDriver", (it) => { expect(yield* fs.exists(legacyRoot)).toBe(false); }), ); + + it.effect.skipIf(windowsHost)( + "forwards thread MCP environment and defers MCP tools for Antigravity sessions", + () => + Effect.gen(function* () { + const h = yield* makeHarness(); + h.controls.selected = yield* h.makeExecutable("runtime http mcp", { httpMcp: true }); + const mcpSessions = yield* McpProviderSessions.McpProviderSessions; + const workspace = yield* h.fs.makeTempDirectoryScoped({ prefix: "t3-ag-mcp-" }); + const threadId = ThreadId.make("thread-antigravity-mcp-env"); + const providerSessionId = ProviderSessionId.make("provider-session-antigravity-mcp-env"); + const endpoint = "http://127.0.0.1:43210/mcp"; + const authorizationHeader = "Bearer test-mcp-token"; + yield* mcpSessions.set({ + environmentId: EnvironmentId.make("env-test"), + threadId, + providerSessionId, + providerInstanceId: h.instance.instanceId, + endpoint, + authorizationHeader, + browserToolsAvailable: false, + }); + const runtimePolicy = ProviderAdapter.ProviderAdapterV2RuntimePolicy.make({ + runtimeMode: "approval-required", + interactionMode: "default", + cwd: workspace, + }); + const modelSelection = { instanceId: h.instance.instanceId, model: "gemini-test-low" }; + const session = yield* h.instance.orchestrationAdapter.openSession({ + threadId, + providerSessionId, + modelSelection, + runtimePolicy, + }); + yield* session.ensureThread({ threadId, modelSelection, runtimePolicy }); + + const launch = h.launches.at(-1); + expect(launch?.threadId).toBe(threadId); + expect(launch?.mcpEndpoint).toBe(endpoint); + expect(launch?.mcpAuthorization).toBe(authorizationHeader); + expect(launch?.mcpNode).toBeTruthy(); + + const newSessionRequest = (yield* h.readRequests).findLast( + (request) => request.method === "session/new", + ); + expect(newSessionRequest?.params?.mcpServers).toEqual([ + { + type: "http", + name: "t3-code", + url: endpoint, + headers: [ + { name: "Authorization", value: authorizationHeader }, + { name: "t3-tool-exposure", value: "deferred" }, + ], + }, + ]); + }), + ); }); diff --git a/apps/server/src/provider/Drivers/AntigravityDriver.ts b/apps/server/src/provider/Drivers/AntigravityDriver.ts index 75bca5acc87be..d2804d74e27d8 100644 --- a/apps/server/src/provider/Drivers/AntigravityDriver.ts +++ b/apps/server/src/provider/Drivers/AntigravityDriver.ts @@ -276,7 +276,12 @@ export const AntigravityDriver: ProviderDriver< installation: executable, profile, cwd: input.cwd, - baseEnv: processEnvironment, + baseEnv: input.processEnvironment + ? { + ...processEnvironment, + ...input.processEnvironment, + } + : processEnvironment, auth, runtimeTempDirectory, }), diff --git a/apps/server/src/provider/acp/AntigravityAcpSupport.ts b/apps/server/src/provider/acp/AntigravityAcpSupport.ts index 7fd64e2534fa7..e042d07e96276 100644 --- a/apps/server/src/provider/acp/AntigravityAcpSupport.ts +++ b/apps/server/src/provider/acp/AntigravityAcpSupport.ts @@ -50,6 +50,7 @@ export interface AntigravityAcpRuntimeInput extends Omit< /** Conversational sessions only; helpers and probes leave the system prompt untouched. */ readonly sessionSystemPromptAppend?: string; readonly authMethod?: AntigravityAuthMethod; + readonly processEnvironment?: NodeJS.ProcessEnv | undefined; } function parseAntigravityVersion(version: string | null | undefined) {