From 2bdc7a6d5776d00cc58555425c31c7213a0204cc Mon Sep 17 00:00:00 2001 From: Ivan Kosilov Date: Thu, 12 Nov 2026 19:19:30 +0500 Subject: [PATCH 1/4] perf(server): store a subagent's record and its item in one commit While a subagent runs, each progress line reaches the server as `subagent.updated` followed by the subagent's `turn_item.updated`, and each landed in its own SQLite commit. On a real database these updates are 13-18% of all events, and every commit pays a fixed set of hot pages (the thread row and its index, projection metadata, sqlite_sequence, page 1, the event log's index leaves). Adapters now attach the item to the record (`turnItem` on `subagent.updated`) when nothing is emitted between them: Claude for progress-only updates, Codex, Pi, Cursor, OpenCode and ACP for every record their item follows. `providerEventMembers` splits it back into the two events, so the session manager and the run consumer give each the handling of a standalone event (routing, delivery filter, child and background-item tracking, ingestion). The run ingests both through `ProviderEventIngestor.ingestTogether`, one transaction, and the child thread's model sync still runs on the consumer right after the commit that stores the record, outside it, as before. Clients receive the same two events with the same contents in the same order. When the child's model needs a sync, its `thread.model-selection-updated` on the child thread now follows the parent's item instead of preceding it; every reader of that event replaces the child thread row from its payload. Replay of recorded windows (counts repeat exactly run to run): - 5,417 events: commits 5,030 -> 4,472, WAL frames per event 29.84 -> 28.55 (-4.3%), checkpoint pages per event 11.33 -> 11.13 - 418 events: commits 392 -> 344, WAL frames per event 31.78 -> 30.31 (-4.6%), checkpoint pages per event 12.69 -> 12.46 - a Claude turn without subagents: unchanged (44 commits) Co-Authored-By: Claude Opus 5.5 --- .../Adapters/AcpAdapterV2.test.ts | 24 +- .../Adapters/AcpRegistryAdapterV2.test.ts | 4 +- .../Adapters/ClaudeAdapterV2.ts | 77 +++-- .../Adapters/CodexAdapterV2.ts | 4 - .../ProviderEventIngestor.test.ts | 317 ++++++++++++++++++ .../orchestration-v2/ProviderEventIngestor.ts | 58 +++- .../ProviderSessionManager.ts | 4 +- .../RunExecutionService.test.ts | 62 +++- .../orchestration-v2/RunExecutionService.ts | 10 +- packages/contracts/src/orchestrationV2.ts | 35 +- packages/provider-acp/src/server/adapter.ts | 8 +- .../src/server/ProviderAdapter.ts | 12 + .../provider-cursor/src/server/adapter.ts | 4 - .../src/server/adapter.test.ts | 3 + .../provider-opencode/src/server/adapter.ts | 4 - .../src/server/v2/adapter.ts | 4 - .../provider-pi/src/server/adapter.test.ts | 15 +- packages/provider-pi/src/server/adapter.ts | 4 - 18 files changed, 533 insertions(+), 116 deletions(-) diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index fb51465629018..fb41197839734 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -6840,7 +6840,9 @@ describe("AcpAdapterV2", () => { }); const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -7024,7 +7026,9 @@ describe("AcpAdapterV2", () => { const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -7215,7 +7219,9 @@ describe("AcpAdapterV2", () => { const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -7460,7 +7466,9 @@ describe("AcpAdapterV2", () => { const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -7740,7 +7748,9 @@ describe("AcpAdapterV2", () => { }); const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -9648,7 +9658,9 @@ describe("AcpAdapterV2", () => { }); const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ diff --git a/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts index 635d63d91473a..f1074d93ddae6 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts @@ -284,7 +284,9 @@ const openClaudeReplaySession = Effect.fn("openClaudeReplaySession")(function* ( } const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy }); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 3e10e09906796..6e7680dfcf783 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -115,6 +115,7 @@ import { type OrchestrationV2RuntimeRequest, type OrchestrationV2UserInputQuestion, type OrchestrationV2Subagent, + type OrchestrationV2SubagentTurnItem, type OrchestrationV2AppThread, type OrchestrationV2SubagentWorkflow, type OrchestrationV2WorkflowRunHandles, @@ -7578,11 +7579,45 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }, }); } + const turnItem = { + id: subagent.turnItemId, + threadId: task.threadId, + runId: task.runId, + nodeId: task.id, + providerThreadId: input.context.input.providerThread.id, + providerTurnId: input.context.providerTurnId, + nativeItemRef: task.nativeTaskRef, + parentItemId: null, + ordinal: subagent.turnItemOrdinal, + status: task.status, + title: task.title, + startedAt: task.startedAt, + completedAt: task.completedAt, + updatedAt: task.updatedAt, + type: "subagent", + subagentId: task.id, + origin: task.origin, + driver: task.driver, + providerInstanceId: task.providerInstanceId, + childThreadId: task.childThreadId, + prompt: task.prompt, + ...(task.progress === undefined ? {} : { progress: task.progress }), + result: task.result, + } satisfies OrchestrationV2SubagentTurnItem; + const promptNativeItemId = + existingSubagent === undefined + ? `${nativeItemId}:prompt` + : resumeToolUseId === null + ? null + : `${nativeItemId}:prompt:${resumeToolUseId}`; + const itemFollowsRecord = + !lifecycleChanged && promptNativeItemId === null && workflowWithThreads === undefined; // Emit before child records so the resumed run can adopt the child thread. yield* emitProviderEvent({ type: "subagent.updated", driver: CLAUDE_PROVIDER, subagent: task, + ...(itemFollowsRecord ? { turnItem } : {}), }); if (lifecycleChanged) { yield* emitProviderEvent({ @@ -7607,12 +7642,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }, }); } - const promptNativeItemId = - existingSubagent === undefined - ? `${nativeItemId}:prompt` - : resumeToolUseId === null - ? null - : `${nativeItemId}:prompt:${resumeToolUseId}`; if (promptNativeItemId !== null) { yield* emitSubagentConversationMessage({ nativeItemId: promptNativeItemId, @@ -7636,35 +7665,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); if ((yield* Ref.get(sessionSubagentsByTaskId)).get(input.taskId) !== subagent) return; } - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: CLAUDE_PROVIDER, - turnItem: { - id: subagent.turnItemId, - threadId: task.threadId, - runId: task.runId, - nodeId: task.id, - providerThreadId: input.context.input.providerThread.id, - providerTurnId: input.context.providerTurnId, - nativeItemRef: task.nativeTaskRef, - parentItemId: null, - ordinal: subagent.turnItemOrdinal, - status: task.status, - title: task.title, - startedAt: task.startedAt, - completedAt: task.completedAt, - updatedAt: task.updatedAt, - type: "subagent", - subagentId: task.id, - origin: task.origin, - driver: task.driver, - providerInstanceId: task.providerInstanceId, - childThreadId: task.childThreadId, - prompt: task.prompt, - ...(task.progress === undefined ? {} : { progress: task.progress }), - result: task.result, - }, - }); + if (!itemFollowsRecord) { + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CLAUDE_PROVIDER, + turnItem, + }); + } if (input.nativeOutcome !== undefined && subagent.confirmedOutcome !== undefined) { const outcome = subagentStopOutcomes.get(input.taskId); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index 4e8fd1ac5f30b..ba7da881c6e4a 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -2767,10 +2767,6 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( type: "subagent.updated", driver: CODEX_PROVIDER, subagent: task, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: CODEX_PROVIDER, turnItem: { id: input.subagent.turnItemId, threadId: task.threadId, diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 520fea7c0575b..ac162187f8eba 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -23,6 +23,7 @@ import { RunAttemptId, RunId, RuntimeRequestId, + type ThreadId, TurnItemId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -33,9 +34,13 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; import * as SqlClient from "effect/sql/SqlClient"; import * as SqlError from "effect/sql/SqlError"; import * as Stream from "effect/Stream"; +import * as Statement from "effect/sql/Statement"; +import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; import { toolOutputImages } from "@t3tools/shared/toolOutput"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; @@ -2345,6 +2350,318 @@ layer("ProviderEventIngestorV2", (it) => { ); }); +/** A native subagent's record with its item, as an adapter emits them when nothing comes between. */ +const pairedSubagentUpdate = (input: { + readonly threadId: ThreadId; + readonly childThreadId: ThreadId; + readonly key: string; + readonly status: "running" | "completed"; + readonly reported: ModelSelection; + readonly now: DateTime.Utc; +}) => { + const id = NodeId.make(`node:${input.key}`); + const common = { + threadId: input.threadId, + runId: null, + driver: CODEX_DRIVER, + providerInstanceId: input.reported.instanceId, + childThreadId: input.childThreadId, + prompt: "Review the design", + title: "review design", + status: input.status, + result: null, + startedAt: input.now, + completedAt: input.status === "running" ? null : input.now, + updatedAt: input.now, + origin: "provider_native" as const, + }; + return { + type: "subagent.updated", + driver: CODEX_DRIVER, + subagent: { + ...common, + id, + parentNodeId: NodeId.make("node:root"), + createdBy: "agent", + providerThreadId: null, + nativeTaskRef: null, + model: input.reported.model, + modelSelection: input.reported, + progress: "Reading the design", + }, + turnItem: { + ...common, + id: TurnItemId.make(`turn-item:${input.key}`), + nodeId: id, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + type: "subagent", + subagentId: id, + progress: "Reading the design", + }, + } satisfies ProviderAdapter.ProviderAdapterV2Event; +}; + +const createChildThread = (key: string) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const eventSink = yield* EventSink.EventSinkV2; + const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const rootEvent = yield* threadCreatedEvent(now); + if (rootEvent.type !== "thread.created") { + throw new Error("Expected a thread.created fixture event"); + } + const childThreadId = idAllocator.derive.threadFromProviderThread({ + driver: CODEX_DRIVER, + nativeThreadId: key, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: rootEvent.threadId, + }); + const ingest = (event: ProviderEventIngestor.ProviderEventIngestInput["event"]) => + ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: rootEvent.threadId, + event, + }); + yield* eventSink.write({ events: [rootEvent] }); + yield* ingest({ + type: "app_thread.created", + driver: CODEX_DRIVER, + appThread: { + ...rootEvent.payload, + id: childThreadId, + title: "review design", + activeProviderThreadId: null, + lineage: { + parentThreadId: rootEvent.threadId, + relationshipToParent: "subagent", + rootThreadId: rootEvent.threadId, + }, + }, + }); + // Every member of an emission goes through the same ingestion as an event of its own. + const ingestEmission = (event: ProviderEventIngestor.ProviderEventIngestInput["event"]) => { + const members = Effect.forEach(ProviderAdapter.providerEventMembers(event), ingest, { + discard: true, + }); + return event.type === "subagent.updated" && event.turnItem !== undefined + ? ingestor.ingestTogether(members) + : members; + }; + return { now, rootThreadId: rootEvent.threadId, childThreadId, ingestEmission }; + }); + +it.effect( + "commits a subagent record and its item at once, then syncs the child's model after it", + () => { + let commits = 0; + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (time, exit) => { + end(time, exit); + if (span.name === "COMMIT") commits++; + }; + return span; + }, + }); + return Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const eventStore = yield* EventStore.EventStoreV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const { now, rootThreadId, childThreadId, ingestEmission } = + yield* createChildThread("native-paired-subagent"); + const reported = { instanceId: modelSelection.instanceId, model: "gpt-6.1-sol" }; + const highWater = yield* eventSink.latestSequence(); + commits = 0; + yield* ingestEmission( + pairedSubagentUpdate({ + threadId: rootThreadId, + childThreadId, + key: "paired-subagent", + status: "running", + reported, + now, + }), + ); + const stored = yield* eventStore.read({ afterSequence: highWater }).pipe(Stream.runCollect); + assert.deepEqual( + stored.map(({ event }) => [event.type, event.threadId]), + [ + ["subagent.updated", rootThreadId], + ["turn-item.updated", rootThreadId], + ["thread.model-selection-updated", childThreadId], + ], + ); + // The record and its item share a commit; the model sync takes the lock after it. + assert.equal(commits, 2); + assert.deepEqual((yield* projectionStore.getThread(childThreadId)).modelSelection, reported); + const item = yield* projectionStore.getTurnItem({ + threadId: rootThreadId, + itemId: TurnItemId.make("turn-item:paired-subagent"), + }); + assert.equal(item?.type === "subagent" ? item.progress : undefined, "Reading the design"); + }).pipe( + Effect.provide(Layer.fresh(layerTest)), + Effect.withTracer(tracer), + Effect.withTracerEnabled(true), + Effect.provideService(Statement.SpanPropagationEnabled, true), + ); + }, +); + +// On a file, lock-free reads see committed state on reader workers while a write holds the writer. +const layerOnFile = (filename: string) => { + const stores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( + Layer.provideMerge( + SqlitePersistence.layerFromPath(filename).pipe(Layer.provide(NodeServices.layer)), + ), + ); + const shared = Layer.mergeAll( + layerEmptyInstances, + stores, + EventSink.layer.pipe(Layer.provide(stores)), + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerGit, + ); + return Layer.mergeAll(shared, ProviderEventIngestor.layer.pipe(Layer.provide(shared))); +}; + +it.effect( + "restores a native subagent's model that a command changed while its update was queued", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "ingestor-queued-model-" }); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const eventStore = yield* EventStore.EventStoreV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor; + const { now, rootThreadId, childThreadId, ingestEmission } = yield* createChildThread( + "native-queued-model-subagent", + ); + const highWater = yield* eventSink.latestSequence(); + + const writerHeld = yield* Deferred.make(); + const release = yield* Deferred.make(); + const pulled = yield* Queue.unbounded(); + const input = yield* Queue.unbounded< + ProviderEventIngestor.ProviderEventIngestInput["event"], + Cause.Done + >(); + const holder: ProviderEventIngestor.ProviderEventIngestInput["event"] = { + type: "message.updated", + driver: CODEX_DRIVER, + message: { + id: MessageId.make("message:writer-holder"), + threadId: rootThreadId, + runId: null, + nodeId: null, + createdBy: "agent", + creationSource: "provider", + role: "assistant", + text: "holds the shared commit", + attachments: [], + streaming: true, + createdAt: now, + updatedAt: now, + }, + }; + const consumer = yield* eventSink + .consumeProviderEvents( + Stream.fromQueue(input).pipe(Stream.tap((event) => Queue.offer(pulled, event.type))), + (event) => + event === holder + ? Deferred.succeed(writerHeld, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ) + : ingestEmission(event), + Effect.succeed(false), + Effect.succeed(true), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Queue.offer(input, holder); + yield* Deferred.await(writerHeld); + + // As thread.model-selection.set does: lock the thread, read it, queue the full row. + const userSelection = { instanceId: modelSelection.instanceId, model: "gpt-5.5" }; + const commandRead = yield* Deferred.make(); + const command = yield* threadCommands + .withLock( + childThreadId, + Effect.gen(function* () { + const thread = yield* projectionStore.getThread(childThreadId); + yield* Deferred.succeed(commandRead, undefined); + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: childThreadId }), + type: "thread.model-selection-updated", + threadId: childThreadId, + occurredAt: now, + payload: { ...thread, modelSelection: userSelection, updatedAt: now }, + }, + ], + }); + }), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(commandRead); + yield* TestClock.adjust(0); + + // The provider still reports the model the thread had before the queued command. + yield* Queue.offer( + input, + pairedSubagentUpdate({ + threadId: rootThreadId, + childThreadId, + key: "queued-model-subagent", + status: "completed", + reported: modelSelection, + now, + }), + ); + assert.deepEqual(yield* Queue.takeN(pulled, 2), ["message.updated", "subagent.updated"]); + yield* TestClock.adjust(0); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(command); + yield* Queue.end(input); + yield* Fiber.join(consumer); + + // As on main, the sync sees the command's model after the record commits and restores it. + const stored = yield* eventStore.read({ afterSequence: highWater }).pipe(Stream.runCollect); + assert.deepEqual( + stored.map(({ event }) => [ + event.type, + event.threadId, + event.type === "thread.model-selection-updated" ? event.payload.modelSelection : null, + ]), + [ + ["thread.model-selection-updated", childThreadId, userSelection], + ["subagent.updated", rootThreadId, null], + ["turn-item.updated", rootThreadId, null], + ["thread.model-selection-updated", childThreadId, modelSelection], + ], + ); + assert.deepEqual( + (yield* projectionStore.getThread(childThreadId)).modelSelection, + modelSelection, + ); + }).pipe(Effect.provide(layerOnFile(`${directory}/statev2.sqlite`))); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + const ingestCompletedProviderTurn = ( guard: Pick< Parameters[0], diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 623fddb491108..f88eaaee0f54a 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -222,8 +222,21 @@ export interface ProviderEventIngestorV2Shape { }; }, ) => Effect.Effect, ProviderEventIngestorV2Error>; + /** + * Commits every write `ingest` makes in one transaction, then runs what they leave for after + * the commit. `ingest` must not take a thread lock: the transaction is open while it runs. + */ + readonly ingestTogether: ( + ingest: Effect.Effect, + ) => Effect.Effect; } +// Taking a thread lock inside a write transaction deadlocks a command that holds the lock and waits +// to write, so ingestTogether runs this work after its commit. +const AfterCommit = Context.Reference< + Array> | undefined +>("t3/orchestration-v2/ProviderEventIngestor/AfterCommit", { defaultValue: () => undefined }); + export class ProviderEventIngestorV2 extends Context.Service< ProviderEventIngestorV2, ProviderEventIngestorV2Shape @@ -699,6 +712,15 @@ export const layer: Layer.Layer< return ProviderEventIngestorV2.of({ normalize, + // Runs for each adapter emission; the provider batch owns the span. + ingestTogether: Effect.fnUntraced(function* (ingest: Effect.Effect) { + const afterCommit: Array> = []; + const result = yield* eventSink.withProviderBatch( + Effect.provideService(ingest, AfterCommit, afterCommit), + ); + for (const work of afterCommit) yield* work; + return result; + }), ingestNormalized: /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced( @@ -802,22 +824,26 @@ export const layer: Layer.Layer< ), ), ), - // @effect-diagnostics-next-line flatMapConditionalToFilterOrFail:off - The predicate narrows the captured event for the subagent update. - Effect.flatMap((storedEvents) => - storedEvents.length === 0 || input.event.type !== "subagent.updated" - ? Effect.succeed(storedEvents) - : syncSubagentThreadModel(input, input.event.subagent).pipe( - Effect.map((synced) => [...storedEvents, ...synced]), - Effect.mapError( - (cause) => - new ProviderEventPublishError({ - providerSessionId: input.providerSessionId, - eventCount: 1, - cause, - }), - ), - ), - ), + Effect.flatMap((storedEvents) => { + if (storedEvents.length === 0 || input.event.type !== "subagent.updated") + return Effect.succeed(storedEvents); + const sync = syncSubagentThreadModel(input, input.event.subagent).pipe( + Effect.mapError( + (cause) => + new ProviderEventPublishError({ + providerSessionId: input.providerSessionId, + eventCount: 1, + cause, + }), + ), + ); + return Effect.flatMap(AfterCommit, (afterCommit) => { + if (afterCommit === undefined) + return Effect.map(sync, (synced) => [...storedEvents, ...synced]); + afterCommit.push(sync); + return Effect.succeed(storedEvents); + }); + }), Effect.tap((storedEvents) => Effect.gen(function* () { if (storedEvents.length === 0 || input.event.type !== "provider_turn.updated") diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index eccbb8157ac46..179cf4cc65d84 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2415,7 +2415,9 @@ export const layerWithOptions = ( readonly armed?: ProviderRetryItem; readonly settled?: ProviderTurnId; } => { - noteThreadActivity(event); + for (const member of ProviderAdapter.providerEventMembers(event)) { + noteThreadActivity(member); + } if (event.type === "turn_item.updated") { const item = event.turnItem; if ( diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 0e584bcf0c0a5..2cf2fc53de609 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -1954,6 +1954,51 @@ it.effect("keeps ingesting a child thread's late background item completion", () }), ); +it.effect.each([false, true])( + "handles a subagent record and its paired item as two events in one write (batched: %s)", + (batched) => + Effect.gen(function* () { + const writes: Array = []; + const observed = yield* runBackgroundItemScenario( + `paired-subagent-item-${batched}`, + (ids) => { + const record = subagentEvent(ids, "running"); + const item = backgroundTurnItemEvent(ids, "subagent", "running", 1); + if (record.type !== "subagent.updated" || item.type !== "turn_item.updated") + throw new Error("Missing subagent fixture"); + return [ + { ...record, turnItem: item.turnItem } as ProviderAdapter.ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + subagentEvent(ids, "completed"), + // Only the paired item's running state keeps the run ingesting until this one. + backgroundTurnItemEvent(ids, "subagent", "completed", 2), + ]; + }, + { + batched, + onIngest: (event, write) => + Effect.sync(() => { + writes.push(`${event.type} ${write.together ?? "alone"}`); + }), + }, + ); + assert.deepEqual(observed, [ + "subagent:running", + "turn_item:running", + "root-finalized", + "subagent:completed", + "turn_item:completed", + ]); + assert.deepEqual(writes, [ + "subagent.updated 1", + "turn_item.updated 1", + "turn.terminal alone", + "subagent.updated alone", + "turn_item.updated alone", + ]); + }), +); + it.effect.each([null, "run:nested-sdk"] as const)( "releases a settled run when its live child transfers to another run (%s)", (nestedRunId) => @@ -3994,7 +4039,7 @@ function runBackgroundItemScenario( options?: { readonly onIngest?: ( event: ProviderAdapter.ProviderAdapterV2Event, - write: { readonly inSharedCommit: boolean }, + write: { readonly inSharedCommit: boolean; readonly together: number | undefined }, ) => Effect.Effect; // Consumes through the real commit coordinator, as the event sink does. readonly batched?: boolean; @@ -4024,6 +4069,8 @@ function runBackgroundItemScenario( Effect.provide(PreviewAgentActivity.layer), ); let inSharedCommit = false; + let togetherCalls = 0; + let together: number | undefined; const coordinator = options?.batched === true ? yield* ProviderCommitBatch.make( @@ -4071,7 +4118,7 @@ function runBackgroundItemScenario( ingestNormalized: (input) => { const write = Effect.gen(function* () { const event = input.event; - yield* options?.onIngest?.(event, { inSharedCommit }) ?? Effect.void; + yield* options?.onIngest?.(event, { inSharedCommit, together }) ?? Effect.void; if (event.type === "turn_item.updated") { yield* Ref.update(observed, (current) => [ ...current, @@ -4092,6 +4139,17 @@ function runBackgroundItemScenario( // A write outside a batch waits for queued snapshots, as the event sink's does. return coordinator === undefined ? write : coordinator.barrier(write); }, + ingestTogether: (ingest) => + Effect.suspend(() => { + together = ++togetherCalls; + return ingest; + }).pipe( + Effect.ensuring( + Effect.sync(() => { + together = undefined; + }), + ), + ), }), ServerSettings.layerTest(), ), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index 612a130e019aa..6b637809bf74f 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1686,7 +1686,15 @@ export const layer: Layer.Layer< event.type === "idle_flush" ? flushIdleAssistantText(event.at) : Effect.gen(function* () { - yield* ingestEvent(yield* routeEvent(event)); + const members = ProviderAdapter.providerEventMembers(event); + const ingestMembers = Effect.forEach( + members, + (member) => Effect.flatMap(routeEvent(member), ingestEvent), + { discard: true }, + ); + yield* members.length === 1 + ? ingestMembers + : providerEventIngestor.ingestTogether(ingestMembers); }), Effect.gen(function* () { if (yield* agentTurnsSettled) yield* endPreviewActivity; diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 7232eb2eb2b53..e2732fce6422f 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1650,6 +1650,24 @@ const OrchestrationV2LinkRequestFields = { failure: Schema.optional(Schema.String), } as const; +export const OrchestrationV2SubagentTurnItem = Schema.Struct({ + ...OrchestrationV2TurnItemBaseFields, + type: Schema.Literal("subagent"), + subagentId: NodeId, + origin: Schema.Literals(["provider_native", "app_owned"]), + driver: ProviderDriverKind, + providerInstanceId: ProviderInstanceId, + childThreadId: Schema.NullOr(ThreadId), + remoteChild: Schema.optional(OrchestrationV2RemoteTaskChild), + prompt: Schema.String, + progress: Schema.optional(Schema.String), + result: Schema.NullOr(Schema.String), + /** Same as the prompt and result of subagent `subagentId`. */ + promptOmitted: DedupedBodyMarker, + resultOmitted: DedupedBodyMarker, +}); +export type OrchestrationV2SubagentTurnItem = typeof OrchestrationV2SubagentTurnItem.Type; + export const OrchestrationV2TurnItem = Schema.Union([ Schema.Struct({ ...OrchestrationV2TurnItemBaseFields, @@ -1852,22 +1870,7 @@ export const OrchestrationV2TurnItem = Schema.Union([ ...OrchestrationV2TurnItemBaseFields, ...OrchestrationV2LinkRequestFields, }), - Schema.Struct({ - ...OrchestrationV2TurnItemBaseFields, - type: Schema.Literal("subagent"), - subagentId: NodeId, - origin: Schema.Literals(["provider_native", "app_owned"]), - driver: ProviderDriverKind, - providerInstanceId: ProviderInstanceId, - childThreadId: Schema.NullOr(ThreadId), - remoteChild: Schema.optional(OrchestrationV2RemoteTaskChild), - prompt: Schema.String, - progress: Schema.optional(Schema.String), - result: Schema.NullOr(Schema.String), - /** Same as the prompt and result of subagent `subagentId`. */ - promptOmitted: DedupedBodyMarker, - resultOmitted: DedupedBodyMarker, - }), + OrchestrationV2SubagentTurnItem, Schema.Struct({ ...OrchestrationV2TurnItemBaseFields, type: Schema.Literal("dynamic_tool"), diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 8278dcc35fdfd..246a6c62fae97 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -3404,10 +3404,10 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( completedAt: subagent.task.completedAt, }, }); - yield* emitProviderEvent({ type: "subagent.updated", driver, subagent: subagent.task }); yield* emitProviderEvent({ - type: "turn_item.updated", + type: "subagent.updated", driver, + subagent: subagent.task, turnItem: { id: turnItemId, threadId: context.input.threadId, @@ -6061,10 +6061,6 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( type: "subagent.updated", driver, subagent: subagent.task, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver, turnItem: { id: subagent.turnItemId, threadId: subagent.task.threadId, diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index 2a2537c8bf866..3ffc4b1c22e5c 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -17,6 +17,7 @@ import { OrchestrationV2ProviderTurn, OrchestrationV2RuntimeRequest, OrchestrationV2Subagent, + OrchestrationV2SubagentTurnItem, OrchestrationV2TurnItem, ProviderApprovalDecision, ProviderInteractionMode, @@ -117,6 +118,8 @@ export const ProviderAdapterV2Event = Schema.Union([ type: Schema.Literal("subagent.updated"), driver: ProviderDriverKind, subagent: OrchestrationV2Subagent, + /** The subagent's own item when nothing comes between them, so both commit in one write. */ + turnItem: Schema.optional(OrchestrationV2SubagentTurnItem), }), Schema.Struct({ type: Schema.Literal("message.updated"), @@ -203,6 +206,15 @@ export const ProviderAdapterV2Event = Schema.Union([ ]); export type ProviderAdapterV2Event = typeof ProviderAdapterV2Event.Type; +/** The events an emission stands for, in order; each gets the per-event handling of its own. */ +export function providerEventMembers( + event: ProviderAdapterV2Event, +): ReadonlyArray { + if (event.type !== "subagent.updated" || event.turnItem === undefined) return [event]; + const { turnItem, ...record } = event; + return [record, { type: "turn_item.updated", driver: event.driver, turnItem }]; +} + export class ProviderAdapterCapabilitiesError extends Schema.TaggedError()( "ProviderAdapterCapabilitiesError", { diff --git a/packages/provider-cursor/src/server/adapter.ts b/packages/provider-cursor/src/server/adapter.ts index 243cf7341d405..507c44813ee5d 100644 --- a/packages/provider-cursor/src/server/adapter.ts +++ b/packages/provider-cursor/src/server/adapter.ts @@ -1853,10 +1853,6 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( type: "subagent.updated", driver: CursorAgentSdk.CURSOR_PROVIDER, subagent: task, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: CursorAgentSdk.CURSOR_PROVIDER, turnItem: { id: subagent.turnItemId, threadId: input.context.input.threadId, diff --git a/packages/provider-opencode/src/server/adapter.test.ts b/packages/provider-opencode/src/server/adapter.test.ts index 91400661e6eed..57a1e66cc3d74 100644 --- a/packages/provider-opencode/src/server/adapter.test.ts +++ b/packages/provider-opencode/src/server/adapter.test.ts @@ -1175,6 +1175,9 @@ describe("OpenCodeAdapterV2", () => { }); yield* harness.startTurn(); const received = yield* harness.runtime.events.pipe( + Stream.flatMap((event) => + Stream.fromIterable(ProviderAdapter.providerEventMembers(event)), + ), Stream.takeUntil( (event) => event.type === "turn_item.updated" && event.turnItem.type === "compaction", ), diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index 4459a219aa7d3..b718f382f39a7 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -1493,10 +1493,6 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function type: "subagent.updated", driver: OPENCODE_PROVIDER, subagent, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: OPENCODE_PROVIDER, turnItem: { id: turnItemId, threadId: turn.threadId, diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index b13f6c361bacf..b50b4a63447e9 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -1519,10 +1519,6 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr completedAt, updatedAt, }, - }); - yield* emit({ - type: "turn_item.updated", - driver, turnItem: { id: idAllocator.derive.turnItemFromProviderItem({ driver, nativeItemId: call.nativeId }), threadId: turn.input.threadId, diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index 65f3cc6c4344d..3cba48fd78999 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -3162,18 +3162,9 @@ describe("PiAdapterV2", () => { assert.isTrue( doneCard.type === "subagent.updated" && doneCard.subagent.result === "repo has one file" && - doneCard.subagent.childThreadId === null, - ); - const subagentItem = yield* takeEvent( - (event) => - event.type === "turn_item.updated" && - event.turnItem.type === "subagent" && - event.turnItem.status === "completed", - ); - assert.isTrue( - subagentItem.type === "turn_item.updated" && - subagentItem.turnItem.type === "subagent" && - subagentItem.turnItem.childThreadId === null, + doneCard.subagent.childThreadId === null && + doneCard.turnItem?.status === "completed" && + doneCard.turnItem.childThreadId === null, ); }).pipe(Effect.scoped, Effect.provide(layerTest)), ); diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index c237095e5e16a..3bb7b1d3e8097 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -1348,10 +1348,6 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( completedAt: finished ? emittedAt : null, updatedAt: emittedAt, }, - }); - yield* emit({ - type: "turn_item.updated", - driver: PI_PROVIDER, turnItem: { ...baseItemFields(turn, nativeTaskId, startedAt, emittedAt), status, From db082eb71df35c14d6ce9bde9d1fc49f2bb5f997 Mon Sep 17 00:00:00 2001 From: Ivan Kosilov Date: Mon, 16 Nov 2026 16:09:39 +0500 Subject: [PATCH 2/4] fix(server): a rolled-back subagent pair leaves the run's open work as stored The run tracked each member of a paired ingestion's open subagents, nodes and items as it ingested it, inside the transaction. When the item's write or the COMMIT failed, the record rolled back while the run had already dropped the settled subagent from the open work that failure finalization ends, so the stored subagent stayed running. Tracking now waits for the commit through the same after-commit queue as the child model sync and is skipped when the transaction fails. Outside a paired ingestion it runs at once, as before. Co-Authored-By: Claude Opus 5.5 --- .../orchestration-v2/ProviderEventIngestor.ts | 26 +++++- .../RunExecutionService.test.ts | 92 +++++++++++++++++-- .../orchestration-v2/RunExecutionService.ts | 10 +- 3 files changed, 110 insertions(+), 18 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 7e6c9818aee04..c9e0a43dfb256 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -239,6 +239,24 @@ const AfterCommit = Context.Reference< Array> | undefined >("t3/orchestration-v2/ProviderEventIngestor/AfterCommit", { defaultValue: () => undefined }); +/** Runs `ingest` through `commit`, then the work it left for after the commit, none if it fails. */ +// Runs for each adapter emission; the provider batch owns the span. +export const commitThenAfterCommit = Effect.fnUntraced(function* ( + commit: (ingest: Effect.Effect) => Effect.Effect, + ingest: Effect.Effect, +) { + const queue: Array> = []; + const result = yield* commit(Effect.provideService(ingest, AfterCommit, queue)); + for (const work of queue) yield* work; + return result; +}); + +/** Runs `work` after the commit of the enclosing `ingestTogether`, or now outside one. */ +export const afterCommit = (work: Effect.Effect) => + Effect.flatMap(AfterCommit, (queue) => + queue === undefined ? work : Effect.sync(() => void queue.push(work)), + ); + export class ProviderEventIngestorV2 extends Context.Service< ProviderEventIngestorV2, ProviderEventIngestorV2Shape @@ -758,12 +776,10 @@ export const layer: Layer.Layer< normalize, // Runs for each adapter emission; the provider batch owns the span. ingestTogether: Effect.fnUntraced(function* (ingest: Effect.Effect) { - const afterCommit: Array> = []; - const result = yield* eventSink.withProviderBatch( - Effect.provideService(ingest, AfterCommit, afterCommit), + return yield* commitThenAfterCommit( + (work: Effect.Effect) => eventSink.withProviderBatch(work), + ingest, ); - for (const work of afterCommit) yield* work; - return result; }), ingestNormalized: /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 9283a9ee47a28..31cab4f9948f8 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -3444,6 +3444,50 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c }), ); +it.effect("ends a subagent whose terminal record was rolled back with its paired item", () => + Effect.gen(function* () { + const key = "paired-terminal-rollback"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const fixture = (status: "running" | "interrupted") => + ({ ids, providerInstanceId, childThreadId: ids.childThreadId, driver, status }) as const; + const pair = (status: "running" | "interrupted") => ({ + type: "subagent.updated" as const, + driver, + subagent: makeRunOwnedSubagentFixture(fixture(status)), + turnItem: makeRunOwnedSubagentTurnItemFixture(fixture(status)), + }); + const { finalEvents } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + events: () => + Stream.make(pair("running"), pair("interrupted"), rootTerminalEvent(ids, "completed")), + ingestNormalized: (request) => + request.event.type === "turn_item.updated" && + request.event.turnItem.status === "interrupted" + ? Effect.fail( + new ProviderEventIngestor.ProviderEventPublishError({ + providerSessionId: ProviderSessionId.make(`session:${key}`), + eventCount: 1, + cause: "disk full", + }), + ) + : Effect.succeed([]), + }); + const subagentId = makeRunOwnedSubagentFixture(fixture("running")).id; + assert.deepEqual( + finalEvents.flatMap((event) => + event.type === "subagent.updated" && event.payload.id === subagentId + ? [`subagent ${event.payload.status}`] + : event.type === "turn-item.updated" && event.payload.type === "subagent" + ? [`item ${event.payload.status}`] + : [], + ), + ["subagent failed", "item failed"], + ); + }), +); + it.effect.each([ { status: "completed", failure: "later-stream", interrupt: "succeeds" }, { status: "failed", failure: "later-stream", interrupt: "succeeds" }, @@ -3998,7 +4042,11 @@ function captureRootRunTermination(input: { return Effect.gen(function* () { const ids = backgroundScenarioIds(input.key); // The rows the provider's events stored, which the run reads back when its runtime is lost. - const stored = { + const stored: { + subagents: Map; + nodes: Map; + turnItems: Map; + } = { subagents: new Map(), nodes: new Map(), turnItems: new Map(), @@ -4109,6 +4157,26 @@ function captureRootRunTermination(input: { store(request.event).pipe( Effect.andThen(input.ingestNormalized?.(request) ?? Effect.succeed([])), ), + // A failed transaction stores none of its rows. + ingestTogether: (ingest) => + ProviderEventIngestor.commitThenAfterCommit( + (work) => + Effect.suspend(() => { + const subagents = new Map(stored.subagents); + const nodes = new Map(stored.nodes); + const turnItems = new Map(stored.turnItems); + return work.pipe( + Effect.onError(() => + Effect.sync(() => { + stored.subagents = subagents; + stored.nodes = nodes; + stored.turnItems = turnItems; + }), + ), + ); + }), + ingest, + ), }), ServerSettings.layerTest(), Layer.succeed(RunFinalizationService.RunFinalizationObserver, { @@ -4681,15 +4749,19 @@ function runBackgroundItemScenario( return coordinator === undefined ? write : coordinator.barrier(write); }, ingestTogether: (ingest) => - Effect.suspend(() => { - together = ++togetherCalls; - return ingest; - }).pipe( - Effect.ensuring( - Effect.sync(() => { - together = undefined; - }), - ), + ProviderEventIngestor.commitThenAfterCommit( + (work) => + Effect.suspend(() => { + together = ++togetherCalls; + return work; + }).pipe( + Effect.ensuring( + Effect.sync(() => { + together = undefined; + }), + ), + ), + ingest, ), }), ServerSettings.layerTest(), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index e0adba4268098..2a6cfa088f450 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1411,8 +1411,8 @@ export const layer: Layer.Layer< const trackChildLifecycle = Effect.fnUntraced(function* ( event: ProviderAdapter.ProviderAdapterV2Event, deliverable: boolean, + routing: ProviderEventRoutingState, ) { - const routing = yield* Ref.get(eventRouting); if (event.type === "provider_turn.updated") { const isRoot = event.providerTurn.runAttemptId === input.attempt.id || @@ -1815,7 +1815,7 @@ export const layer: Layer.Layer< continue; // A failed write fails ingestion like any provider write. yield* ingestDelivered(event, event); - yield* trackChildLifecycle(event, true); + yield* trackChildLifecycle(event, true, yield* Ref.get(eventRouting)); } yield* scheduleIdleFlush; }); @@ -1926,7 +1926,11 @@ export const layer: Layer.Layer< pendingBackgroundProviderTurns.delete(event.providerTurnId); } } - yield* trackChildLifecycle(event, deliveredEvent !== null); + // Failure finalization ends the open work this tracks, so it must match what is stored: + // inside a paired ingestion it waits for the commit and is skipped if that rolls back. + yield* ProviderEventIngestor.afterCommit( + trackChildLifecycle(event, deliveredEvent !== null, yield* Ref.get(eventRouting)), + ); yield* scheduleIdleFlush; }); const providerEventFiber = yield* eventSink From 54ad7ae01b933e31416ff62a7ab8447a81e0d172 Mon Sep 17 00:00:00 2001 From: Ivan Kosilov Date: Mon, 16 Nov 2026 16:20:34 +0500 Subject: [PATCH 3/4] fix(server): a failed child model sync still settles its committed subagent pair After a paired ingestion committed, its after-commit queue ran the child model sync before the members' lifecycle tracking, and stopped at the first failure. A failed sync write left both committed members open in the run's view, so failure finalization wrote the settled item back as failed. The queue now holds the run's bookkeeping, which cannot fail, apart from follow-up writes: once the pair commits, all bookkeeping applies, without an interruption between the commit and it, and only then do the writes run. Co-Authored-By: Claude Opus 5.5 --- .../orchestration-v2/ProviderEventIngestor.ts | 56 +++++++++++++------ .../RunExecutionService.test.ts | 44 +++++++++++++++ 2 files changed, 82 insertions(+), 18 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index c9e0a43dfb256..eeb815f43c243 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -233,28 +233,49 @@ export interface ProviderEventIngestorV2Shape { ) => Effect.Effect; } -// Taking a thread lock inside a write transaction deadlocks a command that holds the lock and waits -// to write, so ingestTogether runs this work after its commit. -const AfterCommit = Context.Reference< - Array> | undefined ->("t3/orchestration-v2/ProviderEventIngestor/AfterCommit", { defaultValue: () => undefined }); +interface AfterCommitQueue { + /** The run's view of what is stored; it cannot fail, so all of it applies once stored. */ + readonly bookkeeping: Array>; + // Taking a thread lock inside a write transaction deadlocks a command that holds the lock and + // waits to write, so these writes wait for the commit. + readonly writes: Array>; +} +const AfterCommit = Context.Reference( + "t3/orchestration-v2/ProviderEventIngestor/AfterCommit", + { defaultValue: () => undefined }, +); -/** Runs `ingest` through `commit`, then the work it left for after the commit, none if it fails. */ +/** + * Runs `ingest` through `commit`. Once it commits, applies all of its bookkeeping, then its + * follow-up writes; nothing if it fails. + */ // Runs for each adapter emission; the provider batch owns the span. export const commitThenAfterCommit = Effect.fnUntraced(function* ( commit: (ingest: Effect.Effect) => Effect.Effect, ingest: Effect.Effect, ) { - const queue: Array> = []; - const result = yield* commit(Effect.provideService(ingest, AfterCommit, queue)); - for (const work of queue) yield* work; + const queue: AfterCommitQueue = { bookkeeping: [], writes: [] }; + const result = yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const committed = yield* commit(restore(Effect.provideService(ingest, AfterCommit, queue))); + for (const work of queue.bookkeeping) yield* work; + return committed; + }), + ); + for (const work of queue.writes) yield* work; return result; }); -/** Runs `work` after the commit of the enclosing `ingestTogether`, or now outside one. */ -export const afterCommit = (work: Effect.Effect) => +/** Applies `work` once the enclosing `ingestTogether` commits, or now outside one. */ +export const afterCommit = (work: Effect.Effect) => Effect.flatMap(AfterCommit, (queue) => - queue === undefined ? work : Effect.sync(() => void queue.push(work)), + queue === undefined ? work : Effect.sync(() => void queue.bookkeeping.push(work)), + ); + +/** Runs `write` after the enclosing `ingestTogether` commits and applies its bookkeeping. */ +export const writeAfterCommit = (write: Effect.Effect) => + Effect.flatMap(AfterCommit, (queue) => + queue === undefined ? Effect.asVoid(write) : Effect.sync(() => void queue.writes.push(write)), ); export class ProviderEventIngestorV2 extends Context.Service< @@ -892,12 +913,11 @@ export const layer: Layer.Layer< }), ), ); - return Effect.flatMap(AfterCommit, (afterCommit) => { - if (afterCommit === undefined) - return Effect.map(sync, (synced) => [...storedEvents, ...synced]); - afterCommit.push(sync); - return Effect.succeed(storedEvents); - }); + return Effect.flatMap(AfterCommit, (afterCommit) => + afterCommit === undefined + ? Effect.map(sync, (synced) => [...storedEvents, ...synced]) + : Effect.as(writeAfterCommit(sync), storedEvents), + ); }), Effect.tap((storedEvents) => Effect.gen(function* () { diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 31cab4f9948f8..9816c3d9b2da5 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -3488,6 +3488,50 @@ it.effect("ends a subagent whose terminal record was rolled back with its paired }), ); +it.effect("keeps a committed subagent pair settled when its child model sync fails", () => + Effect.gen(function* () { + const key = "paired-terminal-sync-failure"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const fixture = (status: "running" | "interrupted") => + ({ ids, providerInstanceId, childThreadId: ids.childThreadId, driver, status }) as const; + const pair = (status: "running" | "interrupted") => ({ + type: "subagent.updated" as const, + driver, + subagent: makeRunOwnedSubagentFixture(fixture(status)), + turnItem: makeRunOwnedSubagentTurnItemFixture(fixture(status)), + }); + const { finalEvents } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + events: () => + Stream.make(pair("running"), pair("interrupted"), rootTerminalEvent(ids, "completed")), + // The ingestor defers a child model sync, which writes, until the pair commits. + ingestNormalized: (request) => + request.event.type === "subagent.updated" && request.event.subagent.status === "interrupted" + ? ProviderEventIngestor.writeAfterCommit( + Effect.fail( + new ProviderEventIngestor.ProviderEventPublishError({ + providerSessionId: ProviderSessionId.make(`session:${key}`), + eventCount: 1, + cause: "disk full", + }), + ), + ).pipe(Effect.as([])) + : Effect.succeed([]), + }); + assert.deepEqual( + finalEvents.flatMap((event) => + event.type === "subagent.updated" || + (event.type === "turn-item.updated" && event.payload.type === "subagent") + ? [event.type] + : [], + ), + [], + ); + }), +); + it.effect.each([ { status: "completed", failure: "later-stream", interrupt: "succeeds" }, { status: "failed", failure: "later-stream", interrupt: "succeeds" }, From ebdfc0cedb0eb94e00fbc2cbae6b4cdc37ca7ecc Mon Sep 17 00:00:00 2001 From: Ivan Kosilov Date: Mon, 16 Nov 2026 16:26:20 +0500 Subject: [PATCH 4/4] test(server): Claude's progress-only subagent update carries its item Co-Authored-By: Claude Opus 5.5 --- .../orchestration-v2/Adapters/ClaudeAdapterV2.test.ts | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index b9e2f977a36ee..8f78d89b3cf83 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -17790,6 +17790,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { event.subagent.progress === "Reading the last commits", ), ); + const progressUpdate = harness.events.find( + (event) => + event.type === "subagent.updated" && + event.subagent.progress === "Reading the last commits", + ); + // A progress-only update carries its item, so ingestion stores both in one commit. + assert.equal( + progressUpdate?.type === "subagent.updated" ? progressUpdate.turnItem?.progress : undefined, + "Reading the last commits", + ); const routing = subagentRouting(harness.events, [STRADDLING_BASH, IDLE_BASH]); assert.deepEqual(routing.assistantTexts(routing.childThreadId), ["Still auditing."]); const straddling = latestToolItem(STRADDLING_BASH);