diff --git a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts index 863f2279ef2a3..b2b4d93e5596d 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpAdapterV2.test.ts @@ -8042,7 +8042,9 @@ describe("AcpAdapterV2", () => { }); const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -8223,7 +8225,9 @@ describe("AcpAdapterV2", () => { const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -8411,7 +8415,9 @@ describe("AcpAdapterV2", () => { const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -8653,7 +8659,9 @@ describe("AcpAdapterV2", () => { const hasPendingBackgroundWork = runtime.hasPendingBackgroundWork; const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -8930,7 +8938,9 @@ describe("AcpAdapterV2", () => { }); const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ @@ -11108,7 +11118,9 @@ describe("AcpAdapterV2", () => { }); const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ diff --git a/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts index ee82cc06de836..2ed9c1c04f50f 100644 --- a/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/AcpRegistryAdapterV2.test.ts @@ -295,7 +295,9 @@ const openClaudeReplaySession = Effect.fn("openClaudeReplaySession")(function* ( } const events = yield* Queue.unbounded(); yield* runtime.events.pipe( - Stream.runForEach((event) => Queue.offer(events, event)), + Stream.runForEach((event) => + Queue.offerAll(events, ProviderAdapter.providerEventMembers(event)), + ), Effect.forkScoped, ); const providerThread = yield* runtime.ensureThread({ threadId, modelSelection, runtimePolicy }); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts index 1d549d7f2c18b..a2e9f7c03195a 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.test.ts @@ -17834,6 +17834,16 @@ describe("ClaudeAdapterV2 background wake turns", () => { event.subagent.progress === "Reading the last commits", ), ); + const progressUpdate = harness.events.find( + (event) => + event.type === "subagent.updated" && + event.subagent.progress === "Reading the last commits", + ); + // A progress-only update carries its item, so ingestion stores both in one commit. + assert.equal( + progressUpdate?.type === "subagent.updated" ? progressUpdate.turnItem?.progress : undefined, + "Reading the last commits", + ); const routing = subagentRouting(harness.events, [STRADDLING_BASH, IDLE_BASH]); assert.deepEqual(routing.assistantTexts(routing.childThreadId), ["Still auditing."]); const straddling = latestToolItem(STRADDLING_BASH); diff --git a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts index 5dcb77f7abddc..03bf0a6b5f25a 100644 --- a/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/ClaudeAdapterV2.ts @@ -117,6 +117,7 @@ import { type OrchestrationV2RuntimeRequest, type OrchestrationV2UserInputQuestion, type OrchestrationV2Subagent, + type OrchestrationV2SubagentTurnItem, type OrchestrationV2AppThread, type OrchestrationV2SubagentWorkflow, type OrchestrationV2WorkflowRunHandles, @@ -8120,11 +8121,45 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }, }); } + const turnItem = { + id: subagent.turnItemId, + threadId: task.threadId, + runId: task.runId, + nodeId: task.id, + providerThreadId: input.context.input.providerThread.id, + providerTurnId: input.context.providerTurnId, + nativeItemRef: task.nativeTaskRef, + parentItemId: null, + ordinal: subagent.turnItemOrdinal, + status: task.status, + title: task.title, + startedAt: task.startedAt, + completedAt: task.completedAt, + updatedAt: task.updatedAt, + type: "subagent", + subagentId: task.id, + origin: task.origin, + driver: task.driver, + providerInstanceId: task.providerInstanceId, + childThreadId: task.childThreadId, + prompt: task.prompt, + ...(task.progress === undefined ? {} : { progress: task.progress }), + result: task.result, + } satisfies OrchestrationV2SubagentTurnItem; + const promptNativeItemId = + existingSubagent === undefined + ? `${nativeItemId}:prompt` + : resumeToolUseId === null + ? null + : `${nativeItemId}:prompt:${resumeToolUseId}`; + const itemFollowsRecord = + !lifecycleChanged && promptNativeItemId === null && workflowWithThreads === undefined; // Emit before child records so the resumed run can adopt the child thread. yield* emitProviderEvent({ type: "subagent.updated", driver: CLAUDE_PROVIDER, subagent: task, + ...(itemFollowsRecord ? { turnItem } : {}), }); if (lifecycleChanged) { yield* emitProviderEvent({ @@ -8149,12 +8184,6 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }, }); } - const promptNativeItemId = - existingSubagent === undefined - ? `${nativeItemId}:prompt` - : resumeToolUseId === null - ? null - : `${nativeItemId}:prompt:${resumeToolUseId}`; if (promptNativeItemId !== null) { yield* emitSubagentConversationMessage({ nativeItemId: promptNativeItemId, @@ -8178,35 +8207,13 @@ export const makeClaudeAdapterV2 = Effect.fn("makeClaudeAdapterV2")(function* ( }); if ((yield* Ref.get(sessionSubagentsByTaskId)).get(input.taskId) !== subagent) return; } - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: CLAUDE_PROVIDER, - turnItem: { - id: subagent.turnItemId, - threadId: task.threadId, - runId: task.runId, - nodeId: task.id, - providerThreadId: input.context.input.providerThread.id, - providerTurnId: input.context.providerTurnId, - nativeItemRef: task.nativeTaskRef, - parentItemId: null, - ordinal: subagent.turnItemOrdinal, - status: task.status, - title: task.title, - startedAt: task.startedAt, - completedAt: task.completedAt, - updatedAt: task.updatedAt, - type: "subagent", - subagentId: task.id, - origin: task.origin, - driver: task.driver, - providerInstanceId: task.providerInstanceId, - childThreadId: task.childThreadId, - prompt: task.prompt, - ...(task.progress === undefined ? {} : { progress: task.progress }), - result: task.result, - }, - }); + if (!itemFollowsRecord) { + yield* emitProviderEvent({ + type: "turn_item.updated", + driver: CLAUDE_PROVIDER, + turnItem, + }); + } if (input.nativeOutcome !== undefined && subagent.confirmedOutcome !== undefined) { const outcome = subagentStopOutcomes.get(input.taskId); diff --git a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts index c464b1e3c287a..7d887589cc33b 100644 --- a/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts +++ b/apps/server/src/orchestration-v2/Adapters/CodexAdapterV2.ts @@ -2972,10 +2972,6 @@ export const makeCodexAdapterV2 = Effect.fn("makeCodexAdapterV2")(function* ( type: "subagent.updated", driver: CODEX_PROVIDER, subagent: task, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: CODEX_PROVIDER, turnItem: { id: input.subagent.turnItemId, threadId: task.threadId, diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts index 7aa18f0956e39..b0bba264d94ae 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.test.ts @@ -23,6 +23,7 @@ import { RunAttemptId, RunId, RuntimeRequestId, + type ThreadId, TurnItemId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; @@ -33,10 +34,14 @@ import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; +import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as SqlError from "effect/sql/SqlError"; import * as Stream from "effect/Stream"; +import * as Statement from "effect/sql/Statement"; +import * as TestClock from "effect/testing/TestClock"; +import * as Tracer from "effect/Tracer"; import { toolOutputImages } from "@t3tools/shared/toolOutput"; import * as SqlitePersistence from "../persistence/Sqlite.ts"; @@ -2447,6 +2452,318 @@ layer("ProviderEventIngestorV2", (it) => { ); }); +/** A native subagent's record with its item, as an adapter emits them when nothing comes between. */ +const pairedSubagentUpdate = (input: { + readonly threadId: ThreadId; + readonly childThreadId: ThreadId; + readonly key: string; + readonly status: "running" | "completed"; + readonly reported: ModelSelection; + readonly now: DateTime.Utc; +}) => { + const id = NodeId.make(`node:${input.key}`); + const common = { + threadId: input.threadId, + runId: null, + driver: CODEX_DRIVER, + providerInstanceId: input.reported.instanceId, + childThreadId: input.childThreadId, + prompt: "Review the design", + title: "review design", + status: input.status, + result: null, + startedAt: input.now, + completedAt: input.status === "running" ? null : input.now, + updatedAt: input.now, + origin: "provider_native" as const, + }; + return { + type: "subagent.updated", + driver: CODEX_DRIVER, + subagent: { + ...common, + id, + parentNodeId: NodeId.make("node:root"), + createdBy: "agent", + providerThreadId: null, + nativeTaskRef: null, + model: input.reported.model, + modelSelection: input.reported, + progress: "Reading the design", + }, + turnItem: { + ...common, + id: TurnItemId.make(`turn-item:${input.key}`), + nodeId: id, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + parentItemId: null, + ordinal: 1, + type: "subagent", + subagentId: id, + progress: "Reading the design", + }, + } satisfies ProviderAdapter.ProviderAdapterV2Event; +}; + +const createChildThread = (key: string) => + Effect.gen(function* () { + const now = yield* DateTime.now; + const eventSink = yield* EventSink.EventSinkV2; + const ingestor = yield* ProviderEventIngestor.ProviderEventIngestorV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const rootEvent = yield* threadCreatedEvent(now); + if (rootEvent.type !== "thread.created") { + throw new Error("Expected a thread.created fixture event"); + } + const childThreadId = idAllocator.derive.threadFromProviderThread({ + driver: CODEX_DRIVER, + nativeThreadId: key, + }); + const providerSessionId = yield* idAllocator.allocate.providerSession({ + providerInstanceId: modelSelection.instanceId, + threadId: rootEvent.threadId, + }); + const ingest = (event: ProviderEventIngestor.ProviderEventIngestInput["event"]) => + ingestor.ingestNormalized({ + providerSessionId, + providerInstanceId: modelSelection.instanceId, + threadId: rootEvent.threadId, + event, + }); + yield* eventSink.write({ events: [rootEvent] }); + yield* ingest({ + type: "app_thread.created", + driver: CODEX_DRIVER, + appThread: { + ...rootEvent.payload, + id: childThreadId, + title: "review design", + activeProviderThreadId: null, + lineage: { + parentThreadId: rootEvent.threadId, + relationshipToParent: "subagent", + rootThreadId: rootEvent.threadId, + }, + }, + }); + // Every member of an emission goes through the same ingestion as an event of its own. + const ingestEmission = (event: ProviderEventIngestor.ProviderEventIngestInput["event"]) => { + const members = Effect.forEach(ProviderAdapter.providerEventMembers(event), ingest, { + discard: true, + }); + return event.type === "subagent.updated" && event.turnItem !== undefined + ? ingestor.ingestTogether(members) + : members; + }; + return { now, rootThreadId: rootEvent.threadId, childThreadId, ingestEmission }; + }); + +it.effect( + "commits a subagent record and its item at once, then syncs the child's model after it", + () => { + let commits = 0; + const tracer = Tracer.make({ + span(options) { + const span = new Tracer.NativeSpan(options); + const end = span.end.bind(span); + span.end = (time, exit) => { + end(time, exit); + if (span.name === "COMMIT") commits++; + }; + return span; + }, + }); + return Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const eventStore = yield* EventStore.EventStoreV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const { now, rootThreadId, childThreadId, ingestEmission } = + yield* createChildThread("native-paired-subagent"); + const reported = { instanceId: modelSelection.instanceId, model: "gpt-6.1-sol" }; + const highWater = yield* eventSink.latestSequence(); + commits = 0; + yield* ingestEmission( + pairedSubagentUpdate({ + threadId: rootThreadId, + childThreadId, + key: "paired-subagent", + status: "running", + reported, + now, + }), + ); + const stored = yield* eventStore.read({ afterSequence: highWater }).pipe(Stream.runCollect); + assert.deepEqual( + stored.map(({ event }) => [event.type, event.threadId]), + [ + ["subagent.updated", rootThreadId], + ["turn-item.updated", rootThreadId], + ["thread.model-selection-updated", childThreadId], + ], + ); + // The record and its item share a commit; the model sync takes the lock after it. + assert.equal(commits, 2); + assert.deepEqual((yield* projectionStore.getThread(childThreadId)).modelSelection, reported); + const item = yield* projectionStore.getTurnItem({ + threadId: rootThreadId, + itemId: TurnItemId.make("turn-item:paired-subagent"), + }); + assert.equal(item?.type === "subagent" ? item.progress : undefined, "Reading the design"); + }).pipe( + Effect.provide(Layer.fresh(layerTest)), + Effect.withTracer(tracer), + Effect.withTracerEnabled(true), + Effect.provideService(Statement.SpanPropagationEnabled, true), + ); + }, +); + +// On a file, lock-free reads see committed state on reader workers while a write holds the writer. +const layerOnFile = (filename: string) => { + const stores = Layer.merge(EventStore.layer, ProjectionStore.layer).pipe( + Layer.provideMerge( + SqlitePersistence.layerFromPath(filename).pipe(Layer.provide(NodeServices.layer)), + ), + ); + const shared = Layer.mergeAll( + layerEmptyInstances, + stores, + EventSink.layer.pipe(Layer.provide(stores)), + IdAllocator.layer, + ThreadCommandExecutor.layer, + layerGit, + ); + return Layer.mergeAll(shared, ProviderEventIngestor.layer.pipe(Layer.provide(shared))); +}; + +it.effect( + "restores a native subagent's model that a command changed while its update was queued", + () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const directory = yield* fs.makeTempDirectoryScoped({ prefix: "ingestor-queued-model-" }); + yield* Effect.gen(function* () { + const eventSink = yield* EventSink.EventSinkV2; + const eventStore = yield* EventStore.EventStoreV2; + const projectionStore = yield* ProjectionStore.ProjectionStoreV2; + const idAllocator = yield* IdAllocator.IdAllocatorV2; + const threadCommands = yield* ThreadCommandExecutor.ThreadCommandExecutor; + const { now, rootThreadId, childThreadId, ingestEmission } = yield* createChildThread( + "native-queued-model-subagent", + ); + const highWater = yield* eventSink.latestSequence(); + + const writerHeld = yield* Deferred.make(); + const release = yield* Deferred.make(); + const pulled = yield* Queue.unbounded(); + const input = yield* Queue.unbounded< + ProviderEventIngestor.ProviderEventIngestInput["event"], + Cause.Done + >(); + const holder: ProviderEventIngestor.ProviderEventIngestInput["event"] = { + type: "message.updated", + driver: CODEX_DRIVER, + message: { + id: MessageId.make("message:writer-holder"), + threadId: rootThreadId, + runId: null, + nodeId: null, + createdBy: "agent", + creationSource: "provider", + role: "assistant", + text: "holds the shared commit", + attachments: [], + streaming: true, + createdAt: now, + updatedAt: now, + }, + }; + const consumer = yield* eventSink + .consumeProviderEvents( + Stream.fromQueue(input).pipe(Stream.tap((event) => Queue.offer(pulled, event.type))), + (event) => + event === holder + ? Deferred.succeed(writerHeld, undefined).pipe( + Effect.andThen(Deferred.await(release)), + ) + : ingestEmission(event), + Effect.succeed(false), + Effect.succeed(true), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Queue.offer(input, holder); + yield* Deferred.await(writerHeld); + + // As thread.model-selection.set does: lock the thread, read it, queue the full row. + const userSelection = { instanceId: modelSelection.instanceId, model: "gpt-5.5" }; + const commandRead = yield* Deferred.make(); + const command = yield* threadCommands + .withLock( + childThreadId, + Effect.gen(function* () { + const thread = yield* projectionStore.getThread(childThreadId); + yield* Deferred.succeed(commandRead, undefined); + yield* eventSink.write({ + events: [ + { + id: yield* idAllocator.allocate.event({ threadId: childThreadId }), + type: "thread.model-selection-updated", + threadId: childThreadId, + occurredAt: now, + payload: { ...thread, modelSelection: userSelection, updatedAt: now }, + }, + ], + }); + }), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(commandRead); + yield* TestClock.adjust(0); + + // The provider still reports the model the thread had before the queued command. + yield* Queue.offer( + input, + pairedSubagentUpdate({ + threadId: rootThreadId, + childThreadId, + key: "queued-model-subagent", + status: "completed", + reported: modelSelection, + now, + }), + ); + assert.deepEqual(yield* Queue.takeN(pulled, 2), ["message.updated", "subagent.updated"]); + yield* TestClock.adjust(0); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(command); + yield* Queue.end(input); + yield* Fiber.join(consumer); + + // As on main, the sync sees the command's model after the record commits and restores it. + const stored = yield* eventStore.read({ afterSequence: highWater }).pipe(Stream.runCollect); + assert.deepEqual( + stored.map(({ event }) => [ + event.type, + event.threadId, + event.type === "thread.model-selection-updated" ? event.payload.modelSelection : null, + ]), + [ + ["thread.model-selection-updated", childThreadId, userSelection], + ["subagent.updated", rootThreadId, null], + ["turn-item.updated", rootThreadId, null], + ["thread.model-selection-updated", childThreadId, modelSelection], + ], + ); + assert.deepEqual( + (yield* projectionStore.getThread(childThreadId)).modelSelection, + modelSelection, + ); + }).pipe(Effect.provide(layerOnFile(`${directory}/statev2.sqlite`))); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + const ingestCompletedProviderTurn = ( guard: Pick< Parameters[0], diff --git a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts index 45ed0d1c78180..eeb815f43c243 100644 --- a/apps/server/src/orchestration-v2/ProviderEventIngestor.ts +++ b/apps/server/src/orchestration-v2/ProviderEventIngestor.ts @@ -224,8 +224,60 @@ export interface ProviderEventIngestorV2Shape { }; }, ) => Effect.Effect, ProviderEventIngestorV2Error>; + /** + * Commits every write `ingest` makes in one transaction, then runs what they leave for after + * the commit. `ingest` must not take a thread lock: the transaction is open while it runs. + */ + readonly ingestTogether: ( + ingest: Effect.Effect, + ) => Effect.Effect; } +interface AfterCommitQueue { + /** The run's view of what is stored; it cannot fail, so all of it applies once stored. */ + readonly bookkeeping: Array>; + // Taking a thread lock inside a write transaction deadlocks a command that holds the lock and + // waits to write, so these writes wait for the commit. + readonly writes: Array>; +} +const AfterCommit = Context.Reference( + "t3/orchestration-v2/ProviderEventIngestor/AfterCommit", + { defaultValue: () => undefined }, +); + +/** + * Runs `ingest` through `commit`. Once it commits, applies all of its bookkeeping, then its + * follow-up writes; nothing if it fails. + */ +// Runs for each adapter emission; the provider batch owns the span. +export const commitThenAfterCommit = Effect.fnUntraced(function* ( + commit: (ingest: Effect.Effect) => Effect.Effect, + ingest: Effect.Effect, +) { + const queue: AfterCommitQueue = { bookkeeping: [], writes: [] }; + const result = yield* Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const committed = yield* commit(restore(Effect.provideService(ingest, AfterCommit, queue))); + for (const work of queue.bookkeeping) yield* work; + return committed; + }), + ); + for (const work of queue.writes) yield* work; + return result; +}); + +/** Applies `work` once the enclosing `ingestTogether` commits, or now outside one. */ +export const afterCommit = (work: Effect.Effect) => + Effect.flatMap(AfterCommit, (queue) => + queue === undefined ? work : Effect.sync(() => void queue.bookkeeping.push(work)), + ); + +/** Runs `write` after the enclosing `ingestTogether` commits and applies its bookkeeping. */ +export const writeAfterCommit = (write: Effect.Effect) => + Effect.flatMap(AfterCommit, (queue) => + queue === undefined ? Effect.asVoid(write) : Effect.sync(() => void queue.writes.push(write)), + ); + export class ProviderEventIngestorV2 extends Context.Service< ProviderEventIngestorV2, ProviderEventIngestorV2Shape @@ -743,6 +795,13 @@ export const layer: Layer.Layer< return ProviderEventIngestorV2.of({ normalize, + // Runs for each adapter emission; the provider batch owns the span. + ingestTogether: Effect.fnUntraced(function* (ingest: Effect.Effect) { + return yield* commitThenAfterCommit( + (work: Effect.Effect) => eventSink.withProviderBatch(work), + ingest, + ); + }), ingestNormalized: /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced( @@ -841,22 +900,25 @@ export const layer: Layer.Layer< ), ), ), - // @effect-diagnostics-next-line flatMapConditionalToFilterOrFail:off - The predicate narrows the captured event for the subagent update. - Effect.flatMap((storedEvents) => - storedEvents.length === 0 || input.event.type !== "subagent.updated" - ? Effect.succeed(storedEvents) - : syncSubagentThreadModel(input, input.event.subagent).pipe( - Effect.map((synced) => [...storedEvents, ...synced]), - Effect.mapError( - (cause) => - new ProviderEventPublishError({ - providerSessionId: input.providerSessionId, - eventCount: 1, - cause, - }), - ), - ), - ), + Effect.flatMap((storedEvents) => { + if (storedEvents.length === 0 || input.event.type !== "subagent.updated") + return Effect.succeed(storedEvents); + const sync = syncSubagentThreadModel(input, input.event.subagent).pipe( + Effect.mapError( + (cause) => + new ProviderEventPublishError({ + providerSessionId: input.providerSessionId, + eventCount: 1, + cause, + }), + ), + ); + return Effect.flatMap(AfterCommit, (afterCommit) => + afterCommit === undefined + ? Effect.map(sync, (synced) => [...storedEvents, ...synced]) + : Effect.as(writeAfterCommit(sync), storedEvents), + ); + }), Effect.tap((storedEvents) => Effect.gen(function* () { if (storedEvents.length === 0 || input.event.type !== "provider_turn.updated") diff --git a/apps/server/src/orchestration-v2/ProviderSessionManager.ts b/apps/server/src/orchestration-v2/ProviderSessionManager.ts index ba9ba7a2bd3f3..fa0bda36cec7e 100644 --- a/apps/server/src/orchestration-v2/ProviderSessionManager.ts +++ b/apps/server/src/orchestration-v2/ProviderSessionManager.ts @@ -2679,7 +2679,9 @@ export const layerWithOptions = ( readonly armed?: ProviderRetryItem; readonly settled?: ProviderTurnId; } => { - noteThreadActivity(event); + for (const member of ProviderAdapter.providerEventMembers(event)) { + noteThreadActivity(member); + } if (event.type === "turn_item.updated") { const item = event.turnItem; if ( diff --git a/apps/server/src/orchestration-v2/RunExecutionService.test.ts b/apps/server/src/orchestration-v2/RunExecutionService.test.ts index 9a35624ecfe5f..9816c3d9b2da5 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.test.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.test.ts @@ -2196,6 +2196,51 @@ it.effect("keeps ingesting a child thread's late background item completion", () }), ); +it.effect.each([false, true])( + "handles a subagent record and its paired item as two events in one write (batched: %s)", + (batched) => + Effect.gen(function* () { + const writes: Array = []; + const observed = yield* runBackgroundItemScenario( + `paired-subagent-item-${batched}`, + (ids) => { + const record = subagentEvent(ids, "running"); + const item = backgroundTurnItemEvent(ids, "subagent", "running", 1); + if (record.type !== "subagent.updated" || item.type !== "turn_item.updated") + throw new Error("Missing subagent fixture"); + return [ + { ...record, turnItem: item.turnItem } as ProviderAdapter.ProviderAdapterV2Event, + rootTerminalEvent(ids, "completed"), + subagentEvent(ids, "completed"), + // Only the paired item's running state keeps the run ingesting until this one. + backgroundTurnItemEvent(ids, "subagent", "completed", 2), + ]; + }, + { + batched, + onIngest: (event, write) => + Effect.sync(() => { + writes.push(`${event.type} ${write.together ?? "alone"}`); + }), + }, + ); + assert.deepEqual(observed, [ + "subagent:running", + "turn_item:running", + "root-finalized", + "subagent:completed", + "turn_item:completed", + ]); + assert.deepEqual(writes, [ + "subagent.updated 1", + "turn_item.updated 1", + "turn.terminal alone", + "subagent.updated alone", + "turn_item.updated alone", + ]); + }), +); + it.effect.each([null, "run:nested-sdk"] as const)( "releases a settled run when its live child transfers to another run (%s)", (nestedRunId) => @@ -3399,6 +3444,94 @@ it.effect.each(["failure", "defect", "interrupt-failure", "ownership-lost"] as c }), ); +it.effect("ends a subagent whose terminal record was rolled back with its paired item", () => + Effect.gen(function* () { + const key = "paired-terminal-rollback"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const fixture = (status: "running" | "interrupted") => + ({ ids, providerInstanceId, childThreadId: ids.childThreadId, driver, status }) as const; + const pair = (status: "running" | "interrupted") => ({ + type: "subagent.updated" as const, + driver, + subagent: makeRunOwnedSubagentFixture(fixture(status)), + turnItem: makeRunOwnedSubagentTurnItemFixture(fixture(status)), + }); + const { finalEvents } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + events: () => + Stream.make(pair("running"), pair("interrupted"), rootTerminalEvent(ids, "completed")), + ingestNormalized: (request) => + request.event.type === "turn_item.updated" && + request.event.turnItem.status === "interrupted" + ? Effect.fail( + new ProviderEventIngestor.ProviderEventPublishError({ + providerSessionId: ProviderSessionId.make(`session:${key}`), + eventCount: 1, + cause: "disk full", + }), + ) + : Effect.succeed([]), + }); + const subagentId = makeRunOwnedSubagentFixture(fixture("running")).id; + assert.deepEqual( + finalEvents.flatMap((event) => + event.type === "subagent.updated" && event.payload.id === subagentId + ? [`subagent ${event.payload.status}`] + : event.type === "turn-item.updated" && event.payload.type === "subagent" + ? [`item ${event.payload.status}`] + : [], + ), + ["subagent failed", "item failed"], + ); + }), +); + +it.effect("keeps a committed subagent pair settled when its child model sync fails", () => + Effect.gen(function* () { + const key = "paired-terminal-sync-failure"; + const ids = backgroundScenarioIds(key); + const providerInstanceId = ProviderInstanceId.make("codex"); + const fixture = (status: "running" | "interrupted") => + ({ ids, providerInstanceId, childThreadId: ids.childThreadId, driver, status }) as const; + const pair = (status: "running" | "interrupted") => ({ + type: "subagent.updated" as const, + driver, + subagent: makeRunOwnedSubagentFixture(fixture(status)), + turnItem: makeRunOwnedSubagentTurnItemFixture(fixture(status)), + }); + const { finalEvents } = yield* captureRootRunTermination({ + key, + shouldFinalizeRun: () => Effect.succeed(true), + events: () => + Stream.make(pair("running"), pair("interrupted"), rootTerminalEvent(ids, "completed")), + // The ingestor defers a child model sync, which writes, until the pair commits. + ingestNormalized: (request) => + request.event.type === "subagent.updated" && request.event.subagent.status === "interrupted" + ? ProviderEventIngestor.writeAfterCommit( + Effect.fail( + new ProviderEventIngestor.ProviderEventPublishError({ + providerSessionId: ProviderSessionId.make(`session:${key}`), + eventCount: 1, + cause: "disk full", + }), + ), + ).pipe(Effect.as([])) + : Effect.succeed([]), + }); + assert.deepEqual( + finalEvents.flatMap((event) => + event.type === "subagent.updated" || + (event.type === "turn-item.updated" && event.payload.type === "subagent") + ? [event.type] + : [], + ), + [], + ); + }), +); + it.effect.each([ { status: "completed", failure: "later-stream", interrupt: "succeeds" }, { status: "failed", failure: "later-stream", interrupt: "succeeds" }, @@ -3953,7 +4086,11 @@ function captureRootRunTermination(input: { return Effect.gen(function* () { const ids = backgroundScenarioIds(input.key); // The rows the provider's events stored, which the run reads back when its runtime is lost. - const stored = { + const stored: { + subagents: Map; + nodes: Map; + turnItems: Map; + } = { subagents: new Map(), nodes: new Map(), turnItems: new Map(), @@ -4064,6 +4201,26 @@ function captureRootRunTermination(input: { store(request.event).pipe( Effect.andThen(input.ingestNormalized?.(request) ?? Effect.succeed([])), ), + // A failed transaction stores none of its rows. + ingestTogether: (ingest) => + ProviderEventIngestor.commitThenAfterCommit( + (work) => + Effect.suspend(() => { + const subagents = new Map(stored.subagents); + const nodes = new Map(stored.nodes); + const turnItems = new Map(stored.turnItems); + return work.pipe( + Effect.onError(() => + Effect.sync(() => { + stored.subagents = subagents; + stored.nodes = nodes; + stored.turnItems = turnItems; + }), + ), + ); + }), + ingest, + ), }), ServerSettings.layerTest(), Layer.succeed(RunFinalizationService.RunFinalizationObserver, { @@ -4529,7 +4686,7 @@ function runBackgroundItemScenario( options?: { readonly onIngest?: ( event: ProviderAdapter.ProviderAdapterV2Event, - write: { readonly inSharedCommit: boolean }, + write: { readonly inSharedCommit: boolean; readonly together: number | undefined }, ) => Effect.Effect; // Consumes through the real commit coordinator, as the event sink does. readonly batched?: boolean; @@ -4561,6 +4718,8 @@ function runBackgroundItemScenario( Effect.provide(PreviewAgentActivity.layer), ); let inSharedCommit = false; + let togetherCalls = 0; + let together: number | undefined; const coordinator = options?.batched === true ? yield* ProviderCommitBatch.make( @@ -4608,7 +4767,7 @@ function runBackgroundItemScenario( ingestNormalized: (input) => { const write = Effect.gen(function* () { const event = input.event; - yield* options?.onIngest?.(event, { inSharedCommit }) ?? Effect.void; + yield* options?.onIngest?.(event, { inSharedCommit, together }) ?? Effect.void; if (event.type === "turn_item.updated") { yield* Ref.update(observed, (current) => [ ...current, @@ -4633,6 +4792,21 @@ function runBackgroundItemScenario( // A write outside a batch waits for queued snapshots, as the event sink's does. return coordinator === undefined ? write : coordinator.barrier(write); }, + ingestTogether: (ingest) => + ProviderEventIngestor.commitThenAfterCommit( + (work) => + Effect.suspend(() => { + together = ++togetherCalls; + return work; + }).pipe( + Effect.ensuring( + Effect.sync(() => { + together = undefined; + }), + ), + ), + ingest, + ), }), ServerSettings.layerTest(), ), diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index d29b47bbbb821..2a6cfa088f450 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1411,8 +1411,8 @@ export const layer: Layer.Layer< const trackChildLifecycle = Effect.fnUntraced(function* ( event: ProviderAdapter.ProviderAdapterV2Event, deliverable: boolean, + routing: ProviderEventRoutingState, ) { - const routing = yield* Ref.get(eventRouting); if (event.type === "provider_turn.updated") { const isRoot = event.providerTurn.runAttemptId === input.attempt.id || @@ -1815,7 +1815,7 @@ export const layer: Layer.Layer< continue; // A failed write fails ingestion like any provider write. yield* ingestDelivered(event, event); - yield* trackChildLifecycle(event, true); + yield* trackChildLifecycle(event, true, yield* Ref.get(eventRouting)); } yield* scheduleIdleFlush; }); @@ -1926,7 +1926,11 @@ export const layer: Layer.Layer< pendingBackgroundProviderTurns.delete(event.providerTurnId); } } - yield* trackChildLifecycle(event, deliveredEvent !== null); + // Failure finalization ends the open work this tracks, so it must match what is stored: + // inside a paired ingestion it waits for the commit and is skipped if that rolls back. + yield* ProviderEventIngestor.afterCommit( + trackChildLifecycle(event, deliveredEvent !== null, yield* Ref.get(eventRouting)), + ); yield* scheduleIdleFlush; }); const providerEventFiber = yield* eventSink @@ -1938,7 +1942,15 @@ export const layer: Layer.Layer< event.type === "idle_flush" ? flushIdleAssistantText(event.at) : Effect.gen(function* () { - yield* ingestEvent(yield* routeEvent(event)); + const members = ProviderAdapter.providerEventMembers(event); + const ingestMembers = Effect.forEach( + members, + (member) => Effect.flatMap(routeEvent(member), ingestEvent), + { discard: true }, + ); + yield* members.length === 1 + ? ingestMembers + : providerEventIngestor.ingestTogether(ingestMembers); }), Effect.gen(function* () { if (yield* agentTurnsSettled) yield* endPreviewActivity; diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.ts index 38d2e98ef72c2..ee7a5caec8388 100644 --- a/packages/contracts/src/orchestrationV2.ts +++ b/packages/contracts/src/orchestrationV2.ts @@ -1701,6 +1701,24 @@ const OrchestrationV2LinkRequestFields = { failure: Schema.optional(Schema.String), } as const; +export const OrchestrationV2SubagentTurnItem = Schema.Struct({ + ...OrchestrationV2TurnItemBaseFields, + type: Schema.Literal("subagent"), + subagentId: NodeId, + origin: Schema.Literals(["provider_native", "app_owned"]), + driver: ProviderDriverKind, + providerInstanceId: ProviderInstanceId, + childThreadId: Schema.NullOr(ThreadId), + remoteChild: Schema.optional(OrchestrationV2RemoteTaskChild), + prompt: Schema.String, + progress: Schema.optional(Schema.String), + result: Schema.NullOr(Schema.String), + /** Same as the prompt and result of subagent `subagentId`. */ + promptOmitted: DedupedBodyMarker, + resultOmitted: DedupedBodyMarker, +}); +export type OrchestrationV2SubagentTurnItem = typeof OrchestrationV2SubagentTurnItem.Type; + export const OrchestrationV2TurnItem = Schema.Union([ Schema.Struct({ ...OrchestrationV2TurnItemBaseFields, @@ -1904,22 +1922,7 @@ export const OrchestrationV2TurnItem = Schema.Union([ ...OrchestrationV2TurnItemBaseFields, ...OrchestrationV2LinkRequestFields, }), - Schema.Struct({ - ...OrchestrationV2TurnItemBaseFields, - type: Schema.Literal("subagent"), - subagentId: NodeId, - origin: Schema.Literals(["provider_native", "app_owned"]), - driver: ProviderDriverKind, - providerInstanceId: ProviderInstanceId, - childThreadId: Schema.NullOr(ThreadId), - remoteChild: Schema.optional(OrchestrationV2RemoteTaskChild), - prompt: Schema.String, - progress: Schema.optional(Schema.String), - result: Schema.NullOr(Schema.String), - /** Same as the prompt and result of subagent `subagentId`. */ - promptOmitted: DedupedBodyMarker, - resultOmitted: DedupedBodyMarker, - }), + OrchestrationV2SubagentTurnItem, Schema.Struct({ ...OrchestrationV2TurnItemBaseFields, type: Schema.Literal("dynamic_tool"), diff --git a/packages/provider-acp/src/server/adapter.ts b/packages/provider-acp/src/server/adapter.ts index 058ff68c4b1f8..383d9f795b7a5 100644 --- a/packages/provider-acp/src/server/adapter.ts +++ b/packages/provider-acp/src/server/adapter.ts @@ -3627,10 +3627,10 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( completedAt: subagent.task.completedAt, }, }); - yield* emitProviderEvent({ type: "subagent.updated", driver, subagent: subagent.task }); yield* emitProviderEvent({ - type: "turn_item.updated", + type: "subagent.updated", driver, + subagent: subagent.task, turnItem: { id: turnItemId, threadId: context.input.threadId, @@ -6376,10 +6376,6 @@ export const makeAcpAdapterV2 = Effect.fn("makeAcpAdapterV2")(function* ( type: "subagent.updated", driver, subagent: subagent.task, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver, turnItem: { id: subagent.turnItemId, threadId: subagent.task.threadId, diff --git a/packages/provider-core/src/server/ProviderAdapter.ts b/packages/provider-core/src/server/ProviderAdapter.ts index ff879ef0f723f..2458f3f734606 100644 --- a/packages/provider-core/src/server/ProviderAdapter.ts +++ b/packages/provider-core/src/server/ProviderAdapter.ts @@ -17,6 +17,7 @@ import { OrchestrationV2ProviderTurn, OrchestrationV2RuntimeRequest, OrchestrationV2Subagent, + OrchestrationV2SubagentTurnItem, OrchestrationV2TurnItem, ProviderApprovalDecision, ProviderInteractionMode, @@ -119,6 +120,8 @@ export const ProviderAdapterV2Event = Schema.Union([ type: Schema.Literal("subagent.updated"), driver: ProviderDriverKind, subagent: OrchestrationV2Subagent, + /** The subagent's own item when nothing comes between them, so both commit in one write. */ + turnItem: Schema.optional(OrchestrationV2SubagentTurnItem), }), Schema.Struct({ type: Schema.Literal("message.updated"), @@ -207,6 +210,15 @@ export const ProviderAdapterV2Event = Schema.Union([ ]); export type ProviderAdapterV2Event = typeof ProviderAdapterV2Event.Type; +/** The events an emission stands for, in order; each gets the per-event handling of its own. */ +export function providerEventMembers( + event: ProviderAdapterV2Event, +): ReadonlyArray { + if (event.type !== "subagent.updated" || event.turnItem === undefined) return [event]; + const { turnItem, ...record } = event; + return [record, { type: "turn_item.updated", driver: event.driver, turnItem }]; +} + export class ProviderAdapterCapabilitiesError extends Schema.TaggedError()( "ProviderAdapterCapabilitiesError", { diff --git a/packages/provider-cursor/src/server/adapter.ts b/packages/provider-cursor/src/server/adapter.ts index 90c55675af296..f5e297e1600fc 100644 --- a/packages/provider-cursor/src/server/adapter.ts +++ b/packages/provider-cursor/src/server/adapter.ts @@ -1911,10 +1911,6 @@ export const makeCursorAdapterV2 = Effect.fn("makeCursorAdapterV2")(function* ( type: "subagent.updated", driver: CursorAgentSdk.CURSOR_PROVIDER, subagent: task, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: CursorAgentSdk.CURSOR_PROVIDER, turnItem: { id: subagent.turnItemId, threadId: input.context.input.threadId, diff --git a/packages/provider-opencode/src/server/adapter.test.ts b/packages/provider-opencode/src/server/adapter.test.ts index e215a2f6fdde2..5fb6ab01fb4f3 100644 --- a/packages/provider-opencode/src/server/adapter.test.ts +++ b/packages/provider-opencode/src/server/adapter.test.ts @@ -1222,6 +1222,9 @@ describe("OpenCodeAdapterV2", () => { }); yield* harness.startTurn(); const received = yield* harness.runtime.events.pipe( + Stream.flatMap((event) => + Stream.fromIterable(ProviderAdapter.providerEventMembers(event)), + ), Stream.takeUntil( (event) => event.type === "turn_item.updated" && event.turnItem.type === "compaction", ), @@ -2695,6 +2698,7 @@ describe("OpenCodeAdapterV2", () => { } yield* harness.startTurn(); const received = yield* harness.runtime.events.pipe( + Stream.flatMap((event) => Stream.fromIterable(ProviderAdapter.providerEventMembers(event))), Stream.takeUntil( (event) => event.type === "turn_item.updated" && diff --git a/packages/provider-opencode/src/server/adapter.ts b/packages/provider-opencode/src/server/adapter.ts index 602977628612d..5216fea0f8b61 100644 --- a/packages/provider-opencode/src/server/adapter.ts +++ b/packages/provider-opencode/src/server/adapter.ts @@ -1647,10 +1647,6 @@ export const makeOpenCodeAdapterV2 = Effect.fn("makeOpenCodeAdapterV2")(function type: "subagent.updated", driver: OPENCODE_PROVIDER, subagent, - }); - yield* emitProviderEvent({ - type: "turn_item.updated", - driver: OPENCODE_PROVIDER, turnItem: { id: turnItemId, threadId: turn.threadId, diff --git a/packages/provider-opencode/src/server/v2/adapter.ts b/packages/provider-opencode/src/server/v2/adapter.ts index 189cc3cd21fef..5622ce1226582 100644 --- a/packages/provider-opencode/src/server/v2/adapter.ts +++ b/packages/provider-opencode/src/server/v2/adapter.ts @@ -1632,10 +1632,6 @@ export const make = Effect.fn("OpenCode2Adapter.make")(function* (instanceId: Pr completedAt, updatedAt, }, - }); - yield* emit({ - type: "turn_item.updated", - driver, turnItem: { id: idAllocator.derive.turnItemFromProviderItem({ driver, nativeItemId: call.nativeId }), threadId: turn.input.threadId, diff --git a/packages/provider-pi/src/server/adapter.test.ts b/packages/provider-pi/src/server/adapter.test.ts index ff9f9c22ef976..43729490b5983 100644 --- a/packages/provider-pi/src/server/adapter.test.ts +++ b/packages/provider-pi/src/server/adapter.test.ts @@ -3565,18 +3565,9 @@ describe("PiAdapterV2", () => { assert.isTrue( doneCard.type === "subagent.updated" && doneCard.subagent.result === "repo has one file" && - doneCard.subagent.childThreadId === null, - ); - const subagentItem = yield* takeEvent( - (event) => - event.type === "turn_item.updated" && - event.turnItem.type === "subagent" && - event.turnItem.status === "completed", - ); - assert.isTrue( - subagentItem.type === "turn_item.updated" && - subagentItem.turnItem.type === "subagent" && - subagentItem.turnItem.childThreadId === null, + doneCard.subagent.childThreadId === null && + doneCard.turnItem?.status === "completed" && + doneCard.turnItem.childThreadId === null, ); }).pipe(Effect.scoped, Effect.provide(layerTest)), ); diff --git a/packages/provider-pi/src/server/adapter.ts b/packages/provider-pi/src/server/adapter.ts index 6694623d98883..e5109fd359b24 100644 --- a/packages/provider-pi/src/server/adapter.ts +++ b/packages/provider-pi/src/server/adapter.ts @@ -1555,10 +1555,6 @@ export const makePiAdapterV2 = Effect.fn("makePiAdapterV2")(function* ( completedAt: finished ? emittedAt : null, updatedAt: emittedAt, }, - }); - yield* emit({ - type: "turn_item.updated", - driver: PI_PROVIDER, turnItem: { ...baseItemFields(turn, nativeTaskId, startedAt, emittedAt), status: task.status,