diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index fff4fd3e3db7..cad0aff3afb1 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -72,12 +72,22 @@ export type EventSinkV2Error = typeof EventSinkV2Error.Type; interface ProviderBatchPublication { readonly publish: Array>; + readonly projection: ProjectionStore.ProviderBatchProjection; + /** The projection high-water mark, recorded once before the batch commits. */ + projectedSequence: number | undefined; eventCount: number; } const CurrentProviderBatch = Context.Reference( "t3/orchestration-v2/EventSink/CurrentProviderBatch", { defaultValue: () => undefined }, ); +// Projecting `provider-thread.updated` also rewrites its app thread's row. +const projectedThreadIds = (events: ReadonlyArray) => + events.flatMap((event) => + event.type === "provider-thread.updated" && event.payload.appThreadId !== null + ? [event.threadId, event.payload.appThreadId] + : [event.threadId], + ); export interface EventSinkV2Shape { /** Items that are not provider snapshots, such as a consumer's own control items, never batch. */ @@ -409,6 +419,30 @@ const layerBase: Layer.Layer< ); }; + // The enclosing write or commit span already traces this projection bookkeeping. + const recordProjectedSequence = Effect.fnUntraced(function* (sequence: number) { + const now = DateTime.formatIso(yield* DateTime.now); + yield* sql` + INSERT INTO orchestration_v2_projection_metadata ( + projection_name, + schema_version, + last_sequence, + updated_at + ) + VALUES ( + 'thread-projections', + ${ProjectionStore.ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION}, + ${sequence}, + ${now} + ) + ON CONFLICT(projection_name) + DO UPDATE SET + schema_version = excluded.schema_version, + last_sequence = excluded.last_sequence, + updated_at = excluded.updated_at + `; + }); + const applyStoredEvents = /* This helper runs for individual orchestration events; the enclosing operation owns the span. */ Effect.fnUntraced(function* (storedEvents: ReadonlyArray) { @@ -417,26 +451,9 @@ const layerBase: Layer.Layer< }); const sequence = storedEvents.at(-1)?.sequence; if (sequence !== undefined) { - const now = DateTime.formatIso(yield* DateTime.now); - yield* sql` - INSERT INTO orchestration_v2_projection_metadata ( - projection_name, - schema_version, - last_sequence, - updated_at - ) - VALUES ( - 'thread-projections', - ${ProjectionStore.ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION}, - ${sequence}, - ${now} - ) - ON CONFLICT(projection_name) - DO UPDATE SET - schema_version = excluded.schema_version, - last_sequence = excluded.last_sequence, - updated_at = excluded.updated_at - `; + const batch = yield* CurrentProviderBatch; + if (batch !== undefined) batch.projectedSequence = sequence; + else yield* recordProjectedSequence(sequence); } // Read lineage here, on the connection this transaction holds; see makeRunOutcomeRecorder. return yield* runOutcomes.prepare(storedEvents); @@ -476,6 +493,7 @@ const layerBase: Layer.Layer< const batch = yield* CurrentProviderBatch; if (batch !== undefined) batch.eventCount += input.events.length; const { committed } = yield* commitThenPublish( + projectedThreadIds(input.events), Effect.gen(function* () { // Check ownership in the same transaction that claims the native session. const imported = input.unlessNativeThreadOwned; @@ -530,6 +548,7 @@ const layerBase: Layer.Layer< }); return yield* commitThenPublish( + [input.threadId, ...projectedThreadIds(input.events)], Effect.gen(function* () { const rows = yield* sql<{ readonly status: string; @@ -611,6 +630,7 @@ const layerBase: Layer.Layer< }); return yield* commitThenPublish( + projectedThreadIds(input.events), Effect.gen(function* () { const rows = yield* sql<{ readonly active_attempt_id: string | null; @@ -664,6 +684,7 @@ const layerBase: Layer.Layer< events: ReadonlyArray, ) => commitThenPublish( + projectedThreadIds(events), Effect.gen(function* () { const rows = yield* readStoredPayload; if (rows[0]?.payload_json !== expectedJson) { @@ -764,6 +785,7 @@ const layerBase: Layer.Layer< input: Parameters[0], ) { const result = yield* commitThenPublish( + [input.threadId, ...projectedThreadIds(input.events)], Effect.gen(function* () { const reserved = yield* commandReceipts.insertIfAbsent({ commandId: input.commandId, @@ -909,6 +931,7 @@ const layerBase: Layer.Layer< const commitProjectCommandEffect = Effect.fn("orchestrationV2.EventSink.commitProjectCommand")( function* (input: Parameters[0]) { const result = yield* commitThenPublish( + [], Effect.gen(function* () { const reserved: CommandReceiptStore.ProjectCommandReceiptV2 = { commandId: input.commandId, @@ -1037,7 +1060,9 @@ const layerBase: Layer.Layer< ); }; + // Waits only for queued provider snapshots of `threadIds`, the threads this write touches. const commitThenPublish = ( + threadIds: ReadonlyArray, transaction: Effect.Effect, publish: (committed: A) => Effect.Effect, ) => @@ -1046,7 +1071,7 @@ const layerBase: Layer.Layer< ? Effect.flatMap(Effect.serviceOption(sql.transactionService), (existing) => Option.isSome(existing) ? commitThenPublishNow(transaction, publish) - : coordinator.barrier(commitThenPublishNow(transaction, publish)), + : coordinator.barrier(commitThenPublishNow(transaction, publish), threadIds), ) : transaction.pipe( Effect.tap((committed) => @@ -1061,12 +1086,33 @@ const layerBase: Layer.Layer< "orchestrationV2.EventSink.providerBatch", )(function* (effect: Effect.Effect) { if ((yield* CurrentProviderBatch) !== undefined) return yield* effect; - const batch: ProviderBatchPublication = { publish: [], eventCount: 0 }; + const batch: ProviderBatchPublication = { + publish: [], + eventCount: 0, + projection: { nodeReads: new Map(), threadTouches: new Map() }, + projectedSequence: undefined, + }; let started = false; const result = yield* commitThenPublishNow( Effect.sync(() => { started = true; - }).pipe(Effect.andThen(Effect.provideService(effect, CurrentProviderBatch, batch))), + }).pipe( + Effect.andThen( + effect.pipe( + Effect.provideService(CurrentProviderBatch, batch), + Effect.provideService(ProjectionStore.ProviderBatchProjectionState, batch.projection), + Effect.tap(() => + ProjectionStore.writeThreadTouches(sql, batch.projection.threadTouches).pipe( + Effect.andThen( + batch.projectedSequence === undefined + ? Effect.void + : recordProjectedSequence(batch.projectedSequence), + ), + ), + ), + ), + ), + ), () => Effect.forEach(batch.publish, (publish) => publish, { discard: true }), ).pipe( // Normalization mutates run-local state, so only retry before its transaction body starts. @@ -1105,12 +1151,19 @@ const layerBase: Layer.Layer< const batch = yield* CurrentProviderBatch; const publicationCount = batch?.publish.length ?? 0; const eventCount = batch?.eventCount ?? 0; + const projectedSequence = batch?.projectedSequence; + const threadTouches = new Map(batch?.projection.threadTouches); return yield* sql.withTransaction(work).pipe( Effect.onError(() => Effect.sync(() => { if (batch !== undefined) { batch.publish.length = publicationCount; batch.eventCount = eventCount; + batch.projectedSequence = projectedSequence; + batch.projection.nodeReads.clear(); + batch.projection.threadTouches.clear(); + for (const [threadId, updatedAt] of threadTouches) + batch.projection.threadTouches.set(threadId, updatedAt); } }), ), diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 14f443aa8cb6..1fbab4085b28 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -419,6 +419,209 @@ it.effect( }).pipe(Effect.provide(Layer.fresh(layerTest))), ); +function nodeComparisonEvent(thread: OrchestrationV2AppThread, now: DateTime.Utc) { + const nodeId = NodeId.make(`node:${thread.id}`); + return { + id: EventId.make(`node-cache:${thread.id}`), + type: "node.updated", + threadId: thread.id, + occurredAt: now, + payload: { + id: nodeId, + threadId: thread.id, + runId: null, + parentNodeId: null, + rootNodeId: nodeId, + kind: "assistant_message", + status: "running", + countsForRun: false, + providerThreadId: null, + providerTurnId: null, + nativeItemRef: null, + runtimeRequestId: null, + checkpointScopeId: null, + startedAt: now, + completedAt: null, + }, + } satisfies OrchestrationV2DomainEvent; +} + +it.effect("commits other threads' writes ahead of a thread's queued provider snapshots", () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const now = yield* DateTime.now; + const [busy, independent] = ["busy", "independent"].map((id) => + makeThread(ThreadId.make(id), now), + ); + yield* sink.write({ + events: [busy!, independent!].map((thread) => + threadCreatedEvent({ id: `created:${thread.id}`, thread, now }), + ), + }); + const highWater = yield* sink.latestSequence(); + const busyNode = nodeComparisonEvent(busy!, now); + const snapshot = { + type: "node.updated" as const, + driver: providerDriver, + node: busyNode.payload, + }; + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + const pulled = yield* Deferred.make(); + let writes = 0; + let pulls = 0; + const input = yield* Queue.unbounded(); + const provider = yield* sink + .consumeProviderEvents( + Stream.fromQueue(input).pipe( + Stream.take(3), + Stream.tap(() => (++pulls === 3 ? Deferred.succeed(pulled, undefined) : Effect.void)), + ), + () => + Effect.gen(function* () { + const id = EventId.make(`busy-provider-${++writes}`); + yield* sink.write({ events: [{ ...busyNode, id }] }); + if (writes > 1) return; + yield* Deferred.succeed(started, undefined); + yield* Deferred.await(release); + }), + Effect.succeed(false), + Effect.succeed(true), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Queue.offer(input, snapshot); + yield* Deferred.await(started); + yield* Queue.offerAll(input, [snapshot, snapshot]); + yield* Deferred.await(pulled); + yield* TestClock.adjust(0); + const writeTo = (event: OrchestrationV2DomainEvent, id: string) => + sink + .write({ events: [{ ...event, id: EventId.make(id) }] }) + .pipe(Effect.forkScoped({ startImmediately: true })); + const sameThread = yield* writeTo(busyNode, "busy-command"); + const otherThread = yield* writeTo(nodeComparisonEvent(independent!, now), "independent"); + const childOfBusy = yield* writeTo( + { + id: EventId.make("unused"), + type: "provider-thread.updated", + threadId: independent!.id, + driver: providerDriver, + providerInstanceId, + occurredAt: now, + payload: { + id: ProviderThreadId.make("provider-thread:busy-child"), + driver: providerDriver, + providerInstanceId, + providerSessionId: null, + appThreadId: busy!.id, + ownerNodeId: null, + nativeThreadRef: null, + nativeConversationHeadRef: null, + status: "idle", + firstRunOrdinal: 1, + lastRunOrdinal: 1, + handoffIds: [], + forkedFrom: null, + createdAt: now, + updatedAt: now, + }, + }, + "busy-child", + ); + yield* TestClock.adjust(0); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(provider); + yield* Fiber.join(sameThread); + yield* Fiber.join(otherThread); + yield* Fiber.join(childOfBusy); + const store = yield* EventStore.EventStoreV2; + const events = yield* store.read({ afterSequence: highWater }).pipe(Stream.runCollect); + assert.deepEqual( + events.map((stored) => stored.event.id), + [ + "busy-provider-1", + "independent", + "busy-provider-2", + "busy-provider-3", + "busy-command", + "busy-child", + ], + ); + }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), +); + +it.effect( + "reuses node comparisons and writes thread bookkeeping once per provider transaction", + () => + Effect.gen(function* () { + const sink = yield* EventSink.EventSinkV2; + const projections = yield* ProjectionStore.ProjectionStoreV2; + const sql = yield* SqlClient.SqlClient; + const now = yield* DateTime.now; + const thread = makeThread(ThreadId.make("node-comparison-cache"), now); + yield* sink.write({ + events: [threadCreatedEvent({ id: "node-cache-created", thread, now })], + }); + const event = nodeComparisonEvent(thread, now); + const statements = { nodeReads: 0, threadTouches: 0, highWaterWrites: 0 }; + const observe: Statement.Transformer = (statement) => { + const text = statement.compile()[0]; + if ( + text.includes("SELECT payload_json FROM orchestration_v2_projection_nodes WHERE node_id") + ) + statements.nodeReads++; + if (text.includes("UPDATE orchestration_v2_projection_threads")) statements.threadTouches++; + if (text.includes("INSERT INTO orchestration_v2_projection_metadata")) + statements.highWaterWrites++; + return Effect.succeed(statement); + }; + const lastOccurredAt = DateTime.add(now, { minutes: 3 }); + yield* sink + .withProviderBatch( + Effect.gen(function* () { + for (let n = 0; n < 21; n++) + assert.isFalse(yield* projections.isNodeStored(event.payload)); + assert.equal(statements.nodeReads, 1); + yield* sink.write({ events: [event] }); + assert.isTrue(yield* projections.isNodeStored(event.payload)); + for (const minutes of [1, 2, 3]) { + yield* sink.write({ + events: [ + { + ...event, + id: EventId.make(`node-cache-touch:${minutes}`), + occurredAt: DateTime.add(now, { minutes }), + }, + ], + }); + } + assert.deepEqual([statements.threadTouches, statements.highWaterWrites], [0, 0]); + }), + ) + .pipe(Effect.provideService(Statement.CurrentTransformer, observe)); + assert.deepEqual([statements.threadTouches, statements.highWaterWrites], [1, 1]); + assert.deepEqual( + DateTime.toEpochMillis((yield* projections.getThread(thread.id)).updatedAt), + DateTime.toEpochMillis(lastOccurredAt), + ); + assert.deepEqual( + yield* sql`SELECT last_sequence FROM orchestration_v2_projection_metadata + WHERE projection_name = 'thread-projections'`, + [{ last_sequence: yield* sink.latestSequence() }], + ); + const completed = { ...event.payload, status: "completed" as const, completedAt: now }; + yield* sink.write({ + events: [{ ...event, id: EventId.make("node-cache-completed"), payload: completed }], + }); + yield* sink.withProviderBatch( + Effect.gen(function* () { + assert.isFalse(yield* projections.isNodeStored(event.payload)); + assert.isTrue(yield* projections.isNodeStored(completed)); + }), + ); + }).pipe(Effect.scoped, Effect.provide(Layer.fresh(layerTest))), +); + it.effect.each( (["none", "duplicate", "after-write"] as const).flatMap((failure) => [1, 10].map((streams) => ({ failure, streams })), @@ -427,7 +630,7 @@ it.effect.each( "shares the writer and isolates failure=$failure across $streams independent consumers", ({ failure, streams }) => { const rollback = failure !== "none"; - const failedIndex = streams - 1; + const failedIndex = failure === "after-write" ? 0 : streams - 1; const jobError = { _tag: "InvalidUpdate" } as const; const writes: number[] = []; const tracer = Tracer.make({ @@ -455,6 +658,8 @@ it.effect.each( threadCreatedEvent({ id: `created:${index}`, thread, now }), ), }); + const node = nodeComparisonEvent(threads[0]!, now); + yield* sink.write({ events: [node] }); const highWater = yield* sink.latestSequence(); const store = yield* EventStore.EventStoreV2; const publications = yield* Queue.unbounded(); @@ -465,7 +670,7 @@ it.effect.each( const pulled = yield* Deferred.make(); let arrivals = 0; const inputs = yield* Effect.forEach(threads, () => - Queue.unbounded(), + Queue.unbounded(), ); const consumers = yield* Effect.forEach(inputs, (input, index) => sink @@ -477,8 +682,11 @@ it.effect.each( ), (raw) => raw.type === "message.updated" - ? sink - .write({ + ? Effect.gen(function* () { + if (failure === "after-write" && index !== failedIndex) { + assert.isTrue(yield* projections.isNodeStored(node.payload)); + } + yield* sink.write({ events: [ { id: EventId.make( @@ -487,19 +695,29 @@ it.effect.each( : `shared-event:${index}`, ), threadId: raw.message.threadId, - occurredAt: now, + occurredAt: DateTime.add(now, { minutes: index + 1 }), type: "message.updated", payload: raw.message, }, + ...(failure === "after-write" && index === failedIndex + ? [ + { + ...node, + id: EventId.make("rolled-back-node"), + occurredAt: DateTime.add(now, { hours: 1 }), + payload: { + ...node.payload, + status: "completed" as const, + completedAt: now, + }, + }, + ] + : []), ], - }) - .pipe( - Effect.andThen( - failure === "after-write" && index === failedIndex - ? Effect.fail(jobError) - : Effect.void, - ), - ) + }); + if (failure === "after-write" && index === failedIndex) + return yield* Effect.fail(jobError); + }) : Effect.void, Effect.succeed(false), Effect.succeed(true), @@ -527,10 +745,10 @@ it.effect.each( }); } yield* Deferred.await(pulled); - yield* TestClock.adjust(0); - yield* sink.write({ events: [] }); + yield* Effect.forEach(inputs, Queue.end, { discard: true }); + const exits = yield* Effect.forEach(consumers, Fiber.join); if (rollback) { - const exit = yield* Fiber.join(consumers[failedIndex]!); + const exit = exits[failedIndex]!; assert.isTrue(Exit.isFailure(exit)); if (failure === "after-write" && Exit.isFailure(exit)) assert.strictEqual(Cause.squash(exit.cause), jobError); @@ -544,10 +762,21 @@ it.effect.each( assert.equal(yield* Queue.size(publications), 0); assert.deepEqual(writes, rollback && streams === 1 ? [] : [streams - (rollback ? 1 : 0)]); assert.equal(yield* sink.latestSequence(), highWater + streams - (rollback ? 1 : 0)); + const sql = yield* SqlClient.SqlClient; + assert.deepEqual( + yield* sql`SELECT last_sequence FROM orchestration_v2_projection_metadata + WHERE projection_name = 'thread-projections'`, + [{ last_sequence: yield* sink.latestSequence() }], + ); for (let index = 0; index < threads.length; index++) { + const failed = rollback && index === failedIndex; assert.lengthOf( (yield* projections.getThreadRecords(threads[index]!.id, ["messages"])).messages, - rollback && index === failedIndex ? 0 : 1, + failed ? 0 : 1, + ); + assert.equal( + DateTime.toEpochMillis((yield* projections.getThread(threads[index]!.id)).updatedAt), + DateTime.toEpochMillis(failed ? now : DateTime.add(now, { minutes: index + 1 })), ); } }).pipe( diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts index 5b8a636cc24f..9462d2b1b6ee 100644 --- a/apps/server/src/orchestration-v2/ProjectionStore.ts +++ b/apps/server/src/orchestration-v2/ProjectionStore.ts @@ -838,6 +838,34 @@ export class ProjectionStoreV2 extends Context.Service; + /** Thread `updated_at` bumps, written once by `writeThreadTouches` before the batch commits. */ + readonly threadTouches: Map; +} +export class ProviderBatchProjectionState extends Context.Reference< + ProviderBatchProjection | undefined +>("t3/orchestration-v2/ProjectionStore/ProviderBatchProjectionState", { + defaultValue: () => undefined, +}) {} + +// Runs per projected event or once per batch; the projection or batch commit span traces it. +export const writeThreadTouches = Effect.fnUntraced(function* ( + sql: SqlClient.SqlClient, + touches: ReadonlyMap, +) { + for (const [threadId, updatedAt] of touches) { + yield* sql` + UPDATE orchestration_v2_projection_threads + SET + updated_at = ${updatedAt}, + payload_json = json_set(payload_json, '$.updatedAt', ${updatedAt}) + WHERE thread_id = ${threadId} + `; + } +}); + export const ORCHESTRATION_V2_PROJECTION_SCHEMA_VERSION = 2; function upsertById(items: ReadonlyArray, next: T): Array { @@ -2347,6 +2375,23 @@ export const layer: Layer.Layer = for (const [key, cached] of timelineIndexes) { if (cached.index.threadIds.includes(event.threadId)) timelineIndexes.delete(key); } + const batch = yield* ProviderBatchProjectionState; + // These rewrite the thread row from what they read, so a deferred bump must land first. + const rewrittenThreadId = event.type.startsWith("thread.") + ? event.threadId + : event.type === "provider-thread.updated" + ? event.payload.appThreadId + : null; + const deferredUpdatedAt = + rewrittenThreadId === null ? undefined : batch?.threadTouches.get(rewrittenThreadId); + if ( + batch !== undefined && + rewrittenThreadId !== null && + deferredUpdatedAt !== undefined + ) { + batch.threadTouches.delete(rewrittenThreadId); + yield* writeThreadTouches(sql, new Map([[rewrittenThreadId, deferredUpdatedAt]])); + } switch (event.type) { case "thread.pull-request-link-updated": { const thread = yield* getThread(event.threadId); @@ -2593,6 +2638,7 @@ export const layer: Layer.Layer = completed_at = excluded.completed_at, payload_json = excluded.payload_json `; + batch?.nodeReads.set(event.payload.id, payloadJson); break; } case "subagent.updated": { @@ -3234,13 +3280,8 @@ export const layer: Layer.Layer = event.type !== "thread.guest-share-updated" ) { const updatedAt = yield* encodeThreadUpdatedAt(event.occurredAt); - yield* sql` - UPDATE orchestration_v2_projection_threads - SET - updated_at = ${updatedAt}, - payload_json = json_set(payload_json, '$.updatedAt', ${updatedAt}) - WHERE thread_id = ${event.threadId} - `; + if (batch !== undefined) batch.threadTouches.set(event.threadId, updatedAt); + else yield* writeThreadTouches(sql, new Map([[event.threadId, updatedAt]])); } }, (effect, event) => @@ -5818,9 +5859,12 @@ export const layer: Layer.Layer = )( function* (node) { const payloadJson = yield* encodeNodePayload(node); + const cache = (yield* ProviderBatchProjectionState)?.nodeReads; + if (cache?.has(node.id)) return cache.get(node.id) === payloadJson; const rows = yield* sql` SELECT payload_json FROM orchestration_v2_projection_nodes WHERE node_id = ${node.id} `; + cache?.set(node.id, rows[0]?.payload_json); return rows[0]?.payload_json === payloadJson; }, (effect, node) => effect.pipe(Effect.mapError(controlReadError(node.threadId))), diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index dd159a2ff14e..2e88f2272dd1 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -404,6 +404,65 @@ it.effect.each([false, true])( }), ); +it.effect.each(["none", "writer", "snapshots"] as const)( + "runs another stream's control during a busy commit; splits slow backlogs for a write waiting on %s", + (waitingOn) => + Effect.gen(function* () { + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + let first = true; + const recorder = yield* recordBatches( + Effect.gen(function* () { + if (!first) return; + first = false; + yield* Deferred.succeed(started, undefined); + yield* Deferred.await(release); + }), + ); + const leading = yield* recorder + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false)) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(started); + yield* recorder.consume(Stream.make(control), recorder.process, Effect.succeed(true)); + assert.deepEqual(yield* Queue.takeAll(recorder.output), [[control]]); + + const pulled = yield* Deferred.make(); + const backlog = [event(0), event(1), event(2)]; + const slow = yield* recorder + .consume( + Stream.fromArray(backlog).pipe( + Stream.tap((raw) => + raw === backlog.at(-1) ? Deferred.succeed(pulled, undefined) : Effect.void, + ), + ), + (raw) => recorder.process(raw).pipe(Effect.andThen(TestClock.adjust("100 millis"))), + Effect.succeed(false), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(pulled); + yield* TestClock.adjust(0); + // An unrelated write stands in for one waiting on the writer the batch holds. + const writerFree = yield* Deferred.make(); + const write = yield* recorder + .barrier( + waitingOn === "writer" ? Deferred.await(writerFree) : Effect.void, + waitingOn === "snapshots" ? ["single-thread"] : ["other-thread"], + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.succeed(release, undefined); + yield* Fiber.join(leading); + yield* Fiber.join(slow); + yield* Deferred.succeed(writerFree, undefined); + yield* Fiber.join(write); + assert.deepEqual( + yield* Queue.takeAll(recorder.output), + waitingOn === "none" + ? [[event(-1)], backlog] + : [[event(-1)], ...backlog.map((raw) => [raw])], + ); + }), +); + it.effect.each(["failure", "stop"] as const)( "flushes the prefix and respects a source %s barrier", (mode) => diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 1a6ca8d28a60..9117653682f3 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -11,6 +11,8 @@ import * as Stream from "effect/Stream"; const MAX_EVENTS = 64; const MAX_BYTES = 1024 * 1024; +// While another write waits, a shared transaction stops taking jobs after this long. +const MAX_HOLD_NANOS = 100_000_000n; const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe( @@ -18,11 +20,28 @@ export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe( Effect.orDie, ); -const isSnapshot = (event: { readonly type: string }) => +type Snapshot = + | { readonly type: "node.updated"; readonly node: { readonly threadId: string } } + | { readonly type: "message.updated"; readonly message: { readonly threadId: string } } + | { readonly type: "turn_item.updated"; readonly turnItem: { readonly threadId: string } } + | { readonly type: "plan.updated"; readonly plan: { readonly threadId: string } }; +const isSnapshot = (event: { readonly type: string }): event is Snapshot => event.type === "node.updated" || event.type === "message.updated" || event.type === "turn_item.updated" || event.type === "plan.updated"; +const snapshotThreadId = (event: Snapshot) => { + switch (event.type) { + case "node.updated": + return event.node.threadId; + case "message.updated": + return event.message.threadId; + case "turn_item.updated": + return event.turnItem.threadId; + case "plan.updated": + return event.plan.threadId; + } +}; export class CommitBatchError extends Schema.TaggedError()("CommitBatchError", { cause: Schema.Defect(), @@ -31,8 +50,8 @@ export const isCommitBatchError = Schema.is(CommitBatchError); interface Job { readonly owner: { active: boolean }; + readonly threadId: string; readonly queuedAt: number; - readonly snapshot: boolean; readonly bytes: number; readonly execute: Effect.Effect; readonly settle: ( @@ -40,11 +59,11 @@ interface Job { preserveFailure: boolean, ) => Effect.Effect; readonly cancel: Effect.Effect; + readonly settled: Effect.Effect; } interface Batch { readonly events: ReadonlyArray; - readonly bytes: number; - readonly reason: "ready" | "limit" | "control"; + readonly reason: "ready" | "limit"; } function collect(queue: Queue.Queue) { @@ -54,20 +73,15 @@ function collect(queue: Queue.Queue) { let events: Array = []; let bytes = 0; const flush = (reason: Batch["reason"]) => { - if (events.length > 0) batches.push({ events, bytes, reason }); + if (events.length > 0) batches.push({ events, reason }); events = []; bytes = 0; }; for (const job of ready) { - if (!job.snapshot) { - flush("control"); - batches.push({ events: [job], bytes: 0, reason: "control" }); - } else { - if (events.length > 0 && bytes + job.bytes > MAX_BYTES) flush("limit"); - events.push(job); - bytes += job.bytes; - if (events.length >= MAX_EVENTS || bytes >= MAX_BYTES) flush("limit"); - } + if (events.length > 0 && bytes + job.bytes > MAX_BYTES) flush("limit"); + events.push(job); + bytes += job.bytes; + if (events.length >= MAX_EVENTS || bytes >= MAX_BYTES) flush("limit"); } flush("ready"); return Stream.fromArray(batches); @@ -92,6 +106,8 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( const queue = yield* Queue.bounded(MAX_EVENTS); const pending = new Set(); let closed = false; + // Writes in `barrier` that have not finished; they wait on the batch or the writer it holds. + let waitingWriters = 0; const shutdown = Effect.gen(function* () { closed = true; yield* Queue.shutdown(queue); @@ -102,7 +118,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( const enqueue = Effect.fnUntraced(function* ( effect: Effect.Effect, token: { active: boolean }, - snapshot: boolean, + threadId: string, bytes: number, ) { const caller = yield* Effect.context(); @@ -111,8 +127,8 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( let result: A; const job: Job = { owner: token, + threadId, queuedAt: yield* Clock.currentTimeMillis, - snapshot, bytes, execute: effect.pipe( Effect.tapCause((cause) => @@ -146,6 +162,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( pending.delete(job); yield* Deferred.interrupt(reply); }), + settled: Deferred.await(reply).pipe(Effect.exit, Effect.asVoid), }; yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { @@ -158,77 +175,111 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ); return Deferred.await(reply); }); + // Commits a prefix of `jobs` in one transaction and returns how many it took. + const commitPrefix = (jobs: ReadonlyArray, reason: Batch["reason"]) => + Effect.uninterruptibleMask((restore) => + Effect.gen(function* () { + const outcomes = new Map>(); + let taken = 0; + const work = Effect.gen(function* () { + // A retried transaction runs this again from the first job. + outcomes.clear(); + taken = 0; + const startedAt = yield* Clock.monotonicTimeNanos; + for (const job of jobs) { + if ( + taken > 0 && + waitingWriters > 0 && + (yield* Clock.monotonicTimeNanos) - startedAt >= MAX_HOLD_NANOS + ) + return; + taken++; + if (!job.owner.active) { + yield* job.cancel; + continue; + } + const outcome = yield* (jobs.length > 1 ? isolate(job.execute) : job.execute).pipe( + Effect.exit, + ); + outcomes.set(job, outcome); + if (Exit.isFailure(outcome)) { + job.owner.active = false; + if (jobs.length === 1) return yield* Effect.failCause(outcome.cause); + } + } + }); + const exit = yield* restore( + commit(work).pipe(Effect.provideService(CurrentWorker, yield* Effect.fiberId)), + ).pipe(Effect.exit); + const committed = taken === 0 ? jobs : jobs.slice(0, taken); + yield* Effect.annotateCurrentSpan({ + "orchestration_v2.work_item_count": committed.length, + "orchestration_v2.deferred_work_item_count": jobs.length - committed.length, + "orchestration_v2.consumer_count": new Set(committed.map((job) => job.owner)).size, + "orchestration_v2.raw_json_bytes": committed.reduce((bytes, job) => bytes + job.bytes, 0), + "orchestration_v2.batch_flush_reason": reason, + "orchestration_v2.backlog_batching": true, + "orchestration_v2.batch_wait_ms": + (yield* Clock.currentTimeMillis) - Math.min(...committed.map((job) => job.queuedAt)), + }); + for (const job of committed) { + const outcome = outcomes.get(job) ?? Exit.void; + yield* job.settle( + Exit.isSuccess(exit) ? outcome : exit, + Exit.isSuccess(exit) || + (Exit.isFailure(exit) && + Exit.isFailure(outcome) && + Cause.squash(exit.cause) === Cause.squash(outcome.cause)), + ); + } + return committed.length; + }), + ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush")); if (enabled) { yield* collect(queue).pipe( - Stream.runForEach((batch) => - Effect.uninterruptibleMask((restore) => - Effect.gen(function* () { - const jobs = batch.events.filter((job) => job.owner.active); - for (const job of batch.events) if (!job.owner.active) yield* job.cancel; + Stream.runForEach( + Effect.fnUntraced(function* (batch: Batch) { + let jobs = batch.events; + while (jobs.length > 0) { + for (const job of jobs) if (!job.owner.active) yield* job.cancel; + jobs = jobs.filter((job) => job.owner.active); if (jobs.length === 0) return; - yield* Effect.annotateCurrentSpan({ - "orchestration_v2.work_item_count": jobs.length, - "orchestration_v2.consumer_count": new Set(jobs.map((job) => job.owner)).size, - "orchestration_v2.raw_json_bytes": batch.bytes, - "orchestration_v2.batch_flush_reason": batch.reason, - "orchestration_v2.backlog_batching": true, - "orchestration_v2.batch_wait_ms": - (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)), - }); - const outcomes = new Map>(); - const work = Effect.forEach( - jobs, - (job) => - Effect.gen(function* () { - if (!job.owner.active) return yield* job.cancel; - const outcome = yield* ( - jobs.length > 1 && job.bytes > 0 ? isolate(job.execute) : job.execute - ).pipe(Effect.exit); - outcomes.set(job, outcome); - if (Exit.isFailure(outcome)) { - job.owner.active = false; - if (jobs.length === 1) return yield* Effect.failCause(outcome.cause); - } - }), - { discard: true }, - ); - const exit = yield* restore( - (jobs.every((job) => job.bytes === 0) ? work : commit(work)).pipe( - Effect.provideService(CurrentWorker, yield* Effect.fiberId), - ), - ).pipe(Effect.exit); - for (const job of jobs) { - const outcome = outcomes.get(job) ?? Exit.void; - yield* job.settle( - Exit.isSuccess(exit) ? outcome : exit, - Exit.isSuccess(exit) || - (Exit.isFailure(exit) && - Exit.isFailure(outcome) && - Cause.squash(exit.cause) === Cause.squash(outcome.cause)), - ); - } - }), - ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush")), + jobs = jobs.slice(yield* commitPrefix(jobs, batch.reason)); + } + }), ), Effect.ensuring(shutdown), Effect.forkScoped, ); } - const barrier = Effect.fnUntraced(function* (effect: Effect.Effect) { + /** + * Runs `effect` after queued snapshots for `threadIds` commit; omitted, after every queued + * snapshot. Other threads' snapshots never delay it. + */ + const barrier = Effect.fnUntraced(function* ( + effect: Effect.Effect, + threadIds?: ReadonlyArray, + ) { if (closed) return yield* Effect.interrupt; - if (!enabled || pending.size === 0 || (yield* CurrentWorker) === (yield* Effect.fiberId)) - return yield* effect; - const token = { active: true }; - return yield* enqueue(Effect.exit(effect), token, false, 0).pipe( - Effect.flatten, - Effect.orDie, - Effect.flatten, - Effect.ensuring( - Effect.sync(() => { - token.active = false; + if (!enabled || (yield* CurrentWorker) === (yield* Effect.fiberId)) return yield* effect; + return yield* Effect.uninterruptibleMask((restore) => { + waitingWriters++; + return restore( + Effect.gen(function* () { + for (const job of Array.from(pending)) { + if (threadIds === undefined || threadIds.includes(job.threadId)) yield* job.settled; + } + if (closed) return yield* Effect.interrupt; + return yield* effect; }), - ), - ); + ).pipe( + Effect.ensuring( + Effect.sync(() => { + waitingWriters--; + }), + ), + ); + }); }); const consume = Effect.fnUntraced(function* ( source: Stream.Stream, @@ -249,29 +300,31 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( Stream.concat(Stream.succeed({ _tag: "End" as const })), ); let resume: Effect.Effect = Effect.void; + // Replies run in order, so EOF, failures and controls follow this stream's earlier snapshots + // without waiting for other streams' work. yield* signals.pipe( Stream.mapEffect((signal) => Effect.gen(function* () { yield* resume; const processed = yield* Deferred.make(); - const event = signal._tag === "Event" ? signal.event : undefined; - // EOF only acknowledges prior work; it can share a commit with other consumers. - const snapshot = - signal._tag === "End" || - (event !== undefined && isSnapshot(event) && (yield* canBatch)); - const bytes = - snapshot && event !== undefined ? Buffer.byteLength(yield* encodeQueue(event)) : 0; - const work: Effect.Effect = - signal._tag === "Failure" - ? Effect.failCause(signal.cause) - : event === undefined - ? Effect.succeed(true) - : process(event).pipe(Effect.as(false)); - const reply = snapshot - ? yield* enqueue(work, token, true, bytes) - : (yield* enqueue(Effect.void, token, false, 0)).pipe(Effect.andThen(work)); - if (!snapshot) resume = Deferred.await(processed); - return { reply, processed, check: !snapshot }; + let reply: Effect.Effect; + let check = false; + if (signal._tag === "End") reply = Effect.succeed(true); + else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause); + else if (isSnapshot(signal.event) && (yield* canBatch)) { + const bytes = Buffer.byteLength(yield* encodeQueue(signal.event)); + reply = yield* enqueue( + process(signal.event).pipe(Effect.as(false)), + token, + snapshotThreadId(signal.event), + bytes, + ); + } else { + reply = process(signal.event).pipe(Effect.as(false)); + check = true; + resume = Deferred.await(processed); + } + return { reply, processed, check }; }), ), Stream.buffer({ capacity: MAX_EVENTS }), diff --git a/packages/shared/src/nodeSqliteClient.test.ts b/packages/shared/src/nodeSqliteClient.test.ts index 5c7b086642a4..24910bc0bf9c 100644 --- a/packages/shared/src/nodeSqliteClient.test.ts +++ b/packages/shared/src/nodeSqliteClient.test.ts @@ -230,6 +230,61 @@ it.effect.each([false, true])("batches worker results without losing rows (reade }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), ); +it.effect("coalesces savepoints with queries and preserves nested rollback", () => + Effect.gen(function* () { + const { filename } = yield* makeTempDatabase; + yield* Effect.gen(function* () { + const sql = yield* SqlClient.SqlClient; + const worker = startedWorkers.filter((entry) => !checkpointWorkers.includes(entry)).at(-1)!; + using messages = vi.spyOn(worker, "postMessage"); + yield* sql.withTransaction( + Effect.forEach(Array.from({ length: 53 }), () => sql.withTransaction(Effect.void)), + ); + assert.equal(messages.mock.calls.length, 2); + + yield* sql.withTransaction( + Effect.gen(function* () { + yield* sql`UPDATE counters SET value = 1 WHERE id = 1`; + const failure = yield* sql + .withTransaction( + sql`UPDATE counters SET value = 2 WHERE id = 1`.pipe( + Effect.andThen(Effect.fail("bad update")), + ), + ) + .pipe(Effect.flip); + assert.equal(failure, "bad update"); + const sent = messages.mock.calls.length; + assert.equal( + yield* sql.withTransaction(Effect.fail("empty update")).pipe(Effect.flip), + "empty update", + ); + assert.equal(messages.mock.calls.length, sent); + assert.deepEqual(yield* sql`SELECT value FROM counters WHERE id = 1`, [{ value: 1 }]); + assert.deepEqual(messages.mock.calls.at(-1)?.[0].savepoints, []); + yield* sql.withTransaction( + sql + .withTransaction( + sql`UPDATE counters SET value = 3 WHERE id = 1`.pipe( + Effect.andThen(Effect.fail("nested update")), + ), + ) + .pipe(Effect.flip), + ); + assert.deepEqual(yield* sql`SELECT value FROM counters WHERE id = 1`, [{ value: 1 }]); + + yield* sql.withTransaction(sql`UPDATE counters SET value = 4 WHERE id = 1`); + yield* sql`UPDATE counters SET value = 5 WHERE id = 1`; + const unsent = yield* sql + .withTransaction(sql.unsafe("SELECT ?", [Symbol("uncloneable")])) + .pipe(Effect.exit); + assert.isTrue(Exit.isFailure(unsent)); + }), + ); + assert.deepEqual(yield* sql`SELECT value FROM counters WHERE id = 1`, [{ value: 5 }]); + }).pipe(Effect.provide(SqliteClient.layer({ filename, writerWorker: true }))); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), +); + it.effect("reuses a worker query after an iterator constraint failure", () => Effect.gen(function* () { const { filename } = yield* makeTempDatabase; diff --git a/packages/shared/src/nodeSqliteClient.ts b/packages/shared/src/nodeSqliteClient.ts index 51e47c4bb553..da521a7c6036 100644 --- a/packages/shared/src/nodeSqliteClient.ts +++ b/packages/shared/src/nodeSqliteClient.ts @@ -247,11 +247,13 @@ parentPort.on("message", (request) => { parentPort.close(); return; } - const { id, sql, params, cached, values, raw, safeIntegers, received } = request; + const { id, sql, params, cached, values, raw, safeIntegers, received, savepoints } = request; let operation = "open"; let iterator; try { if (db === undefined) throw new Error("database connection is unavailable"); + operation = "execute"; + for (const savepoint of savepoints) db.exec(savepoint); operation = "prepare"; const statement = prepare(sql, cached); operation = "execute"; @@ -370,6 +372,8 @@ const makeWorkerConnection = Effect.fnUntraced(function* (options: { } | undefined; let nextRequestId = 0; + // Materialize nested savepoints with the next statement; empty jobs need no worker round trip. + const pendingSavepoints: string[] = []; // A replacement writer would lose the pragmas callers set on it, such as foreign_keys. let writerStopped: string | undefined; @@ -467,8 +471,18 @@ const makeWorkerConnection = Effect.fnUntraced(function* (options: { }); if (pending.size === 1) worker.ref(); const cached = input.cached && !hasLargeBinding(input.params); + const message = { + id, + safeIntegers, + ...input, + cached, + received, + savepoints: [...pendingSavepoints], + }; // oxlint-disable-next-line unicorn/require-post-message-target-origin -- Node workers do not accept a target origin. - worker.postMessage({ id, safeIntegers, ...input, cached, received }); + worker.postMessage(message); + // An unsent savepoint must stay queued, or its rollback would target an older sibling's. + pendingSavepoints.length = 0; }); // A posted write runs to the end, as it did on the event loop; the caller and its lock // must not move on while the worker is still applying it. Reads stay cancellable. @@ -492,12 +506,27 @@ const makeWorkerConnection = Effect.fnUntraced(function* (options: { executeUnprepared(sql, params, rowTransform) { // Read snapshots need deferred BEGIN; readonly connections reject IMMEDIATE and EXCLUSIVE. const statement = readOnly && /^\s*BEGIN\b/i.test(sql) ? "BEGIN" : sql; - const effect = request({ - sql: statement, - params: params ?? [], - cached: false, - values: false, - raw: false, + const effect = Effect.suspend(() => { + if (!readOnly && /^SAVEPOINT effect_sql_\d+$/.test(statement)) { + pendingSavepoints.push(statement); + return Effect.succeed([]); + } + const rollback = /^ROLLBACK TO SAVEPOINT (effect_sql_\d+)$/.exec(statement); + if (rollback !== null) { + const pending = pendingSavepoints.lastIndexOf(`SAVEPOINT ${rollback[1]}`); + if (pending !== -1) { + pendingSavepoints.splice(pending); + return Effect.succeed([]); + } + } + if (statement === "COMMIT" || statement === "ROLLBACK") pendingSavepoints.length = 0; + return request({ + sql: statement, + params: params ?? [], + cached: false, + values: false, + raw: false, + }); }); return rowTransform ? Effect.map(effect, rowTransform) : effect; },