diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index cad0aff3afb19..3f6744e576edd 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -15,6 +15,7 @@ import { ThreadId, } from "@t3tools/contracts"; import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; +import * as ThreadViewers from "./ThreadViewers.ts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -96,6 +97,7 @@ export interface EventSinkV2Shape { process: (event: A) => Effect.Effect, after: Effect.Effect, canBatch: Effect.Effect, + priority?: ProviderCommitBatch.StreamPriority, ) => Effect.Effect< void, E | E2 | ProviderCommitBatch.CommitBatchError | Schema.SchemaError, @@ -277,10 +279,12 @@ const layerBase: Layer.Layer< | ProjectStore.ProjectStoreV2 | SqlClient.SqlClient | TurnItemPositionStore.TurnItemPositionStoreV2 + | ThreadViewers.ThreadViewers > = Layer.effect( EventSinkV2, Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const threadViewers = yield* ThreadViewers.ThreadViewers; const commandReceipts = yield* CommandReceiptStore.CommandReceiptStoreV2; const effectOutbox = yield* EffectOutbox.EffectOutboxV2; const eventStore = yield* EventStore.EventStoreV2; @@ -1174,6 +1178,7 @@ const layerBase: Layer.Layer< ), ); }), + threadViewers.isViewed, ); return EventSinkV2.of({ @@ -1327,6 +1332,7 @@ export const layer: Layer.Layer< EffectOutbox.layer, ProjectStore.layer, TurnItemPositionStore.layer, + ThreadViewers.layer, ), ), ); diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 1fbab4085b283..5e208fc7fd658 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -64,6 +64,7 @@ import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; +import * as ThreadViewers from "./ThreadViewers.ts"; import * as EventStore from "./EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; @@ -491,7 +492,13 @@ it.effect("commits other threads' writes ahead of a thread's queued provider sna .pipe(Effect.forkScoped({ startImmediately: true })); yield* Queue.offer(input, snapshot); yield* Deferred.await(started); - yield* Queue.offerAll(input, [snapshot, snapshot]); + yield* Queue.offerAll( + input, + [2, 3].map((n) => ({ + ...snapshot, + node: { ...busyNode.payload, id: NodeId.make(`node:busy:${n}`) }, + })), + ); yield* Deferred.await(pulled); yield* TestClock.adjust(0); const writeTo = (event: OrchestrationV2DomainEvent, id: string) => @@ -4310,6 +4317,7 @@ it.effect.each(["direct", "provider-batch", "cancel-provider-batch"] as const)( layerCommandReceiptStoreProvided, ProjectStore.layer.pipe(Layer.provide(layerDatabase)), TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)), + ThreadViewers.layer, ), ), ); @@ -4608,6 +4616,7 @@ it.effect.each(["cancel transaction", "fail transaction", "cancel publication"] layerCommandReceiptStoreProvided, ProjectStore.layer.pipe(Layer.provide(layerDatabase)), TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)), + ThreadViewers.layer, ), ), ); diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index 2e88f2272dd14..86323b0bcafbc 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -21,7 +21,7 @@ function event(index: number, text = `delta:${index}`): ProviderAdapter.Provider type: "message.updated", driver, message: { - id: MessageId.make("single-message"), + id: MessageId.make(`message:${index}`), threadId, runId: null, nodeId: null, @@ -36,6 +36,15 @@ function event(index: number, text = `delta:${index}`): ProviderAdapter.Provider }, }; } +function snapshot(id: string, text: string, threadId = "single-thread") { + const base = event(0, text); + return base.type === "message.updated" + ? { + ...base, + message: { ...base.message, id: MessageId.make(id), threadId: ThreadId.make(threadId) }, + } + : base; +} const control: ProviderAdapter.ProviderAdapterV2Event = { type: "background_work.updated", driver, @@ -45,6 +54,7 @@ const control: ProviderAdapter.ProviderAdapterV2Event = { const recordBatches = Effect.fn("test.recordProviderBatches")(function* ( beforeCommit: Effect.Effect = Effect.void, + isViewed?: (threadId: string) => boolean, ) { const output = yield* Queue.unbounded>(); let current: ProviderAdapter.ProviderAdapterV2Event[] = []; @@ -57,6 +67,8 @@ const recordBatches = Effect.fn("test.recordProviderBatches")(function* ( if (current.length > 0) yield* Queue.offer(output, current); }), true, + undefined, + isViewed, ); const process = (raw: ProviderAdapter.ProviderAdapterV2Event) => raw.type === "background_work.updated" @@ -463,6 +475,150 @@ it.effect.each(["none", "writer", "snapshots"] as const)( }), ); +const blockFirstCommit = Effect.gen(function* () { + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + let first = true; + const block = Effect.gen(function* () { + if (!first) return; + first = false; + yield* Deferred.succeed(started, undefined); + yield* Deferred.await(release); + }); + return { started, release, block }; +}); + +it.effect.each([ + { keys: ["a", "b", "a", "b", "a", "b"], kept: [0, 4, 5] }, + // A first appearance between two snapshots of an entity keeps the earlier snapshot. + { keys: ["a", "b", "a", "b", "c", "a", "b"], kept: [0, 2, 3, 4, 5, 6] }, +])("combines superseded snapshots queued behind a busy commit: $keys", ({ keys, kept }) => + Effect.gen(function* () { + const writer = yield* blockFirstCommit; + const recorder = yield* recordBatches(writer.block); + const leading = yield* recorder + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false)) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(writer.started); + const snapshots = keys.map((key, index) => snapshot(key, `${key}:${index}`)); + const pulled = yield* Deferred.make(); + const consumer = yield* recorder + .consume( + Stream.fromIterable(snapshots).pipe( + Stream.tap((raw) => + raw === snapshots.at(-1) ? Deferred.succeed(pulled, undefined) : Effect.void, + ), + ), + recorder.process, + Effect.succeed(false), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(pulled); + yield* TestClock.adjust(0); + yield* Deferred.succeed(writer.release, undefined); + yield* Fiber.join(leading); + yield* Fiber.join(consumer); + assert.deepEqual(yield* Queue.takeAll(recorder.output), [ + [event(-1)], + kept.map((index) => snapshots[index]!), + ]); + }), +); + +it.effect("makes streams wait to read more once the shared backlog is full", () => + Effect.gen(function* () { + const writer = yield* blockFirstCommit; + const recorder = yield* recordBatches(writer.block); + const leading = yield* recorder + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false)) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(writer.started); + let arrivals = 0; + const streams = Array.from({ length: 5 }, (_, stream) => + Array.from({ length: 64 }, (_, n) => snapshot(`${stream}:${n}`, `${stream}:${n}`)), + ); + const consumers = yield* Effect.forEach(streams, (snapshots) => + recorder + .consume( + Stream.fromIterable(snapshots).pipe( + Stream.tap(() => + Effect.sync(() => { + arrivals++; + }), + ), + ), + recorder.process, + Effect.succeed(false), + ) + .pipe(Effect.forkScoped({ startImmediately: true })), + ); + yield* TestClock.adjust(0); + // 256 queued, plus at most one read per stream waiting for room. + assert.isAtMost(arrivals, 256 + streams.length); + yield* Deferred.succeed(writer.release, undefined); + yield* Fiber.join(leading); + yield* Effect.forEach(consumers, Fiber.join, { discard: true }); + assert.lengthOf((yield* Queue.takeAll(recorder.output)).flat(), 1 + 5 * 64); + }), +); + +it.effect.each([false, true])( + "orders a backlog by focus, thread kind and a finishing control with starved=%s", + (starved) => + Effect.gen(function* () { + const writer = yield* blockFirstCommit; + const recorder = yield* recordBatches(writer.block, (threadId) => threadId === "focused"); + const leading = yield* recorder + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false)) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(writer.started); + const streams = [ + { name: "subagent", delegated: true, finishing: false }, + { name: "main", delegated: false, finishing: false }, + { name: "finishing", delegated: false, finishing: true }, + { name: "focused", delegated: false, finishing: false }, + ]; + const pulled = yield* Deferred.make(); + let arrivals = 0; + const consumers = yield* Effect.forEach(streams, ({ name, delegated, finishing }) => + recorder + .consume( + // Open streams, so only the control marks a stream as finishing; EOF would too. + Stream.concat( + Stream.make(snapshot(name, name, name), ...(finishing ? [control] : [])), + Stream.never, + ).pipe( + Stream.tap(() => + ++arrivals === streams.length + 1 + ? Deferred.succeed(pulled, undefined) + : Effect.void, + ), + ), + recorder.process, + Effect.succeed(false), + Effect.succeed(true), + { threadId: name, delegated }, + ) + .pipe(Effect.forkScoped({ startImmediately: true })), + ); + yield* Deferred.await(pulled); + yield* TestClock.adjust(starved ? "30 seconds" : 0); + yield* Deferred.succeed(writer.release, undefined); + yield* Fiber.join(leading); + const outputs = yield* Queue.takeN(recorder.output, 3); + yield* Effect.forEach(consumers, Fiber.interrupt, { discard: true }); + // Waiting 30 seconds ranks a subagent like a main thread. + const order = starved + ? ["focused", "finishing", "subagent", "main"] + : ["focused", "finishing", "main", "subagent"]; + assert.deepEqual(Array.from(outputs), [ + [event(-1)], + order.map((name) => snapshot(name, name, name)), + [control], + ]); + }), +); + it.effect.each(["failure", "stop"] as const)( "flushes the prefix and respects a source %s barrier", (mode) => diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 9117653682f31..1fa7eac5ecd37 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -7,12 +7,17 @@ import * as Deferred from "effect/Deferred"; import * as Exit from "effect/Exit"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; const MAX_EVENTS = 64; const MAX_BYTES = 1024 * 1024; // While another write waits, a shared transaction stops taking jobs after this long. const MAX_HOLD_NANOS = 100_000_000n; +// A stream waiting this long ranks like a main thread, so subagents cannot starve. +const STARVATION_MILLIS = 30_000; +// Bounds snapshots held across stream queues; a full backlog makes streams wait to read more. +const MAX_QUEUED = 4 * MAX_EVENTS; const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe( @@ -21,35 +26,66 @@ export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe( ); type Snapshot = - | { readonly type: "node.updated"; readonly node: { readonly threadId: string } } - | { readonly type: "message.updated"; readonly message: { readonly threadId: string } } - | { readonly type: "turn_item.updated"; readonly turnItem: { readonly threadId: string } } - | { readonly type: "plan.updated"; readonly plan: { readonly threadId: string } }; + | { + readonly type: "node.updated"; + readonly node: { readonly id: string; readonly threadId: string }; + } + | { + readonly type: "message.updated"; + readonly message: { readonly id: string; readonly threadId: string }; + } + | { + readonly type: "turn_item.updated"; + readonly turnItem: { readonly id: string; readonly threadId: string }; + } + | { + readonly type: "plan.updated"; + readonly plan: { readonly id: string; readonly threadId: string }; + }; const isSnapshot = (event: { readonly type: string }): event is Snapshot => event.type === "node.updated" || event.type === "message.updated" || event.type === "turn_item.updated" || event.type === "plan.updated"; -const snapshotThreadId = (event: Snapshot) => { - switch (event.type) { - case "node.updated": - return event.node.threadId; - case "message.updated": - return event.message.threadId; - case "turn_item.updated": - return event.turnItem.threadId; - case "plan.updated": - return event.plan.threadId; - } +const snapshotEntity = (event: Snapshot) => { + const entity = + event.type === "node.updated" + ? event.node + : event.type === "message.updated" + ? event.message + : event.type === "turn_item.updated" + ? event.turnItem + : event.plan; + return { key: `${event.type}:${entity.id}`, threadId: entity.threadId }; }; +/** Who a provider stream writes for; ranks its snapshots against other streams'. */ +export interface StreamPriority { + readonly threadId: string; + /** A subagent or delegated task, which yields to main threads. */ + readonly delegated: boolean; +} + export class CommitBatchError extends Schema.TaggedError()("CommitBatchError", { cause: Schema.Defect(), }) {} export const isCommitBatchError = Schema.is(CommitBatchError); -interface Job { +interface StreamState { readonly owner: { active: boolean }; + readonly priority: StreamPriority | undefined; + /** Queued snapshots not yet taken into a batch, in stream order. */ + readonly jobs: Array; + unsettled: number; + servedAt: number; + /** A control, failure or EOF of this stream waits for its queued snapshots. */ + finishing: boolean; +} +interface Job { + readonly stream: StreamState; + /** Still in its stream's queue, holding one of the backlog's `MAX_QUEUED` slots. */ + queued: boolean; + readonly key: string; readonly threadId: string; readonly queuedAt: number; readonly bytes: number; @@ -58,34 +94,51 @@ interface Job { exit: Exit.Exit, preserveFailure: boolean, ) => Effect.Effect; + /** Settles a job a later snapshot of the same entity replaced, with that snapshot's outcome. */ + readonly skip: (exit: Exit.Exit) => Effect.Effect; readonly cancel: Effect.Effect; readonly settled: Effect.Effect; } +interface Unit { + readonly job: Job; + readonly superseded: ReadonlyArray; +} interface Batch { - readonly events: ReadonlyArray; + readonly units: ReadonlyArray; readonly reason: "ready" | "limit"; + readonly dequeued: number; } -function collect(queue: Queue.Queue) { - return Stream.fromEffectRepeat(Queue.takeBetween(queue, 1, MAX_EVENTS)).pipe( - Stream.flatMap((ready) => { - const batches: Array = []; - let events: Array = []; - let bytes = 0; - const flush = (reason: Batch["reason"]) => { - if (events.length > 0) batches.push({ events, reason }); - events = []; - bytes = 0; - }; - for (const job of ready) { - if (events.length > 0 && bytes + job.bytes > MAX_BYTES) flush("limit"); - events.push(job); - bytes += job.bytes; - if (events.length >= MAX_EVENTS || bytes >= MAX_BYTES) flush("limit"); - } - flush("ready"); - return Stream.fromArray(batches); - }), +/** + * Snapshots carry an entity's full state, so an earlier one is dropped when a later one of the + * same entity follows it in this stream with no first appearance of another entity in between. + * Kept jobs stay in stream order, and entities first appear in the same order as before. + */ +function coalesce(jobs: ReadonlyArray): Array { + const firstSeen = new Map(); + jobs.forEach((job, index) => { + if (!firstSeen.has(job.key)) firstSeen.set(job.key, index); + }); + const survivor: Array = Array.from({ length: jobs.length }, (_, index) => index); + const nextOfKey = new Map(); + for (let index = jobs.length - 1; index >= 0; index--) { + const job = jobs[index]!; + const next = nextOfKey.get(job.key); + nextOfKey.set(job.key, index); + if (next === undefined) continue; + let replaced = true; + for (let between = index + 1; replaced && between < next; between++) { + if (firstSeen.get(jobs[between]!.key)! > index) replaced = false; + } + if (replaced) survivor[index] = survivor[next]!; + } + const superseded = new Map>(); + jobs.forEach((job, index) => { + const kept = survivor[index]!; + if (kept !== index) superseded.set(kept, [...(superseded.get(kept) ?? []), job]); + }); + return jobs.flatMap((job, index) => + survivor[index] === index ? [{ job, superseded: superseded.get(index) ?? [] }] : [], ); } @@ -102,32 +155,44 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( isolate: ( work: Effect.Effect, ) => Effect.Effect = (work) => work, + /** Whether a client has the thread open; its streams rank first. */ + isViewed: (threadId: string) => boolean = () => false, ) { - const queue = yield* Queue.bounded(MAX_EVENTS); + const wake = yield* Queue.sliding(1); + const backlog = yield* Semaphore.make(MAX_QUEUED); + const streams = new Set(); const pending = new Set(); let closed = false; + let servings = 0; // Writes in `barrier` that have not finished; they wait on the batch or the writer it holds. let waitingWriters = 0; const shutdown = Effect.gen(function* () { closed = true; - yield* Queue.shutdown(queue); + yield* Queue.shutdown(wake); for (const job of pending) yield* job.cancel; + yield* backlog.releaseAll; }); yield* Effect.addFinalizer(() => shutdown); // These helpers run once per raw event; the shared transaction owns the tracing span. const enqueue = Effect.fnUntraced(function* ( effect: Effect.Effect, - token: { active: boolean }, - threadId: string, + stream: StreamState, + entity: { readonly key: string; readonly threadId: string }, bytes: number, + skipped: A, ) { const caller = yield* Effect.context(); const reply = yield* Deferred.make(); let ownFailure: Cause.Cause | undefined; let result: A; + const release = () => { + if (pending.delete(job)) stream.unsettled--; + }; const job: Job = { - owner: token, - threadId, + stream, + queued: false, + key: entity.key, + threadId: entity.threadId, queuedAt: yield* Clock.currentTimeMillis, bytes, execute: effect.pipe( @@ -149,44 +214,111 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( exit: Exit.Exit, preserveFailure: boolean, ) { - pending.delete(job); + release(); if (Exit.isFailure(exit)) { - token.active = false; + stream.owner.active = false; yield* Deferred.failCause( reply, preserveFailure && ownFailure !== undefined ? ownFailure : exit.cause, ); } else yield* Deferred.succeed(reply, result); }), - cancel: Effect.gen(function* () { - pending.delete(job); - yield* Deferred.interrupt(reply); + skip: (exit) => + Exit.isFailure(exit) + ? job.settle(exit, false) + : Effect.suspend(() => { + release(); + return Deferred.succeed(reply, skipped); + }), + cancel: Effect.suspend(() => { + release(); + const dequeued = job.queued; + job.queued = false; + return Deferred.interrupt(reply).pipe( + Effect.andThen(dequeued ? backlog.release(1) : Effect.void), + ); }), settled: Deferred.await(reply).pipe(Effect.exit, Effect.asVoid), }; yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { + if (closed) return yield* Deferred.interrupt(reply); + yield* restore(backlog.take(1)); + if (closed) { + yield* backlog.release(1); + return yield* Deferred.interrupt(reply); + } + job.queued = true; pending.add(job); - const accepted = yield* restore(Queue.offer(queue, job)).pipe( - Effect.onError(() => job.cancel), - ); - if (!accepted) yield* job.cancel; + stream.unsettled++; + stream.jobs.push(job); + yield* Queue.offer(wake, undefined); }), ); return Deferred.await(reply); }); - // Commits a prefix of `jobs` in one transaction and returns how many it took. - const commitPrefix = (jobs: ReadonlyArray, reason: Batch["reason"]) => + const rank = (stream: StreamState, now: number) => { + const tier = + stream.priority === undefined + ? 1 + : isViewed(stream.priority.threadId) + ? 0 + : stream.priority.delegated + ? 2 + : 1; + const starved = now - stream.jobs[0]!.queuedAt >= STARVATION_MILLIS; + return [starved ? Math.min(tier, 1) : tier, stream.finishing ? 0 : 1, stream.servedAt]; + }; + // Viewed streams first, then main threads, then subagents; within a tier, streams finishing a + // control first, then the stream served longest ago. + const nextBatch = (now: number): Batch | undefined => { + const ready = Array.from(streams, (stream) => + stream.jobs.length === 0 ? [] : [{ stream, rank: rank(stream, now) }], + ) + .flat() + .sort((left, right) => { + for (let index = 0; index < left.rank.length; index++) { + const order = left.rank[index]! - right.rank[index]!; + if (order !== 0) return order; + } + return 0; + }); + if (ready.length === 0) return undefined; + const units: Array = []; + let bytes = 0; + let dequeued = 0; + let reason: Batch["reason"] = "ready"; + for (const { stream } of ready) { + let take = 0; + while (take < stream.jobs.length && take < MAX_EVENTS - units.length) { + const next = stream.jobs[take]!.bytes; + if (units.length + take > 0 && bytes + next > MAX_BYTES) break; + bytes += next; + take++; + } + if (take < stream.jobs.length) reason = "limit"; + if (take === 0) break; + const taken = stream.jobs.splice(0, take); + for (const job of taken) job.queued = false; + dequeued += take; + units.push(...coalesce(taken)); + stream.servedAt = ++servings; + if (units.length >= MAX_EVENTS || bytes >= MAX_BYTES) break; + } + return { units, reason, dequeued }; + }; + // Commits a prefix of `units` in one transaction and returns how many it took. + const commitPrefix = (units: ReadonlyArray, reason: Batch["reason"]) => Effect.uninterruptibleMask((restore) => Effect.gen(function* () { const outcomes = new Map>(); let taken = 0; const work = Effect.gen(function* () { - // A retried transaction runs this again from the first job. + // A retried transaction runs this again from the first unit. outcomes.clear(); taken = 0; const startedAt = yield* Clock.monotonicTimeNanos; - for (const job of jobs) { + for (const { job, superseded } of units) { if ( taken > 0 && waitingWriters > 0 && @@ -194,63 +326,62 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ) return; taken++; - if (!job.owner.active) { + if (!job.stream.owner.active) { yield* job.cancel; + for (const replaced of superseded) yield* replaced.cancel; continue; } - const outcome = yield* (jobs.length > 1 ? isolate(job.execute) : job.execute).pipe( + const outcome = yield* (units.length > 1 ? isolate(job.execute) : job.execute).pipe( Effect.exit, ); outcomes.set(job, outcome); if (Exit.isFailure(outcome)) { - job.owner.active = false; - if (jobs.length === 1) return yield* Effect.failCause(outcome.cause); + job.stream.owner.active = false; + if (units.length === 1) return yield* Effect.failCause(outcome.cause); } } }); const exit = yield* restore( commit(work).pipe(Effect.provideService(CurrentWorker, yield* Effect.fiberId)), ).pipe(Effect.exit); - const committed = taken === 0 ? jobs : jobs.slice(0, taken); + const committed = taken === 0 ? units : units.slice(0, taken); + const jobs = committed.flatMap((unit) => [unit.job, ...unit.superseded]); yield* Effect.annotateCurrentSpan({ "orchestration_v2.work_item_count": committed.length, - "orchestration_v2.deferred_work_item_count": jobs.length - committed.length, - "orchestration_v2.consumer_count": new Set(committed.map((job) => job.owner)).size, - "orchestration_v2.raw_json_bytes": committed.reduce((bytes, job) => bytes + job.bytes, 0), + "orchestration_v2.coalesced_work_item_count": jobs.length - committed.length, + "orchestration_v2.deferred_work_item_count": units.length - committed.length, + "orchestration_v2.consumer_count": new Set(jobs.map((job) => job.stream)).size, + "orchestration_v2.raw_json_bytes": jobs.reduce((bytes, job) => bytes + job.bytes, 0), "orchestration_v2.batch_flush_reason": reason, "orchestration_v2.backlog_batching": true, "orchestration_v2.batch_wait_ms": - (yield* Clock.currentTimeMillis) - Math.min(...committed.map((job) => job.queuedAt)), + (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)), }); - for (const job of committed) { + for (const { job, superseded } of committed) { const outcome = outcomes.get(job) ?? Exit.void; + const final = Exit.isSuccess(exit) ? outcome : exit; yield* job.settle( - Exit.isSuccess(exit) ? outcome : exit, + final, Exit.isSuccess(exit) || (Exit.isFailure(exit) && Exit.isFailure(outcome) && Cause.squash(exit.cause) === Cause.squash(outcome.cause)), ); + for (const replaced of superseded) yield* replaced.skip(final); } return committed.length; }), ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush")); if (enabled) { - yield* collect(queue).pipe( - Stream.runForEach( - Effect.fnUntraced(function* (batch: Batch) { - let jobs = batch.events; - while (jobs.length > 0) { - for (const job of jobs) if (!job.owner.active) yield* job.cancel; - jobs = jobs.filter((job) => job.owner.active); - if (jobs.length === 0) return; - jobs = jobs.slice(yield* commitPrefix(jobs, batch.reason)); - } - }), - ), - Effect.ensuring(shutdown), - Effect.forkScoped, - ); + yield* Effect.forever( + Effect.gen(function* () { + const batch = nextBatch(yield* Clock.currentTimeMillis); + if (batch === undefined) return yield* Queue.take(wake); + yield* backlog.release(batch.dequeued); + let units = batch.units; + while (units.length > 0) units = units.slice(yield* commitPrefix(units, batch.reason)); + }), + ).pipe(Effect.ensuring(shutdown), Effect.forkScoped); } /** * Runs `effect` after queued snapshots for `threadIds` commit; omitted, after every queued @@ -286,6 +417,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( process: (event: A) => Effect.Effect, after: Effect.Effect, canBatch: Effect.Effect = Effect.succeed(true), + priority?: StreamPriority, ) { if (!enabled) return yield* source.pipe( @@ -293,7 +425,15 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( Stream.takeUntil(Boolean), Stream.runDrain, ); - const token = { active: true }; + const stream: StreamState = { + owner: { active: true }, + priority, + jobs: [], + unsettled: 0, + servedAt: 0, + finishing: false, + }; + streams.add(stream); const signals = source.pipe( Stream.map((event) => ({ _tag: "Event" as const, event })), Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure" as const, cause })), @@ -309,20 +449,31 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( const processed = yield* Deferred.make(); let reply: Effect.Effect; let check = false; - if (signal._tag === "End") reply = Effect.succeed(true); - else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause); - else if (isSnapshot(signal.event) && (yield* canBatch)) { - const bytes = Buffer.byteLength(yield* encodeQueue(signal.event)); + const snapshot = + signal._tag === "Event" && isSnapshot(signal.event) && (yield* canBatch) + ? signal.event + : undefined; + if (snapshot !== undefined) { + const bytes = Buffer.byteLength(yield* encodeQueue(snapshot)); reply = yield* enqueue( - process(signal.event).pipe(Effect.as(false)), - token, - snapshotThreadId(signal.event), + process(snapshot).pipe(Effect.as(false)), + stream, + snapshotEntity(snapshot), bytes, + false, ); } else { - reply = process(signal.event).pipe(Effect.as(false)); - check = true; - resume = Deferred.await(processed); + if (stream.unsettled > 0) stream.finishing = true; + if (signal._tag === "End") reply = Effect.succeed(true); + else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause); + else { + const event = signal.event; + reply = Effect.sync(() => { + stream.finishing = false; + }).pipe(Effect.andThen(process(event)), Effect.as(false)); + check = true; + resume = Deferred.await(processed); + } } return { reply, processed, check }; }), @@ -335,7 +486,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ), Effect.tap((stop) => Effect.sync(() => { - if (stop) token.active = false; + if (stop) stream.owner.active = false; }), ), Effect.ensuring(Deferred.succeed(processed, undefined)), @@ -344,8 +495,10 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( Stream.takeUntil(Boolean), Stream.runDrain, Effect.ensuring( - Effect.sync(() => { - token.active = false; + Effect.gen(function* () { + stream.owner.active = false; + streams.delete(stream); + for (const job of stream.jobs.splice(0)) yield* job.cancel; }), ), ); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index da162282719aa..af167bd76a101 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1,6 +1,7 @@ import { makeAssistantStreamingFilter } from "./assistantStreaming.ts"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { + isDelegatedAppThread, isOrchestrationV2WorkActive, CommandId, type EventId, @@ -1988,6 +1989,7 @@ export const layer: Layer.Layer< return yield* shouldStopProviderEventIngestion; }), Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)), + { threadId: input.run.threadId, delegated: isDelegatedAppThread(input.appThread) }, ) .pipe( Effect.ensuring(closeIdleFlush), diff --git a/apps/server/src/orchestration-v2/ThreadViewers.ts b/apps/server/src/orchestration-v2/ThreadViewers.ts new file mode 100644 index 0000000000000..68d6cf319280f --- /dev/null +++ b/apps/server/src/orchestration-v2/ThreadViewers.ts @@ -0,0 +1,33 @@ +import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import type * as Scope from "effect/Scope"; + +export class ThreadViewers extends Context.Service< + ThreadViewers, + { + /** Counts a client viewing `threadId` until the enclosing scope closes. */ + readonly view: (threadId: string) => Effect.Effect; + readonly isViewed: (threadId: string) => boolean; + } +>()("t3/orchestration-v2/ThreadViewers") {} + +export const layer = Layer.sync(ThreadViewers, () => { + const viewers = new Map(); + return ThreadViewers.of({ + view: Effect.fn("ThreadViewers.view")(function* (threadId: string) { + yield* Effect.acquireRelease( + Effect.sync(() => { + viewers.set(threadId, (viewers.get(threadId) ?? 0) + 1); + }), + () => + Effect.sync(() => { + const remaining = (viewers.get(threadId) ?? 1) - 1; + if (remaining > 0) viewers.set(threadId, remaining); + else viewers.delete(threadId); + }), + ); + }), + isViewed: (threadId) => viewers.has(threadId), + }); +}); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index da06fd60e9000..9477f58c73061 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -28,6 +28,7 @@ import * as ContextHandoffService from "./ContextHandoffService.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; +import * as ThreadViewers from "./ThreadViewers.ts"; import * as EventStore from "./EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts"; @@ -86,7 +87,10 @@ const layerStores = Layer.mergeAll( TurnItemPositionStore.layer, ); -export const layerEventSink = EventSink.layerFromStores.pipe(Layer.provide(layerStores)); +// The server provides this same layer to thread subscriptions; memoized, they share one registry. +export const layerEventSink = EventSink.layerFromStores.pipe( + Layer.provide(Layer.merge(layerStores, ThreadViewers.layer)), +); const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe( Layer.provide(layerStores), ); diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 1e69421513a8b..2c977140a5a23 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -37,6 +37,7 @@ import * as ContextHandoffService from "../ContextHandoffService.ts"; import * as EffectOutbox from "../EffectOutbox.ts"; import * as EffectWorker from "../EffectWorker.ts"; import * as EventSink from "../EventSink.ts"; +import * as ThreadViewers from "../ThreadViewers.ts"; import * as EventStore from "../EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as Orchestrator from "../Orchestrator.ts"; @@ -349,7 +350,7 @@ export function layerWithRegistry( TurnItemPositionStore.layer, ).pipe(Layer.provide(layerDatabase)); const layerEventSinkProvided = EventSink.layerFromStores.pipe( - Layer.provide(Layer.mergeAll(layerStores, layerDatabase)), + Layer.provide(Layer.mergeAll(layerStores, layerDatabase, ThreadViewers.layer)), ); const layerRuntime = RuntimePolicy.layerEnforcingCeiling.pipe( Layer.provide(Layer.merge(layerRuntimeBase, layerServerSettings)), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 1c97c4629738e..35f63e9c44a60 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -215,6 +215,7 @@ import * as CursorUsageAccounts from "@t3tools/provider-cursor/server/CursorUsag import * as UsageService from "./usage/UsageService.ts"; import * as CodexUsageHistory from "./usage/CodexUsageHistory.ts"; import * as RuntimeLayer from "./orchestration-v2/runtimeLayer.ts"; +import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; import * as ThreadSettlementService from "./orchestration-v2/ThreadSettlementService.ts"; @@ -814,6 +815,8 @@ const layerRuntimeCoreDependencies = layerRuntimeCoreDependenciesBase.pipe( const layerRuntimeDependencies = layerRuntimeCoreDependencies.pipe( Layer.provideMerge(layerUsage), Layer.provideMerge(layerBackground), + // The same layer the event sink uses; memoized, subscriptions and batch ranking share it. + Layer.provideMerge(ThreadViewers.layer), Layer.provideMerge(layerResourceDiagnostics), Layer.provideMerge(TraceDiagnostics.layer), Layer.provideMerge(TriageService.layer.pipe(Layer.provide(GitVcsDriver.layer))), diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index 26456c0da320d..c96497c76a2c3 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -82,6 +82,7 @@ import * as BackgroundPolicy from "./background/BackgroundPolicy.ts"; import * as CheckpointDiffQuery from "./checkpointing/CheckpointDiffQuery.ts"; import * as NativeChildWorkQuery from "./orchestration-v2/NativeChildWorkQuery.ts"; import * as CommandOutputHub from "./orchestration-v2/CommandOutputHub.ts"; +import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts"; import * as CodexInstallation from "./provider/CodexInstallation.ts"; import * as DeviceService from "./device/DeviceService.ts"; import * as DocumentText from "./provider/DocumentText.ts"; @@ -832,6 +833,7 @@ const layerUnusedConfigRpcServices = Layer.mergeAll( Layer.mock(CheckpointDiffQuery.CheckpointDiffQuery)({}), Layer.mock(NativeChildWorkQuery.NativeChildWorkQuery)({}), Layer.mock(CommandOutputHub.CommandOutputHub)({}), + ThreadViewers.layer, Layer.mock(CodexInstallation.CodexInstallation)({ managedDirectory: "/unused" }), Layer.mock(DeviceService.DeviceService)({}), Layer.mock(EnvironmentTheme.EnvironmentThemeService)({}), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 9408127f5a2b3..7b4c232b92680 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -119,6 +119,7 @@ import * as EnvironmentTheme from "./environmentTheme.ts"; import * as Keybindings from "./keybindings.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; import * as ThreadManagementService from "./orchestration-v2/ThreadManagementService.ts"; +import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts"; import * as CommandOutputHub from "./orchestration-v2/CommandOutputHub.ts"; import * as McpAppRequests from "./mcpApps/McpAppRequests.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; @@ -1480,6 +1481,7 @@ export const layerWsRpc = ( }; const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService; const commandOutputHub = yield* CommandOutputHub.CommandOutputHub; + const threadViewers = yield* ThreadViewers.ThreadViewers; const providerSessionManager = yield* ProviderSessionManager.ProviderSessionManagerV2; const scheduledTasks = yield* ScheduledTasks.ScheduledTaskService; const secretRequests = yield* SecretRequests.SecretRequests; @@ -2322,6 +2324,7 @@ export const layerWsRpc = ( [ORCHESTRATION_V2_WS_METHODS.subscribeThread]: (input) => Stream.unwrap( Effect.annotateCurrentSpan({ "orchestration_v2.thread_id": input.threadId }).pipe( + Effect.andThen(threadViewers.view(input.threadId)), Effect.andThen(subscribeOrchestrationV2Thread(input)), ), ),