From 991cfa3b9d3e1b30ced1253ae679084d822c136d Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 19 Nov 2026 02:31:19 -0600 Subject: [PATCH 1/3] perf(server): rank queued provider snapshots by focus and combine superseded ones Agent streams shared one first-in-first-out snapshot queue, so a viewed thread's last updates waited behind every other stream's backlog, and each queued snapshot was written even when a later one replaced it. Each stream now keeps its own queue. Batches take viewed threads first, then main threads, then subagents; within a tier, streams finishing a control or EOF go first, then the stream served longest ago. A stream waiting 30 seconds ranks like a main thread. Within a batch, an earlier snapshot of an entity is skipped when a later one follows with no other entity first appearing in between, so kept work and first appearances stay in stream order. Implemented with Claude Opus 5.5 through Cursor in T3 Code. Co-authored-by: Cursor --- apps/server/src/orchestration-v2/EventSink.ts | 1 + .../FoundationPersistence.test.ts | 8 +- .../ProviderCommitBatch.test.ts | 123 ++++++- .../orchestration-v2/ProviderCommitBatch.ts | 324 ++++++++++++------ .../orchestration-v2/RunExecutionService.ts | 2 + .../src/orchestration-v2/ThreadViewers.ts | 31 ++ apps/server/src/ws.ts | 7 + 7 files changed, 397 insertions(+), 99 deletions(-) create mode 100644 apps/server/src/orchestration-v2/ThreadViewers.ts diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index cad0aff3afb19..4d0e32a9e28e1 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -96,6 +96,7 @@ export interface EventSinkV2Shape { process: (event: A) => Effect.Effect, after: Effect.Effect, canBatch: Effect.Effect, + priority?: ProviderCommitBatch.StreamPriority, ) => Effect.Effect< void, E | E2 | ProviderCommitBatch.CommitBatchError | Schema.SchemaError, diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 1fbab4085b283..20d8ab364ea7c 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -491,7 +491,13 @@ it.effect("commits other threads' writes ahead of a thread's queued provider sna .pipe(Effect.forkScoped({ startImmediately: true })); yield* Queue.offer(input, snapshot); yield* Deferred.await(started); - yield* Queue.offerAll(input, [snapshot, snapshot]); + yield* Queue.offerAll( + input, + [2, 3].map((n) => ({ + ...snapshot, + node: { ...busyNode.payload, id: NodeId.make(`node:busy:${n}`) }, + })), + ); yield* Deferred.await(pulled); yield* TestClock.adjust(0); const writeTo = (event: OrchestrationV2DomainEvent, id: string) => diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index 2e88f2272dd14..6d4326e84897c 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -12,6 +12,7 @@ import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; +import * as ThreadViewers from "./ThreadViewers.ts"; const driver = ProviderDriverKind.make("codex"); function event(index: number, text = `delta:${index}`): ProviderAdapter.ProviderAdapterV2Event { @@ -21,7 +22,7 @@ function event(index: number, text = `delta:${index}`): ProviderAdapter.Provider type: "message.updated", driver, message: { - id: MessageId.make("single-message"), + id: MessageId.make(`message:${index}`), threadId, runId: null, nodeId: null, @@ -36,6 +37,15 @@ function event(index: number, text = `delta:${index}`): ProviderAdapter.Provider }, }; } +function snapshot(id: string, text: string, threadId = "single-thread") { + const base = event(0, text); + return base.type === "message.updated" + ? { + ...base, + message: { ...base.message, id: MessageId.make(id), threadId: ThreadId.make(threadId) }, + } + : base; +} const control: ProviderAdapter.ProviderAdapterV2Event = { type: "background_work.updated", driver, @@ -463,6 +473,117 @@ it.effect.each(["none", "writer", "snapshots"] as const)( }), ); +const blockFirstCommit = Effect.gen(function* () { + const started = yield* Deferred.make(); + const release = yield* Deferred.make(); + let first = true; + const block = Effect.gen(function* () { + if (!first) return; + first = false; + yield* Deferred.succeed(started, undefined); + yield* Deferred.await(release); + }); + return { started, release, block }; +}); + +it.effect.each([ + { keys: ["a", "b", "a", "b", "a", "b"], kept: [0, 4, 5] }, + // A first appearance between two snapshots of an entity keeps the earlier snapshot. + { keys: ["a", "b", "a", "b", "c", "a", "b"], kept: [0, 2, 3, 4, 5, 6] }, +])("combines superseded snapshots queued behind a busy commit: $keys", ({ keys, kept }) => + Effect.gen(function* () { + const writer = yield* blockFirstCommit; + const recorder = yield* recordBatches(writer.block); + const leading = yield* recorder + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false)) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(writer.started); + const snapshots = keys.map((key, index) => snapshot(key, `${key}:${index}`)); + const pulled = yield* Deferred.make(); + const consumer = yield* recorder + .consume( + Stream.fromIterable(snapshots).pipe( + Stream.tap((raw) => + raw === snapshots.at(-1) ? Deferred.succeed(pulled, undefined) : Effect.void, + ), + ), + recorder.process, + Effect.succeed(false), + ) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(pulled); + yield* TestClock.adjust(0); + yield* Deferred.succeed(writer.release, undefined); + yield* Fiber.join(leading); + yield* Fiber.join(consumer); + assert.deepEqual(yield* Queue.takeAll(recorder.output), [ + [event(-1)], + kept.map((index) => snapshots[index]!), + ]); + }), +); + +it.effect.each([false, true])( + "orders a backlog by focus, thread kind and a finishing control with starved=%s", + (starved) => + Effect.gen(function* () { + const viewers = ThreadViewers.make(); + viewers.view("focused"); + const writer = yield* blockFirstCommit; + const recorder = yield* recordBatches(writer.block).pipe( + Effect.provideService(ThreadViewers.ThreadViewers, viewers), + ); + const leading = yield* recorder + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false)) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(writer.started); + const streams = [ + { name: "subagent", delegated: true, finishing: false }, + { name: "main", delegated: false, finishing: false }, + { name: "finishing", delegated: false, finishing: true }, + { name: "focused", delegated: false, finishing: false }, + ]; + const pulled = yield* Deferred.make(); + let arrivals = 0; + const consumers = yield* Effect.forEach(streams, ({ name, delegated, finishing }) => + recorder + .consume( + // Open streams, so only the control marks a stream as finishing; EOF would too. + Stream.concat( + Stream.make(snapshot(name, name, name), ...(finishing ? [control] : [])), + Stream.never, + ).pipe( + Stream.tap(() => + ++arrivals === streams.length + 1 + ? Deferred.succeed(pulled, undefined) + : Effect.void, + ), + ), + recorder.process, + Effect.succeed(false), + Effect.succeed(true), + { threadId: name, delegated }, + ) + .pipe(Effect.forkScoped({ startImmediately: true })), + ); + yield* Deferred.await(pulled); + yield* TestClock.adjust(starved ? "30 seconds" : 0); + yield* Deferred.succeed(writer.release, undefined); + yield* Fiber.join(leading); + const outputs = yield* Queue.takeN(recorder.output, 3); + yield* Effect.forEach(consumers, Fiber.interrupt, { discard: true }); + // Waiting 30 seconds ranks a subagent like a main thread. + const order = starved + ? ["focused", "finishing", "subagent", "main"] + : ["focused", "finishing", "main", "subagent"]; + assert.deepEqual(Array.from(outputs), [ + [event(-1)], + order.map((name) => snapshot(name, name, name)), + [control], + ]); + }), +); + it.effect.each(["failure", "stop"] as const)( "flushes the prefix and respects a source %s barrier", (mode) => diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 9117653682f31..6bc69ac3c0dd4 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -9,10 +9,14 @@ import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; import * as Stream from "effect/Stream"; +import { ThreadViewers } from "./ThreadViewers.ts"; + const MAX_EVENTS = 64; const MAX_BYTES = 1024 * 1024; // While another write waits, a shared transaction stops taking jobs after this long. const MAX_HOLD_NANOS = 100_000_000n; +// A stream waiting this long ranks like a main thread, so subagents cannot starve. +const STARVATION_MILLIS = 30_000; const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe( @@ -21,35 +25,64 @@ export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe( ); type Snapshot = - | { readonly type: "node.updated"; readonly node: { readonly threadId: string } } - | { readonly type: "message.updated"; readonly message: { readonly threadId: string } } - | { readonly type: "turn_item.updated"; readonly turnItem: { readonly threadId: string } } - | { readonly type: "plan.updated"; readonly plan: { readonly threadId: string } }; + | { + readonly type: "node.updated"; + readonly node: { readonly id: string; readonly threadId: string }; + } + | { + readonly type: "message.updated"; + readonly message: { readonly id: string; readonly threadId: string }; + } + | { + readonly type: "turn_item.updated"; + readonly turnItem: { readonly id: string; readonly threadId: string }; + } + | { + readonly type: "plan.updated"; + readonly plan: { readonly id: string; readonly threadId: string }; + }; const isSnapshot = (event: { readonly type: string }): event is Snapshot => event.type === "node.updated" || event.type === "message.updated" || event.type === "turn_item.updated" || event.type === "plan.updated"; -const snapshotThreadId = (event: Snapshot) => { - switch (event.type) { - case "node.updated": - return event.node.threadId; - case "message.updated": - return event.message.threadId; - case "turn_item.updated": - return event.turnItem.threadId; - case "plan.updated": - return event.plan.threadId; - } +const snapshotEntity = (event: Snapshot) => { + const entity = + event.type === "node.updated" + ? event.node + : event.type === "message.updated" + ? event.message + : event.type === "turn_item.updated" + ? event.turnItem + : event.plan; + return { key: `${event.type}:${entity.id}`, threadId: entity.threadId }; }; +/** Who a provider stream writes for; ranks its snapshots against other streams'. */ +export interface StreamPriority { + readonly threadId: string; + /** A subagent or delegated task, which yields to main threads. */ + readonly delegated: boolean; +} + export class CommitBatchError extends Schema.TaggedError()("CommitBatchError", { cause: Schema.Defect(), }) {} export const isCommitBatchError = Schema.is(CommitBatchError); -interface Job { +interface StreamState { readonly owner: { active: boolean }; + readonly priority: StreamPriority | undefined; + /** Queued snapshots not yet taken into a batch, in stream order. */ + readonly jobs: Array; + unsettled: number; + servedAt: number; + /** A control, failure or EOF of this stream waits for its queued snapshots. */ + finishing: boolean; +} +interface Job { + readonly stream: StreamState; + readonly key: string; readonly threadId: string; readonly queuedAt: number; readonly bytes: number; @@ -58,34 +91,50 @@ interface Job { exit: Exit.Exit, preserveFailure: boolean, ) => Effect.Effect; + /** Settles a job a later snapshot of the same entity replaced, with that snapshot's outcome. */ + readonly skip: (exit: Exit.Exit) => Effect.Effect; readonly cancel: Effect.Effect; readonly settled: Effect.Effect; } +interface Unit { + readonly job: Job; + readonly superseded: ReadonlyArray; +} interface Batch { - readonly events: ReadonlyArray; + readonly units: ReadonlyArray; readonly reason: "ready" | "limit"; } -function collect(queue: Queue.Queue) { - return Stream.fromEffectRepeat(Queue.takeBetween(queue, 1, MAX_EVENTS)).pipe( - Stream.flatMap((ready) => { - const batches: Array = []; - let events: Array = []; - let bytes = 0; - const flush = (reason: Batch["reason"]) => { - if (events.length > 0) batches.push({ events, reason }); - events = []; - bytes = 0; - }; - for (const job of ready) { - if (events.length > 0 && bytes + job.bytes > MAX_BYTES) flush("limit"); - events.push(job); - bytes += job.bytes; - if (events.length >= MAX_EVENTS || bytes >= MAX_BYTES) flush("limit"); - } - flush("ready"); - return Stream.fromArray(batches); - }), +/** + * Snapshots carry an entity's full state, so an earlier one is dropped when a later one of the + * same entity follows it in this stream with no first appearance of another entity in between. + * Kept jobs stay in stream order, and entities first appear in the same order as before. + */ +function coalesce(jobs: ReadonlyArray): Array { + const firstSeen = new Map(); + jobs.forEach((job, index) => { + if (!firstSeen.has(job.key)) firstSeen.set(job.key, index); + }); + const survivor: Array = Array.from({ length: jobs.length }, (_, index) => index); + const nextOfKey = new Map(); + for (let index = jobs.length - 1; index >= 0; index--) { + const job = jobs[index]!; + const next = nextOfKey.get(job.key); + nextOfKey.set(job.key, index); + if (next === undefined) continue; + let replaced = true; + for (let between = index + 1; replaced && between < next; between++) { + if (firstSeen.get(jobs[between]!.key)! > index) replaced = false; + } + if (replaced) survivor[index] = survivor[next]!; + } + const superseded = new Map>(); + jobs.forEach((job, index) => { + const kept = survivor[index]!; + if (kept !== index) superseded.set(kept, [...(superseded.get(kept) ?? []), job]); + }); + return jobs.flatMap((job, index) => + survivor[index] === index ? [{ job, superseded: superseded.get(index) ?? [] }] : [], ); } @@ -103,31 +152,39 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( work: Effect.Effect, ) => Effect.Effect = (work) => work, ) { - const queue = yield* Queue.bounded(MAX_EVENTS); + const viewers = yield* ThreadViewers; + const wake = yield* Queue.sliding(1); + const streams = new Set(); const pending = new Set(); let closed = false; + let servings = 0; // Writes in `barrier` that have not finished; they wait on the batch or the writer it holds. let waitingWriters = 0; const shutdown = Effect.gen(function* () { closed = true; - yield* Queue.shutdown(queue); + yield* Queue.shutdown(wake); for (const job of pending) yield* job.cancel; }); yield* Effect.addFinalizer(() => shutdown); // These helpers run once per raw event; the shared transaction owns the tracing span. const enqueue = Effect.fnUntraced(function* ( effect: Effect.Effect, - token: { active: boolean }, - threadId: string, + stream: StreamState, + entity: { readonly key: string; readonly threadId: string }, bytes: number, + skipped: A, ) { const caller = yield* Effect.context(); const reply = yield* Deferred.make(); let ownFailure: Cause.Cause | undefined; let result: A; + const release = () => { + if (pending.delete(job)) stream.unsettled--; + }; const job: Job = { - owner: token, - threadId, + stream, + key: entity.key, + threadId: entity.threadId, queuedAt: yield* Clock.currentTimeMillis, bytes, execute: effect.pipe( @@ -149,44 +206,97 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( exit: Exit.Exit, preserveFailure: boolean, ) { - pending.delete(job); + release(); if (Exit.isFailure(exit)) { - token.active = false; + stream.owner.active = false; yield* Deferred.failCause( reply, preserveFailure && ownFailure !== undefined ? ownFailure : exit.cause, ); } else yield* Deferred.succeed(reply, result); }), - cancel: Effect.gen(function* () { - pending.delete(job); - yield* Deferred.interrupt(reply); + skip: (exit) => + Exit.isFailure(exit) + ? job.settle(exit, false) + : Effect.suspend(() => { + release(); + return Deferred.succeed(reply, skipped); + }), + cancel: Effect.suspend(() => { + release(); + return Deferred.interrupt(reply); }), settled: Deferred.await(reply).pipe(Effect.exit, Effect.asVoid), }; - yield* Effect.uninterruptibleMask((restore) => + yield* Effect.uninterruptibleMask(() => Effect.gen(function* () { + if (closed) return yield* Deferred.interrupt(reply); pending.add(job); - const accepted = yield* restore(Queue.offer(queue, job)).pipe( - Effect.onError(() => job.cancel), - ); - if (!accepted) yield* job.cancel; + stream.unsettled++; + stream.jobs.push(job); + yield* Queue.offer(wake, undefined); }), ); return Deferred.await(reply); }); - // Commits a prefix of `jobs` in one transaction and returns how many it took. - const commitPrefix = (jobs: ReadonlyArray, reason: Batch["reason"]) => + const rank = (stream: StreamState, now: number) => { + const tier = + stream.priority === undefined + ? 1 + : viewers.isViewed(stream.priority.threadId) + ? 0 + : stream.priority.delegated + ? 2 + : 1; + const starved = now - stream.jobs[0]!.queuedAt >= STARVATION_MILLIS; + return [starved ? Math.min(tier, 1) : tier, stream.finishing ? 0 : 1, stream.servedAt]; + }; + // Viewed streams first, then main threads, then subagents; within a tier, streams finishing a + // control first, then the stream served longest ago. + const nextBatch = (now: number): Batch | undefined => { + const ready = Array.from(streams, (stream) => + stream.jobs.length === 0 ? [] : [{ stream, rank: rank(stream, now) }], + ) + .flat() + .sort((left, right) => { + for (let index = 0; index < left.rank.length; index++) { + const order = left.rank[index]! - right.rank[index]!; + if (order !== 0) return order; + } + return 0; + }); + if (ready.length === 0) return undefined; + const units: Array = []; + let bytes = 0; + let reason: Batch["reason"] = "ready"; + for (const { stream } of ready) { + let take = 0; + while (take < stream.jobs.length && take < MAX_EVENTS - units.length) { + const next = stream.jobs[take]!.bytes; + if (units.length + take > 0 && bytes + next > MAX_BYTES) break; + bytes += next; + take++; + } + if (take < stream.jobs.length) reason = "limit"; + if (take === 0) break; + units.push(...coalesce(stream.jobs.splice(0, take))); + stream.servedAt = ++servings; + if (units.length >= MAX_EVENTS || bytes >= MAX_BYTES) break; + } + return { units, reason }; + }; + // Commits a prefix of `units` in one transaction and returns how many it took. + const commitPrefix = (units: ReadonlyArray, reason: Batch["reason"]) => Effect.uninterruptibleMask((restore) => Effect.gen(function* () { const outcomes = new Map>(); let taken = 0; const work = Effect.gen(function* () { - // A retried transaction runs this again from the first job. + // A retried transaction runs this again from the first unit. outcomes.clear(); taken = 0; const startedAt = yield* Clock.monotonicTimeNanos; - for (const job of jobs) { + for (const { job, superseded } of units) { if ( taken > 0 && waitingWriters > 0 && @@ -194,63 +304,61 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ) return; taken++; - if (!job.owner.active) { + if (!job.stream.owner.active) { yield* job.cancel; + for (const replaced of superseded) yield* replaced.cancel; continue; } - const outcome = yield* (jobs.length > 1 ? isolate(job.execute) : job.execute).pipe( + const outcome = yield* (units.length > 1 ? isolate(job.execute) : job.execute).pipe( Effect.exit, ); outcomes.set(job, outcome); if (Exit.isFailure(outcome)) { - job.owner.active = false; - if (jobs.length === 1) return yield* Effect.failCause(outcome.cause); + job.stream.owner.active = false; + if (units.length === 1) return yield* Effect.failCause(outcome.cause); } } }); const exit = yield* restore( commit(work).pipe(Effect.provideService(CurrentWorker, yield* Effect.fiberId)), ).pipe(Effect.exit); - const committed = taken === 0 ? jobs : jobs.slice(0, taken); + const committed = taken === 0 ? units : units.slice(0, taken); + const jobs = committed.flatMap((unit) => [unit.job, ...unit.superseded]); yield* Effect.annotateCurrentSpan({ "orchestration_v2.work_item_count": committed.length, - "orchestration_v2.deferred_work_item_count": jobs.length - committed.length, - "orchestration_v2.consumer_count": new Set(committed.map((job) => job.owner)).size, - "orchestration_v2.raw_json_bytes": committed.reduce((bytes, job) => bytes + job.bytes, 0), + "orchestration_v2.coalesced_work_item_count": jobs.length - committed.length, + "orchestration_v2.deferred_work_item_count": units.length - committed.length, + "orchestration_v2.consumer_count": new Set(jobs.map((job) => job.stream)).size, + "orchestration_v2.raw_json_bytes": jobs.reduce((bytes, job) => bytes + job.bytes, 0), "orchestration_v2.batch_flush_reason": reason, "orchestration_v2.backlog_batching": true, "orchestration_v2.batch_wait_ms": - (yield* Clock.currentTimeMillis) - Math.min(...committed.map((job) => job.queuedAt)), + (yield* Clock.currentTimeMillis) - Math.min(...jobs.map((job) => job.queuedAt)), }); - for (const job of committed) { + for (const { job, superseded } of committed) { const outcome = outcomes.get(job) ?? Exit.void; + const final = Exit.isSuccess(exit) ? outcome : exit; yield* job.settle( - Exit.isSuccess(exit) ? outcome : exit, + final, Exit.isSuccess(exit) || (Exit.isFailure(exit) && Exit.isFailure(outcome) && Cause.squash(exit.cause) === Cause.squash(outcome.cause)), ); + for (const replaced of superseded) yield* replaced.skip(final); } return committed.length; }), ).pipe(Effect.withSpan("orchestrationV2.ProviderCommitBatch.flush")); if (enabled) { - yield* collect(queue).pipe( - Stream.runForEach( - Effect.fnUntraced(function* (batch: Batch) { - let jobs = batch.events; - while (jobs.length > 0) { - for (const job of jobs) if (!job.owner.active) yield* job.cancel; - jobs = jobs.filter((job) => job.owner.active); - if (jobs.length === 0) return; - jobs = jobs.slice(yield* commitPrefix(jobs, batch.reason)); - } - }), - ), - Effect.ensuring(shutdown), - Effect.forkScoped, - ); + yield* Effect.forever( + Effect.gen(function* () { + const batch = nextBatch(yield* Clock.currentTimeMillis); + if (batch === undefined) return yield* Queue.take(wake); + let units = batch.units; + while (units.length > 0) units = units.slice(yield* commitPrefix(units, batch.reason)); + }), + ).pipe(Effect.ensuring(shutdown), Effect.forkScoped); } /** * Runs `effect` after queued snapshots for `threadIds` commit; omitted, after every queued @@ -286,6 +394,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( process: (event: A) => Effect.Effect, after: Effect.Effect, canBatch: Effect.Effect = Effect.succeed(true), + priority?: StreamPriority, ) { if (!enabled) return yield* source.pipe( @@ -293,7 +402,15 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( Stream.takeUntil(Boolean), Stream.runDrain, ); - const token = { active: true }; + const stream: StreamState = { + owner: { active: true }, + priority, + jobs: [], + unsettled: 0, + servedAt: 0, + finishing: false, + }; + streams.add(stream); const signals = source.pipe( Stream.map((event) => ({ _tag: "Event" as const, event })), Stream.catchCause((cause) => Stream.succeed({ _tag: "Failure" as const, cause })), @@ -309,20 +426,31 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( const processed = yield* Deferred.make(); let reply: Effect.Effect; let check = false; - if (signal._tag === "End") reply = Effect.succeed(true); - else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause); - else if (isSnapshot(signal.event) && (yield* canBatch)) { - const bytes = Buffer.byteLength(yield* encodeQueue(signal.event)); + const snapshot = + signal._tag === "Event" && isSnapshot(signal.event) && (yield* canBatch) + ? signal.event + : undefined; + if (snapshot !== undefined) { + const bytes = Buffer.byteLength(yield* encodeQueue(snapshot)); reply = yield* enqueue( - process(signal.event).pipe(Effect.as(false)), - token, - snapshotThreadId(signal.event), + process(snapshot).pipe(Effect.as(false)), + stream, + snapshotEntity(snapshot), bytes, + false, ); } else { - reply = process(signal.event).pipe(Effect.as(false)); - check = true; - resume = Deferred.await(processed); + if (stream.unsettled > 0) stream.finishing = true; + if (signal._tag === "End") reply = Effect.succeed(true); + else if (signal._tag === "Failure") reply = Effect.failCause(signal.cause); + else { + const event = signal.event; + reply = Effect.sync(() => { + stream.finishing = false; + }).pipe(Effect.andThen(process(event)), Effect.as(false)); + check = true; + resume = Deferred.await(processed); + } } return { reply, processed, check }; }), @@ -335,7 +463,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ), Effect.tap((stop) => Effect.sync(() => { - if (stop) token.active = false; + if (stop) stream.owner.active = false; }), ), Effect.ensuring(Deferred.succeed(processed, undefined)), @@ -344,8 +472,10 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( Stream.takeUntil(Boolean), Stream.runDrain, Effect.ensuring( - Effect.sync(() => { - token.active = false; + Effect.gen(function* () { + stream.owner.active = false; + streams.delete(stream); + for (const job of stream.jobs.splice(0)) yield* job.cancel; }), ), ); diff --git a/apps/server/src/orchestration-v2/RunExecutionService.ts b/apps/server/src/orchestration-v2/RunExecutionService.ts index f1d21f4a6bad5..9437e6433e6e9 100644 --- a/apps/server/src/orchestration-v2/RunExecutionService.ts +++ b/apps/server/src/orchestration-v2/RunExecutionService.ts @@ -1,6 +1,7 @@ import { makeAssistantStreamingFilter } from "./assistantStreaming.ts"; import { resolveProjectSettings } from "@t3tools/shared/projectSettings"; import { + isDelegatedAppThread, isOrchestrationV2WorkActive, CommandId, type EventId, @@ -1988,6 +1989,7 @@ export const layer: Layer.Layer< return yield* shouldStopProviderEventIngestion; }), Ref.get(rootTerminalSeen).pipe(Effect.map((seen) => !seen)), + { threadId: input.run.threadId, delegated: isDelegatedAppThread(input.appThread) }, ) .pipe( Effect.ensuring(closeIdleFlush), diff --git a/apps/server/src/orchestration-v2/ThreadViewers.ts b/apps/server/src/orchestration-v2/ThreadViewers.ts new file mode 100644 index 0000000000000..cad1c139dd261 --- /dev/null +++ b/apps/server/src/orchestration-v2/ThreadViewers.ts @@ -0,0 +1,31 @@ +import * as Context from "effect/Context"; + +export interface ThreadViewersShape { + /** Counts a client viewing `threadId` until the returned release runs. */ + readonly view: (threadId: string) => () => void; + readonly isViewed: (threadId: string) => boolean; +} + +export const make = (): ThreadViewersShape => { + const viewers = new Map(); + return { + view: (threadId) => { + viewers.set(threadId, (viewers.get(threadId) ?? 0) + 1); + let released = false; + return () => { + if (released) return; + released = true; + const remaining = (viewers.get(threadId) ?? 1) - 1; + if (remaining > 0) viewers.set(threadId, remaining); + else viewers.delete(threadId); + }; + }, + isViewed: (threadId) => viewers.has(threadId), + }; +}; + +// The default is one shared registry, so thread subscriptions and the provider batch agree. +export class ThreadViewers extends Context.Reference( + "t3/orchestration-v2/ThreadViewers", + { defaultValue: make }, +) {} diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index 63d45c48816e4..62fc43b076716 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -118,6 +118,7 @@ import * as EnvironmentTheme from "./environmentTheme.ts"; import * as Keybindings from "./keybindings.ts"; import * as ExternalLauncher from "./process/externalLauncher.ts"; import * as ThreadManagementService from "./orchestration-v2/ThreadManagementService.ts"; +import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts"; import * as CommandOutputHub from "./orchestration-v2/CommandOutputHub.ts"; import * as McpAppRequests from "./mcpApps/McpAppRequests.ts"; import * as ProviderSessionManager from "./orchestration-v2/ProviderSessionManager.ts"; @@ -2317,6 +2318,12 @@ export const layerWsRpc = ( Stream.unwrap( Effect.annotateCurrentSpan({ "orchestration_v2.thread_id": input.threadId }).pipe( Effect.andThen(subscribeOrchestrationV2Thread(input)), + Effect.flatMap((stream) => + Effect.gen(function* () { + const release = (yield* ThreadViewers.ThreadViewers).view(input.threadId); + return stream.pipe(Stream.ensuring(Effect.sync(release))); + }), + ), ), ), [ORCHESTRATION_V2_WS_METHODS.subscribeCommandOutput]: (input) => From 96f5ec94120fd42fc1afc50ef03da37381abdade Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 19 Nov 2026 21:27:12 -0600 Subject: [PATCH 2/3] fix(server): bound the provider snapshot backlog across streams Per-stream queues dropped the shared queue's backpressure, so each stream could hold a full buffer of snapshots in memory. Streams now wait to read more once 256 snapshots are queued across all of them. Co-authored-by: Cursor --- .../ProviderCommitBatch.test.ts | 37 +++++++++++++++++++ .../orchestration-v2/ProviderCommitBatch.ts | 32 ++++++++++++++-- 2 files changed, 65 insertions(+), 4 deletions(-) diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index 6d4326e84897c..6055266c17875 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -523,6 +523,43 @@ it.effect.each([ }), ); +it.effect("makes streams wait to read more once the shared backlog is full", () => + Effect.gen(function* () { + const writer = yield* blockFirstCommit; + const recorder = yield* recordBatches(writer.block); + const leading = yield* recorder + .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false)) + .pipe(Effect.forkScoped({ startImmediately: true })); + yield* Deferred.await(writer.started); + let arrivals = 0; + const streams = Array.from({ length: 5 }, (_, stream) => + Array.from({ length: 64 }, (_, n) => snapshot(`${stream}:${n}`, `${stream}:${n}`)), + ); + const consumers = yield* Effect.forEach(streams, (snapshots) => + recorder + .consume( + Stream.fromIterable(snapshots).pipe( + Stream.tap(() => + Effect.sync(() => { + arrivals++; + }), + ), + ), + recorder.process, + Effect.succeed(false), + ) + .pipe(Effect.forkScoped({ startImmediately: true })), + ); + yield* TestClock.adjust(0); + // 256 queued, plus at most one read per stream waiting for room. + assert.isAtMost(arrivals, 256 + streams.length); + yield* Deferred.succeed(writer.release, undefined); + yield* Fiber.join(leading); + yield* Effect.forEach(consumers, Fiber.join, { discard: true }); + assert.lengthOf((yield* Queue.takeAll(recorder.output)).flat(), 1 + 5 * 64); + }), +); + it.effect.each([false, true])( "orders a backlog by focus, thread kind and a finishing control with starved=%s", (starved) => diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 6bc69ac3c0dd4..994fb59583d9d 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -7,6 +7,7 @@ import * as Deferred from "effect/Deferred"; import * as Exit from "effect/Exit"; import * as Queue from "effect/Queue"; import * as Schema from "effect/Schema"; +import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; import { ThreadViewers } from "./ThreadViewers.ts"; @@ -17,6 +18,8 @@ const MAX_BYTES = 1024 * 1024; const MAX_HOLD_NANOS = 100_000_000n; // A stream waiting this long ranks like a main thread, so subagents cannot starve. const STARVATION_MILLIS = 30_000; +// Bounds snapshots held across stream queues; a full backlog makes streams wait to read more. +const MAX_QUEUED = 4 * MAX_EVENTS; const encodeQueue = Schema.encodeEffect(Schema.fromJsonString(Schema.Unknown)); export const enabled = Config.Boolean("T3CODE_PROVIDER_COMMIT_BATCHING").pipe( @@ -82,6 +85,8 @@ interface StreamState { } interface Job { readonly stream: StreamState; + /** Still in its stream's queue, holding one of the backlog's `MAX_QUEUED` slots. */ + queued: boolean; readonly key: string; readonly threadId: string; readonly queuedAt: number; @@ -103,6 +108,7 @@ interface Unit { interface Batch { readonly units: ReadonlyArray; readonly reason: "ready" | "limit"; + readonly dequeued: number; } /** @@ -154,6 +160,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( ) { const viewers = yield* ThreadViewers; const wake = yield* Queue.sliding(1); + const backlog = yield* Semaphore.make(MAX_QUEUED); const streams = new Set(); const pending = new Set(); let closed = false; @@ -164,6 +171,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( closed = true; yield* Queue.shutdown(wake); for (const job of pending) yield* job.cancel; + yield* backlog.releaseAll; }); yield* Effect.addFinalizer(() => shutdown); // These helpers run once per raw event; the shared transaction owns the tracing span. @@ -183,6 +191,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( }; const job: Job = { stream, + queued: false, key: entity.key, threadId: entity.threadId, queuedAt: yield* Clock.currentTimeMillis, @@ -224,13 +233,23 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( }), cancel: Effect.suspend(() => { release(); - return Deferred.interrupt(reply); + const dequeued = job.queued; + job.queued = false; + return Deferred.interrupt(reply).pipe( + Effect.andThen(dequeued ? backlog.release(1) : Effect.void), + ); }), settled: Deferred.await(reply).pipe(Effect.exit, Effect.asVoid), }; - yield* Effect.uninterruptibleMask(() => + yield* Effect.uninterruptibleMask((restore) => Effect.gen(function* () { if (closed) return yield* Deferred.interrupt(reply); + yield* restore(backlog.take(1)); + if (closed) { + yield* backlog.release(1); + return yield* Deferred.interrupt(reply); + } + job.queued = true; pending.add(job); stream.unsettled++; stream.jobs.push(job); @@ -268,6 +287,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( if (ready.length === 0) return undefined; const units: Array = []; let bytes = 0; + let dequeued = 0; let reason: Batch["reason"] = "ready"; for (const { stream } of ready) { let take = 0; @@ -279,11 +299,14 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( } if (take < stream.jobs.length) reason = "limit"; if (take === 0) break; - units.push(...coalesce(stream.jobs.splice(0, take))); + const taken = stream.jobs.splice(0, take); + for (const job of taken) job.queued = false; + dequeued += take; + units.push(...coalesce(taken)); stream.servedAt = ++servings; if (units.length >= MAX_EVENTS || bytes >= MAX_BYTES) break; } - return { units, reason }; + return { units, reason, dequeued }; }; // Commits a prefix of `units` in one transaction and returns how many it took. const commitPrefix = (units: ReadonlyArray, reason: Batch["reason"]) => @@ -355,6 +378,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( Effect.gen(function* () { const batch = nextBatch(yield* Clock.currentTimeMillis); if (batch === undefined) return yield* Queue.take(wake); + yield* backlog.release(batch.dequeued); let units = batch.units; while (units.length > 0) units = units.slice(yield* commitPrefix(units, batch.reason)); }), From 17735eb08d7ddb8e777f9dcbd26549ce9aea5b78 Mon Sep 17 00:00:00 2001 From: Yash Singh Date: Thu, 19 Nov 2026 21:34:36 -0600 Subject: [PATCH 3/3] refactor(server): make thread viewers a scoped service ThreadViewers was a Context.Reference with a shared mutable default, and the subscribeThread handler attached its release only once the stream started, so an interruption in between could leave a thread ranked as viewed. It is now a service with a layer: view registers with acquireRelease in the stream's scope, the event sink passes isViewed to the batch coordinator, and the server provides the same memoized layer to subscriptions and the event sink. Co-authored-by: Cursor --- apps/server/src/orchestration-v2/EventSink.ts | 5 ++ .../FoundationPersistence.test.ts | 3 ++ .../ProviderCommitBatch.test.ts | 10 ++-- .../orchestration-v2/ProviderCommitBatch.ts | 7 ++- .../src/orchestration-v2/ThreadViewers.ts | 54 ++++++++++--------- .../src/orchestration-v2/runtimeLayer.ts | 6 ++- .../testkit/ProviderReplayHarness.ts | 3 +- apps/server/src/server.ts | 3 ++ apps/server/src/ws.test.ts | 2 + apps/server/src/ws.ts | 8 +-- 10 files changed, 57 insertions(+), 44 deletions(-) diff --git a/apps/server/src/orchestration-v2/EventSink.ts b/apps/server/src/orchestration-v2/EventSink.ts index 4d0e32a9e28e1..3f6744e576edd 100644 --- a/apps/server/src/orchestration-v2/EventSink.ts +++ b/apps/server/src/orchestration-v2/EventSink.ts @@ -15,6 +15,7 @@ import { ThreadId, } from "@t3tools/contracts"; import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; +import * as ThreadViewers from "./ThreadViewers.ts"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Effect from "effect/Effect"; @@ -278,10 +279,12 @@ const layerBase: Layer.Layer< | ProjectStore.ProjectStoreV2 | SqlClient.SqlClient | TurnItemPositionStore.TurnItemPositionStoreV2 + | ThreadViewers.ThreadViewers > = Layer.effect( EventSinkV2, Effect.gen(function* () { const sql = yield* SqlClient.SqlClient; + const threadViewers = yield* ThreadViewers.ThreadViewers; const commandReceipts = yield* CommandReceiptStore.CommandReceiptStoreV2; const effectOutbox = yield* EffectOutbox.EffectOutboxV2; const eventStore = yield* EventStore.EventStoreV2; @@ -1175,6 +1178,7 @@ const layerBase: Layer.Layer< ), ); }), + threadViewers.isViewed, ); return EventSinkV2.of({ @@ -1328,6 +1332,7 @@ export const layer: Layer.Layer< EffectOutbox.layer, ProjectStore.layer, TurnItemPositionStore.layer, + ThreadViewers.layer, ), ), ); diff --git a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts index 20d8ab364ea7c..5e208fc7fd658 100644 --- a/apps/server/src/orchestration-v2/FoundationPersistence.test.ts +++ b/apps/server/src/orchestration-v2/FoundationPersistence.test.ts @@ -64,6 +64,7 @@ import * as CommandReceiptStore from "./CommandReceiptStore.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; +import * as ThreadViewers from "./ThreadViewers.ts"; import * as EventStore from "./EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProjectionMaintenance from "./ProjectionMaintenance.ts"; @@ -4316,6 +4317,7 @@ it.effect.each(["direct", "provider-batch", "cancel-provider-batch"] as const)( layerCommandReceiptStoreProvided, ProjectStore.layer.pipe(Layer.provide(layerDatabase)), TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)), + ThreadViewers.layer, ), ), ); @@ -4614,6 +4616,7 @@ it.effect.each(["cancel transaction", "fail transaction", "cancel publication"] layerCommandReceiptStoreProvided, ProjectStore.layer.pipe(Layer.provide(layerDatabase)), TurnItemPositionStore.layer.pipe(Layer.provide(layerDatabase)), + ThreadViewers.layer, ), ), ); diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts index 6055266c17875..86323b0bcafbc 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.test.ts @@ -12,7 +12,6 @@ import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as TestClock from "effect/testing/TestClock"; import * as ProviderCommitBatch from "./ProviderCommitBatch.ts"; -import * as ThreadViewers from "./ThreadViewers.ts"; const driver = ProviderDriverKind.make("codex"); function event(index: number, text = `delta:${index}`): ProviderAdapter.ProviderAdapterV2Event { @@ -55,6 +54,7 @@ const control: ProviderAdapter.ProviderAdapterV2Event = { const recordBatches = Effect.fn("test.recordProviderBatches")(function* ( beforeCommit: Effect.Effect = Effect.void, + isViewed?: (threadId: string) => boolean, ) { const output = yield* Queue.unbounded>(); let current: ProviderAdapter.ProviderAdapterV2Event[] = []; @@ -67,6 +67,8 @@ const recordBatches = Effect.fn("test.recordProviderBatches")(function* ( if (current.length > 0) yield* Queue.offer(output, current); }), true, + undefined, + isViewed, ); const process = (raw: ProviderAdapter.ProviderAdapterV2Event) => raw.type === "background_work.updated" @@ -564,12 +566,8 @@ it.effect.each([false, true])( "orders a backlog by focus, thread kind and a finishing control with starved=%s", (starved) => Effect.gen(function* () { - const viewers = ThreadViewers.make(); - viewers.view("focused"); const writer = yield* blockFirstCommit; - const recorder = yield* recordBatches(writer.block).pipe( - Effect.provideService(ThreadViewers.ThreadViewers, viewers), - ); + const recorder = yield* recordBatches(writer.block, (threadId) => threadId === "focused"); const leading = yield* recorder .consume(Stream.make(event(-1)), recorder.process, Effect.succeed(false)) .pipe(Effect.forkScoped({ startImmediately: true })); diff --git a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts index 994fb59583d9d..1fa7eac5ecd37 100644 --- a/apps/server/src/orchestration-v2/ProviderCommitBatch.ts +++ b/apps/server/src/orchestration-v2/ProviderCommitBatch.ts @@ -10,8 +10,6 @@ import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import * as Stream from "effect/Stream"; -import { ThreadViewers } from "./ThreadViewers.ts"; - const MAX_EVENTS = 64; const MAX_BYTES = 1024 * 1024; // While another write waits, a shared transaction stops taking jobs after this long. @@ -157,8 +155,9 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( isolate: ( work: Effect.Effect, ) => Effect.Effect = (work) => work, + /** Whether a client has the thread open; its streams rank first. */ + isViewed: (threadId: string) => boolean = () => false, ) { - const viewers = yield* ThreadViewers; const wake = yield* Queue.sliding(1); const backlog = yield* Semaphore.make(MAX_QUEUED); const streams = new Set(); @@ -262,7 +261,7 @@ export const make = Effect.fn("ProviderCommitBatch.make")(function* ( const tier = stream.priority === undefined ? 1 - : viewers.isViewed(stream.priority.threadId) + : isViewed(stream.priority.threadId) ? 0 : stream.priority.delegated ? 2 diff --git a/apps/server/src/orchestration-v2/ThreadViewers.ts b/apps/server/src/orchestration-v2/ThreadViewers.ts index cad1c139dd261..68d6cf319280f 100644 --- a/apps/server/src/orchestration-v2/ThreadViewers.ts +++ b/apps/server/src/orchestration-v2/ThreadViewers.ts @@ -1,31 +1,33 @@ import * as Context from "effect/Context"; +import * as Effect from "effect/Effect"; +import * as Layer from "effect/Layer"; +import type * as Scope from "effect/Scope"; -export interface ThreadViewersShape { - /** Counts a client viewing `threadId` until the returned release runs. */ - readonly view: (threadId: string) => () => void; - readonly isViewed: (threadId: string) => boolean; -} +export class ThreadViewers extends Context.Service< + ThreadViewers, + { + /** Counts a client viewing `threadId` until the enclosing scope closes. */ + readonly view: (threadId: string) => Effect.Effect; + readonly isViewed: (threadId: string) => boolean; + } +>()("t3/orchestration-v2/ThreadViewers") {} -export const make = (): ThreadViewersShape => { +export const layer = Layer.sync(ThreadViewers, () => { const viewers = new Map(); - return { - view: (threadId) => { - viewers.set(threadId, (viewers.get(threadId) ?? 0) + 1); - let released = false; - return () => { - if (released) return; - released = true; - const remaining = (viewers.get(threadId) ?? 1) - 1; - if (remaining > 0) viewers.set(threadId, remaining); - else viewers.delete(threadId); - }; - }, + return ThreadViewers.of({ + view: Effect.fn("ThreadViewers.view")(function* (threadId: string) { + yield* Effect.acquireRelease( + Effect.sync(() => { + viewers.set(threadId, (viewers.get(threadId) ?? 0) + 1); + }), + () => + Effect.sync(() => { + const remaining = (viewers.get(threadId) ?? 1) - 1; + if (remaining > 0) viewers.set(threadId, remaining); + else viewers.delete(threadId); + }), + ); + }), isViewed: (threadId) => viewers.has(threadId), - }; -}; - -// The default is one shared registry, so thread subscriptions and the provider batch agree. -export class ThreadViewers extends Context.Reference( - "t3/orchestration-v2/ThreadViewers", - { defaultValue: make }, -) {} + }); +}); diff --git a/apps/server/src/orchestration-v2/runtimeLayer.ts b/apps/server/src/orchestration-v2/runtimeLayer.ts index da06fd60e9000..9477f58c73061 100644 --- a/apps/server/src/orchestration-v2/runtimeLayer.ts +++ b/apps/server/src/orchestration-v2/runtimeLayer.ts @@ -28,6 +28,7 @@ import * as ContextHandoffService from "./ContextHandoffService.ts"; import * as EffectOutbox from "./EffectOutbox.ts"; import * as EffectWorker from "./EffectWorker.ts"; import * as EventSink from "./EventSink.ts"; +import * as ThreadViewers from "./ThreadViewers.ts"; import * as EventStore from "./EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as LegacyV1ThreadImporter from "./legacy/LegacyV1ThreadImporter.ts"; @@ -86,7 +87,10 @@ const layerStores = Layer.mergeAll( TurnItemPositionStore.layer, ); -export const layerEventSink = EventSink.layerFromStores.pipe(Layer.provide(layerStores)); +// The server provides this same layer to thread subscriptions; memoized, they share one registry. +export const layerEventSink = EventSink.layerFromStores.pipe( + Layer.provide(Layer.merge(layerStores, ThreadViewers.layer)), +); const layerProjectionMaintenanceProvided = ProjectionMaintenance.layer.pipe( Layer.provide(layerStores), ); diff --git a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts index 1e69421513a8b..2c977140a5a23 100644 --- a/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts +++ b/apps/server/src/orchestration-v2/testkit/ProviderReplayHarness.ts @@ -37,6 +37,7 @@ import * as ContextHandoffService from "../ContextHandoffService.ts"; import * as EffectOutbox from "../EffectOutbox.ts"; import * as EffectWorker from "../EffectWorker.ts"; import * as EventSink from "../EventSink.ts"; +import * as ThreadViewers from "../ThreadViewers.ts"; import * as EventStore from "../EventStore.ts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as Orchestrator from "../Orchestrator.ts"; @@ -349,7 +350,7 @@ export function layerWithRegistry( TurnItemPositionStore.layer, ).pipe(Layer.provide(layerDatabase)); const layerEventSinkProvided = EventSink.layerFromStores.pipe( - Layer.provide(Layer.mergeAll(layerStores, layerDatabase)), + Layer.provide(Layer.mergeAll(layerStores, layerDatabase, ThreadViewers.layer)), ); const layerRuntime = RuntimePolicy.layerEnforcingCeiling.pipe( Layer.provide(Layer.merge(layerRuntimeBase, layerServerSettings)), diff --git a/apps/server/src/server.ts b/apps/server/src/server.ts index 1c97c4629738e..35f63e9c44a60 100644 --- a/apps/server/src/server.ts +++ b/apps/server/src/server.ts @@ -215,6 +215,7 @@ import * as CursorUsageAccounts from "@t3tools/provider-cursor/server/CursorUsag import * as UsageService from "./usage/UsageService.ts"; import * as CodexUsageHistory from "./usage/CodexUsageHistory.ts"; import * as RuntimeLayer from "./orchestration-v2/runtimeLayer.ts"; +import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts"; import * as ProjectStore from "./orchestration-v2/ProjectStore.ts"; import * as ThreadSearch from "./orchestration-v2/ThreadSearch.ts"; import * as ThreadSettlementService from "./orchestration-v2/ThreadSettlementService.ts"; @@ -814,6 +815,8 @@ const layerRuntimeCoreDependencies = layerRuntimeCoreDependenciesBase.pipe( const layerRuntimeDependencies = layerRuntimeCoreDependencies.pipe( Layer.provideMerge(layerUsage), Layer.provideMerge(layerBackground), + // The same layer the event sink uses; memoized, subscriptions and batch ranking share it. + Layer.provideMerge(ThreadViewers.layer), Layer.provideMerge(layerResourceDiagnostics), Layer.provideMerge(TraceDiagnostics.layer), Layer.provideMerge(TriageService.layer.pipe(Layer.provide(GitVcsDriver.layer))), diff --git a/apps/server/src/ws.test.ts b/apps/server/src/ws.test.ts index 26456c0da320d..c96497c76a2c3 100644 --- a/apps/server/src/ws.test.ts +++ b/apps/server/src/ws.test.ts @@ -82,6 +82,7 @@ import * as BackgroundPolicy from "./background/BackgroundPolicy.ts"; import * as CheckpointDiffQuery from "./checkpointing/CheckpointDiffQuery.ts"; import * as NativeChildWorkQuery from "./orchestration-v2/NativeChildWorkQuery.ts"; import * as CommandOutputHub from "./orchestration-v2/CommandOutputHub.ts"; +import * as ThreadViewers from "./orchestration-v2/ThreadViewers.ts"; import * as CodexInstallation from "./provider/CodexInstallation.ts"; import * as DeviceService from "./device/DeviceService.ts"; import * as DocumentText from "./provider/DocumentText.ts"; @@ -832,6 +833,7 @@ const layerUnusedConfigRpcServices = Layer.mergeAll( Layer.mock(CheckpointDiffQuery.CheckpointDiffQuery)({}), Layer.mock(NativeChildWorkQuery.NativeChildWorkQuery)({}), Layer.mock(CommandOutputHub.CommandOutputHub)({}), + ThreadViewers.layer, Layer.mock(CodexInstallation.CodexInstallation)({ managedDirectory: "/unused" }), Layer.mock(DeviceService.DeviceService)({}), Layer.mock(EnvironmentTheme.EnvironmentThemeService)({}), diff --git a/apps/server/src/ws.ts b/apps/server/src/ws.ts index c9cd604a35b4d..7b4c232b92680 100644 --- a/apps/server/src/ws.ts +++ b/apps/server/src/ws.ts @@ -1481,6 +1481,7 @@ export const layerWsRpc = ( }; const threadLaunch = yield* ThreadLaunchService.ThreadLaunchService; const commandOutputHub = yield* CommandOutputHub.CommandOutputHub; + const threadViewers = yield* ThreadViewers.ThreadViewers; const providerSessionManager = yield* ProviderSessionManager.ProviderSessionManagerV2; const scheduledTasks = yield* ScheduledTasks.ScheduledTaskService; const secretRequests = yield* SecretRequests.SecretRequests; @@ -2323,13 +2324,8 @@ export const layerWsRpc = ( [ORCHESTRATION_V2_WS_METHODS.subscribeThread]: (input) => Stream.unwrap( Effect.annotateCurrentSpan({ "orchestration_v2.thread_id": input.threadId }).pipe( + Effect.andThen(threadViewers.view(input.threadId)), Effect.andThen(subscribeOrchestrationV2Thread(input)), - Effect.flatMap((stream) => - Effect.gen(function* () { - const release = (yield* ThreadViewers.ThreadViewers).view(input.threadId); - return stream.pipe(Stream.ensuring(Effect.sync(release))); - }), - ), ), ), [ORCHESTRATION_V2_WS_METHODS.subscribeCommandOutput]: (input) =>