From 6528ae7159729571f7a9ea0e4bedb415ccd9893e Mon Sep 17 00:00:00 2001 From: John Doe <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 14:21:12 +0100 Subject: [PATCH 1/4] Fork: one first-come line in the slot gate; idle compaction retried while due (#738) The slot gate keeps one line of waiting starts, oldest first. Every wake-up that asks again while refused keeps its place from its first ask (question wakes, provider continuations, handoffs, idle compaction, the PR-watch wake-up, the usage-limit resume, a wait's message), and every slot wait joins at its own age. A start without a place, and every younger waiter, counts the waiters ahead of it, so no waiter is passed over without limit. A refused idle compaction is held in upstream's timer and asked again every 5 s while the fork reads that the thread is still idle and its cache window open. Held-wakes file: a failing check or read is an error, never "missing": nothing is written over it, and it is read again every 5 s. Saves are synced before the rename, and the folder after it. Round-2 minors of #735: continuations ask off upstream's queue loop, each in its own fiber; a granted slot is held until upstream's dispatch ends; a refused handoff builds its text only with a slot; the orphan scan logs a failed send as a failed send. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/fork/agentSlotRoutes.ts | 20 +- apps/server/src/fork/continuationSlot.test.ts | 118 +++++- apps/server/src/fork/continuationSlot.ts | 124 ++++--- .../DelegatedOrphanScan.test.ts | 64 +++- .../delegatedResults/DelegatedOrphanScan.ts | 38 +- .../src/fork/handoff/HandoffService.test.ts | 45 ++- .../server/src/fork/handoff/HandoffService.ts | 77 ++-- .../src/fork/handoff/handoffDispatch.ts | 28 +- apps/server/src/fork/limitResumeSlot.test.ts | 5 +- apps/server/src/fork/limitResumeSlot.ts | 2 + .../src/fork/pullRequestWatchSlot.test.ts | 3 + apps/server/src/fork/pullRequestWatchSlot.ts | 6 + apps/server/src/fork/workerWakeSlots.test.ts | 342 +++++++++++++++--- apps/server/src/fork/workerWakeSlots.ts | 254 +++++++++++-- apps/server/src/slots/SlotGate.test.ts | 200 ++++++++++ apps/server/src/slots/SlotGate.ts | 253 ++++++++++++- apps/server/src/waits/WaitDelivery.ts | 2 + .../src/waits/WaitService.slots.test.ts | 3 + apps/server/src/waits/WaitService.ts | 11 +- docs/fork/README.md | 54 +-- 20 files changed, 1422 insertions(+), 227 deletions(-) diff --git a/apps/server/src/fork/agentSlotRoutes.ts b/apps/server/src/fork/agentSlotRoutes.ts index 15980ca85fa4e..e5fe77649dcb2 100644 --- a/apps/server/src/fork/agentSlotRoutes.ts +++ b/apps/server/src/fork/agentSlotRoutes.ts @@ -59,6 +59,8 @@ export interface RouteStart { readonly intoThreadId?: ThreadId | undefined; /** The caller is the woken thread itself: its held slot stays held (`SlotGate.acquire`). */ readonly keepsHeld?: boolean | undefined; + /** Its place in the gate's one line: a start that asks again while refused. */ + readonly place?: SlotGate.SlotPlace | undefined; } const ENDED: RouteSlot = { type: "allow", end: () => Effect.void }; @@ -83,6 +85,7 @@ const admit = ( ...(start.alsoNeeds === undefined ? {} : { alsoNeeds: start.alsoNeeds }), ...(start.intoThreadId === undefined ? {} : { intoThreadId: start.intoThreadId }), ...(start.keepsHeld === undefined ? {} : { keepsHeld: start.keepsHeld }), + ...(start.place === undefined ? {} : { place: start.place }), }); if (outcome.type === "refuse") return outcome; const token = outcome.token; @@ -181,8 +184,22 @@ export interface SendSlot { readonly instanceId: ProviderInstanceId; /** The receiver is the caller (a wait's message): its held slot stays held. */ readonly keepsHeld?: boolean | undefined; + /** A wake-up that asks again while refused: its place in the gate's one line. */ + readonly place?: SlotGate.SlotPlace | undefined; + /** + * The caller ends the slot (`end`) when its dispatch ended. Until then the start counts whatever + * the start grace, so a send that waits after the gate step still has its slot when it starts. + */ + readonly endsItself?: boolean | undefined; } +/** Waiters that no longer need a slot leave the gate's line (nothing without a gate). */ +export const leaveLine = ( + gate: Option.Option, + keys: ReadonlyArray, +): Effect.Effect => + Option.match(gate, { onNone: () => Effect.void, onSome: (slots) => slots.leave(keys) }); + /** * A message into an existing thread: into one that shows a run it only queues; into one that * shows none it takes a slot like `delegate_task`. One gate step decides both, so a run that @@ -196,8 +213,9 @@ export const admitSend = ( callerThreadId: send.callerThreadId, targetInstanceId: send.instanceId, intoThreadId: send.threadId, - childThreadId: send.threadId, + ...(send.endsItself === true ? {} : { childThreadId: send.threadId }), ...(send.keepsHeld === undefined ? {} : { keepsHeld: send.keepsHeld }), + ...(send.place === undefined ? {} : { place: send.place }), }); /** diff --git a/apps/server/src/fork/continuationSlot.test.ts b/apps/server/src/fork/continuationSlot.test.ts index d253830334f10..2d891cc5f5a7a 100644 --- a/apps/server/src/fork/continuationSlot.test.ts +++ b/apps/server/src/fork/continuationSlot.test.ts @@ -15,6 +15,7 @@ import { } from "@t3tools/contracts"; import * as IdAllocator from "@t3tools/provider-core/server/IdAllocator"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; +import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; @@ -54,8 +55,14 @@ interface World { /** Threads the gate refuses whatever is free. */ full: Set; asks: Array; + /** The gate's calls after an ask: `started `, `release `, `leave `. */ + calls: Array; + /** Threads whose ask takes this long (a slow count read), then is refused. */ + slow: Set; /** Threads gone (`null` shell), and whether a shell read fails. */ gone: Set; + /** Threads the gate sees live but upstream's worker then reads archived. */ + archivedForUpstream: Set; unreadable: boolean; logs: Array; } @@ -64,7 +71,10 @@ const newWorld = (free: number): World => ({ free, full: new Set(), asks: [], + calls: [], + slow: new Set(), gone: new Set(), + archivedForUpstream: new Set(), unreadable: false, logs: [], }); @@ -72,19 +82,25 @@ const newWorld = (free: number): World => ({ const gateOf = (world: World) => Layer.mock(SlotGate.SlotGate)({ acquire: (input) => - Effect.sync((): SlotGate.SlotAcquire => { + Effect.gen(function* () { world.asks.push(input); - if (world.free === 0 || world.full.has(input.callerThreadId!)) - return { - type: "refuse", - message: "full", - refusal: { type: "unread", problem: "timeout" }, - }; + const refused: SlotGate.SlotAcquire = { + type: "refuse", + message: "full", + refusal: { type: "unread", problem: "timeout" }, + }; + if (world.slow.has(input.callerThreadId!)) { + yield* Effect.sleep("5 seconds"); + return refused; + } + if (world.free === 0 || world.full.has(input.callerThreadId!)) return refused; world.free -= 1; - return { type: "allow", token: world.asks.length }; + return { type: "allow", token: world.asks.length } satisfies SlotGate.SlotAcquire; }), - started: () => Effect.void, - release: () => Effect.void, + started: (token, childThreadId) => + Effect.sync(() => world.calls.push(`started ${childThreadId} ${token}`)), + release: (token) => Effect.sync(() => world.calls.push(`release ${token}`)), + leave: (keys) => Effect.sync(() => world.calls.push(...keys.map((key) => `leave ${key}`))), }); const projection = { @@ -115,7 +131,7 @@ const server = ( ), getThreadRecords: (threadId) => Effect.succeed( - world.gone.has(threadId) + world.gone.has(threadId) || world.archivedForUpstream.has(threadId) ? { ...projection, thread: { archivedAt: "2026-11-21T00:00:00.000Z", deletedAt: null } } : projection, ) as never, @@ -152,6 +168,12 @@ const offer = (value: ProviderContinuationRequests.ProviderContinuationRequest) Effect.flatMap((requests) => requests.offer(value)), ); +/** Lets forked fibers run (scheduler turns, no clock moved). */ +const settle = Effect.forEach(Array.from({ length: 50 }), () => Effect.yieldNow, { discard: true }); + +/** The gate's slot calls, without the line's. */ +const slotCalls = (world: World) => world.calls.filter((call) => !call.startsWith("leave ")); + const waitLogs = (world: World) => world.logs.filter((log) => log.includes("provider continuation waits for an agent slot")); @@ -182,7 +204,10 @@ describe("provider continuations take an agent slot", () => { targetInstanceId: INSTANCE, intoThreadId: A, keepsHeld: true, + place: { key: "continuation:0" }, }); + // Every ask in its one place in the gate's line. + assert.isTrue(world.asks.every((asked) => asked.place?.key === "continuation:0")); world.free = 1; yield* TestClock.adjust("5 seconds"); const sent = yield* Queue.take(dispatched); @@ -214,6 +239,69 @@ describe("provider continuations take an agent slot", () => { }), ); + it.effect("a slow count read for one thread holds up no other thread's continuation", () => + Effect.gen(function* () { + const world = newWorld(5); + world.slow.add(A); + const dispatched = yield* Queue.unbounded(); + yield* Effect.gen(function* () { + yield* offer(request(A)); + yield* offer(request(B)); + // No time passes: B is not queued behind A's 5-second ask. + yield* settle; + assert.strictEqual(yield* Queue.size(dispatched), 1); + const sent = yield* Queue.take(dispatched); + assert.strictEqual(sent.type === "message.dispatch" && sent.threadId, B); + // A's ask ends (it cannot be cut off half-way), so the server can stop. + yield* TestClock.adjust("5 seconds"); + }).pipe(Effect.provide(server(world, dispatched)), Effect.scoped); + }), + ); + + it.effect( + "a granted slot is held until upstream's dispatch of it ends, however long it waits", + () => + Effect.gen(function* () { + const world = newWorld(1); + const dispatched = yield* Queue.unbounded(); + const lockFree = yield* Deferred.make(); + yield* Effect.gen(function* () { + yield* offer( + request(A, { + // Behind its own thread's earlier dispatch (its parent's lock) for 2 minutes. + dispatchIfCurrent: (effect) => + Deferred.await(lockFree).pipe(Effect.andThen(Effect.asSome(effect))), + }), + ); + yield* TestClock.adjust("2 minutes"); + assert.lengthOf(world.asks, 1); + // Asked and granted, not ended: the start counts whatever the start grace. + assert.deepStrictEqual(slotCalls(world), []); + yield* Deferred.succeed(lockFree, undefined); + yield* Queue.take(dispatched); + yield* settle; + assert.deepStrictEqual(slotCalls(world), [`started ${A} 1`, "release 1"]); + }).pipe(Effect.provide(server(world, dispatched)), Effect.scoped); + }), + ); + + it.effect("a granted request upstream drops (its thread archived meanwhile) frees its slot", () => + Effect.gen(function* () { + const world = newWorld(1); + const dispatched = yield* Queue.unbounded(); + const cleared: Array = []; + yield* Effect.gen(function* () { + // Granted on a live shell; upstream then reads the thread archived and clears the latch. + world.archivedForUpstream.add(A); + yield* offer(request(A, { clearIfCurrent: () => Effect.sync(() => cleared.push(A)) })); + yield* settle; + assert.deepStrictEqual(cleared, [A]); + assert.strictEqual(yield* Queue.size(dispatched), 0); + assert.deepStrictEqual(slotCalls(world), [`started ${A} 1`, "release 1"]); + }).pipe(Effect.provide(server(world, dispatched)), Effect.scoped); + }), + ); + it.effect("a thread that cannot be read waits, then is sent once", () => Effect.gen(function* () { const world = newWorld(5); @@ -241,7 +329,7 @@ describe("provider continuations take an agent slot", () => { const cleared: Array = []; yield* Effect.gen(function* () { yield* offer(request(A, { clearIfCurrent: () => Effect.sync(() => cleared.push(A)) })); - yield* TestClock.adjust("0 seconds"); + yield* settle; assert.deepStrictEqual(cleared, [A]); assert.deepStrictEqual(world.asks, []); assert.strictEqual(yield* Queue.size(dispatched), 0); @@ -254,7 +342,7 @@ describe("provider continuations take an agent slot", () => { const world = newWorld(0); const inner = yield* Queue.unbounded(); - const gated = ContinuationSlot.gateContinuations( + const gated = yield* ContinuationSlot.gateContinuations( { offer: (value) => Queue.offer(inner, value).pipe(Effect.asVoid), take: Queue.take(inner), @@ -291,6 +379,8 @@ describe("provider continuations take an agent slot", () => { yield* TestClock.adjust("12 seconds"); }).pipe(Effect.provide(server(world, dispatched)), Effect.scoped); const asked = world.asks.length; + // Its place in the line goes with it. + assert.include(world.calls, "leave continuation:0"); world.free = 5; yield* TestClock.adjust("1 minute"); assert.strictEqual(yield* Queue.size(dispatched), 0); @@ -308,7 +398,7 @@ describe("provider continuations take an agent slot", () => { }).pipe(Effect.provide(server(world, dispatched, null)), Effect.scoped); const inner = { offer: () => Effect.void, take: Effect.never }; assert.strictEqual( - ContinuationSlot.gateContinuations( + yield* ContinuationSlot.gateContinuations( inner, Option.none(), () => Effect.succeed(null), diff --git a/apps/server/src/fork/continuationSlot.ts b/apps/server/src/fork/continuationSlot.ts index ec0f5a85475f5..8bf6e77b14e34 100644 --- a/apps/server/src/fork/continuationSlot.ts +++ b/apps/server/src/fork/continuationSlot.ts @@ -1,7 +1,7 @@ /** - * Fork: a provider continuation into an idle thread takes an agent slot. Asked at the queue, not - * around the dispatch: an adapter's `dispatchIfCurrent` holds its own lock while it dispatches. - * A refused request goes back on the queue; the adapter's latch stays set, so it offers no second. + * Fork: a provider continuation into an idle thread takes an agent slot, asked off upstream's queue + * (never inside `dispatchIfCurrent`, which holds the adapter's lock), each request in its own fiber + * and place in the gate's line. The slot is held until upstream's dispatch of it ends. */ import type { ProviderInstanceId, ThreadId } from "@t3tools/contracts"; import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests"; @@ -9,11 +9,12 @@ import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; +import * as Queue from "effect/Queue"; import type * as Scope from "effect/Scope"; import * as ThreadManagement from "../orchestration-v2/ThreadManagementService.ts"; import * as SlotGate from "../slots/SlotGate.ts"; -import { admitSend, reasonHere } from "./agentSlotRoutes.ts"; +import { admitSend, leaveLine, reasonHere } from "./agentSlotRoutes.ts"; type Request = ProviderContinuationRequests.ProviderContinuationRequest; interface Requests { @@ -27,69 +28,104 @@ export const CONTINUATION_SLOT_RETRY = Duration.seconds(5); /** `null`: no such thread, or it is gone. */ export type ContinuationThread = { readonly instanceId: ProviderInstanceId } | null; +/** + * The request holding its slot (`end`) until upstream's dispatch of it ends, whichever way, so a + * run that waits behind its thread's earlier dispatch still has it. One never dispatched frees it + * at the gate's dispatch limit. + */ +const holdingSlot = ( + request: Request, + end: (childThreadId: ThreadId | null) => Effect.Effect, +): Request => { + const dispatchIfCurrent = request.dispatchIfCurrent; + const clearIfCurrent = request.clearIfCurrent; + const ended = end(request.threadId); + return { + ...request, + dispatchIfCurrent: (effect: Effect.Effect) => + (dispatchIfCurrent === undefined ? Effect.asSome(effect) : dispatchIfCurrent(effect)).pipe( + Effect.ensuring(ended), + ), + ...(clearIfCurrent === undefined + ? {} + : { clearIfCurrent: () => clearIfCurrent().pipe(Effect.ensuring(ended)) }), + }; +}; + /** * `take` hands out a request into an idle thread only with a slot. Upstream's at once: a delegated * result (docs/fork/README.md) and a gone thread (upstream drops it and clears the adapter's latch). - * An unreadable thread waits, as a refused one does. + * An unreadable thread waits, as a refused one does. No ask ever holds up upstream's queue: each + * request asks in its own fiber, so a slow count delays only that request. */ export const gateContinuations = ( requests: Requests, gate: Option.Option, threadOf: (threadId: ThreadId) => Effect.Effect, scope: Scope.Scope, -): Requests => { - if (Option.isNone(gate)) return requests; - /** Logged once until handed out. */ - const waiting = new Set(); +): Effect.Effect => { + if (Option.isNone(gate)) return Effect.succeed(requests); + let places = 0; - /** `null` when the request may go to upstream now, else why it waits. */ - const waitsFor = Effect.fn("ContinuationSlot.waitsFor")(function* (request: Request) { - if (request.delegatedCompletion !== undefined) return null; + /** The request for upstream when it may go now (holding its slot), else why it waits. */ + const ask = Effect.fn("ContinuationSlot.ask")(function* ( + request: Request, + place: SlotGate.SlotPlace, + ) { const thread = yield* threadOf(request.threadId).pipe( Effect.orElseSucceed(() => "unreadable" as const), ); if (thread === "unreadable") return "the thread could not be read to check for a slot"; - if (thread === null) return null; + if (thread === null) return request; const slot = yield* admitSend(gate, { callerThreadId: request.threadId, threadId: request.threadId, instanceId: thread.instanceId, keepsHeld: true, + place, + endsItself: true, }); - return slot.type === "refuse" ? reasonHere(slot.refusal) : null; + return slot.type === "refuse" ? reasonHere(slot.refusal) : holdingSlot(request, slot.end); }); - const putBack = Effect.fn("ContinuationSlot.putBack")(function* ( - request: Request, - detail: string, - ) { - if (!waiting.has(request)) { - waiting.add(request); - yield* Effect.logInfo("provider continuation waits for an agent slot", { - threadId: request.threadId, - providerThreadId: request.providerThreadId, - detail, - }); - } - yield* Effect.sleep(CONTINUATION_SLOT_RETRY).pipe( - Effect.andThen(requests.offer(request)), - Effect.forkIn(scope), - ); - }); - - const take: Effect.Effect = Effect.gen(function* () { - while (true) { - const request = yield* requests.take; - const detail = yield* waitsFor(request); - if (detail === null) { - waiting.delete(request); - return request; + /** One request, until it may go: asked again every `CONTINUATION_SLOT_RETRY`, logged once. */ + const untilSlot = Effect.fn("ContinuationSlot.untilSlot")( + function* (request: Request, place: SlotGate.SlotPlace, ready: Queue.Queue) { + let logged = false; + while (true) { + const answer = yield* ask(request, place); + if (typeof answer !== "string") { + yield* Queue.offer(ready, answer); + return; + } + if (!logged) { + logged = true; + yield* Effect.logInfo("provider continuation waits for an agent slot", { + threadId: request.threadId, + providerThreadId: request.providerThreadId, + detail: answer, + }); + } + yield* Effect.sleep(CONTINUATION_SLOT_RETRY); } - yield* putBack(request, detail); - } - }); + }, + // Stopped while it waits (the server stops): nothing is sent later, and its place goes. + (effect, _request: Request, place: SlotGate.SlotPlace) => + Effect.ensuring(effect, leaveLine(gate, [place.key])), + ); - return { offer: requests.offer, take }; + return Effect.gen(function* () { + const ready = yield* Queue.unbounded(); + // Upstream's queue is only read here, and never waits on an ask. + const handOut = (request: Request): Effect.Effect => + request.delegatedCompletion !== undefined + ? Effect.asVoid(Queue.offer(ready, request)) + : Effect.asVoid( + Effect.forkIn(untilSlot(request, { key: `continuation:${places++}` }, ready), scope), + ); + yield* requests.take.pipe(Effect.flatMap(handOut), Effect.forever, Effect.forkIn(scope)); + return { offer: requests.offer, take: Queue.take(ready) }; + }); }; /** The queue the adapters offer to (memoized by reference), gated for upstream's worker only. */ @@ -99,7 +135,7 @@ export const layerOver = Layer.effect( const requests = yield* ProviderContinuationRequests.ProviderContinuationRequests; const threads = yield* ThreadManagement.ThreadManagementService; const gate = yield* Effect.serviceOption(SlotGate.SlotGate); - return gateContinuations( + return yield* gateContinuations( requests, gate, (threadId) => diff --git a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts index d40299302da1e..a1dd3b1c2c9a5 100644 --- a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts +++ b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts @@ -80,6 +80,8 @@ const makeWorld = (waitFor: { readonly enabled: boolean } = { enabled: true }) = /** Answers its first page only. */ const cutShort = new Set(); const linksOff = { value: false }; + /** Every send fails as a store that cannot be read now does (tried again, same id). */ + const sendFails = { value: false }; const pageReads: Array<{ environmentId: EnvironmentId; cursor: number }> = []; const sent: Array<{ threadId: ThreadId; messageId: string; text: string }> = []; const receipts = new Map(); @@ -110,6 +112,7 @@ const makeWorld = (waitFor: { readonly enabled: boolean } = { enabled: true }) = silent, cutShort, linksOff, + sendFails, pageReads, sent, receipts, @@ -132,11 +135,23 @@ const makeWorld = (waitFor: { readonly enabled: boolean } = { enabled: true }) = ), ), sendToThread: (input) => - Effect.sync(() => { - sent.push({ threadId: input.threadId, messageId: input.messageId!, text: input.text }); - receipts.set(input.commandId, input.threadId); - return {} as never; - }), + sendFails.value + ? Effect.fail( + new ThreadManagement.ThreadManagementProjectionLoadError({ + projectId: input.projectId, + threadId: input.threadId, + cause: new Error("store busy"), + }), + ) + : Effect.sync(() => { + sent.push({ + threadId: input.threadId, + messageId: input.messageId!, + text: input.text, + }); + receipts.set(input.commandId, input.threadId); + return {} as never; + }), }), Layer.mock(CommandReceiptStore.CommandReceiptStoreV2)({ getByCommandId: (commandId: CommandId) => @@ -663,6 +678,8 @@ describe("DelegatedOrphanScan and agent slots", () => { callerThreadId: PARENT, intoThreadId: PARENT, keepsHeld: true, + // Asked again every 5 s through the wait service's delivery, in one place in line. + place: { key: expect.stringMatching(/^wait-delivery:/) }, }), ]); }), @@ -817,6 +834,43 @@ describe("DelegatedOrphanScan and agent slots", () => { }), ); + it.effect("a send that keeps failing is logged as a failed send, not an unreadable parent", () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + world.slots.state.all = true; + const warnings: Array = []; + const logger = Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(JSON.stringify(message)); + }); + const fiber = yield* (yield* service(world)) + .run() + .pipe( + Effect.provide(Logger.layer([logger], { mergeWithExisting: false })), + Effect.forkChild, + ); + yield* settle; + yield* TestClock.adjust(OPTIONS.delay); + yield* settle; + // A slot frees, but every send fails. + world.slots.state.all = false; + world.sendFails.value = true; + yield* TestClock.adjust(SLOT_RETRY); + yield* settle; + for (let step = 0; step < OPTIONS.attempts; step++) { + yield* TestClock.adjust(OPTIONS.retryDelay); + yield* settle; + } + expect(fiber.pollUnsafe()).toBeDefined(); + expect(warnings.filter((warning) => warning.includes("its send failed"))).toHaveLength(1); + expect(warnings.filter((warning) => warning.includes("cannot be read"))).toHaveLength(0); + expect( + warnings.filter((warning) => warning.includes("left to the next start: its send kept")), + ).toHaveLength(1); + expect(world.sent).toHaveLength(0); + }), + ); + it.effect("a restart while one waits loses nothing: the next start names it once", () => Effect.gen(function* () { yield* setClock; diff --git a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts index 51fe2fb84b5ab..33a429c849bb1 100644 --- a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts +++ b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.ts @@ -75,7 +75,8 @@ export type ParentOutcome = /** No orphan, or every one was named before. */ | { readonly type: "none" } | { readonly type: "gone" } - | { readonly type: "unreadable" } + /** The parent could not be read, or (`sendFailed`, why) a reminder's send failed: tried again. */ + | { readonly type: "unreadable"; readonly sendFailed?: string } /** * The agent slot gate refused (nothing of that message was sent): `run` asks again for this * parent alone, with the threads this scan found. @@ -158,7 +159,7 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( // A busy parent is not held: its run's end changes nothing about an orphan, and the message // waits in its queue until that run (a question to a person included) is over. const sent: Array = []; - let retry = false; + let sendFailed: string | undefined; let refused: string | undefined; for (const batch of batches) { const ids = orphanBatchIds(parentThreadId, batch); @@ -185,14 +186,14 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( refused = result.detail; break; case "retry": - retry = true; + sendFailed = result.detail; break; case "waitsForSlot": // The next batches go to the same idle thread: they wait too. Sent ones are recorded. return { type: "waitsForSlot", detail: result.detail, linked } satisfies ParentOutcome; } } - if (retry) return { type: "unreadable" } satisfies ParentOutcome; + if (sendFailed !== undefined) return { type: "unreadable", sendFailed } satisfies ParentOutcome; if (refused !== undefined) return { type: "held", detail: refused } satisfies ParentOutcome; return { type: "sent", threadIds: sent } satisfies ParentOutcome; }); @@ -265,9 +266,14 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( return { type: "scanned", unreachable, cutShort, partial, parents } satisfies ScanOutcome; }); - const unreadableParents = (outcome: ScanOutcome) => + /** Parents tried again: unreadable ones, or (`sendFailed`) ones whose reminder's send failed. */ + const unreadableParents = (outcome: ScanOutcome, sendFailed?: boolean) => [...outcome.parents.entries()] - .filter(([, parent]) => parent.type === "unreadable") + .filter( + ([, parent]) => + parent.type === "unreadable" && + (sendFailed === undefined || (parent.sendFailed !== undefined) === sendFailed), + ) .map(([threadId]) => threadId); /** Another pass while an environment did not answer (fully) or a parent could not be read. */ @@ -303,13 +309,22 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( } if (outcome.type !== "waitsForSlot" && outcome.type !== "unreadable") return; unreadable = outcome.type === "unreadable" ? unreadable + 1 : 0; + // A failed send is told as one, not as a parent that cannot be read. + const sendFailed = outcome.type === "unreadable" ? outcome.sendFailed : undefined; if (unreadable === 1) - yield* Effect.logWarning("delegated orphan reminder: its parent cannot be read", { - threadId: parentThreadId, - }); + yield* sendFailed === undefined + ? Effect.logWarning("delegated orphan reminder: its parent cannot be read", { + threadId: parentThreadId, + }) + : Effect.logWarning("delegated orphan reminder: its send failed", { + threadId: parentThreadId, + detail: sendFailed, + }); if (unreadable >= options.attempts) return yield* Effect.logWarning( - "delegated orphan reminder left to the next start: its parent stayed unreadable", + sendFailed === undefined + ? "delegated orphan reminder left to the next start: its parent stayed unreadable" + : "delegated orphan reminder left to the next start: its send kept failing", { threadId: parentThreadId, tries: unreadable }, ); yield* Effect.sleep(outcome.type === "waitsForSlot" ? slotRetryDelay : options.retryDelay); @@ -355,7 +370,8 @@ export const make = Effect.fn("DelegatedOrphanScan.make")(function* ( scans: options.attempts, unansweredEnvironmentIds: outcome.unreachable, cutShortEnvironmentIds: outcome.cutShort, - unreadableParentThreadIds: unreadableParents(outcome), + unreadableParentThreadIds: unreadableParents(outcome, false), + failedSendParentThreadIds: unreadableParents(outcome, true), }, ); } diff --git a/apps/server/src/fork/handoff/HandoffService.test.ts b/apps/server/src/fork/handoff/HandoffService.test.ts index 1b8aa914c2f0c..88afa16844f97 100644 --- a/apps/server/src/fork/handoff/HandoffService.test.ts +++ b/apps/server/src/fork/handoff/HandoffService.test.ts @@ -102,7 +102,8 @@ const harness = (state: { yield* TestClock.setTime(NOW_MS); const events = yield* Queue.unbounded(); const sent: Array = []; - const reads = { count: 0 }; + /** `text`: the reads that build a handoff's text (its transcript part). */ + const reads = { count: 0, text: 0 }; const ended = new Map(); const milestones: Array<{ readonly name: string; @@ -147,9 +148,10 @@ const harness = (state: { }), Layer.mock(ProjectionStore.ProjectionStoreV2)({ getThread: () => Effect.succeed(state.data.thread), - getThreadRecords: () => + getThreadRecords: (_threadId, fields) => Effect.sync(() => { reads.count++; + if ((fields as ReadonlyArray).includes("attempts")) reads.text++; return state.data; }), }), @@ -529,7 +531,11 @@ describe("HandoffService", () => { /** A gate with `free` slots: each allowed start takes one; the test frees them. */ const countingGate = (free: number) => { - const slots = { free, asks: [] as Array }; + const slots = { + free, + asks: [] as Array, + left: [] as Array, + }; const layer = Layer.mock(SlotGate.SlotGate)({ acquire: (input) => Effect.sync((): SlotGate.SlotAcquire => { @@ -545,6 +551,7 @@ describe("HandoffService", () => { }), started: () => Effect.void, release: () => Effect.void, + leave: (keys) => Effect.sync(() => slots.left.push(...keys)), }); return { slots, layer }; }; @@ -553,25 +560,53 @@ describe("HandoffService", () => { Effect.gen(function* () { const gate = countingGate(0); const state = { data: projection({ usedTokens: 160_000 }), settings: on, gate: gate.layer }; - const { sent, emit } = yield* harness(state); + const { sent, emit, reads } = yield* harness(state); yield* emit(runUpdated(run()), "readThread", 1); yield* settle; yield* TestClock.adjust(MINUTE_MS); expect(sent).toEqual([]); - // Asked again every 5 s, each time after the busy check, as the thread's own start. + // Asked again every 5 s, each time after the busy check, as the thread's own start, in + // its one place in the gate's line. expect(gate.slots.asks.length).toBeGreaterThanOrEqual(12); + const place = { key: `handoff:server:handoff:${threadId}:${runId}` }; expect(gate.slots.asks[0]).toEqual({ callerThreadId: threadId, targetInstanceId: modelSelection.instanceId, intoThreadId: threadId, keepsHeld: true, + place, }); + expect(gate.slots.asks.every((asked) => asked.place?.key === place.key)).toBe(true); + // Refused: its text (the transcript read) is not built at all. + expect(reads.text).toBe(0); gate.slots.free = 1; yield* TestClock.adjust(HandoffService.HANDOFF_SLOT_RETRY); yield* emit(null, "handOffWhenDue", 1); expect(messages(sent).map((message) => message.commandId)).toEqual([ `server:handoff:${threadId}:${runId}`, ]); + // Built once, with the slot. + expect(reads.text).toBe(1); + }), + ); + + it.effect("a waiting handoff whose thread gets busy, or a newer run, leaves its place", () => + Effect.gen(function* () { + const gate = countingGate(0); + const data = projection({ usedTokens: 160_000 }); + const state = { data, settings: on, gate: gate.layer }; + const { sent, emit, events } = yield* harness(state); + yield* emit(runUpdated(run()), "readThread", 1); + yield* settle; + yield* TestClock.adjust(MINUTE_MS); + expect(gate.slots.left).toEqual([]); + // A newer run completes: its own timer replaces the waiting one, which leaves the line. + const newer = run({ id: RunId.make("run-2"), ordinal: 2 }); + state.data = { ...data, runs: [...data.runs, newer] }; + yield* Queue.offer(events, runUpdated(newer)); + yield* settle; + expect(gate.slots.left).toContain(`handoff:server:handoff:${threadId}:${runId}`); + expect(sent).toEqual([]); }), ); diff --git a/apps/server/src/fork/handoff/HandoffService.ts b/apps/server/src/fork/handoff/HandoffService.ts index 6e18013461b75..ac9a66ab5e2a3 100644 --- a/apps/server/src/fork/handoff/HandoffService.ts +++ b/apps/server/src/fork/handoff/HandoffService.ts @@ -30,8 +30,10 @@ import { forkParked } from "../../serverActivation.ts"; import * as ServerSettings from "../../serverSettings.ts"; import * as SlotGate from "../../slots/SlotGate.ts"; import { type HandoffTrigger, handoffDecision } from "./handoffDecision.ts"; +import { leaveLine } from "../agentSlotRoutes.ts"; import { dispatchHandoff, + handoffPlace, type HandoffWaitsForSlot, readHandoffRecords, } from "./handoffDispatch.ts"; @@ -104,15 +106,6 @@ export const make = Effect.gen(function* () { ids: { readonly commandId: string; readonly messageId: string }, read: NonNullable>>, ) { - const recent = read.records.runs - .toSorted((left, right) => right.ordinal - left.ordinal) - .slice(0, TRANSCRIPT_RUNS) - .map((recentRun) => recentRun.id); - const more = yield* projections.getThreadRecords( - threadId, - ["attempts", "subagents", "turnItems"], - { turnItemRunIds: recent, omitSettledSubagents: true }, - ); const firstSends = new Set( read.records.runs.flatMap((other) => { const parsed = parseHandoffMessageId(threadId, other.userMessageId); @@ -127,12 +120,24 @@ export const make = Effect.gen(function* () { commandId: CommandId.make(ids.commandId), messageId: MessageId.make(ids.messageId), threadId, - text: handoffMessageText({ - records: { ...read.records, ...more }, - sourceRunId: run.id, - number: firstSends.size + 1, - nowIso: DateTime.formatIso(yield* DateTime.now), - modelSelection, + // Built only with a slot: a refused handoff asks again every 5 s and reads nothing more. + text: Effect.gen(function* () { + const recent = read.records.runs + .toSorted((left, right) => right.ordinal - left.ordinal) + .slice(0, TRANSCRIPT_RUNS) + .map((recentRun) => recentRun.id); + const more = yield* projections.getThreadRecords( + threadId, + ["attempts", "subagents", "turnItems"], + { turnItemRunIds: recent, omitSettledSubagents: true }, + ); + return handoffMessageText({ + records: { ...read.records, ...more }, + sourceRunId: run.id, + number: firstSends.size + 1, + nowIso: DateTime.formatIso(yield* DateTime.now), + modelSelection, + }); }), attachments: [], createdBy: "agent", @@ -144,8 +149,19 @@ export const make = Effect.gen(function* () { ); }); - const handOffWhenDue = Effect.fn("HandoffService.handOffWhenDue")( - function* (threadId: ThreadId, sourceRunId: RunId) { + /** A handoff's place in the gate's line while it waits for a slot; left when it stops waiting. */ + interface Waiting { + key: string | undefined; + } + const leavePlace = (waiting: Waiting) => + Effect.suspend(() => { + const key = waiting.key; + waiting.key = undefined; + return key === undefined ? Effect.void : leaveLine(gate, [key]); + }); + + const handOffLoop = Effect.fn("HandoffService.handOffWhenDue")( + function* (threadId: ThreadId, sourceRunId: RunId, waiting: Waiting) { let runId = sourceRunId; let trigger: HandoffTrigger = "turn-end"; let slotWaitLogged = false; @@ -202,6 +218,8 @@ export const make = Effect.gen(function* () { nowMs, }); if (decision.kind !== "handoff") yield* endSettle; + // Not due now: it needs no slot, so it holds no place from the waiters behind it. + if (decision.kind !== "handoff") yield* leavePlace(waiting); if (decision.kind === "skip") return; if (decision.kind === "sleep") { yield* Effect.sleep(Duration.millis(decision.untilMs - nowMs)); @@ -256,6 +274,8 @@ export const make = Effect.gen(function* () { read, ); if (!waitsForSlot(outcome)) return; + if (waiting.key !== handoffPlace(decision.commandId).key) yield* leavePlace(waiting); + waiting.key = handoffPlace(decision.commandId).key; // Settled already: it decides again and checks the relay's view, without a new settle. if (!slotWaitLogged) yield* logSlotWait(threadId, outcome); slotWaitLogged = true; @@ -270,6 +290,12 @@ export const make = Effect.gen(function* () { ), ); + /** Replaced by a newer run's timer, or stopped: its place in the line goes with it. */ + const handOffWhenDue = (threadId: ThreadId, sourceRunId: RunId) => { + const waiting: Waiting = { key: undefined }; + return handOffLoop(threadId, sourceRunId, waiting).pipe(Effect.ensuring(leavePlace(waiting))); + }; + /** A handoff run that ended: keep the thread read, or send the same text once more if it failed. */ const handoffRunEnded = Effect.fn("HandoffService.handoffRunEnded")( function* ( @@ -345,11 +371,18 @@ export const make = Effect.gen(function* () { // The busy check runs again at each try: a thread that moved on drops it. let outcome = yield* resend; if (waitsForSlot(outcome)) yield* logSlotWait(threadId, outcome); - while (waitsForSlot(outcome)) { - yield* Effect.sleep(HANDOFF_SLOT_RETRY); - if ((yield* readThread(threadId)) === null) return; - outcome = yield* resend; - } + // Its place in the line goes when it stops waiting, whichever way. + yield* Effect.gen(function* () { + while (waitsForSlot(outcome)) { + yield* Effect.sleep(HANDOFF_SLOT_RETRY); + if ((yield* readThread(threadId)) === null) return; + outcome = yield* resend; + } + }).pipe( + Effect.ensuring( + leaveLine(gate, [handoffPlace(`server:${run.userMessageId}${HANDOFF_RETRY_SUFFIX}`).key]), + ), + ); }, (effect, run) => effect.pipe( diff --git a/apps/server/src/fork/handoff/handoffDispatch.ts b/apps/server/src/fork/handoff/handoffDispatch.ts index a6e9e8f643f96..4e9f1826724ec 100644 --- a/apps/server/src/fork/handoff/handoffDispatch.ts +++ b/apps/server/src/fork/handoff/handoffDispatch.ts @@ -9,7 +9,7 @@ import * as Effect from "effect/Effect"; import * as Orchestrator from "../../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../../orchestration-v2/ProjectionStore.ts"; import * as SlotGate from "../../slots/SlotGate.ts"; -import { admitSend, reasonHere } from "../agentSlotRoutes.ts"; +import { admitSend, leaveLine, reasonHere } from "../agentSlotRoutes.ts"; import { type HandoffRecords, handoffDispatchDropped } from "./handoffFacts.ts"; export type HandoffDispatchCommand = Extract< @@ -59,30 +59,46 @@ export interface HandoffWaitsForSlot { readonly detail: string; } +/** A handoff's place in the gate's line, while it waits for a slot. */ +export const handoffPlace = (commandId: string): SlotGate.SlotPlace => ({ + key: `handoff:${commandId}`, +}); + /** * Sends a handoff message only to a quiet thread at its source run (see the file comment), with an * agent slot when the server has a gate. One uninterruptible step: a replaced timer or a shutdown - * cannot stop it between the send and the cancel. + * cannot stop it between the send and the cancel. A text given as an effect is built only once + * the gate gave a slot, so a refused handoff reads nothing more. */ -export const dispatchHandoff = Effect.fn("HandoffDispatch.dispatch")(function* ( - command: HandoffDispatchCommand, +export const dispatchHandoff = Effect.fn("HandoffDispatch.dispatch")(function* < + E = never, + R = never, +>( + command: Omit & { + readonly text: string | Effect.Effect; + }, ) { const orchestrator = yield* Orchestrator.OrchestratorV2; const projections = yield* ProjectionStore.ProjectionStoreV2; + const gate = yield* Effect.serviceOption(SlotGate.SlotGate); + const place = handoffPlace(command.commandId); const records = yield* readHandoffRecords(command.threadId); if (handoffDispatchDropped(command, records)) { + yield* leaveLine(gate, [place.key]); yield* Effect.logInfo("handoff dropped: the thread moved on", { threadId: command.threadId }); return "dropped" as const; } - const slot = yield* admitSend(yield* Effect.serviceOption(SlotGate.SlotGate), { + const slot = yield* admitSend(gate, { callerThreadId: command.threadId, threadId: command.threadId, instanceId: (command.modelSelection ?? records.thread.modelSelection).instanceId, keepsHeld: true, + place, }); if (slot.type === "refuse") return { type: "waitsForSlot", detail: reasonHere(slot.refusal) } satisfies HandoffWaitsForSlot; - yield* orchestrator.dispatch(command); + const text = typeof command.text === "string" ? command.text : yield* command.text; + yield* orchestrator.dispatch({ ...command, text }); const { runs } = yield* projections.getThreadRecords(command.threadId, ["runs"], { runStatuses: ["queued"], }); diff --git a/apps/server/src/fork/limitResumeSlot.test.ts b/apps/server/src/fork/limitResumeSlot.test.ts index 76ab6107774d3..6c4a95dfcc0ae 100644 --- a/apps/server/src/fork/limitResumeSlot.test.ts +++ b/apps/server/src/fork/limitResumeSlot.test.ts @@ -170,12 +170,15 @@ describe("usage-limit resumes take an agent slot", () => { (thread) => limitRecoveryCommand(thread, true, RESET_MS)?.commandId, ), ); - // Each asked as the thread's own start. + // Each asked as the thread's own start, in its place in the gate's line. expect(gate.state.asks[0]).toEqual({ callerThreadId: ThreadId.make("thread:a"), targetInstanceId: INSTANCE, intoThreadId: ThreadId.make("thread:a"), keepsHeld: true, + place: { + key: `limit-resume:${limitRecoveryCommand(armed("a"), true, RESET_MS)?.commandId}`, + }, }); }).pipe( Effect.provide( diff --git a/apps/server/src/fork/limitResumeSlot.ts b/apps/server/src/fork/limitResumeSlot.ts index 11cf0948510a1..5a5e781b6bea5 100644 --- a/apps/server/src/fork/limitResumeSlot.ts +++ b/apps/server/src/fork/limitResumeSlot.ts @@ -67,6 +67,8 @@ export const gateLimitResumes = ( threadId: command.threadId, instanceId, keepsHeld: true, + // Asked again every tick (5 s) while refused: it keeps its place in the line. + place: { key: `limit-resume:${command.commandId}` }, }); if (slot.type === "refuse") return yield* skipped(command.commandId, command.threadId, reasonHere(slot.refusal)); diff --git a/apps/server/src/fork/pullRequestWatchSlot.test.ts b/apps/server/src/fork/pullRequestWatchSlot.test.ts index 4e2ecb75e15a2..93283035a2835 100644 --- a/apps/server/src/fork/pullRequestWatchSlot.test.ts +++ b/apps/server/src/fork/pullRequestWatchSlot.test.ts @@ -26,6 +26,7 @@ import { ThreadId, } from "@t3tools/contracts"; import * as Cause from "effect/Cause"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Exit from "effect/Exit"; import * as FileSystem from "effect/FileSystem"; @@ -164,6 +165,8 @@ describe("pull request watch wake-ups take an agent slot", () => { targetInstanceId: INSTANCE, intoThreadId: THREAD, keepsHeld: true, + // Its place in the gate's line, kept over 2 missed sweeps (2 min each). + place: { key: `pull-request-watch:${THREAD}`, staleAfter: Duration.minutes(5) }, }, ]); }), diff --git a/apps/server/src/fork/pullRequestWatchSlot.ts b/apps/server/src/fork/pullRequestWatchSlot.ts index 0227bb9e66d57..7c992b37315de 100644 --- a/apps/server/src/fork/pullRequestWatchSlot.ts +++ b/apps/server/src/fork/pullRequestWatchSlot.ts @@ -10,6 +10,7 @@ import type { ThreadPullRequestKey, } from "@t3tools/contracts"; import * as Clock from "effect/Clock"; +import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; @@ -38,6 +39,9 @@ export class PullRequestWatchWakeWaitsForSlot extends Schema.TaggedError( threadId: command.threadId, instanceId, keepsHeld: true, + // Asked again at upstream's next sweep (2 min) while refused: it keeps its place in line. + place: { key: `pull-request-watch:${command.threadId}`, staleAfter: WATCH_PLACE_STALE }, }); const keys = watchesOf(command).map((watch) => watchKey(command.threadId, watch)); if (slot.type === "refuse") { diff --git a/apps/server/src/fork/workerWakeSlots.test.ts b/apps/server/src/fork/workerWakeSlots.test.ts index e6d025cbd8264..070eedc5fd8be 100644 --- a/apps/server/src/fork/workerWakeSlots.test.ts +++ b/apps/server/src/fork/workerWakeSlots.test.ts @@ -15,6 +15,7 @@ import { RuntimeRequestId, ThreadId, } from "@t3tools/contracts"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -23,6 +24,7 @@ import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import * as PlatformError from "effect/PlatformError"; import * as Queue from "effect/Queue"; import * as Stream from "effect/Stream"; import * as Tracer from "effect/Tracer"; @@ -56,6 +58,8 @@ const WAKE_COMMAND = `server:delegated-task-question:${PARENT}:${REQUEST}`; interface World { free: number; asks: Array; + /** Keys that left the gate's line. */ + left: Array; /** The parent: idle, gone (archived), or not readable now. */ parent: "idle" | "gone" | "unreadable"; requestStatus: "pending" | "resolved"; @@ -68,6 +72,7 @@ interface World { const newWorld = (free: number): World => ({ free, asks: [], + left: [], parent: "idle", requestStatus: "pending", taskStatus: "running", @@ -91,8 +96,11 @@ const gateOf = (world: World) => }), started: () => Effect.void, release: () => Effect.void, + leave: (keys) => Effect.sync(() => world.left.push(...keys)), }); +const QUESTION_PLACE = { key: `question-wake:${WAKE_COMMAND}` }; + const logsOf = (world: World) => Logger.layer( [ @@ -166,16 +174,82 @@ const upstreamOf = (world: World) => }), ); -/** One server: the wrapper over upstream, with the state folder `dir` (a restart reuses it). */ -const server = (world: World, dir: string) => +/** One server: the wrapper over upstream, on `disk` (a restart reuses it). */ +const server = (world: World, disk: Disk) => WorkerWakeSlots.layerOver.pipe( + // The held-wakes file is on the test's disk; the server's folders are real. + Layer.provide(disk.layer), Layer.provideMerge( - Layer.mergeAll(upstreamOf(world), gateOf(world), ServerConfig.layerTest(process.cwd(), dir)), + Layer.mergeAll( + upstreamOf(world), + gateOf(world), + ServerConfig.layerTest(process.cwd(), disk.dir), + ServerSettings.layerTest({}), + ), ), Layer.provideMerge(logsOf(world)), Layer.provideMerge(NodeServices.layer), ); +/** + * The wrapper's file system: in memory, so the test clock alone orders its reads and writes. The + * server's folders are real (`dir`). `exists` of the held-wakes file fails while `failing.exists` + * is set; every rename and sync is recorded in `ops` (by base name). + */ +interface Disk { + readonly dir: string; + readonly files: Map; + readonly ops: Array; + readonly failing: { exists: boolean }; + readonly layer: Layer.Layer; +} + +const baseName = (file: string) => file.slice(file.lastIndexOf("/") + 1); +const diskError = (tag: "NotFound" | "PermissionDenied", method: string, file: string) => + PlatformError.systemError({ _tag: tag, module: "FileSystem", method, pathOrDescriptor: file }); + +const diskOver = (dir: string): Disk => { + const files = new Map(); + const ops: Array = []; + const failing = { exists: false }; + const layer = FileSystem.layerNoop({ + exists: (file) => + failing.exists && baseName(file) === WorkerWakeSlots.HELD_WAKES_FILE + ? Effect.fail(diskError("PermissionDenied", "exists", file)) + : Effect.succeed(files.has(file)), + readFileString: (file) => + Effect.suspend(() => { + const text = files.get(file); + return text === undefined + ? Effect.fail(diskError("NotFound", "readFileString", file)) + : Effect.succeed(text); + }), + rename: (from, to) => + Effect.suspend(() => { + const text = files.get(from); + if (text === undefined) return Effect.fail(diskError("NotFound", "rename", from)); + ops.push(`rename ${baseName(from)} ${baseName(to)}`); + files.delete(from); + files.set(to, text); + return Effect.void; + }), + open: (file, options) => + Effect.sync(() => { + if (options?.flag === "w") files.set(file, ""); + return { + writeAll: (bytes: Uint8Array) => + Effect.sync(() => { + files.set(file, (files.get(file) ?? "") + new TextDecoder().decode(bytes)); + }), + sync: Effect.sync(() => { + ops.push(`sync ${baseName(file)}`); + }), + } as unknown as FileSystem.File; + }), + }); + return { dir, files, ops, failing, layer }; +}; + const questionAsked: OrchestrationV2DomainEvent = { id: EventId.make("event:question"), type: "runtime-request.updated", @@ -193,28 +267,27 @@ const ask = DelegatedTaskQuestionReactor.DelegatedTaskQuestionReactor.pipe( const wakes = (world: World) => world.sent.filter((command) => command.commandId === WAKE_COMMAND); -const heldFile = (dir: string) => +const heldFile = (disk: Disk) => Effect.gen(function* () { const path = yield* Path.Path; const config = yield* ServerConfig.pipe( - Effect.provide(ServerConfig.layerTest(process.cwd(), dir)), + Effect.provide(ServerConfig.layerTest(process.cwd(), disk.dir)), ); return path.join(config.stateDir, WorkerWakeSlots.HELD_WAKES_FILE); }).pipe(Effect.provide(NodeServices.layer)); -const heldIds = (dir: string) => +const heldIds = (disk: Disk) => Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem; - const file = yield* heldFile(dir); - if (!(yield* fs.exists(file))) return []; - const parsed = JSON.parse(yield* fs.readFileString(file)) as { - wakes: ReadonlyArray<{ commandId: string }>; - }; + const text = disk.files.get(yield* heldFile(disk)); + if (text === undefined) return []; + const parsed = JSON.parse(text) as { wakes: ReadonlyArray<{ commandId: string }> }; return parsed.wakes.map((wake) => wake.commandId); - }).pipe(Effect.provide(NodeServices.layer)); + }); +/** A real folder for the server's own directories, and the held-wakes file in memory. */ const tempDir = FileSystem.FileSystem.pipe( Effect.flatMap((fs) => fs.makeTempDirectoryScoped({ prefix: "t3-fork-held-wakes-" })), + Effect.map(diskOver), Effect.provide(NodeServices.layer), ); @@ -237,12 +310,15 @@ describe("a delegated task's question to an idle parent takes an agent slot", () targetInstanceId: ProviderInstanceId.make("claudeAgent"), intoThreadId: PARENT, keepsHeld: true, + place: QUESTION_PLACE, }); // Upstream's reactor asks again (a later event): still one held, never two. yield* ask; yield* TestClock.adjust("1 minute"); assert.deepStrictEqual(wakes(world), []); assert.isAtLeast(world.asks.length, 12); + // Every ask in its one place in the gate's line. + assert.isTrue(world.asks.every((asked) => asked.place?.key === QUESTION_PLACE.key)); assert.lengthOf(waitLogs(world), 1); world.free = 1; yield* TestClock.adjust("5 seconds"); @@ -281,7 +357,8 @@ describe("a delegated task's question to an idle parent takes an agent slot", () ); assert.deepStrictEqual(wakes(world), []); world.free = 1; - yield* TestClock.adjust("5 seconds").pipe( + // The next server asks at once, so the file's wakes take their places in line first. + yield* TestClock.adjust("0 seconds").pipe( Effect.provide(server(world, dir)), Effect.scoped, ); @@ -304,15 +381,14 @@ describe("a delegated task's question to an idle parent takes an agent slot", () const world = newWorld(0); yield* ask.pipe(Effect.provide(server(world, dir)), Effect.scoped); const file = yield* heldFile(dir); - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const held = yield* fs.readFileString(file).pipe(Effect.provide(NodeServices.layer)); + const held = dir.files.get(file)!; world.free = 1; yield* TestClock.adjust("5 seconds").pipe( Effect.provide(server(world, dir)), Effect.scoped, ); // The write after the send is lost: the file still names it. - yield* fs.writeFileString(file, held).pipe(Effect.provide(NodeServices.layer)); + dir.files.set(file, held); world.free = 1; yield* TestClock.adjust("5 seconds").pipe( Effect.provide(server(world, dir)), @@ -342,6 +418,8 @@ describe("a delegated task's question to an idle parent takes an agent slot", () world.free = 5; yield* TestClock.adjust("5 seconds"); assert.deepStrictEqual(yield* heldIds(dir), []); + // It holds no place in the line either. + assert.include(world.left, QUESTION_PLACE.key); yield* TestClock.adjust("1 minute"); assert.deepStrictEqual(wakes(world), []); }).pipe(Effect.provide(server(world, dir)), Effect.scoped); @@ -374,8 +452,6 @@ describe("a delegated task's question to an idle parent takes an agent slot", () "old, empty, broken or missing file: nothing sent from it; a bad one is kept; a new hold writes it", () => Effect.gen(function* () { - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); for (const contents of [ null, "", @@ -386,8 +462,7 @@ describe("a delegated task's question to an idle parent takes an agent slot", () const dir = yield* tempDir; const world = newWorld(0); const file = yield* heldFile(dir); - if (contents !== null) - yield* fs.writeFileString(file, contents).pipe(Effect.provide(NodeServices.layer)); + if (contents !== null) dir.files.set(file, contents); yield* Effect.gen(function* () { yield* TestClock.adjust("10 seconds"); assert.deepStrictEqual(world.sent, []); @@ -395,19 +470,16 @@ describe("a delegated task's question to an idle parent takes an agent slot", () assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); }).pipe(Effect.provide(server(world, dir)), Effect.scoped); // A file that could not be read is kept beside the new one, and named in the log. - const kept = (yield* fs - .readDirectory(path.dirname(file)) - .pipe(Effect.provide(NodeServices.layer))).filter((name) => - name.startsWith(`${WorkerWakeSlots.HELD_WAKES_FILE}.corrupt-`), - ); + const kept = [...dir.files.keys()] + .map(baseName) + .filter((name) => name.startsWith(`${WorkerWakeSlots.HELD_WAKES_FILE}.corrupt-`)); if (contents === null || contents.startsWith('{"version":1')) { assert.deepStrictEqual(kept, []); continue; } assert.lengthOf(kept, 1); - const keptAs = path.join(path.dirname(file), kept[0]!); assert.strictEqual( - yield* fs.readFileString(keptAs).pipe(Effect.provide(NodeServices.layer)), + dir.files.get(`${file}.corrupt-${kept[0]!.split("-").at(-1)}`), contents, ); assert.isTrue( @@ -461,6 +533,7 @@ describe("a delegated task's question to an idle parent takes an agent slot", () }), threadOf: () => Effect.succeed({ instanceId: ProviderInstanceId.make("claudeAgent") }), questionOpen: () => Effect.succeed(true), + compactionDue: () => Effect.succeed(false), store: { load: Effect.succeed([w1]), save: (held) => @@ -502,16 +575,8 @@ describe("a delegated task's question to an idle parent takes an agent slot", () const world = newWorld(0); yield* ask.pipe(Effect.provide(server(world, dir)), Effect.scoped); const file = yield* heldFile(dir); - const fs = yield* FileSystem.FileSystem.pipe(Effect.provide(NodeServices.layer)); - const stored = JSON.parse( - yield* fs.readFileString(file).pipe(Effect.provide(NodeServices.layer)), - ) as { version: 1; wakes: Array }; - yield* fs - .writeFileString( - file, - JSON.stringify({ ...stored, wakes: [{ type: "?" }, ...stored.wakes] }), - ) - .pipe(Effect.provide(NodeServices.layer)); + const stored = JSON.parse(dir.files.get(file)!) as { version: 1; wakes: Array }; + dir.files.set(file, JSON.stringify({ ...stored, wakes: [{ type: "?" }, ...stored.wakes] })); world.free = 1; yield* TestClock.adjust("5 seconds").pipe(Effect.provide(server(world, dir)), Effect.scoped); assert.lengthOf(wakes(world), 1); @@ -519,6 +584,124 @@ describe("a delegated task's question to an idle parent takes an agent slot", () }).pipe(Effect.scoped), ); + it.effect( + "a file whose check fails is not missing: nothing written over it, sent once it reads", + () => + Effect.gen(function* () { + const dir = yield* tempDir; + const world = newWorld(0); + yield* ask.pipe(Effect.provide(server(world, dir)), Effect.scoped); + assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); + dir.failing.exists = true; + world.free = 1; + yield* Effect.gen(function* () { + yield* TestClock.adjust("20 seconds"); + assert.deepStrictEqual(wakes(world), []); + assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); + assert.lengthOf( + world.logs.filter((log) => log.includes("held wakes file could not be read")), + 1, + ); + dir.failing.exists = false; + yield* TestClock.adjust("5 seconds"); + assert.lengthOf(wakes(world), 1); + assert.deepStrictEqual(yield* heldIds(dir), []); + }).pipe(Effect.provide(server(world, dir)), Effect.scoped); + }).pipe(Effect.scoped), + ); + + it.effect( + "a file that cannot be read yet: a new hold is kept in memory, the file's own wakes go first", + () => + Effect.gen(function* () { + const wake = (name: string) => + ({ + type: "message.dispatch", + commandId: `server:delegated-task-question:${PARENT}:${name}`, + messageId: `message:${name}`, + threadId: PARENT, + }) as never as Extract; + const [fromFile, held] = [wake("from-file"), wake("held")]; + const world = newWorld(0); + let readable = false; + const saves: Array> = []; + const slots = yield* WorkerWakeSlots.makeWorkerWakeSlots({ + orchestrator: { dispatch: () => Effect.die("unused") } as never, + gate: Option.some(yield* SlotGate.SlotGate.pipe(Effect.provide(gateOf(world)))), + threadOf: () => Effect.succeed({ instanceId: ProviderInstanceId.make("claudeAgent") }), + questionOpen: () => Effect.succeed(true), + compactionDue: () => Effect.succeed(false), + store: { + load: Effect.suspend(() => + readable + ? Effect.succeed([fromFile]) + : Effect.fail(diskError("PermissionDenied", "exists", "held")), + ), + save: (wakes) => Effect.sync(() => saves.push(wakes.map((one) => one.commandId))), + }, + }); + yield* slots.orchestrator.dispatch(held); + assert.deepStrictEqual( + slots.held().map((one) => one.commandId), + [held.commandId], + ); + assert.deepStrictEqual(saves, []); + readable = true; + yield* slots.retry; + assert.deepStrictEqual( + slots.held().map((one) => one.commandId), + [fromFile.commandId, held.commandId], + ); + assert.deepStrictEqual(saves.at(-1), [fromFile.commandId, held.commandId]); + }).pipe(Effect.provide(logsOf(newWorld(0)))), + ); + + it.effect("a save is synced before its rename, and the folder after it", () => + Effect.gen(function* () { + const dir = yield* tempDir; + const world = newWorld(0); + const ops = dir.ops; + yield* ask.pipe(Effect.provide(server(world, dir)), Effect.scoped); + const file = WorkerWakeSlots.HELD_WAKES_FILE; + const rename = ops.indexOf(`rename ${file}.tmp ${file}`); + assert.isAbove(rename, -1); + assert.include(ops.slice(0, rename), `sync ${file}.tmp`); + const path = yield* Path.Path.pipe(Effect.provide(NodeServices.layer)); + const folder = baseName(path.dirname(yield* heldFile(dir))); + assert.include(ops.slice(rename + 1), `sync ${folder}`); + assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); + }).pipe(Effect.scoped), + ); + + it.effect("on the real disk: a save is read back whole, and a missing file is none", () => + Effect.gen(function* () { + const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; + const folder = yield* fs.makeTempDirectoryScoped({ prefix: "t3-fork-held-wakes-real-" }); + const store = yield* WorkerWakeSlots.fileStore( + path.join(folder, WorkerWakeSlots.HELD_WAKES_FILE), + ); + assert.deepStrictEqual(yield* store.load, []); + const wake = { + type: "message.dispatch", + commandId: WAKE_COMMAND, + messageId: `message:delegated-task-question:${PARENT}:${REQUEST}`, + threadId: PARENT, + text: "A question waits.", + attachments: [], + createdBy: "agent", + creationSource: "server", + dispatchMode: { type: "start_immediately" }, + } as never as Extract; + yield* store.save([wake]); + assert.deepStrictEqual( + (yield* store.load).map((one) => one.commandId), + [WAKE_COMMAND], + ); + assert.deepStrictEqual(yield* fs.readDirectory(folder), [WorkerWakeSlots.HELD_WAKES_FILE]); + }).pipe(Effect.scoped, Effect.provide(NodeServices.layer)), + ); + it.effect("no gate: upstream's own orchestrator", () => Effect.gen(function* () { const inner = { dispatch: () => Effect.die("unused") } as never; @@ -527,6 +710,7 @@ describe("a delegated task's question to an idle parent takes an agent slot", () gate: Option.none(), threadOf: () => Effect.succeed(null), questionOpen: () => Effect.succeed(true), + compactionDue: () => Effect.succeed(false), store: { load: Effect.succeed([]), save: () => Effect.void }, }); assert.strictEqual(slots.orchestrator, inner); @@ -534,8 +718,12 @@ describe("a delegated task's question to an idle parent takes an agent slot", () ); }); -/** Upstream's idle compaction worker for the handoff test thread, behind the wrapper. */ -const compaction = (world: World) => +/** + * Upstream's idle compaction worker for the handoff test thread, behind the wrapper. Its window + * runs from now to 2 minutes on; `thread.data` is what both upstream and the fork's check read. + * Returns when upstream's worker is done with the run. + */ +const compaction = (world: World, thread: { data: ReturnType }) => Effect.gen(function* () { yield* TestClock.setTime(NOW_MS); const decided = yield* Deferred.make(); @@ -560,7 +748,6 @@ const compaction = (world: World) => occurredAt: at(), payload: run(), } as OrchestrationV2DomainEvent); - const data = projection({ usedTokens: 250_000, cacheExpiresInMs: 2 * MINUTE_MS }); const slots = yield* WorkerWakeSlots.makeWorkerWakeSlots({ orchestrator: { streamDomainEvents: Stream.fromQueue(events), @@ -571,8 +758,12 @@ const compaction = (world: World) => }), } as never, gate: Option.some(yield* SlotGate.SlotGate.pipe(Effect.provide(gateOf(world)))), - threadOf: () => Effect.succeed({ instanceId: data.thread.modelSelection.instanceId }), + threadOf: () => Effect.succeed({ instanceId: thread.data.thread.modelSelection.instanceId }), questionOpen: () => Effect.succeed(true), + compactionDue: (command) => + Clock.currentTimeMillis.pipe( + Effect.map((nowMs) => WorkerWakeSlots.compactionWindowOpen(thread.data, command, nowMs)), + ), store: { load: Effect.succeed([]), save: () => Effect.void }, }); yield* IdleCompactionService.make.pipe( @@ -581,8 +772,8 @@ const compaction = (world: World) => Effect.provide( Layer.mergeAll( Layer.mock(ProjectionStore.ProjectionStoreV2)({ - getThread: () => Effect.succeed(data.thread), - getThreadRecords: () => Effect.succeed(data), + getThread: () => Effect.sync(() => thread.data.thread), + getThreadRecords: () => Effect.sync(() => thread.data), }), Layer.mock(ServerSettings.ServerSettingsService)({ getSettings: Effect.succeed({ @@ -595,33 +786,86 @@ const compaction = (world: World) => ), Effect.withTracer(tracer), ); - yield* Deferred.await(decided); + yield* TestClock.adjust("0 seconds"); + return decided; }); +const COMPACTION_PLACE = { key: `idle-compaction:server:idle-compact:${threadId}:${runId}` }; +const warm = () => ({ data: projection({ usedTokens: 250_000, cacheExpiresInMs: 2 * MINUTE_MS }) }); + describe("upstream's idle compaction takes an agent slot", () => { it.effect("a free slot: compacted as upstream does, asked as the thread's own start", () => Effect.gen(function* () { const world = newWorld(1); - yield* compaction(world); + yield* Deferred.await(yield* compaction(world, warm())); assert.lengthOf(world.sent, 1); assert.deepStrictEqual(world.asks[0], { callerThreadId: threadId, targetInstanceId: ProviderInstanceId.make("claudeAgent"), intoThreadId: threadId, keepsHeld: true, + place: COMPACTION_PLACE, }); }), ); - it.effect("full: skipped and logged, never sent late (the prompt cache it saves expires)", () => + it.effect( + "full, then a slot frees while the window is open: asked every 5 s in its place, sent once", + () => + Effect.gen(function* () { + const world = newWorld(0); + const decided = yield* compaction(world, warm()); + yield* TestClock.adjust("30 seconds"); + assert.deepStrictEqual(world.sent, []); + assert.isAtLeast(world.asks.length, 6); + assert.isTrue(world.asks.every((asked) => asked.place?.key === COMPACTION_PLACE.key)); + assert.lengthOf( + world.logs.filter((log) => log.includes("idle compaction waits for an agent slot")), + 1, + ); + world.free = 1; + yield* TestClock.adjust("5 seconds"); + yield* Deferred.await(decided); + assert.lengthOf(world.sent, 1); + yield* TestClock.adjust("1 hour"); + assert.lengthOf(world.sent, 1); + }), + ); + + it.effect("full until the cache window closes: skipped, logged, never sent late", () => Effect.gen(function* () { const world = newWorld(0); - yield* compaction(world); + const decided = yield* compaction(world, warm()); + yield* TestClock.adjust("2 minutes"); + yield* Deferred.await(decided); assert.deepStrictEqual(world.sent, []); - assert.isTrue(world.logs.some((log) => log.includes("idle compaction skipped"))); + assert.isTrue( + world.logs.some((log) => + log.includes("idle compaction skipped: no agent slot while its cache window was open"), + ), + ); + assert.include(world.left, COMPACTION_PLACE.key); world.free = 5; yield* TestClock.adjust("1 hour"); assert.deepStrictEqual(world.sent, []); }), ); + + it.effect("full, and the thread is no longer idle: skipped at the next check, never sent", () => + Effect.gen(function* () { + const world = newWorld(0); + const thread = warm(); + const decided = yield* compaction(world, thread); + // A question to the person is open now: no window, so no compaction. + thread.data = { + ...thread.data, + runtimeRequests: [{ ...question(world), status: "pending" }] as never, + }; + world.free = 5; + yield* TestClock.adjust("5 seconds"); + yield* Deferred.await(decided); + assert.deepStrictEqual(world.sent, []); + assert.include(world.left, COMPACTION_PLACE.key); + }), + ); }); diff --git a/apps/server/src/fork/workerWakeSlots.ts b/apps/server/src/fork/workerWakeSlots.ts index 9ac8820677adf..6642cdd2ebe6e 100644 --- a/apps/server/src/fork/workerWakeSlots.ts +++ b/apps/server/src/fork/workerWakeSlots.ts @@ -1,13 +1,17 @@ /** * Fork: upstream's idle compaction and a delegated task's question to its idle parent start a run, - * so each takes an agent slot. A refused compaction is skipped (it only pays before the prompt - * cache expires). A refused question wake is kept in `HELD_WAKES_FILE` and sent once with a slot. + * so each takes an agent slot, with a place in the gate's line. A refused compaction asks again + * while the thread stays idle and its prompt cache window is open, then is skipped. A refused + * question wake is kept in `HELD_WAKES_FILE` and sent once with a slot. */ import { type CommandId, OrchestrationV2Command, type OrchestrationV2ServerCommand, type ProviderInstanceId, + resolveIdleCompaction, + resolveProjectProviderInstanceEnabled, + type RunId, type RuntimeRequestId, type ThreadId, } from "@t3tools/contracts"; @@ -18,15 +22,21 @@ import * as FileSystem from "effect/FileSystem"; import * as Layer from "effect/Layer"; import * as Option from "effect/Option"; import * as Path from "effect/Path"; +import type * as PlatformError from "effect/PlatformError"; import * as Schema from "effect/Schema"; import * as Semaphore from "effect/Semaphore"; import { ServerConfig } from "../config.ts"; +import { + idleCompactionWindow, + type IdleCompactionRecords, +} from "../orchestration-v2/IdleCompaction.ts"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import { forkParked } from "../serverActivation.ts"; +import * as ServerSettings from "../serverSettings.ts"; import * as SlotGate from "../slots/SlotGate.ts"; -import { admitSend, reasonHere } from "./agentSlotRoutes.ts"; +import { admitSend, leaveLine, reasonHere } from "./agentSlotRoutes.ts"; type MessageDispatch = Extract; @@ -34,6 +44,9 @@ type MessageDispatch = Extract command.type === "message.dispatch" && command.commandId.startsWith(QUESTION_COMMAND); +/** The completed run an idle compaction is for, from upstream's command id. */ +const compactionRunOf = (command: MessageDispatch): RunId | null => { + const prefix = `${COMPACTION_COMMAND}${command.threadId}:`; + return command.commandId.startsWith(prefix) + ? (command.commandId.slice(prefix.length) as RunId) + : null; +}; + +/** + * A refused idle compaction is still due: its thread is idle at its run, and the prompt cache + * window is open, by the rule upstream's worker sent it by (`idleCompactionWindow`). + */ +export const compactionWindowOpen = ( + records: IdleCompactionRecords, + command: MessageDispatch, + nowMs: number, +): boolean => { + const runId = compactionRunOf(command); + const window = idleCompactionWindow(records, nowMs); + return runId !== null && window?.runId === runId && nowMs < window.endMs; +}; + +/** Places in the gate's line: one per held question wake and per waiting compaction. */ +const questionPlace = (command: MessageDispatch): SlotGate.SlotPlace => ({ + key: `question-wake:${command.commandId}`, +}); +const compactionPlace = (command: MessageDispatch): SlotGate.SlotPlace => ({ + key: `idle-compaction:${command.commandId}`, +}); + /** The question a wake is about, from upstream's ids (`DelegatedTaskQuestionReactor.ts`). */ const questionOf = (command: MessageDispatch) => { const source = command.notification?.source; @@ -70,13 +113,17 @@ const encodeCommand = Schema.encodeSync(OrchestrationV2Command); /** The parent as the gate needs it: `null` when gone (deleted or archived) or unknown. */ export type WakeThread = { readonly instanceId: ProviderInstanceId } | null; -/** Where the held wakes are kept; `null` keeps them in memory only (tests). */ +/** Where the held wakes are kept. */ export interface HeldWakesStore { - readonly load: Effect.Effect>; + /** + * What is held; none when there is no file. Fails when the file cannot be checked or read (an + * I/O error, never "missing"): its wakes may be there, so nothing is written over it. + */ + readonly load: Effect.Effect, PlatformError.PlatformError>; readonly save: (wakes: ReadonlyArray) => Effect.Effect; } -export interface WorkerWakeInput { +export interface WorkerWakeInput { readonly orchestrator: Orchestrator.OrchestratorV2["Service"]; readonly gate: Option.Option; readonly threadOf: (threadId: ThreadId) => Effect.Effect; @@ -85,6 +132,11 @@ export interface WorkerWakeInput { parentThreadId: ThreadId, question: NonNullable>, ) => Effect.Effect; + /** + * Whether a refused idle compaction is still due: its thread idle at its run, and the prompt + * cache window open (`idleCompactionWindow`, as upstream's worker reads it). + */ + readonly compactionDue: (command: MessageDispatch) => Effect.Effect; readonly store: HeldWakesStore; } @@ -92,21 +144,54 @@ export interface WorkerWakeInput { * Upstream's orchestrator with the two gated, and `retry`, which sends the held wakes that have a * slot now (the layer runs it every `HELD_WAKE_RETRY`). With no gate it is upstream's own. */ -export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* ( - input: WorkerWakeInput, +export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* ( + input: WorkerWakeInput, ) { const { orchestrator, gate } = input; if (Option.isNone(gate)) return { orchestrator, retry: Effect.void, held: (): ReadonlyArray => [] }; const held = new Map(); - for (const wake of yield* input.store.load) held.set(wake.commandId, wake); - if (held.size > 0) - yield* Effect.logInfo("held delegated task questions loaded: sent when a slot frees", { - count: held.size, - }); + /** + * The file was read (or there is none). Until then nothing is written over it: a file that + * cannot be checked or read may hold wakes. Each retry reads it again. + */ + let loaded = false; + let loadLogged = false; const saving = yield* Semaphore.make(1); // One save at a time, its snapshot taken under the lock: the last write is what is held now. - const save = saving.withPermits(1)(Effect.suspend(() => input.store.save([...held.values()]))); + const save = saving.withPermits(1)( + Effect.suspend(() => (loaded ? input.store.save([...held.values()]) : Effect.void)), + ); + const load = Effect.gen(function* () { + const fromFile = yield* input.store.load.pipe( + Effect.tapCause((cause) => + loadLogged + ? Effect.void + : Effect.logWarning( + "held wakes file could not be read: nothing is written over it, read again in 5 s", + { cause }, + ), + ), + Effect.option, + ); + if (Option.isNone(fromFile)) { + loadLogged = true; + return; + } + // The file's wakes first, in its order (their place in the gate's line), then any held since. + const since = [...held.values()]; + held.clear(); + for (const wake of [...fromFile.value, ...since]) { + if (!held.has(wake.commandId)) held.set(wake.commandId, wake); + } + loaded = true; + if (fromFile.value.length > 0) + yield* Effect.logInfo("held delegated task questions loaded: sent when a slot frees", { + count: fromFile.value.length, + }); + if (since.length > 0) yield* save; + }); + yield* load; /** Held wakes whose parent or question could not be read: logged once. */ const unreadLogged = new Set(); const unread = Effect.fn("WorkerWakeSlots.unread")(function* (command: MessageDispatch) { @@ -119,36 +204,65 @@ export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* < return false; }); - /** `null`: it may start now; else why it waits. */ - const slotFor = Effect.fn("WorkerWakeSlots.slotFor")(function* (command: MessageDispatch) { + /** `null`: it may start now; else why it waits (in its place in the gate's line). */ + const slotFor = Effect.fn("WorkerWakeSlots.slotFor")(function* ( + command: MessageDispatch, + place: SlotGate.SlotPlace, + ) { const thread = yield* input .threadOf(command.threadId) .pipe(Effect.orElseSucceed(() => "unreadable" as const)); if (thread === "unreadable") return "the thread could not be read to check for a slot"; - if (thread === null) return null; + if (thread === null) { + yield* leaveLine(gate, [place.key]); + return null; + } const slot = yield* admitSend(gate, { callerThreadId: command.threadId, threadId: command.threadId, instanceId: thread.instanceId, keepsHeld: true, + place, }); return slot.type === "refuse" ? reasonHere(slot.refusal) : null; }); - const compaction = Effect.fn("WorkerWakeSlots.compaction")(function* (command: MessageDispatch) { - const detail = yield* slotFor(command); - if (detail === null) return yield* orchestrator.dispatch(command); - yield* Effect.logInfo("idle compaction skipped: it waits for no agent slot", { - threadId: command.threadId, - detail, - }); - return NOTHING_STORED; - }); + /** + * Held in upstream's own timer (a newer completed run replaces it, and a stop ends it): asked + * again every `COMPACTION_SLOT_RETRY` while it is still due, else skipped. Late is worse than + * never: after the window a compaction pays a full uncached read. + */ + const compaction = Effect.fn("WorkerWakeSlots.compaction")( + function* (command: MessageDispatch) { + const place = compactionPlace(command); + let detail = yield* slotFor(command, place); + if (detail === null) return yield* orchestrator.dispatch(command); + yield* Effect.logInfo("idle compaction waits for an agent slot", { + threadId: command.threadId, + detail, + }); + while (detail !== null) { + yield* Effect.sleep(COMPACTION_SLOT_RETRY); + // Checked again in the fork before each ask: still idle at its run, window still open. + const due = yield* input.compactionDue(command).pipe(Effect.orElseSucceed(() => false)); + if (!due) { + yield* Effect.logInfo( + "idle compaction skipped: no agent slot while its cache window was open", + { threadId: command.threadId }, + ); + return NOTHING_STORED; + } + detail = yield* slotFor(command, place); + } + return yield* orchestrator.dispatch(command); + }, + (effect, command) => Effect.ensuring(effect, leaveLine(gate, [compactionPlace(command).key])), + ); const question = Effect.fn("WorkerWakeSlots.question")(function* (command: MessageDispatch) { // Held already: the retry sends it, once. if (held.has(command.commandId)) return NOTHING_STORED; - const detail = yield* slotFor(command); + const detail = yield* slotFor(command, questionPlace(command)); if (detail === null) return yield* orchestrator.dispatch(command); held.set(command.commandId, command); yield* save; @@ -161,16 +275,19 @@ export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* < /** One held wake: `true` when it leaves the file (sent, or no longer owed). */ const retryOne = Effect.fn("WorkerWakeSlots.retryOne")(function* (command: MessageDispatch) { + const place = questionPlace(command); + // No longer owed: it leaves the line too, so it holds no slot from the waiters behind it. + const owedNoMore = leaveLine(gate, [place.key]).pipe(Effect.as(true)); const thread = yield* input.threadOf(command.threadId).pipe(Effect.option); if (Option.isNone(thread)) return yield* unread(command); - if (thread.value === null) return true; + if (thread.value === null) return yield* owedNoMore; const asked = questionOf(command); if (asked !== null) { const open = yield* input.questionOpen(command.threadId, asked).pipe(Effect.option); if (Option.isNone(open)) return yield* unread(command); - if (!open.value) return true; + if (!open.value) return yield* owedNoMore; } - if ((yield* slotFor(command)) !== null) return false; + if ((yield* slotFor(command, place)) !== null) return false; yield* orchestrator.dispatch(command).pipe( Effect.catchCause((cause) => Effect.logWarning("waking the parent of a delegated task question failed", { @@ -183,6 +300,7 @@ export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* < }); const retry = Effect.gen(function* () { + if (!loaded) yield* load; let changed = false; for (const command of held.values()) { if (!(yield* retryOne(command))) continue; @@ -209,16 +327,16 @@ export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* < }); /** - * The held wakes in `filePath`. Missing is none. A file or entry that cannot be read is logged and - * left out: the next save writes what is held then. A file that cannot be read is first moved to - * `.corrupt-`, so nothing is written over it. + * The held wakes in `filePath`: missing is none, a failed check or read is an error. A file that + * does not decode is moved to `.corrupt-`; a bad entry is logged and left out. A save is + * synced before its rename and the folder after, so a power loss leaves one whole file. */ -const fileStore = Effect.fn("WorkerWakeSlots.fileStore")(function* (filePath: string) { +export const fileStore = Effect.fn("WorkerWakeSlots.fileStore")(function* (filePath: string) { const fs = yield* FileSystem.FileSystem; + const path = yield* Path.Path; const load = Effect.gen(function* () { - if (!(yield* fs.exists(filePath).pipe(Effect.orElseSucceed(() => false)))) return []; - const text = yield* fs.readFileString(filePath).pipe(Effect.option); - const file = Option.flatMap(text, decodeFile); + if (!(yield* fs.exists(filePath))) return []; + const file = decodeFile(yield* fs.readFileString(filePath)); if (Option.isNone(file)) { const keptAs = `${filePath}.corrupt-${yield* Clock.currentTimeMillis}`; const kept = yield* fs.rename(filePath, keptAs).pipe( @@ -252,8 +370,20 @@ const fileStore = Effect.fn("WorkerWakeSlots.fileStore")(function* (filePath: st version: 1, wakes: wakes.map((wake) => encodeCommand(wake)), }); - yield* fs.writeFileString(temporary, text); + yield* Effect.scoped( + Effect.gen(function* () { + const file = yield* fs.open(temporary, { flag: "w" }); + yield* file.writeAll(new TextEncoder().encode(text)); + yield* file.sync; + }), + ); yield* fs.rename(temporary, filePath); + // The rename itself, on disk. A folder cannot be opened on every system (Windows): skipped. + yield* Effect.scoped( + fs + .open(path.dirname(filePath), { flag: "r" }) + .pipe(Effect.flatMap((folder) => folder.sync)), + ).pipe(Effect.ignore); }, // Not written: they are still held in memory and written with the next change. Effect.catchCause((cause) => Effect.logWarning("held wakes not written", { filePath, cause })), @@ -268,6 +398,7 @@ export const layerOver = Layer.effect( const orchestrator = yield* Orchestrator.OrchestratorV2; const projections = yield* ProjectionStore.ProjectionStoreV2; const config = yield* ServerConfig; + const settings = yield* ServerSettings.ServerSettingsService; const path = yield* Path.Path; const slots = yield* makeWorkerWakeSlots({ orchestrator, @@ -290,12 +421,55 @@ export const layerOver = Layer.effect( (task) => asked.taskIds.includes(task.id) && SETTLED_TASK.has(task.status), ); }), + compactionDue: Effect.fn("WorkerWakeSlots.compactionDue")(function* (command) { + const runId = compactionRunOf(command); + const current = yield* settings.getSettings; + const thread = yield* projections.getThread(command.threadId); + if ( + runId === null || + resolveIdleCompaction(current, thread.providerInstanceId) !== "beforeCacheExpires" || + !resolveProjectProviderInstanceEnabled( + current, + thread.projectId, + thread.providerInstanceId, + ) + ) + return false; + const active = + thread.activeProviderThreadId === null ? [] : [thread.activeProviderThreadId]; + // The records upstream's idle compaction worker reads for its window. + const records = yield* projections.getThreadRecords( + command.threadId, + [ + "runs", + "messages", + "providerThreads", + "providerTurns", + "runtimeRequests", + "turnItems", + "contextTransfers", + ], + { + messageRunIds: [runId], + turnItemStatuses: ["pending", "running", "waiting"], + runStatuses: ["preparing", "starting", "running", "rolled_back"], + includeLatestRun: true, + providerThreadIds: active, + providerTurnThreadIds: active, + latestProviderTurnUsageOnly: true, + runtimeRequestStatuses: ["pending"], + contextTransferStatuses: ["pending"], + }, + ); + return compactionWindowOpen(records, command, yield* Clock.currentTimeMillis); + }), store: yield* fileStore(path.join(config.stateDir, HELD_WAKES_FILE)), }); // A stop waits for a send's file write: a send it cut off would be sent again (receipts dedupe). + // The first pass runs at once, so wakes from the file take their places in line first. yield* forkParked( - Effect.sleep(HELD_WAKE_RETRY).pipe( - Effect.andThen(Effect.uninterruptible(slots.retry)), + Effect.uninterruptible(slots.retry).pipe( + Effect.andThen(Effect.sleep(HELD_WAKE_RETRY)), Effect.forever, ), ); diff --git a/apps/server/src/slots/SlotGate.test.ts b/apps/server/src/slots/SlotGate.test.ts index aaade95d2eafe..34ce414d18bd7 100644 --- a/apps/server/src/slots/SlotGate.test.ts +++ b/apps/server/src/slots/SlotGate.test.ts @@ -15,6 +15,7 @@ import { import { type AgentSlotsSettings, WaitToolFailure } from "@t3tools/contracts/waitFor"; import type { DeepPartial } from "@t3tools/shared/Struct"; import { DISPATCH_SAFETY_MS, type SlotReservation } from "@t3tools/shared/agentSlots"; +import * as Clock from "effect/Clock"; import * as Deferred from "effect/Deferred"; import * as Effect from "effect/Effect"; import * as Fiber from "effect/Fiber"; @@ -2160,3 +2161,202 @@ describe("slot waits granted in one sweep: every wake-up granted is let through" }), ); }); + +describe("SlotGate: one line for every waiting start (issue 738)", () => { + const waitsOn = (patch: Partial) => ({ + ...on(patch), + waitFor: { enabled: true }, + }); + /** A wake-up of `thread` that asks again while refused, in its place `key` (or none). */ + const wake = (thread: string, key: string | null, targetInstanceId = MAIN) => + SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ + callerThreadId: ThreadId.make(thread), + targetInstanceId, + intoThreadId: ThreadId.make(thread), + keepsHeld: true, + ...(key === null ? {} : { place: { key } }), + }), + ), + ); + /** A start without a place (`delegate_task` from `thread`). */ + const start = (thread: string, targetInstanceId = MAIN) => + SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ callerThreadId: ThreadId.make(thread), targetInstanceId }), + ), + ); + const line = SlotGate.SlotGate.pipe(Effect.flatMap((gate) => gate.line)); + + it.effect("steady load: the oldest waiter starts first, never one that asks first", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + for (const name of ["w0", "w1", "w2"]) + assert.strictEqual((yield* wake(name, name)).type, "refuse"); + const started: Array = []; + for (let round = 0; round < 6; round += 1) { + // The last run ends; a new waiter joins every round (steady load). + world.threads = []; + const waiting = yield* line; + // Everyone else asks first: a plain start, then the younger waiters, newest first. + assert.strictEqual((yield* start("newcomer")).type, "refuse"); + for (const name of waiting.slice(1).toReversed()) + assert.strictEqual((yield* wake(name, name)).type, "refuse"); + const oldest = waiting[0]!; + assert.strictEqual((yield* wake(oldest, oldest)).type, "allow"); + started.push(oldest); + world.threads = [{ id: oldest, instanceId: MAIN }]; + assert.strictEqual((yield* wake(`w${round + 3}`, `w${round + 3}`)).type, "refuse"); + } + assert.deepStrictEqual(started, ["w0", "w1", "w2", "w3", "w4", "w5"]); + }), + ); + }), + ); + + it.effect("a waiter whose own account is full holds no slot from the others", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: SECOND }]; + yield* run( + world, + on({ maxPerMachine: 3, maxPerAccount: { [SECOND]: 1 } }), + Effect.gen(function* () { + assert.strictEqual((yield* wake("w", "w", SECOND)).type, "refuse"); + // Two machine slots are free; the waiter can take neither, so both stay free. + assert.strictEqual((yield* start("a")).type, "allow"); + assert.strictEqual((yield* start("b")).type, "allow"); + assert.strictEqual((yield* start("c")).type, "refuse"); + }), + ); + }), + ); + + it.effect("a slot held for a slot wait is its thread's: a waiter behind does not take it", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + const holder = ThreadId.make("thread:holder"); + yield* run( + world, + waitsOn({ maxPerMachine: 2, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const [held] = yield* gate.grant([{ key: "hold", threadId: holder }]); + assert.strictEqual(held?.type, "granted"); + yield* gate.keep(["hold"]); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // The holder's agent starts on its own slot, though a waiter is in line. + assert.strictEqual((yield* start(holder)).type, "allow"); + }), + ); + }), + ); + + it.effect("slot waits share the line, each at its own age", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + waitsOn({ maxPerMachine: 1, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const nowMs = yield* Clock.currentTimeMillis; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + const younger = { key: "young", threadId: ThreadId.make("young"), sinceMs: nowMs + 1 }; + const older = { key: "old", threadId: ThreadId.make("old"), sinceMs: nowMs - 1 }; + assert.deepStrictEqual( + (yield* gate.grant([older, younger])).map((grant) => grant.type), + ["full", "full"], + ); + assert.deepStrictEqual(yield* line, ["old", "w", "young"]); + world.threads = []; + // The waiter asks first, but an older slot wait is in line: it waits. + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + assert.deepStrictEqual( + (yield* gate.grant([older])).map((grant) => grant.type), + ["granted"], + ); + assert.deepStrictEqual(yield* line, ["w", "young"]); + }), + ); + }), + ); + + it.effect("a waiter that left, or stopped asking, or shows a run, holds nothing", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + // Left: its wake is gone (its thread deleted, its question answered). + assert.strictEqual((yield* wake("gone", "gone")).type, "refuse"); + yield* gate.leave(["gone"]); + assert.deepStrictEqual(yield* line, []); + // Stopped asking (a route that died): it leaves after LINE_STALE. + assert.strictEqual((yield* wake("quiet", "quiet")).type, "refuse"); + world.threads = []; + yield* TestClock.adjust("29 seconds"); + assert.strictEqual((yield* start("a")).type, "refuse"); + yield* TestClock.adjust("2 seconds"); + assert.deepStrictEqual(yield* line, []); + // Shows a run: its message only queues, so it needs no slot. + world.threads = [{ id: "busy", instanceId: MAIN }]; + assert.strictEqual((yield* wake("running", "running")).type, "refuse"); + world.threads = [{ id: "running", instanceId: MAIN }]; + assert.deepStrictEqual(yield* line, ["running"]); + world.threads = []; + assert.strictEqual((yield* start("b")).type, "refuse"); + world.threads = [{ id: "running", instanceId: MAIN }]; + yield* TestClock.adjust("31 seconds"); + world.threads = []; + assert.strictEqual((yield* start("b")).type, "allow"); + }), + ); + }), + ); + + it.effect( + "restart: the line is memory only; a new gate has none, and a waiter joins at its next ask", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run(world, on({ maxPerMachine: 1 }), wake("w", "w")); + world.threads = []; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + assert.deepStrictEqual(yield* line, []); + assert.strictEqual((yield* wake("w", "w")).type, "allow"); + assert.deepStrictEqual(yield* line, []); + }), + ); + }), + ); + + it.effect("off or no limit: a waiter is let through and leaves the line", () => + Effect.gen(function* () { + const world = newWorld(); + yield* run( + world, + on({}), + Effect.gen(function* () { + assert.deepStrictEqual(yield* wake("w", "w"), { type: "allow", token: null }); + assert.deepStrictEqual(yield* line, []); + }), + ); + }), + ); +}); diff --git a/apps/server/src/slots/SlotGate.ts b/apps/server/src/slots/SlotGate.ts index f1e2c43ce1b2b..1e8795b681694 100644 --- a/apps/server/src/slots/SlotGate.ts +++ b/apps/server/src/slots/SlotGate.ts @@ -106,6 +106,8 @@ export interface SlotRequest { * it needs no wake-up, so its hold counts no wake-up still to start (`pendingWakes`). */ readonly awake?: boolean | undefined; + /** When the wait was made: its place in the gate's one line (`SlotPlace`); else now. */ + readonly sinceMs?: number | undefined; } export type SlotGrant = @@ -133,6 +135,20 @@ export interface SlotUsage { readonly freeMemoryMb: number | null; } +/** + * A start that asks again while it is refused (a wake-up of an idle thread): its place in the + * gate's one line. Its first ask joins the line; every later ask keeps that place, so a slot that + * frees goes to the oldest waiter, never to whoever asks first. It leaves when it is allowed, by + * `leave`, or when it has not asked for `staleAfter` (default `LINE_STALE`). + */ +export interface SlotPlace { + readonly key: string; + readonly staleAfter?: Duration.Input | undefined; +} + +/** A waiter that has not asked again for this long has left the line (6 asks of 5 s). */ +const LINE_STALE = Duration.seconds(30); + export interface SlotAcquireInput { readonly callerThreadId: ThreadId | null; readonly targetInstanceId: ProviderInstanceId; @@ -152,6 +168,11 @@ export interface SlotAcquireInput { * against it and are not used up, so the agent it wakes up for still has its slot. */ readonly keepsHeld?: boolean | undefined; + /** + * Its place in the line, for a start that asks again while refused. Without one the start asks + * from the back of the line: it never takes a slot a waiter in line is owed. + */ + readonly place?: SlotPlace | undefined; } export class SlotGate extends Context.Service< @@ -192,6 +213,15 @@ export class SlotGate extends Context.Service< * the thread held before counts again. */ readonly drop: (keys: ReadonlyArray) => Effect.Effect; + /** Waiters that no longer need a slot (gone, answered, stopped) leave the line now. */ + readonly leave: (keys: ReadonlyArray) => Effect.Effect; + /** + * Every slot wait still waiting, by key (the wait service's whole list, at each sweep): any + * other slot wait in the line ended (met, cancelled, handed over) and leaves it. + */ + readonly slotWaitsWaiting: (keys: ReadonlyArray) => Effect.Effect; + /** The line's keys, oldest first (waiters that went stale are left out). */ + readonly line: Effect.Effect>; /** * For `wait_list`: null while slots are off, or when the running agents or the held slots * cannot be read. @@ -231,6 +261,23 @@ type CountRead = } | { readonly type: "unread"; readonly problem: CountReadProblem }; +/** One waiter in the line. `instanceId` null: a slot wait with no account (the machine only). */ +interface LineEntry { + readonly key: string; + /** A route's wake-up, or a slot wait (granted only on a whole read of the held slots). */ + readonly kind: "wake" | "slotWait"; + readonly threadId: string; + readonly instanceId: string | null; + readonly sinceMs: number; + /** Joins at the same time keep their order. */ + readonly order: number; + lastAskMs: number; + staleMs: number; +} + +const staleMsOf = (place: SlotPlace) => + Duration.toMillis(Duration.fromInputUnsafe(place.staleAfter ?? LINE_STALE)); + const make = Effect.gen(function* () { const serverSettings = yield* ServerSettings.ServerSettingsService; const activeRunShells = yield* ActiveRunShells.ActiveRunShells; @@ -268,6 +315,111 @@ const make = Effect.gen(function* () { const usedLater = new Set(); /** Why the store last could not be read; warned once until the reason changes or a read works. */ let storeProblem: string | null = null; + /** + * The one line of starts that wait for a slot, by key: every route's waiter (`SlotPlace`) and + * every slot wait (`grant`), oldest first. Memory only: after a restart each waiter joins again + * at its next ask (a slot wait at its own age). + */ + const line = new Map(); + let lineOrder = 0; + + /** Joins the line, or keeps the place it has (asked again). */ + const joinLine = (entry: { + readonly key: string; + readonly kind: LineEntry["kind"]; + readonly threadId: string; + readonly instanceId: string | null; + readonly sinceMs: number; + readonly nowMs: number; + readonly staleMs: number; + }): LineEntry => { + const seen = line.get(entry.key); + if (seen !== undefined) { + seen.lastAskMs = entry.nowMs; + seen.staleMs = entry.staleMs; + return seen; + } + const joined: LineEntry = { + key: entry.key, + kind: entry.kind, + threadId: entry.threadId, + instanceId: entry.instanceId, + sinceMs: entry.sinceMs, + order: lineOrder++, + lastAskMs: entry.nowMs, + staleMs: entry.staleMs, + }; + line.set(entry.key, joined); + return joined; + }; + + /** The live line, oldest first; a waiter that stopped asking has left it. */ + const lineInOrder = (nowMs: number) => { + for (const [key, entry] of line) { + if (nowMs - entry.lastAskMs > entry.staleMs) line.delete(key); + } + return [...line.values()].sort( + (left, right) => left.sinceMs - right.sinceMs || left.order - right.order, + ); + }; + + /** + * The waiters ahead of a start, oldest first, as starts: each that would get a slot now keeps it + * from those behind; one that would not (its account full) holds none. Left out: the asker's own + * thread, a thread that shows a run (it only queues), and one counted already (held, or named). + */ + const lineAhead = (input: { + readonly ahead: ReadonlyArray; + readonly threadId: string | null; + readonly limits: SlotLimits; + readonly running: ReadonlyArray; + readonly visible: ReadonlySet; + readonly kept: ReadonlyArray; + readonly held: HeldRead; + readonly accountOnly: (threadId: string) => ReadonlyArray; + readonly freeMemoryBytes: number | null; + readonly nowMs: number; + }): ReadonlyArray => { + const counted = new Set([ + ...input.held.held.map((reservation) => reservation.threadId), + ...input.kept.flatMap((entry) => + [entry.childThreadId, entry.queuedInto].filter((id) => id !== undefined), + ), + ]); + if (input.threadId !== null) counted.add(input.threadId); + const taken: Array = []; + for (const entry of input.ahead) { + if (counted.has(entry.threadId) || input.visible.has(entry.threadId)) continue; + // A slot wait is granted only when the held slots read whole: until then it takes none. + if (entry.kind === "slotWait" && !input.held.whole) continue; + counted.add(entry.threadId); + const others = onePerThread( + input.held.held.filter((reservation) => reservation.threadId !== entry.threadId), + ); + const decision = slotDecision({ + limits: entry.instanceId === null ? { ...input.limits, account: () => null } : input.limits, + running: input.running.filter((agent) => agent.threadId !== entry.threadId), + starting: [ + ...input.kept, + ...pendingWakes(others, input.visible, input.kept, input.held.woken), + ...taken, + ], + reservations: others, + accountOnly: input.accountOnly(entry.threadId), + targetInstanceId: entry.instanceId ?? "", + freeMemoryBytes: input.freeMemoryBytes, + }); + if (decision.type === "allow") { + taken.push({ + token: -10_000 - taken.length, + instanceId: entry.instanceId ?? "", + atMs: input.nowMs, + childThreadId: entry.threadId, + }); + } + } + return taken; + }; /** * The one count read in flight, joined by every caller that needs the count while it runs. @@ -622,13 +774,33 @@ const make = Effect.gen(function* () { const limits = Option.isNone(settings) ? null : effectiveSlotLimits(settings.value.agentSlots); + const place = input.place; if (Option.isNone(settings) || limits === null) { + if (place !== undefined) line.delete(place.key); return asDecided({ type: "allow", token: null }, []); } const waitsOn = settings.value.waitFor.enabled; const nowMs = yield* Clock.currentTimeMillis; const counted = limitsCount(limits, input.targetInstanceId); const into = input.intoThreadId; + const askingThread = into ?? input.callerThreadId; + // A waiter keeps its place from its first ask, refused or not. + const own = + place === undefined || askingThread === null + ? undefined + : joinLine({ + key: place.key, + kind: "wake", + threadId: askingThread, + instanceId: input.targetInstanceId, + sinceMs: nowMs, + nowMs, + staleMs: staleMsOf(place), + }); + const allowed = (outcome: SlotAcquire, used: ReadonlyArray) => { + if (place !== undefined) line.delete(place.key); + return asDecided(outcome, used); + }; const count = counted || into !== undefined ? yield* readCount : undefined; // Fails closed: a count it needs but cannot read refuses before anything is // recorded, so the refused call starts nothing, uses no held slot and writes nothing. @@ -651,7 +823,7 @@ const make = Effect.gen(function* () { ...current, { token, instanceId: input.targetInstanceId, atMs: nowMs, queuedInto: into }, ]); - return asDecided({ type: "allow", token }, []); + return allowed({ type: "allow", token }, []); } // The running agents only where a count limit applies (no count means none to check). const read = counted ? shown : undefined; @@ -668,6 +840,23 @@ const make = Effect.gen(function* () { // Other held threads not woken yet: each wake-up is a run of its own. const wakes = pendingWakes(others, visible, kept, heldRead.woken); const memoryBytes = yield* freeMemoryBytes(limits); + // The waiters ahead of it in the line (all of them for a start without a place) come first. + const ordered = read === undefined ? [] : lineInOrder(nowMs); + const ahead = + read === undefined + ? [] + : lineAhead({ + ahead: own === undefined ? ordered : ordered.slice(0, ordered.indexOf(own)), + threadId: askingThread, + limits, + running: read.running, + visible, + kept, + held: heldRead, + accountOnly: (threadId) => hiddenHeld(waitsOn, nowMs, threadId), + freeMemoryBytes: memoryBytes, + nowMs, + }); const decisionWith = ( targetInstanceId: ProviderInstanceId, starting: ReadonlyArray, @@ -682,7 +871,7 @@ const make = Effect.gen(function* () { targetInstanceId, freeMemoryBytes: memoryBytes, }); - const decision = decisionWith(input.targetInstanceId, [...kept, ...wakes]); + const decision = decisionWith(input.targetInstanceId, [...kept, ...wakes, ...ahead]); const alsoNeeds = input.alsoNeeds ?? []; const full = ( refused: Extract, @@ -702,6 +891,7 @@ const make = Effect.gen(function* () { const starting: Array = [ ...kept, ...wakes, + ...ahead, { token: -1, instanceId: input.targetInstanceId, atMs: nowMs }, ]; for (const [index, instanceId] of alsoNeeds.entries()) { @@ -726,7 +916,7 @@ const make = Effect.gen(function* () { const caller = input.keepsHeld === true ? null : input.callerThreadId; const used = caller === null ? [] : yield* useUp(reservations, caller); if (!heldRead.whole && caller !== null) unreadStarts.add(caller); - return asDecided({ type: "allow", token }, used); + return allowed({ type: "allow", token }, used); }, (decided, input) => observeBeside(decided, (result) => [result.outcome.type], { @@ -777,7 +967,23 @@ const make = Effect.gen(function* () { let reservations = heldRead.held; const grants: Array = []; const grantedThreads = new Set(); - for (const request of requests) { + // Every slot wait is in the line at its own age; one the sweep stopped asking for leaves it. + const staleMs = Math.max( + Duration.toMillis(LINE_STALE), + 3 * settings.value.waitFor.unitSweepSeconds * 1000, + ); + const places = requests.map((request) => + joinLine({ + key: request.key, + kind: "slotWait", + threadId: request.threadId, + instanceId: request.account ?? null, + sinceMs: request.sinceMs ?? nowMs, + nowMs, + staleMs, + }), + ); + for (const [index, request] of requests.entries()) { // One wake per thread in a call: its other slot waits ask again next time. if (grantedThreads.has(request.threadId)) { grants.push({ type: "sameThread" }); @@ -798,13 +1004,33 @@ const make = Effect.gen(function* () { reservations.filter((reservation) => reservation.threadId !== request.threadId), ); const visible = read?.visible ?? new Set(); + // The waiters ahead of it in the line (route wake-ups and older slot waits) come first. + const ordered = read === undefined ? [] : lineInOrder(nowMs); + const place = places[index]; + const ahead = + read === undefined || place === undefined || !ordered.includes(place) + ? [] + : lineAhead({ + ahead: ordered.slice(0, ordered.indexOf(place)), + threadId: request.threadId, + // Each waiter ahead by its own limits (its account's too), not this wait's. + limits, + running: read.running, + visible, + kept, + held: { ...heldRead, held: reservations }, + accountOnly: (threadId) => + hiddenHeld(settings.value.waitFor.enabled, nowMs, threadId), + freeMemoryBytes: memoryBytes, + nowMs, + }); const decision = slotDecision({ limits: own, running: (read?.running ?? []).filter((agent) => agent.threadId !== request.threadId), // Every other held thread not woken yet needs a slot for its wake-up as well as its // agent, the ones granted earlier in this call too. Its own wake-up is // not counted, as a thread's own run never counts against its own start. - starting: [...kept, ...pendingWakes(others, visible, kept, heldRead.woken)], + starting: [...kept, ...pendingWakes(others, visible, kept, heldRead.woken), ...ahead], // Its own held slot is not counted: a grant moves it here, so the thread keeps one. reservations: others, // Read again for each request: a grant earlier in this call may have hidden holds. @@ -849,6 +1075,7 @@ const make = Effect.gen(function* () { // A slot held for 0 s is not held after this call; the wait is still woken. if (reserveMs > 0) yield* Ref.update(held, (current) => [...current, reservation]); if (request.awake === true) awakeKeys.set(request.key, untilMs); + line.delete(request.key); grantedThreads.add(request.threadId); grants.push({ type: "granted", untilMs }); } @@ -959,6 +1186,22 @@ const make = Effect.gen(function* () { keep: Effect.fn("SlotGate.keep")(function* (keys: ReadonlyArray) { yield* clearStored(yield* keepLocked(keys)); }), + leave: Effect.fn("SlotGate.leave")((keys: ReadonlyArray) => + Effect.sync(() => { + for (const key of keys) line.delete(key); + }), + ), + slotWaitsWaiting: Effect.fn("SlotGate.slotWaitsWaiting")((keys: ReadonlyArray) => + Effect.sync(() => { + const waiting = new Set(keys); + for (const [key, entry] of line) { + if (entry.kind === "slotWait" && !waiting.has(key)) line.delete(key); + } + }), + ), + line: Clock.currentTimeMillis.pipe( + Effect.map((nowMs) => lineInOrder(nowMs).map((entry) => entry.key)), + ), drop: Effect.fn("SlotGate.drop")(function* (keys: ReadonlyArray) { if (keys.length === 0) return; for (const key of keys) { diff --git a/apps/server/src/waits/WaitDelivery.ts b/apps/server/src/waits/WaitDelivery.ts index 2f41beb46e3b6..b0ac9d75b7acb 100644 --- a/apps/server/src/waits/WaitDelivery.ts +++ b/apps/server/src/waits/WaitDelivery.ts @@ -180,6 +180,8 @@ export const make = Effect.fn("WaitDelivery.make")(function* ( threadId: message.threadId, instanceId: shell!.modelSelection.instanceId, keepsHeld: true, + // Asked again every WAKE_SLOT_RETRY_MS while refused: it keeps its place in the line. + place: { key: `wait-delivery:${message.commandId}` }, }); if (slot.type === "refuse") return { type: "waitsForSlot", detail: reasonHere(slot.refusal) }; return yield* threads diff --git a/apps/server/src/waits/WaitService.slots.test.ts b/apps/server/src/waits/WaitService.slots.test.ts index e0ebf8e07d313..a55c0a6463686 100644 --- a/apps/server/src/waits/WaitService.slots.test.ts +++ b/apps/server/src/waits/WaitService.slots.test.ts @@ -543,6 +543,9 @@ describe("WaitService slot waits", () => { }), ); yield* gate.release(allowed.type === "allow" ? allowed.token : null); + // The probe for T2 above is no stored wait: the sweep's list says so, and it leaves + // the gate's line (a real waiting slot wait would go first, issue 738). + yield* gate.slotWaitsWaiting([]); // T1's start used its slot up: the slot is free for another thread, not held twice. assert.strictEqual(yield* delegate(OUTSIDER), "allow"); // Cleared in the file too, not only hidden in this gate's memory. diff --git a/apps/server/src/waits/WaitService.ts b/apps/server/src/waits/WaitService.ts index 195bc875f7c91..f91fc8a1b09bb 100644 --- a/apps/server/src/waits/WaitService.ts +++ b/apps/server/src/waits/WaitService.ts @@ -1731,10 +1731,10 @@ export const make = Effect.fn("WaitService.make")(function* (input: { const grantSweep = Effect.fnUntraced(function* (settings: ServerSettings) { const gate = input.slots; if (gate === undefined) return 0; - const waiting = yield* lock.withPermits(1)( + const read = yield* lock.withPermits(1)( Effect.gen(function* () { const loaded = yield* ensureLoaded(true); - if (!loaded.ok) return []; + if (!loaded.ok) return null; return loaded.contents.waits .flatMap((wait) => (wait.kind === "slot" && wait.state === "waiting" ? [wait] : [])) .sort((a, b) => @@ -1748,6 +1748,11 @@ export const make = Effect.fn("WaitService.make")(function* (input: { ); }), ); + // Unread: the gate's line keeps the slot waits it has (they end by time if they ended). + if (read === null) return 0; + const waiting = read; + // The slot waits that ended since (met, cancelled, handed over) leave the gate's line. + yield* gate.slotWaitsWaiting(waiting.map((wait) => wait.id)); if (waiting.length === 0) return 0; // Outside the lock: the gate reads the store's held slots under it. const grants = yield* gate.grant( @@ -1755,6 +1760,8 @@ export const make = Effect.fn("WaitService.make")(function* (input: { key: wait.id, threadId: wait.threadId, account: wait.spec.account, + // Its place in the gate's one line, beside the routes' waiting wake-ups. + sinceMs: Date.parse(wait.createdAt), })), ); const answers = waiting.map((wait, index) => { diff --git a/docs/fork/README.md b/docs/fork/README.md index 2bccdad52688b..3a80477c5f395 100644 --- a/docs/fork/README.md +++ b/docs/fork/README.md @@ -449,9 +449,17 @@ These routes start a new agent run and go through the same gate, with the same c failed and armed, and upstream's recovery worker asks again on its next tick (every 5 s; logged once per resume). An armed resume has no time limit, a restart finds it from the stored thread, and its id is fixed per failed run and reset, so it is sent once. -- There is no order between wake-ups, and none between them and other routes: a wake-up holds no - place in line and asks again on its own timer, so a busy host can pass one over again and again - while other starts take the slots that free. +- One line, oldest first ([#738](https://github.com/scuttletoby/t3code/issues/738)): every + wake-up that asks again while refused keeps a place in the gate's line from its first ask (the + PR-watch wake-up, the usage-limit resume, a wait's message and the four below), and so does + every slot wait, at its own age. A slot that frees goes to the oldest waiter that can use it: a + start without a place (`delegate_task`, a send, a scheduled run) and every younger waiter count + the waiters ahead of them as starts. A waiter whose own account is full holds nothing from the + others, a thread that shows a run only queues, and a thread with a held slot starts on it. A + waiter leaves the line when it is let through, when its wake is no longer owed (its thread gone, + its question answered, a handoff that is no longer due), or when it has not asked again for 30 s + (a PR-watch wake-up: 5 minutes; a slot wait: 3 sweeps). The line is memory only: after a restart + each waiter joins again at its next ask, held question wakes first, in their file's order. - The bound-thread and message routes decide "running or idle" from the gate's own count in the same step that takes the slot. When the thread's run ends between that check and the send, the queued message starts the next run: the call then holds a slot until that run shows, for the @@ -483,17 +491,20 @@ only queues: no slot. - The fork's context handoff, and its one resend of a failed handoff run (`apps/server/src/fork/handoff/handoffDispatch.ts`, after its busy check). Refused, nothing is sent: it asks again every 5 s or at the thread's next event (logged once), deciding again each - time without a new settle, and a thread that moved on drops it. Like every handoff timer it is + time without a new settle, and a thread that moved on drops it. Its text (a read of the last + runs' turns) is built only once it has a slot. Like every handoff timer it is memory only: a restart drops a waiting handoff, and the thread's next turn end decides again. A waiting resend of a failed handoff run is dropped too, with no log: the failed run's text stays in the thread, and the next turn end decides again. - Upstream's provider continuation ("Background task completed": an agent that ended its turn while its own background work ran starts again when that work ends; - `apps/server/src/fork/continuationSlot.ts`). The slot is asked when upstream's worker takes the - request from its queue, never inside an adapter's `dispatchIfCurrent`, which holds the - adapter's own lock. Refused, the request goes back on the queue and is asked again every 5 s - (logged once); the adapter offers no second one meanwhile, and other threads' continuations - pass. A delegated result is left as above. It waits in memory, as upstream's queue does: a + `apps/server/src/fork/continuationSlot.ts`). The slot is asked off upstream's queue, each + request in a fiber of its own, never inside an adapter's `dispatchIfCurrent`, which holds the + adapter's own lock: a slow count read delays only that request, never upstream's queue. Refused, + it is asked again every 5 s (logged once); the adapter offers no second one meanwhile. A granted + slot is held until upstream's dispatch of that request ends (it may wait behind its own thread's + earlier dispatch), then counts until its run shows, for the start grace at most; a request + upstream never dispatches holds it 10 minutes at most. A delegated result is left as above. It waits in memory, as upstream's queue does: a restart loses it, as the restart ends the provider session whose background work it reports (Claude's buffered output lives in that process). - A delegated task's question to its idle parent (upstream's question reactor; @@ -502,19 +513,18 @@ only queues: no slot. leaves the file once sent, or when the question is answered, its task ends or the parent is archived or deleted. A restart reads the file and sends it once; upstream's fixed command id keeps a send cut off before the file write from making a second message. Saves run one at a - time, each writing what is held then. A file that cannot be read is moved to - `fork-held-wakes.json.corrupt-` beside it and logged, so the next save does not write over - it; an entry that cannot be read is logged and left out. A held wake whose parent or question - cannot be read waits, logged once. -- Upstream's idle compaction (same file), after the handoff owner's check. Refused, it is skipped - (logged), not held: it only pays off while the prompt cache is warm, minutes before it expires, - and a late one costs a full uncached read. The next completed run arms a new one, as after a - restart. It is asked once: a thread whose window always falls while the slots are full does not - compact. - -Like the wake-ups above, the handoff, the continuation and the question wake hold no place in -line: each asks again on its own 5 s timer, so under steady load a slot that frees between two -asks can go to another start, and one can wait without limit. Nothing is lost or sent twice. + time, each writing what is held then, synced to disk before the rename and the folder after it. + A file that does not decode is moved to `fork-held-wakes.json.corrupt-` beside it and + logged, so the next save does not write over it; an entry that does not decode is logged and + left out. A file that cannot be checked or read (an I/O error, not "missing") is logged once and + never written over: new holds stay in memory, and every 5 s it is read again, its wakes then + going first. A held wake whose parent or question cannot be read waits, logged once. +- Upstream's idle compaction (same file), after the handoff owner's check. Refused, it is held in + upstream's own timer and asked again every 5 s in its place in line (logged once), while the + fork reads again that the thread is still idle at that run and its prompt cache window is still + open (upstream's `idleCompactionWindow`). Once not, it is skipped (logged): a late compaction + costs a full uncached read. A newer completed run replaces it, as upstream's timer does; a + restart drops it (upstream). The calling thread is left out of the count (it waits on its child). A start no thread on this machine asks for (a scheduled run, the owner's client, a call from another machine) has no caller, so From 4bf1f7698993903f268bdf0c0b28dd13083ee7d0 Mon Sep 17 00:00:00 2001 From: John Doe <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 14:32:13 +0100 Subject: [PATCH 2/4] Fork: pin a younger slot wait behind a wake-up, and a busy handoff leaving the line (#738) Two more tests, each red when its fix line is removed: a slot wait asked after a waiting wake-up waits behind it (grant counts the waiters ahead), and a waiting handoff whose thread gets busy leaves the gate's line. Co-Authored-By: Claude Opus 5.5 --- .../src/fork/handoff/HandoffService.test.ts | 26 ++++++++++++++++++- apps/server/src/slots/SlotGate.test.ts | 24 +++++++++++++++++ 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/apps/server/src/fork/handoff/HandoffService.test.ts b/apps/server/src/fork/handoff/HandoffService.test.ts index 88afa16844f97..dc9fccf66d3d0 100644 --- a/apps/server/src/fork/handoff/HandoffService.test.ts +++ b/apps/server/src/fork/handoff/HandoffService.test.ts @@ -590,7 +590,31 @@ describe("HandoffService", () => { }), ); - it.effect("a waiting handoff whose thread gets busy, or a newer run, leaves its place", () => + it.effect("a waiting handoff whose thread gets busy leaves its place, and waits", () => + Effect.gen(function* () { + const gate = countingGate(0); + const data = projection({ usedTokens: 160_000 }); + const state = { data, settings: on, gate: gate.layer }; + const { sent, emit } = yield* harness(state); + yield* emit(runUpdated(run()), "readThread", 1); + yield* settle; + yield* TestClock.adjust(MINUTE_MS); + expect(gate.slots.left).toEqual([]); + // A question to the person opens: not due now, so it needs no slot and holds no place. + state.data = { + ...data, + runtimeRequests: [ + { id: RuntimeRequestId.make("request-1"), status: "pending" }, + ] as unknown as OrchestrationV2ThreadProjection["runtimeRequests"], + }; + yield* emit(threadChanged(state.data), "readThread", 14); + yield* settle; + expect(gate.slots.left).toEqual([`handoff:server:handoff:${threadId}:${runId}`]); + expect(sent).toEqual([]); + }), + ); + + it.effect("a waiting handoff replaced by a newer run's timer leaves its place", () => Effect.gen(function* () { const gate = countingGate(0); const data = projection({ usedTokens: 160_000 }); diff --git a/apps/server/src/slots/SlotGate.test.ts b/apps/server/src/slots/SlotGate.test.ts index 34ce414d18bd7..74f58d30c2922 100644 --- a/apps/server/src/slots/SlotGate.test.ts +++ b/apps/server/src/slots/SlotGate.test.ts @@ -2290,6 +2290,30 @@ describe("SlotGate: one line for every waiting start (issue 738)", () => { }), ); + it.effect("a slot wait younger than a waiting wake-up waits behind it", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + waitsOn({ maxPerMachine: 1, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const nowMs = yield* Clock.currentTimeMillis; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + const younger = { key: "young", threadId: ThreadId.make("young"), sinceMs: nowMs + 1 }; + world.threads = []; + // The sweep asks first, but the wake-up is older: the slot is its. + assert.deepStrictEqual( + (yield* gate.grant([younger])).map((grant) => grant.type), + ["full"], + ); + assert.strictEqual((yield* wake("w", "w")).type, "allow"); + }), + ); + }), + ); + it.effect("a waiter that left, or stopped asking, or shows a run, holds nothing", () => Effect.gen(function* () { const world = newWorld(); From d49963299f2437e17cda6895ecad2e0a94cf7503 Mon Sep 17 00:00:00 2001 From: John Doe <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 15:15:33 +0100 Subject: [PATCH 3/4] Fork: scheduled runs and webhook deliveries keep a place in the slot line; a free slot is never held long (#738) - A run on its schedule keeps a place keyed on its task, at its due time; a webhook delivery one keyed on its delivery id, at the time it was stored. The gate drops it when the run starts; the delivery wait drops it when given up or stopped; a 5 s sweep drops places of tasks no longer due (paused, deleted, moved) and deliveries no longer pending. - A free slot kept for the oldest waiter is held 15 s without it asking (a slot wait: 2 sweeps); then the waiters behind it go until it asks again, in its place. A PR-watch place goes when its thread is gone. - A thread's places count from its oldest; stale places are dropped at every ask under any limit; a refusal names the starts waiting in line apart from the running agents. - A retried idle compaction needs a minute of its cache window left; a failed due check is logged with its cause; a broken held-wakes file that cannot be set aside is kept and not written over. The server's due check is a function of its own, tested. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/fork/agentSlotRoutes.test.ts | 169 +++++++++++- apps/server/src/fork/agentSlotRoutes.ts | 176 ++++++++++--- .../DelegatedOrphanScan.test.ts | 36 +++ .../delegatedResults.testkit.ts | 5 +- .../src/fork/pullRequestWatchSlot.test.ts | 2 + apps/server/src/fork/pullRequestWatchSlot.ts | 14 +- apps/server/src/fork/workerWakeSlots.test.ts | 187 +++++++++++++- apps/server/src/fork/workerWakeSlots.ts | 124 +++++---- apps/server/src/slots/SlotGate.test.ts | 243 +++++++++++++++++- apps/server/src/slots/SlotGate.ts | 129 +++++++--- docs/fork/README.md | 46 ++-- packages/shared/src/agentSlots.ts | 25 +- 12 files changed, 1009 insertions(+), 147 deletions(-) diff --git a/apps/server/src/fork/agentSlotRoutes.test.ts b/apps/server/src/fork/agentSlotRoutes.test.ts index 7204f51438a6c..e7c3c1e1c87e8 100644 --- a/apps/server/src/fork/agentSlotRoutes.test.ts +++ b/apps/server/src/fork/agentSlotRoutes.test.ts @@ -19,6 +19,7 @@ import { import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; import * as Logger from "effect/Logger"; +import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; import * as TestClock from "effect/testing/TestClock"; @@ -32,6 +33,7 @@ import * as ScheduledTaskService from "../scheduledTasks/ScheduledTaskService.ts import * as Scheduler from "../scheduling/Scheduler.ts"; import * as SecretRequests from "../secrets/SecretRequests.ts"; import * as ServerSettings from "../serverSettings.ts"; +import * as SlotGate from "../slots/SlotGate.ts"; import * as AgentSlotRoutes from "./agentSlotRoutes.ts"; import { type SlotWorld, slotGateOver, slotWorld } from "./slotWorld.testkit.ts"; @@ -82,6 +84,8 @@ interface Harness { readonly webhookWorker: Effect.Effect; /** One pass of the fork's waiting-note cleanup. */ readonly notesWorker: Effect.Effect; + /** The server's gate the runs ask (with \`slots\`). */ + readonly gate: Option.Option; } interface Options { @@ -165,7 +169,9 @@ const boot = (body: (harness: Harness) => Effect.Effect, options: Op return yield* Effect.gen(function* () { const service = yield* ScheduledTaskService.ScheduledTaskService; const sql = yield* SqlClient.SqlClient; + const gate = yield* Effect.serviceOption(SlotGate.SlotGate); return yield* body({ + gate, service, sql, started, @@ -174,7 +180,9 @@ const boot = (body: (harness: Harness) => Effect.Effect, options: Op notesWorker: Effect.suspend(() => workers.get("agent-slot-notes")!), }); }).pipe( - Effect.provide(Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provide(dependencies)))), + Effect.provide( + Layer.fresh(ScheduledTaskService.layer.pipe(Layer.provideMerge(dependencies))), + ), ); }); @@ -824,7 +832,7 @@ describe("scheduled runs through the agent slot gate (#706)", () => { ({ name, stop }) => { const slots = slotWorld([BUSY]); return withService( - ({ service, started, webhookWorker }) => + ({ service, started, webhookWorker, gate }) => Effect.gen(function* () { yield* TestClock.setTime(START); const { task } = yield* service.upsert( @@ -839,6 +847,8 @@ describe("scheduled runs through the agent slot gate (#706)", () => { slots.running = []; yield* workerRounds(webhookWorker, 4); assert.deepEqual(started, []); + // Given up: its place in the gate's line is gone too. + assert.deepEqual(yield* Option.getOrThrow(gate).line, []); if (name === "paused") { const { delivery } = yield* service.getWebhookDelivery({ id: task.id, @@ -924,6 +934,161 @@ describe("scheduled runs through the agent slot gate (#706)", () => { ); }); +describe("a scheduled run or webhook delivery that waits keeps its place in the gate's line (#738)", () => { + const LATER = ThreadId.make("thread:later-waiter"); + /** A wake-up of another idle thread that joins the line after the run (asks every 5 s). */ + const laterWaiter = (gate: Harness["gate"]) => + Option.getOrThrow(gate).acquire({ + callerThreadId: LATER, + targetInstanceId: INSTANCE, + intoThreadId: LATER, + keepsHeld: true, + place: { key: "later-waiter" }, + }); + const lineOf = (gate: Harness["gate"]) => Option.getOrThrow(gate).line; + + it.effect( + "full: a waiter that joins later does not pass a scheduled run; the run starts first", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, poll, started, gate }) => + Effect.gen(function* () { + const task = yield* dueTask(service); + yield* poll; + assert.deepEqual(started, []); + assert.deepEqual(yield* lineOf(gate), [`scheduled-run:${task.id}`]); + yield* TestClock.adjust("5 seconds"); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + // Runs again: each poll keeps its place (no second place). + yield* poll; + assert.deepEqual(yield* lineOf(gate), [`scheduled-run:${task.id}`, "later-waiter"]); + // A slot frees and the later waiter asks first: the run is ahead of it. + slots.running = []; + yield* TestClock.adjust("1 second"); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* TestClock.adjust("4 seconds"); + yield* poll; + assert.deepEqual(launches(started).length, 1); + // Let through: its place is gone; the waiter is next. + assert.deepEqual(yield* lineOf(gate), ["later-waiter"]); + }), + { slots }, + ); + }, + ); + + it.effect( + "a run due before a waiter joined is ahead of it, though the poll asks after it", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, poll, started, gate }) => + Effect.gen(function* () { + const task = yield* dueTask(service); + // The waiter asks a second after the due time, before the poll does. + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* poll; + assert.deepEqual(yield* lineOf(gate), [`scheduled-run:${task.id}`, "later-waiter"]); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* poll; + assert.equal(launches(started).length, 1); + }), + { slots }, + ); + }, + ); + + it.effect("full: a waiter that joins later does not pass a stored webhook delivery", () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, started, webhookWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + const accepted = yield* service.triggerWebhook(webhookRequest(task)); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + // Refused and stored; the worker holds it, asking every 5 s in its place. + yield* workerRounds(webhookWorker, 2); + assert.deepEqual(yield* lineOf(gate), [`webhook-delivery:${accepted.deliveryId}`]); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* until(() => started.length === 1, "the delivery started"); + assert.deepEqual(yield* lineOf(gate), ["later-waiter"]); + }), + { slots }, + ); + }); + + it.effect( + "a stored delivery's place goes when its task is deleted before the worker runs it", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, webhookWorker, notesWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + const accepted = yield* service.triggerWebhook(webhookRequest(task)); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + yield* settle; + // Refused and stored: its place waits for the worker. + assert.deepEqual(yield* lineOf(gate), [`webhook-delivery:${accepted.deliveryId}`]); + yield* service.delete({ id: task.id } as never); + // The worker drops it without asking the gate; the sweep sees it is no longer pending. + yield* webhookWorker; + yield* settle; + yield* notesWorker; + assert.deepEqual(yield* lineOf(gate), []); + }), + { slots }, + ); + }, + ); + + it.effect( + "a waiting run's place goes when its task is deleted or paused (no slot kept for it)", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, poll, notesWorker, gate }) => + Effect.gen(function* () { + const deleted = yield* dueTask(service); + const paused = yield* service.upsert( + yield* taskInput({ + title: "Second", + schedule: { type: "interval", everyMs: 60_000 }, + }), + ); + yield* TestClock.adjust("2 minutes"); + yield* poll; + assert.sameMembers( + [...(yield* lineOf(gate))], + [`scheduled-run:${deleted.id}`, `scheduled-run:${paused.task.id}`], + ); + // Still due: the sweep keeps both. + yield* notesWorker; + assert.lengthOf(yield* lineOf(gate), 2); + yield* service.delete({ id: deleted.id } as never); + yield* service.setEnabled({ id: paused.task.id, enabled: false } as never); + yield* notesWorker; + assert.deepEqual(yield* lineOf(gate), []); + // A later start is not held back by them. + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "allow"); + }), + { slots }, + ); + }, + ); +}); + describe("refusal texts fit who reads them (#706)", () => { const full = { type: "full" as const, diff --git a/apps/server/src/fork/agentSlotRoutes.ts b/apps/server/src/fork/agentSlotRoutes.ts index e5fe77649dcb2..9e5247175fdf9 100644 --- a/apps/server/src/fork/agentSlotRoutes.ts +++ b/apps/server/src/fork/agentSlotRoutes.ts @@ -13,7 +13,7 @@ import { ScheduledTaskError, type ThreadId, } from "@t3tools/contracts"; -import { slotRefusalReason, type SlotDecision } from "@t3tools/shared/agentSlots"; +import { slotRefusalReason, slotRunningText, type SlotDecision } from "@t3tools/shared/agentSlots"; import * as Context from "effect/Context"; import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; @@ -102,8 +102,6 @@ const admit = ( const slotsFull = (message: string) => new OrchestratorMcpFailure({ code: "slots_full", message }); -const agents = (count: number) => (count === 1 ? "1 agent is" : `${count} agents are`); - const unreadReason = (problem: SlotGate.CountReadProblem) => problem === "timeout" ? `the read took longer than ${Duration.toSeconds(SlotGate.COUNT_READ_TIMEOUT)} seconds` @@ -112,9 +110,9 @@ const unreadReason = (problem: SlotGate.CountReadProblem) => /** {@link slotRefusalReason}, for a reader on another machine ("there", not "this machine"). */ const reasonThere = (decision: FullDecision) => decision.rule === "machine" - ? `${agents(decision.used)} running there, and the limit is ${decision.max}` + ? `${slotRunningText(decision, " there")}, and the limit is ${decision.max}` : decision.rule === "account" - ? `${agents(decision.used)} running there on account ${decision.instanceId}, and the limit is ${decision.max}` + ? `${slotRunningText(decision, ` there on account ${decision.instanceId}`)}, and the limit is ${decision.max}` : `only ${decision.used} MB of memory is free there, and a new agent needs at least ${decision.max} MB`; /** Why, in one clause, for a reader on this machine (a scheduled task's history, run now, logs). */ @@ -378,6 +376,15 @@ const CurrentDirectDelivery = Context.Reference( { defaultValue: () => undefined }, ); +/** + * Places in the gate's line (`SlotGate.SlotPlace`): a scheduled run that waits keeps one per task, + * at its due time; a webhook delivery one per delivery, at the time it was stored (direct: its + * first ask). Each leaves when its run starts (the gate), when it is given up or its task deleted + * (`waitForWebhookSlot`, and `leaveEndedPlaces` every 5 s). + */ +const SCHEDULED_PLACE = "scheduled-run:"; +const DELIVERY_PLACE = "webhook-delivery:"; + /** A stored delivery's request, in the shape the stored-delivery worker reads (`decodePendingRequest`). */ const encodeStoredRequest = Schema.encodeEffect( Schema.fromJsonString( @@ -435,16 +442,99 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( * The slot is named for the run's thread when its dispatch ends (`end`), bound or new, so a * dispatch that failed frees it at once; `ended` frees it on a path that never got there. */ - const take = (target: { - readonly instanceId: ProviderInstanceId; - readonly intoThreadId: ThreadId | undefined; - }) => + const take = ( + target: { + readonly instanceId: ProviderInstanceId; + readonly intoThreadId: ThreadId | undefined; + }, + place: SlotGate.SlotPlace | undefined, + ) => admit(gate, { callerThreadId: null, targetInstanceId: target.instanceId, intoThreadId: target.intoThreadId, + ...(place === undefined ? {} : { place }), }); + /** Webhook deliveries waiting in `waitForWebhookSlot` now: their wait leaves the line itself. */ + const waitingDeliveries = new Set(); + + /** + * The run's place in line: a run on its schedule at its due time, a webhook delivery at the + * time it was stored. "Run now" asks once and has none. A stored delivery is the one its task's + * worker runs now (one at a time per task, in the worker's order); unreadable, it asks with none. + */ + const placeOf = Effect.fnUntraced(function* ( + task: ScheduledTask, + trigger: "scheduled" | "manual" | "webhook", + ) { + if (trigger === "manual") return undefined; + if (trigger === "scheduled") { + const dueMs = task.nextRunAt === null ? Number.NaN : Date.parse(task.nextRunAt); + return { + key: `${SCHEDULED_PLACE}${task.id}`, + ...(Number.isFinite(dueMs) ? { sinceMs: dueMs } : {}), + } satisfies SlotGate.SlotPlace; + } + const direct = yield* CurrentDirectDelivery; + if (direct !== undefined) return { key: `${DELIVERY_PLACE}${direct.deliveryId}` }; + const [stored] = yield* sql<{ delivery_id: string; due_at: number }>` + SELECT delivery_id, due_at FROM scheduled_task_webhook_pending + WHERE task_id = ${task.id} AND state IN ('dispatching', 'running', 'completed', 'failed') + ORDER BY due_at, rowid LIMIT 1 + `.pipe(Effect.orElseSucceed(() => [])); + return stored === undefined + ? undefined + : ({ + key: `${DELIVERY_PLACE}${stored.delivery_id}`, + sinceMs: Number(stored.due_at), + } satisfies SlotGate.SlotPlace); + }); + + const leave = (place: SlotGate.SlotPlace | undefined) => + place === undefined ? Effect.void : leaveLine(gate, [place.key]); + + /** + * Places whose wait ended without the gate seeing it: a run on its schedule no longer due + * (paused, deleted, edited, rescheduled), a stored delivery no longer pending (its task deleted). + */ + const leaveEndedPlaces = Effect.gen(function* () { + if (Option.isNone(gate)) return; + const keys = yield* gate.value.line; + const taskIds = keys.flatMap((key) => + key.startsWith(SCHEDULED_PLACE) ? [key.slice(SCHEDULED_PLACE.length)] : [], + ); + const deliveryIds = keys.flatMap((key) => + key.startsWith(DELIVERY_PLACE) && !waitingDeliveries.has(key) + ? [key.slice(DELIVERY_PLACE.length)] + : [], + ); + if (taskIds.length === 0 && deliveryIds.length === 0) return; + const nowIso = DateTime.formatIso(DateTime.toUtc(yield* DateTime.now)); + const due = new Set( + taskIds.length === 0 + ? [] + : (yield* sql<{ task_id: string }>` + SELECT task_id FROM scheduled_tasks + WHERE task_id IN ${sql.in(taskIds)} AND enabled = 1 AND next_run_at IS NOT NULL + AND next_run_at <= ${nowIso} AND last_run_status <> 'running' + `).map((row) => row.task_id), + ); + const pending = new Set( + deliveryIds.length === 0 + ? [] + : (yield* sql<{ delivery_id: string }>` + SELECT delivery_id FROM scheduled_task_webhook_pending + WHERE delivery_id IN ${sql.in(deliveryIds)} AND state IN ('pending', 'dispatching') + `).map((row) => row.delivery_id), + ); + const ended = [ + ...taskIds.filter((id) => !due.has(id)).map((id) => `${SCHEDULED_PLACE}${id}`), + ...deliveryIds.filter((id) => !pending.has(id)).map((id) => `${DELIVERY_PLACE}${id}`), + ]; + if (ended.length > 0) yield* gate.value.leave(ended); + }); + const allowed = ( task: ScheduledTask, end: (threadId: ThreadId | null) => Effect.Effect, @@ -558,23 +648,38 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( * (its pending row stays as it is, so a restart picks it up again, once). It is dropped only for * the reasons a queued delivery is (the task deleted, replaced, paused or no longer a webhook). */ - const waitForWebhookSlot = Effect.fnUntraced(function* (task: ScheduledTask, first: Refused) { - yield* Effect.logInfo("Webhook delivery waits for an agent slot", { - taskId: task.id, - reason: reasonHere(first.refusal), - }); - while (true) { - yield* Effect.sleep(WEBHOOK_SLOT_RETRY); - const gone = yield* webhookGone(task).pipe(Effect.orElseSucceed(() => null)); - if (gone !== null) return yield* Effect.fail(new WebhookSkipped({ reason: gone })); - // Checked again as it would start: the owner may have widened the project meanwhile. - const refused = task.threadId === null ? yield* profiles.runRefusal(task) : null; - if (refused !== null) return yield* Effect.fail(new WebhookSkipped({ reason: refused })); - // A bound thread that shows a run only queues: the gate lets it pass (`intoThreadId`). - const slot = yield* take(yield* targetOf(task)); - if (slot.type === "allow") return allowed(task, slot.end); - } - }); + const waitForWebhookSlot = Effect.fnUntraced( + function* (task: ScheduledTask, first: Refused, place: SlotGate.SlotPlace | undefined) { + yield* Effect.logInfo("Webhook delivery waits for an agent slot", { + taskId: task.id, + reason: reasonHere(first.refusal), + }); + while (true) { + yield* Effect.sleep(WEBHOOK_SLOT_RETRY); + const gone = yield* webhookGone(task).pipe(Effect.orElseSucceed(() => null)); + if (gone !== null) return yield* Effect.fail(new WebhookSkipped({ reason: gone })); + // Checked again as it would start: the owner may have widened the project meanwhile. + const refused = task.threadId === null ? yield* profiles.runRefusal(task) : null; + if (refused !== null) return yield* Effect.fail(new WebhookSkipped({ reason: refused })); + // A bound thread that shows a run only queues: the gate lets it pass (`intoThreadId`). + // Asked in its place, so a waiter that joined later never passes it. + const slot = yield* take(yield* targetOf(task), place); + if (slot.type === "allow") return allowed(task, slot.end); + } + }, + // Skipped, failed or stopped: its place goes too (let through, the gate dropped it already). + (effect, _task, _first, place) => + place === undefined + ? effect + : Effect.acquireUseRelease( + Effect.sync(() => waitingDeliveries.add(place.key)), + () => effect, + () => + Effect.sync(() => waitingDeliveries.delete(place.key)).pipe( + Effect.andThen(leave(place)), + ), + ), + ); /** * Last before the run is marked running. Refused, a run on its schedule stays due (asked again @@ -596,6 +701,8 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( // A new thread wider than the profile the task was saved with starts nothing: on its schedule // a failed run with the reason (moved to its next time), "run now" fails, a webhook skips. const profileRefusal = task.threadId === null ? yield* profiles.runRefusal(task) : null; + // Refused for its profile, a place it had from an earlier ask goes at the next sweep (its task + // is no longer due, its delivery no longer pending: `leaveEndedPlaces`). if (profileRefusal !== null) { if (trigger === "manual") { return yield* new ScheduledTaskError({ message: profileRefusal, taskId: task.id }); @@ -616,7 +723,8 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( } as ScheduledSlot; } if (Option.isNone(gate)) return { skipped: null, end: () => Effect.void } as ScheduledSlot; - const slot = yield* take(yield* targetOf(task)); + const place = yield* placeOf(task, trigger); + const slot = yield* take(yield* targetOf(task), place); if (slot.type === "allow") return allowed(task, slot.end); if (trigger === "manual") { return yield* new ScheduledTaskError({ @@ -630,7 +738,7 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( skipped: { task, threadId: null, dispatchFailed: false }, end: () => Effect.void, } as ScheduledSlot; - return yield* waitForWebhookSlot(task, slot); + return yield* waitForWebhookSlot(task, slot, place); } yield* keepDue(task, now, slot.refusal).pipe( Effect.catchCause((cause) => @@ -671,7 +779,17 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( AND (enabled = 0 OR next_run_at IS NULL OR next_run_at > ${nowIso}) `; }); - yield* scheduler.register("agent-slot-notes", clearStaleNotes); + yield* scheduler.register( + "agent-slot-notes", + Effect.andThen( + leaveEndedPlaces.pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not check the line places of waiting scheduled runs", { cause }), + ), + ), + clearStaleNotes, + ), + ); return { admit: admitRun, diff --git a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts index a1dd3b1c2c9a5..92a9851c3689d 100644 --- a/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts +++ b/apps/server/src/fork/delegatedResults/DelegatedOrphanScan.test.ts @@ -871,6 +871,42 @@ describe("DelegatedOrphanScan and agent slots", () => { }), ); + it.effect( + "the give-up warning lists a parent whose send kept failing apart from unread ones", + () => + Effect.gen(function* () { + yield* setClock; + const world = makeWorld(); + // A slot is free, but every send to the parent fails: each scan tries it again. + world.sendFails.value = true; + const warnings: Array = []; + const logger = Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(JSON.stringify(message)); + }); + const fiber = yield* (yield* service(world)) + .run() + .pipe( + Effect.provide(Logger.layer([logger], { mergeWithExisting: false })), + Effect.forkChild, + ); + yield* settle; + yield* TestClock.adjust(OPTIONS.delay); + for (let attempt = 1; attempt <= OPTIONS.attempts; attempt += 1) { + yield* settle; + if (attempt < OPTIONS.attempts) yield* TestClock.adjust(OPTIONS.retryDelay); + } + yield* Fiber.join(fiber); + const left = warnings.filter((warning) => warning.includes("work unchecked")); + expect(left).toHaveLength(1); + const fields = (JSON.parse(left[0]!) as Array).find( + (part): part is Record => typeof part === "object" && part !== null, + ); + expect(fields?.failedSendParentThreadIds).toEqual([PARENT]); + expect(fields?.unreadableParentThreadIds).toEqual([]); + expect(world.sent).toHaveLength(0); + }), + ); + it.effect("a restart while one waits loses nothing: the next start names it once", () => Effect.gen(function* () { yield* setClock; diff --git a/apps/server/src/fork/delegatedResults/delegatedResults.testkit.ts b/apps/server/src/fork/delegatedResults/delegatedResults.testkit.ts index 920e9e866b909..bbf2cfd369117 100644 --- a/apps/server/src/fork/delegatedResults/delegatedResults.testkit.ts +++ b/apps/server/src/fork/delegatedResults/delegatedResults.testkit.ts @@ -40,6 +40,8 @@ export const makeSlots = () => { const full = new Set(); const state = { all: false }; const asks: Array = []; + /** Keys that left the gate's line. */ + const left: Array = []; const layer = Layer.mock(SlotGate.SlotGate)({ acquire: (input) => Effect.sync((): SlotGate.SlotAcquire => { @@ -52,8 +54,9 @@ export const makeSlots = () => { }), started: () => Effect.void, release: () => Effect.void, + leave: (keys) => Effect.sync(() => left.push(...keys)), }); - return { full, state, asks, layer }; + return { full, state, asks, left, layer }; }; /** diff --git a/apps/server/src/fork/pullRequestWatchSlot.test.ts b/apps/server/src/fork/pullRequestWatchSlot.test.ts index 93283035a2835..2c70a715cb4dc 100644 --- a/apps/server/src/fork/pullRequestWatchSlot.test.ts +++ b/apps/server/src/fork/pullRequestWatchSlot.test.ts @@ -295,6 +295,8 @@ describe("pull request watch wake-ups take an agent slot", () => { yield* gone.wrapped.dispatch(command); expect(gone.sent).toEqual([command]); expect(gone.slots.asks).toEqual([]); + // Its wake is not owed any more: its place in the gate's line goes with it. + expect(gone.slots.left).toEqual([`pull-request-watch:${THREAD}`]); }), ); diff --git a/apps/server/src/fork/pullRequestWatchSlot.ts b/apps/server/src/fork/pullRequestWatchSlot.ts index 7c992b37315de..76a63217451c0 100644 --- a/apps/server/src/fork/pullRequestWatchSlot.ts +++ b/apps/server/src/fork/pullRequestWatchSlot.ts @@ -19,7 +19,7 @@ import * as Schema from "effect/Schema"; import * as Orchestrator from "../orchestration-v2/Orchestrator.ts"; import * as ProjectionStore from "../orchestration-v2/ProjectionStore.ts"; import * as SlotGate from "../slots/SlotGate.ts"; -import { admitSend, reasonHere } from "./agentSlotRoutes.ts"; +import { admitSend, leaveLine, reasonHere } from "./agentSlotRoutes.ts"; type WatchSync = Extract< OrchestrationV2ServerCommand, @@ -93,7 +93,12 @@ export const gatePullRequestWatchWakes = ( const instanceId = yield* instanceOf(command.threadId).pipe( Effect.orElseSucceed(() => "unreadable" as const), ); - if (instanceId === null) return yield* orchestrator.dispatch(command); + const place = { key: `pull-request-watch:${command.threadId}`, staleAfter: WATCH_PLACE_STALE }; + // No such thread: its wake is not owed, so its place goes (upstream refuses the command). + if (instanceId === null) + return yield* leaveLine(gate, [place.key]).pipe( + Effect.andThen(orchestrator.dispatch(command)), + ); if (instanceId === "unreadable") return yield* waitsForSlot(command, "the thread could not be read to check for a slot"); const slot = yield* admitSend(gate, { @@ -101,8 +106,9 @@ export const gatePullRequestWatchWakes = ( threadId: command.threadId, instanceId, keepsHeld: true, - // Asked again at upstream's next sweep (2 min) while refused: it keeps its place in line. - place: { key: `pull-request-watch:${command.threadId}`, staleAfter: WATCH_PLACE_STALE }, + // Asked again at upstream's next sweep (2 min) while refused: it keeps its place in line, + // but holds a free slot only `SlotGate.LINE_HOLD` (15 s) without asking. + place, }); const keys = watchesOf(command).map((watch) => watchKey(command.threadId, watch)); if (slot.type === "refuse") { diff --git a/apps/server/src/fork/workerWakeSlots.test.ts b/apps/server/src/fork/workerWakeSlots.test.ts index 070eedc5fd8be..38a9bb31a10f2 100644 --- a/apps/server/src/fork/workerWakeSlots.test.ts +++ b/apps/server/src/fork/workerWakeSlots.test.ts @@ -200,7 +200,7 @@ interface Disk { readonly dir: string; readonly files: Map; readonly ops: Array; - readonly failing: { exists: boolean }; + readonly failing: { exists: boolean; rename: boolean }; readonly layer: Layer.Layer; } @@ -211,7 +211,7 @@ const diskError = (tag: "NotFound" | "PermissionDenied", method: string, file: s const diskOver = (dir: string): Disk => { const files = new Map(); const ops: Array = []; - const failing = { exists: false }; + const failing = { exists: false, rename: false }; const layer = FileSystem.layerNoop({ exists: (file) => failing.exists && baseName(file) === WorkerWakeSlots.HELD_WAKES_FILE @@ -226,6 +226,8 @@ const diskOver = (dir: string): Disk => { }), rename: (from, to) => Effect.suspend(() => { + if (failing.rename && baseName(from) === WorkerWakeSlots.HELD_WAKES_FILE) + return Effect.fail(diskError("PermissionDenied", "rename", from)); const text = files.get(from); if (text === undefined) return Effect.fail(diskError("NotFound", "rename", from)); ops.push(`rename ${baseName(from)} ${baseName(to)}`); @@ -610,6 +612,36 @@ describe("a delegated task's question to an idle parent takes an agent slot", () }).pipe(Effect.scoped), ); + it.effect( + "a broken file that cannot be set aside is kept as it is: nothing written over it, read again", + () => + Effect.gen(function* () { + const dir = yield* tempDir; + const world = newWorld(0); + const file = yield* heldFile(dir); + dir.files.set(file, "{not json"); + dir.failing.rename = true; + yield* Effect.gen(function* () { + yield* ask; + yield* TestClock.adjust("20 seconds"); + // Held in memory; the broken file is untouched, and the failure logged once. + assert.strictEqual(dir.files.get(file), "{not json"); + assert.lengthOf( + world.logs.filter((log) => log.includes("held wakes file could not be read")), + 1, + ); + // It can be set aside now: kept as .corrupt-, and the held wake is written. + dir.failing.rename = false; + yield* TestClock.adjust("5 seconds"); + const kept = [...dir.files.keys()] + .map(baseName) + .filter((name) => name.startsWith(`${WorkerWakeSlots.HELD_WAKES_FILE}.corrupt-`)); + assert.lengthOf(kept, 1); + assert.deepStrictEqual(yield* heldIds(dir), [WAKE_COMMAND]); + }).pipe(Effect.provide(server(world, dir)), Effect.scoped); + }).pipe(Effect.scoped), + ); + it.effect( "a file that cannot be read yet: a new hold is kept in memory, the file's own wakes go first", () => @@ -723,7 +755,13 @@ describe("a delegated task's question to an idle parent takes an agent slot", () * runs from now to 2 minutes on; `thread.data` is what both upstream and the fork's check read. * Returns when upstream's worker is done with the run. */ -const compaction = (world: World, thread: { data: ReturnType }) => +const compaction = ( + world: World, + thread: { data: ReturnType }, + due?: ( + command: Extract, + ) => Effect.Effect, +) => Effect.gen(function* () { yield* TestClock.setTime(NOW_MS); const decided = yield* Deferred.make(); @@ -760,10 +798,14 @@ const compaction = (world: World, thread: { data: ReturnType gate: Option.some(yield* SlotGate.SlotGate.pipe(Effect.provide(gateOf(world)))), threadOf: () => Effect.succeed({ instanceId: thread.data.thread.modelSelection.instanceId }), questionOpen: () => Effect.succeed(true), - compactionDue: (command) => - Clock.currentTimeMillis.pipe( - Effect.map((nowMs) => WorkerWakeSlots.compactionWindowOpen(thread.data, command, nowMs)), - ), + compactionDue: + due ?? + ((command) => + Clock.currentTimeMillis.pipe( + Effect.map((nowMs) => + WorkerWakeSlots.compactionWindowOpen(thread.data, command, nowMs), + ), + )), store: { load: Effect.succeed([]), save: () => Effect.void }, }); yield* IdleCompactionService.make.pipe( @@ -851,6 +893,38 @@ describe("upstream's idle compaction takes an agent slot", () => { }), ); + it.effect("full until under a minute of its window is left: skipped then, not sent cold", () => + Effect.gen(function* () { + const world = newWorld(0); + const decided = yield* compaction(world, warm()); + // The window ends 2 minutes on; a slot that frees with less than a minute left is not used. + yield* TestClock.adjust("55 seconds"); + assert.isFalse(world.left.includes(COMPACTION_PLACE.key)); + world.free = 5; + yield* TestClock.adjust("5 seconds"); + yield* Deferred.await(decided); + assert.deepStrictEqual(world.sent, []); + assert.include(world.left, COMPACTION_PLACE.key); + }), + ); + + it.effect("a check that fails: skipped, logged with its cause, not as a full gate", () => + Effect.gen(function* () { + const world = newWorld(0); + const decided = yield* compaction(world, warm(), () => Effect.fail("records unreadable")); + yield* TestClock.adjust("5 seconds"); + yield* Deferred.await(decided); + assert.deepStrictEqual(world.sent, []); + assert.isTrue( + world.logs.some((log) => + log.includes("idle compaction skipped: whether it is still due could not be read"), + ), + ); + assert.isFalse(world.logs.some((log) => log.includes("no agent slot while its cache"))); + assert.include(world.left, COMPACTION_PLACE.key); + }), + ); + it.effect("full, and the thread is no longer idle: skipped at the next check, never sent", () => Effect.gen(function* () { const world = newWorld(0); @@ -869,3 +943,102 @@ describe("upstream's idle compaction takes an agent slot", () => { }), ); }); + +describe("the server's check that a waiting idle compaction is still due", () => { + const commandFor = (run: string) => + ({ + type: "message.dispatch", + commandId: `server:idle-compact:${threadId}:${run}`, + threadId, + }) as never as Extract; + + it("open only for the run upstream's window is for, with a minute of it left", () => { + const records = warm().data; + // The window runs to 2 minutes on (NOW_MS + 2 min). + assert.isTrue(WorkerWakeSlots.compactionWindowOpen(records, commandFor(runId), NOW_MS)); + // Another run's compaction (a newer run completed since): not this window's. + assert.isFalse(WorkerWakeSlots.compactionWindowOpen(records, commandFor("run-older"), NOW_MS)); + const endMs = NOW_MS + 2 * MINUTE_MS; + const minLeft = WorkerWakeSlots.COMPACTION_MIN_LEFT_MS; + assert.isTrue( + WorkerWakeSlots.compactionWindowOpen(records, commandFor(runId), endMs - minLeft - 1), + ); + assert.isFalse( + WorkerWakeSlots.compactionWindowOpen(records, commandFor(runId), endMs - minLeft), + ); + }); + + it.effect("reads the settings, the project, and upstream's records as its worker does", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW_MS); + const thread = warm(); + const queries: Array = []; + let idleCompaction: "beforeCacheExpires" | "off" = "beforeCacheExpires"; + let projectOff = false; + const projections = { + getThread: () => Effect.sync(() => thread.data.thread), + getThreadRecords: (id: ThreadId, kinds: unknown, options: unknown) => + Effect.sync(() => { + queries.push({ id, kinds, options }); + return thread.data; + }), + } as never as ProjectionStore.ProjectionStoreV2["Service"]; + const settings = { + getSettings: Effect.sync(() => ({ + ...DEFAULT_SERVER_SETTINGS, + idleCompaction, + ...(projectOff + ? { + projectSettingsOverrides: { + [thread.data.thread.projectId]: { + providerInstances: { + [thread.data.thread.providerInstanceId]: { enabled: false }, + }, + }, + }, + } + : {}), + })), + } as never as ServerSettings.ServerSettingsService["Service"]; + const due = WorkerWakeSlots.compactionDueOver(settings, projections); + assert.isTrue(yield* due(commandFor(runId))); + assert.isFalse(yield* due(commandFor("run-older"))); + // Upstream's query for its window: the same records, kinds and filters. + const active = [thread.data.thread.activeProviderThreadId]; + assert.deepStrictEqual(queries[0], { + id: threadId, + kinds: [ + "runs", + "messages", + "providerThreads", + "providerTurns", + "runtimeRequests", + "turnItems", + "contextTransfers", + ], + options: { + messageRunIds: [runId], + turnItemStatuses: ["pending", "running", "waiting"], + runStatuses: ["preparing", "starting", "running", "rolled_back"], + includeLatestRun: true, + providerThreadIds: active, + providerTurnThreadIds: active, + latestProviderTurnUsageOnly: true, + runtimeRequestStatuses: ["pending"], + contextTransferStatuses: ["pending"], + }, + }); + // Idle compaction turned off since: no longer due, and nothing more read. + idleCompaction = "off"; + const before = queries.length; + assert.isFalse(yield* due(commandFor(runId))); + assert.strictEqual(queries.length, before); + idleCompaction = "beforeCacheExpires"; + projectOff = true; + const enabled = yield* due(commandFor(runId)); + projectOff = false; + assert.isFalse(enabled); + assert.isTrue(yield* due(commandFor(runId))); + }), + ); +}); diff --git a/apps/server/src/fork/workerWakeSlots.ts b/apps/server/src/fork/workerWakeSlots.ts index 6642cdd2ebe6e..44858ca5cb09b 100644 --- a/apps/server/src/fork/workerWakeSlots.ts +++ b/apps/server/src/fork/workerWakeSlots.ts @@ -67,9 +67,17 @@ const compactionRunOf = (command: MessageDispatch): RunId | null => { : null; }; +/** + * A retried compaction needs this much of its cache window left: half the lead upstream's worker + * sends it with (`LEAD_MS`, 2 min, in `orchestration-v2/IdleCompaction.ts`), the time a session + * takes to resume and send it while the cache is warm. Later it would most likely read cold. + */ +export const COMPACTION_MIN_LEFT_MS = 60_000; + /** * A refused idle compaction is still due: its thread is idle at its run, and the prompt cache - * window is open, by the rule upstream's worker sent it by (`idleCompactionWindow`). + * window is open, by the rule upstream's worker sent it by (`idleCompactionWindow`), with at + * least `COMPACTION_MIN_LEFT_MS` of it left. */ export const compactionWindowOpen = ( records: IdleCompactionRecords, @@ -78,7 +86,7 @@ export const compactionWindowOpen = ( ): boolean => { const runId = compactionRunOf(command); const window = idleCompactionWindow(records, nowMs); - return runId !== null && window?.runId === runId && nowMs < window.endMs; + return runId !== null && window?.runId === runId && nowMs < window.endMs - COMPACTION_MIN_LEFT_MS; }; /** Places in the gate's line: one per held question wake and per waiting compaction. */ @@ -244,7 +252,18 @@ export const makeWorkerWakeSlots = Effect.fn("WorkerWakeSlots.make")(function* < while (detail !== null) { yield* Effect.sleep(COMPACTION_SLOT_RETRY); // Checked again in the fork before each ask: still idle at its run, window still open. - const due = yield* input.compactionDue(command).pipe(Effect.orElseSucceed(() => false)); + // A check that fails skips it too (fail-safe), logged as what it is, with its cause. + const due = yield* input + .compactionDue(command) + .pipe( + Effect.catchCause((cause) => + Effect.logWarning( + "idle compaction skipped: whether it is still due could not be read", + { threadId: command.threadId, cause }, + ).pipe(Effect.as("unread" as const)), + ), + ); + if (due === "unread") return NOTHING_STORED; if (!due) { yield* Effect.logInfo( "idle compaction skipped: no agent slot while its cache window was open", @@ -339,13 +358,11 @@ export const fileStore = Effect.fn("WorkerWakeSlots.fileStore")(function* (fileP const file = decodeFile(yield* fs.readFileString(filePath)); if (Option.isNone(file)) { const keptAs = `${filePath}.corrupt-${yield* Clock.currentTimeMillis}`; - const kept = yield* fs.rename(filePath, keptAs).pipe( - Effect.as(keptAs), - Effect.orElseSucceed(() => null), - ); + // Not set aside: a read error (the caller keeps it, saves nothing over it, reads it again). + yield* fs.rename(filePath, keptAs); yield* Effect.logWarning("held wakes file unreadable; its wakes are not sent", { filePath, - keptAs: kept, + keptAs, }); return []; } @@ -391,6 +408,54 @@ export const fileStore = Effect.fn("WorkerWakeSlots.fileStore")(function* (fileP return { load, save } satisfies HeldWakesStore; }); +/** + * The server's check that a refused idle compaction is still due: idle compaction on for the + * thread's provider (`beforeCacheExpires`), the provider on for its project, and the window open + * over the records upstream's worker reads (a copy of its query in + * `orchestration-v2/IdleCompactionService.ts`: keep the two the same). + */ +export const compactionDueOver = ( + settings: ServerSettings.ServerSettingsService["Service"], + projections: ProjectionStore.ProjectionStoreV2["Service"], +) => + Effect.fn("WorkerWakeSlots.compactionDue")(function* (command: MessageDispatch) { + const runId = compactionRunOf(command); + const current = yield* settings.getSettings; + const thread = yield* projections.getThread(command.threadId); + if ( + runId === null || + resolveIdleCompaction(current, thread.providerInstanceId) !== "beforeCacheExpires" || + !resolveProjectProviderInstanceEnabled(current, thread.projectId, thread.providerInstanceId) + ) + return false; + const active = thread.activeProviderThreadId === null ? [] : [thread.activeProviderThreadId]; + // The records upstream's idle compaction worker reads for its window. + const records = yield* projections.getThreadRecords( + command.threadId, + [ + "runs", + "messages", + "providerThreads", + "providerTurns", + "runtimeRequests", + "turnItems", + "contextTransfers", + ], + { + messageRunIds: [runId], + turnItemStatuses: ["pending", "running", "waiting"], + runStatuses: ["preparing", "starting", "running", "rolled_back"], + includeLatestRun: true, + providerThreadIds: active, + providerTurnThreadIds: active, + latestProviderTurnUsageOnly: true, + runtimeRequestStatuses: ["pending"], + contextTransferStatuses: ["pending"], + }, + ); + return compactionWindowOpen(records, command, yield* Clock.currentTimeMillis); + }); + /** Upstream's orchestrator for the runtime's workers, gated as the file's doc says. */ export const layerOver = Layer.effect( Orchestrator.OrchestratorV2, @@ -421,48 +486,7 @@ export const layerOver = Layer.effect( (task) => asked.taskIds.includes(task.id) && SETTLED_TASK.has(task.status), ); }), - compactionDue: Effect.fn("WorkerWakeSlots.compactionDue")(function* (command) { - const runId = compactionRunOf(command); - const current = yield* settings.getSettings; - const thread = yield* projections.getThread(command.threadId); - if ( - runId === null || - resolveIdleCompaction(current, thread.providerInstanceId) !== "beforeCacheExpires" || - !resolveProjectProviderInstanceEnabled( - current, - thread.projectId, - thread.providerInstanceId, - ) - ) - return false; - const active = - thread.activeProviderThreadId === null ? [] : [thread.activeProviderThreadId]; - // The records upstream's idle compaction worker reads for its window. - const records = yield* projections.getThreadRecords( - command.threadId, - [ - "runs", - "messages", - "providerThreads", - "providerTurns", - "runtimeRequests", - "turnItems", - "contextTransfers", - ], - { - messageRunIds: [runId], - turnItemStatuses: ["pending", "running", "waiting"], - runStatuses: ["preparing", "starting", "running", "rolled_back"], - includeLatestRun: true, - providerThreadIds: active, - providerTurnThreadIds: active, - latestProviderTurnUsageOnly: true, - runtimeRequestStatuses: ["pending"], - contextTransferStatuses: ["pending"], - }, - ); - return compactionWindowOpen(records, command, yield* Clock.currentTimeMillis); - }), + compactionDue: compactionDueOver(settings, projections), store: yield* fileStore(path.join(config.stateDir, HELD_WAKES_FILE)), }); // A stop waits for a send's file write: a send it cut off would be sent again (receipts dedupe). diff --git a/apps/server/src/slots/SlotGate.test.ts b/apps/server/src/slots/SlotGate.test.ts index 74f58d30c2922..c9bd67daffb19 100644 --- a/apps/server/src/slots/SlotGate.test.ts +++ b/apps/server/src/slots/SlotGate.test.ts @@ -2304,10 +2304,10 @@ describe("SlotGate: one line for every waiting start (issue 738)", () => { const younger = { key: "young", threadId: ThreadId.make("young"), sinceMs: nowMs + 1 }; world.threads = []; // The sweep asks first, but the wake-up is older: the slot is its. - assert.deepStrictEqual( - (yield* gate.grant([younger])).map((grant) => grant.type), - ["full"], - ); + const [refused] = yield* gate.grant([younger]); + assert.strictEqual(refused?.type, "full"); + // Its refusal counts the wake-up as waiting in line, not as a running agent. + if (refused?.type === "full") assert.strictEqual(refused.decision.inLine, 1); assert.strictEqual((yield* wake("w", "w")).type, "allow"); }), ); @@ -2370,6 +2370,241 @@ describe("SlotGate: one line for every waiting start (issue 738)", () => { }), ); + it.effect( + "a free slot kept for a waiter that does not come back within LINE_HOLD goes to the next; the first keeps its place", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + // A waiter that asks seldom (a pull request watch: every 2 min), with a long stale time. + const seldom = SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ + callerThreadId: ThreadId.make("seldom"), + targetInstanceId: MAIN, + intoThreadId: ThreadId.make("seldom"), + keepsHeld: true, + place: { key: "seldom", staleAfter: "5 minutes" }, + }), + ), + ); + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + assert.strictEqual((yield* seldom).type, "refuse"); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // The slot frees: it is kept for the oldest waiter, which does not ask now. + world.threads = []; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + yield* TestClock.adjust(SlotGate.LINE_HOLD); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // Not back within its hold: the next waiter takes the slot, not left unused. + yield* TestClock.adjust("1 second"); + assert.strictEqual((yield* wake("w", "w")).type, "allow"); + // It keeps its place: first again once it asks, before a waiter that joined later. + assert.deepStrictEqual(yield* line, ["seldom"]); + world.threads = [{ id: "w", instanceId: MAIN }]; + assert.strictEqual((yield* wake("later", "later")).type, "refuse"); + assert.strictEqual((yield* seldom).type, "refuse"); + world.threads = []; + assert.strictEqual((yield* wake("later", "later")).type, "refuse"); + assert.strictEqual((yield* seldom).type, "allow"); + }), + ); + }), + ); + + it.effect( + "two places of one thread count from its oldest: a waiter between them never blocks it", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + // Thread t waits twice (a question wake and a continuation), x between the two. + assert.strictEqual((yield* wake("t", "t:first")).type, "refuse"); + assert.strictEqual((yield* wake("x", "x")).type, "refuse"); + assert.strictEqual((yield* wake("t", "t:second")).type, "refuse"); + world.threads = []; + // x waits behind t; t's younger place is let through at t's oldest place in line. + assert.strictEqual((yield* wake("x", "x")).type, "refuse"); + assert.strictEqual((yield* wake("t", "t:second")).type, "allow"); + }), + ); + }), + ); + + it.effect("a waiter that could not use a slot for a while is kept a fresh one when it can", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + const seldom = SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ + callerThreadId: ThreadId.make("seldom"), + targetInstanceId: MAIN, + intoThreadId: ThreadId.make("seldom"), + keepsHeld: true, + place: { key: "seldom", staleAfter: "5 minutes" }, + }), + ), + ); + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + assert.strictEqual((yield* seldom).type, "refuse"); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // A slot frees for a moment (kept for the oldest), then is taken again. + world.threads = []; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + world.threads = [{ id: "busy", instanceId: MAIN }]; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // Much later a slot frees again: its 15 s start now, not at the first offer. + yield* TestClock.adjust("20 seconds"); + world.threads = []; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + }), + ); + }), + ); + + it.effect("a slot wait counts from its thread's oldest place in line", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + waitsOn({ maxPerMachine: 1, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const nowMs = yield* Clock.currentTimeMillis; + // Thread t waits as a wake-up, x after it, then t's slot wait (younger than x). + assert.strictEqual((yield* wake("t", "t:wake")).type, "refuse"); + assert.strictEqual((yield* wake("x", "x")).type, "refuse"); + const slotWait = { key: "t:slot", threadId: ThreadId.make("t"), sinceMs: nowMs + 1 }; + assert.strictEqual((yield* gate.grant([slotWait]))[0]?.type, "full"); + world.threads = []; + // x asks first; the free slot is t's (its oldest place), and its slot wait takes it. + assert.strictEqual((yield* wake("x", "x")).type, "refuse"); + assert.strictEqual((yield* gate.grant([slotWait]))[0]?.type, "granted"); + }), + ); + }), + ); + + it.effect( + "a free slot kept for a slot wait is held over the next sweep (2 sweeps), then not", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + waitsOn({ maxPerMachine: 1, reserveSeconds: 120 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const nowMs = yield* Clock.currentTimeMillis; + const older = { key: "old", threadId: ThreadId.make("old"), sinceMs: nowMs - 1 }; + assert.strictEqual((yield* gate.grant([older]))[0]?.type, "full"); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + world.threads = []; + // Kept for the slot wait, asked at sweeps (every 15 s): 20 s on it is still its. + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + yield* TestClock.adjust("20 seconds"); + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + // Two sweeps (30 s) without it: the waiter behind goes. + yield* TestClock.adjust("11 seconds"); + assert.strictEqual((yield* wake("w", "w")).type, "allow"); + }), + ); + }), + ); + + it.effect("starts that make a new thread (scheduled runs) each have a place of their own", () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + const newRun = (key: string) => + SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => + gate.acquire({ callerThreadId: null, targetInstanceId: MAIN, place: { key } }), + ), + ); + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + assert.strictEqual((yield* newRun("run-a")).type, "refuse"); + assert.strictEqual((yield* newRun("run-b")).type, "refuse"); + world.threads = []; + // The younger run asks first: the older is ahead of it, not the same start. + assert.strictEqual((yield* newRun("run-b")).type, "refuse"); + assert.strictEqual((yield* newRun("run-a")).type, "allow"); + }), + ); + }), + ); + + it.effect( + "a refusal tells the waiters in line apart from the running agents (as wait_list)", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [{ id: "busy", instanceId: MAIN }]; + yield* run( + world, + on({ maxPerMachine: 1 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + world.threads = []; + const refused = yield* start("a"); + assert.strictEqual(refused.type, "refuse"); + if (refused.type !== "refuse") return; + assert.strictEqual( + refused.message, + "No free agent slot: 0 agents are running on this machine and 1 start is waiting in line ahead of this one, and the limit is 1. Try again later. Or do the work yourself.", + ); + assert.deepStrictEqual((yield* gate.usage)?.machine, { used: 0, max: 1 }); + }), + ); + }), + ); + + it.effect( + "memory floor only: a waiter that stops asking is dropped from memory at the next ask", + () => + Effect.gen(function* () { + const world = newWorld(); + world.freeMemoryBytes = 500 * MB; + yield* run( + world, + on({ minFreeMemoryMb: 1000 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + assert.strictEqual((yield* wake("w", "w")).type, "refuse"); + assert.strictEqual(yield* gate.lineKept, 1); + // It never asks again (its route ended without leaving); another start asks later. + yield* TestClock.adjust("31 seconds"); + world.freeMemoryBytes = 16_000 * MB; + assert.strictEqual((yield* start("a")).type, "allow"); + assert.strictEqual(yield* gate.lineKept, 0); + // A sweep of slot waits drops one the same way. + world.freeMemoryBytes = 500 * MB; + assert.strictEqual((yield* wake("v", "v")).type, "refuse"); + yield* TestClock.adjust("31 seconds"); + yield* gate.grant([{ key: "s", threadId: ThreadId.make("s") }]); + assert.strictEqual(yield* gate.lineKept, 1); + }), + ); + }), + ); + it.effect("off or no limit: a waiter is let through and leaves the line", () => Effect.gen(function* () { const world = newWorld(); diff --git a/apps/server/src/slots/SlotGate.ts b/apps/server/src/slots/SlotGate.ts index 1e8795b681694..babd76cc77343 100644 --- a/apps/server/src/slots/SlotGate.ts +++ b/apps/server/src/slots/SlotGate.ts @@ -136,18 +136,25 @@ export interface SlotUsage { } /** - * A start that asks again while it is refused (a wake-up of an idle thread): its place in the - * gate's one line. Its first ask joins the line; every later ask keeps that place, so a slot that - * frees goes to the oldest waiter, never to whoever asks first. It leaves when it is allowed, by - * `leave`, or when it has not asked for `staleAfter` (default `LINE_STALE`). + * A start that asks again while refused (a wake-up, a scheduled run, a webhook delivery): its place + * in the gate's one line, kept from its first ask, so a freed slot goes to the oldest waiter. It + * leaves when allowed, by `leave`, or after `staleAfter` without asking (default `LINE_STALE`). */ export interface SlotPlace { readonly key: string; readonly staleAfter?: Duration.Input | undefined; + /** When it became due (a scheduled run's due time, a stored delivery's): its age; else now. */ + readonly sinceMs?: number | undefined; } /** A waiter that has not asked again for this long has left the line (6 asks of 5 s). */ const LINE_STALE = Duration.seconds(30); +/** + * A waiter a free slot is kept for holds it this long without asking again (3 asks of 5 s); then + * the waiters behind it go first until it asks again. It keeps its place. A waiter that asks + * seldom (a pull request watch: every 2 min) or that died so leaves no slot idle for long. + */ +export const LINE_HOLD = Duration.seconds(15); export interface SlotAcquireInput { readonly callerThreadId: ThreadId | null; @@ -222,6 +229,8 @@ export class SlotGate extends Context.Service< readonly slotWaitsWaiting: (keys: ReadonlyArray) => Effect.Effect; /** The line's keys, oldest first (waiters that went stale are left out). */ readonly line: Effect.Effect>; + /** How many places memory keeps, stale ones not dropped yet too (dropped at every ask). */ + readonly lineKept: Effect.Effect; /** * For `wait_list`: null while slots are off, or when the running agents or the held slots * cannot be read. @@ -273,11 +282,32 @@ interface LineEntry { readonly order: number; lastAskMs: number; staleMs: number; + /** How long a free slot is kept for it while it does not ask (`LINE_HOLD`). */ + holdMs: number; + /** Since when a free slot is kept for it, untouched by its own ask since; `null`: none is. */ + offeredAtMs: number | null; } const staleMsOf = (place: SlotPlace) => Duration.toMillis(Duration.fromInputUnsafe(place.staleAfter ?? LINE_STALE)); +/** + * A count refusal says how many of its `used` are waiters in line ahead of it (`inLine`), as + * `wait_list` shows only what runs, starts or is held. + */ +const withLine = ( + refused: Extract, + ahead: ReadonlyArray, +): Extract => { + const inLine = + refused.rule === "machine" + ? ahead.length + : refused.rule === "account" + ? ahead.filter((entry) => entry.instanceId === refused.instanceId).length + : 0; + return inLine === 0 ? refused : { ...refused, inLine }; +}; + const make = Effect.gen(function* () { const serverSettings = yield* ServerSettings.ServerSettingsService; const activeRunShells = yield* ActiveRunShells.ActiveRunShells; @@ -332,11 +362,15 @@ const make = Effect.gen(function* () { readonly sinceMs: number; readonly nowMs: number; readonly staleMs: number; + readonly holdMs: number; }): LineEntry => { const seen = line.get(entry.key); if (seen !== undefined) { seen.lastAskMs = entry.nowMs; seen.staleMs = entry.staleMs; + seen.holdMs = entry.holdMs; + // It came back: a slot kept for it is its own ask's now. + seen.offeredAtMs = null; return seen; } const joined: LineEntry = { @@ -348,25 +382,36 @@ const make = Effect.gen(function* () { order: lineOrder++, lastAskMs: entry.nowMs, staleMs: entry.staleMs, + holdMs: entry.holdMs, + offeredAtMs: null, }; line.set(entry.key, joined); return joined; }; - /** The live line, oldest first; a waiter that stopped asking has left it. */ + const isStale = (entry: LineEntry, nowMs: number) => nowMs - entry.lastAskMs > entry.staleMs; + /** A waiter that stopped asking has left the line: dropped at every ask, whatever the limits. */ + const pruneLine = (nowMs: number) => { + for (const [key, entry] of line) if (isStale(entry, nowMs)) line.delete(key); + }; + /** The live line, oldest first. */ const lineInOrder = (nowMs: number) => { - for (const [key, entry] of line) { - if (nowMs - entry.lastAskMs > entry.staleMs) line.delete(key); - } + pruneLine(nowMs); return [...line.values()].sort( (left, right) => left.sinceMs - right.sinceMs || left.order - right.order, ); }; + /** The waiters ahead of `threadId`'s oldest place (one thread, one place in line). */ + const beforeThread = (ordered: ReadonlyArray, threadId: string) => { + const first = ordered.findIndex((entry) => entry.threadId === threadId); + return first < 0 ? ordered : ordered.slice(0, first); + }; /** * The waiters ahead of a start, oldest first, as starts: each that would get a slot now keeps it * from those behind; one that would not (its account full) holds none. Left out: the asker's own - * thread, a thread that shows a run (it only queues), and one counted already (held, or named). + * thread, a thread that shows a run (it only queues), one counted already (held, or named), and + * one a slot was kept for longer than its hold without it asking (`LINE_HOLD`). */ const lineAhead = (input: { readonly ahead: ReadonlyArray; @@ -392,6 +437,8 @@ const make = Effect.gen(function* () { if (counted.has(entry.threadId) || input.visible.has(entry.threadId)) continue; // A slot wait is granted only when the held slots read whole: until then it takes none. if (entry.kind === "slotWait" && !input.held.whole) continue; + // A slot kept for it that it did not come back for: the next waiter goes (till it asks). + if (entry.offeredAtMs !== null && input.nowMs - entry.offeredAtMs > entry.holdMs) continue; counted.add(entry.threadId); const others = onePerThread( input.held.held.filter((reservation) => reservation.threadId !== entry.threadId), @@ -416,6 +463,9 @@ const make = Effect.gen(function* () { atMs: input.nowMs, childThreadId: entry.threadId, }); + entry.offeredAtMs ??= input.nowMs; + } else { + entry.offeredAtMs = null; } } return taken; @@ -784,18 +834,21 @@ const make = Effect.gen(function* () { const counted = limitsCount(limits, input.targetInstanceId); const into = input.intoThreadId; const askingThread = into ?? input.callerThreadId; - // A waiter keeps its place from its first ask, refused or not. + pruneLine(nowMs); + // A waiter keeps its place from its first ask, refused or not. One for a thread not made yet + // (a scheduled run's new thread) is that start's own, by its key. const own = - place === undefined || askingThread === null + place === undefined ? undefined : joinLine({ key: place.key, kind: "wake", - threadId: askingThread, + threadId: askingThread ?? place.key, instanceId: input.targetInstanceId, - sinceMs: nowMs, + sinceMs: place.sinceMs ?? nowMs, nowMs, staleMs: staleMsOf(place), + holdMs: Duration.toMillis(LINE_HOLD), }); const allowed = (outcome: SlotAcquire, used: ReadonlyArray) => { if (place !== undefined) line.delete(place.key); @@ -846,8 +899,9 @@ const make = Effect.gen(function* () { read === undefined ? [] : lineAhead({ - ahead: own === undefined ? ordered : ordered.slice(0, ordered.indexOf(own)), - threadId: askingThread, + // From its thread's oldest place: two places of one thread never block each other. + ahead: own === undefined ? ordered : beforeThread(ordered, own.threadId), + threadId: own?.threadId ?? askingThread, limits, running: read.running, visible, @@ -876,16 +930,19 @@ const make = Effect.gen(function* () { const full = ( refused: Extract, free: number, - ): SlotAcquire => ({ - type: "refuse", - message: slotRefusalText(refused, waitsOn), - refusal: { - type: "full", - decision: refused, - waitsOn, - ...(alsoNeeds.length === 0 ? {} : { batch: { needed: alsoNeeds.length + 1, free } }), - }, - }); + ): SlotAcquire => { + const decision = withLine(refused, ahead); + return { + type: "refuse", + message: slotRefusalText(decision, waitsOn), + refusal: { + type: "full", + decision, + waitsOn, + ...(alsoNeeds.length === 0 ? {} : { batch: { needed: alsoNeeds.length + 1, free } }), + }, + }; + }; if (decision.type === "refuse") return asDecided(full(decision, 0), []); // The later starts of the batch, counted as starting one after another (nothing recorded). const starting: Array = [ @@ -949,6 +1006,7 @@ const make = Effect.gen(function* () { } if (requests.length === 0) return []; const nowMs = yield* Clock.currentTimeMillis; + pruneLine(nowMs); /** A wait without an account waits for the machine limit and the memory floor only. */ const limitsFor = (request: SlotRequest): SlotLimits => request.account === undefined ? { ...limits, account: () => null } : limits; @@ -968,10 +1026,10 @@ const make = Effect.gen(function* () { const grants: Array = []; const grantedThreads = new Set(); // Every slot wait is in the line at its own age; one the sweep stopped asking for leaves it. - const staleMs = Math.max( - Duration.toMillis(LINE_STALE), - 3 * settings.value.waitFor.unitSweepSeconds * 1000, - ); + const sweepMs = settings.value.waitFor.unitSweepSeconds * 1000; + const staleMs = Math.max(Duration.toMillis(LINE_STALE), 3 * sweepMs); + // Asked at each sweep: a slot kept for it is held over one sweep it may just miss. + const holdMs = Math.max(Duration.toMillis(LINE_HOLD), 2 * sweepMs); const places = requests.map((request) => joinLine({ key: request.key, @@ -981,6 +1039,7 @@ const make = Effect.gen(function* () { sinceMs: request.sinceMs ?? nowMs, nowMs, staleMs, + holdMs, }), ); for (const [index, request] of requests.entries()) { @@ -1011,7 +1070,7 @@ const make = Effect.gen(function* () { read === undefined || place === undefined || !ordered.includes(place) ? [] : lineAhead({ - ahead: ordered.slice(0, ordered.indexOf(place)), + ahead: beforeThread(ordered, request.threadId), threadId: request.threadId, // Each waiter ahead by its own limits (its account's too), not this wait's. limits, @@ -1040,7 +1099,7 @@ const make = Effect.gen(function* () { }); if (decision.type === "refuse") { // Refused: a slot it already holds stays as it is. - grants.push({ type: "full", decision }); + grants.push({ type: "full", decision: withLine(decision, ahead) }); continue; } // Asking again moves its hold but never extends it: a thread that keeps asking without @@ -1200,8 +1259,14 @@ const make = Effect.gen(function* () { }), ), line: Clock.currentTimeMillis.pipe( - Effect.map((nowMs) => lineInOrder(nowMs).map((entry) => entry.key)), + Effect.map((nowMs) => + [...line.values()] + .filter((entry) => !isStale(entry, nowMs)) + .sort((left, right) => left.sinceMs - right.sinceMs || left.order - right.order) + .map((entry) => entry.key), + ), ), + lineKept: Effect.sync(() => line.size), drop: Effect.fn("SlotGate.drop")(function* (keys: ReadonlyArray) { if (keys.length === 0) return; for (const key of keys) { diff --git a/docs/fork/README.md b/docs/fork/README.md index 3a80477c5f395..71d302d0563b4 100644 --- a/docs/fork/README.md +++ b/docs/fork/README.md @@ -387,8 +387,11 @@ These routes start a new agent run and go through the same gate, with the same c deleted, replaced or its trigger changed while it waits (other edits are not seen: it runs with the task as it was when it began to wait). Every waiting delivery is in the store, so a restart keeps it and it runs once: a direct one (no coalescing window) is moved there when it is first - refused. A scheduled run or a webhook takes a slot only when no agent that waits for one holds - it; there is no order between routes, so on a host kept full it can wait long. + refused. A run on its schedule and a webhook delivery that wait keep a place in the gate's one + line (below), a run at its due time and a delivery at the time it was stored, so a waiter that + joins later never passes them. The place goes when the run starts, when the delivery is given up + (task paused, deleted, replaced or retriggered), and within 5 s of its task being paused, + deleted or no longer due (moved to a later time). "Run now" asks once, from the back of the line. - A task bound to a thread whose run is active only queues its prompt there: no new run, so no slot and no wait. A bound thread that is idle starts a run and is gated, counted on the thread's current provider. @@ -451,15 +454,26 @@ These routes start a new agent run and go through the same gate, with the same c and its id is fixed per failed run and reset, so it is sent once. - One line, oldest first ([#738](https://github.com/scuttletoby/t3code/issues/738)): every wake-up that asks again while refused keeps a place in the gate's line from its first ask (the - PR-watch wake-up, the usage-limit resume, a wait's message and the four below), and so does - every slot wait, at its own age. A slot that frees goes to the oldest waiter that can use it: a - start without a place (`delegate_task`, a send, a scheduled run) and every younger waiter count - the waiters ahead of them as starts. A waiter whose own account is full holds nothing from the - others, a thread that shows a run only queues, and a thread with a held slot starts on it. A - waiter leaves the line when it is let through, when its wake is no longer owed (its thread gone, - its question answered, a handoff that is no longer due), or when it has not asked again for 30 s - (a PR-watch wake-up: 5 minutes; a slot wait: 3 sweeps). The line is memory only: after a restart - each waiter joins again at its next ask, held question wakes first, in their file's order. + PR-watch wake-up, the usage-limit resume, a wait's message, a waiting scheduled run or webhook + delivery, and the four below), and so does every slot wait, at its own age. A slot that frees + goes to the oldest waiter that can use it: a start without a place (`delegate_task`, a send, + "run now") and every younger waiter count the waiters ahead of them as starts. A thread with two + places (a question wake and a continuation) counts from its oldest. A waiter whose own account is + full holds nothing from the others, a thread that shows a run only queues, and a thread with a + held slot starts on it. A free slot is kept for the oldest waiter for 15 s (a slot wait: 2 + sweeps) without it asking; if it does not come back by then, the waiters behind it go first until + it asks again, still in its place. So a waiter that asks seldom (the PR-watch wake-up, at + upstream's 2-minute sweep) or that died leaves a slot idle 15 s at most; the PR-watch wake-up + gets a slot only when one is free as it asks, as before the line. A waiter leaves the line when + it is let through, when its wake is no longer owed (its thread gone, its question answered, a + handoff that is no longer due), or when it has not asked again for 30 s (a PR-watch wake-up: 5 + minutes; a slot wait: 3 sweeps); places that stopped asking are dropped at the next ask, under + any limit. A refusal names the waiters in line ahead apart from the running agents ("2 agents + are running on this machine and 1 start is waiting in line ahead of this one"), since + `wait_list` shows only what runs, starts or is held. The line is memory only: after a restart + each waiter joins again at its next ask. Held question wakes ask at once, in their file's order, + but at the time of that ask (the file keeps no age): a slot wait made before the restart, or a + scheduled run due before it, comes first. - The bound-thread and message routes decide "running or idle" from the gate's own count in the same step that takes the slot. When the thread's run ends between that check and the send, the queued message starts the next run: the call then holds a slot until that run shows, for the @@ -515,15 +529,17 @@ only queues: no slot. keeps a send cut off before the file write from making a second message. Saves run one at a time, each writing what is held then, synced to disk before the rename and the folder after it. A file that does not decode is moved to `fork-held-wakes.json.corrupt-` beside it and - logged, so the next save does not write over it; an entry that does not decode is logged and + logged, so the next save does not write over it (when that move fails, the file is handled as + one that cannot be read, below: kept as it is, nothing written over it); an entry that does not decode is logged and left out. A file that cannot be checked or read (an I/O error, not "missing") is logged once and never written over: new holds stay in memory, and every 5 s it is read again, its wakes then going first. A held wake whose parent or question cannot be read waits, logged once. - Upstream's idle compaction (same file), after the handoff owner's check. Refused, it is held in upstream's own timer and asked again every 5 s in its place in line (logged once), while the - fork reads again that the thread is still idle at that run and its prompt cache window is still - open (upstream's `idleCompactionWindow`). Once not, it is skipped (logged): a late compaction - costs a full uncached read. A newer completed run replaces it, as upstream's timer does; a + fork reads again that the thread is still idle at that run and at least a minute of its prompt + cache window is left (upstream's `idleCompactionWindow`; half the 2-minute lead upstream sends + it with). Once not, it is skipped (logged): a late compaction costs a full uncached read. A + check that cannot be read skips it too, logged with its cause. A newer completed run replaces it, as upstream's timer does; a restart drops it (upstream). The calling thread is left out of the count (it waits on its child). A start no thread on this diff --git a/packages/shared/src/agentSlots.ts b/packages/shared/src/agentSlots.ts index aa0558972148a..cf5753d4f495a 100644 --- a/packages/shared/src/agentSlots.ts +++ b/packages/shared/src/agentSlots.ts @@ -247,6 +247,11 @@ export type SlotDecision = /** The limit (count rules) or the floor in MB (memory). */ readonly max: number; readonly instanceId: string; + /** + * Of `used`, the starts waiting in the gate's line ahead of this one (count rules): they are + * not running yet, so `wait_list` does not show them. + */ + readonly inLine?: number | undefined; }; export interface SlotDecisionInput { @@ -325,15 +330,29 @@ export function slotDecision(input: SlotDecisionInput): SlotDecision { return { type: "allow" }; } +/** + * "N agents are running{where}", and the starts waiting in line ahead of this one apart + * (`inLine`), so the count matches `wait_list`. + */ +export function slotRunningText( + decision: Extract, + where: string, +): string { + const agents = (count: number) => (count === 1 ? "1 agent is" : `${count} agents are`); + const inLine = decision.inLine ?? 0; + if (inLine === 0) return `${agents(decision.used)} running${where}`; + const starts = inLine === 1 ? "1 start is" : `${inLine} starts are`; + return `${agents(decision.used - inLine)} running${where} and ${starts} waiting in line ahead of this one`; +} + /** Why a slot is not free, without a full stop (held slots count as running agents). */ export function slotRefusalReason( decision: Extract, ): string { - const agents = (count: number) => (count === 1 ? "1 agent is" : `${count} agents are`); return decision.rule === "machine" - ? `${agents(decision.used)} running on this machine, and the limit is ${decision.max}` + ? `${slotRunningText(decision, " on this machine")}, and the limit is ${decision.max}` : decision.rule === "account" - ? `${agents(decision.used)} running on account ${decision.instanceId}, and the limit is ${decision.max}` + ? `${slotRunningText(decision, ` on account ${decision.instanceId}`)}, and the limit is ${decision.max}` : `only ${decision.used} MB of memory is free, and a new agent needs at least ${decision.max} MB`; } From 8a368e5170175c8268154591b44d50c363bba311 Mon Sep 17 00:00:00 2001 From: John Doe <123843277+scuttletoby@users.noreply.github.com> Date: Sat, 21 Nov 2026 15:38:07 +0100 Subject: [PATCH 4/4] Fork: a waiter in the slot line is modelled on its last ask's account; an unreadable stored delivery keeps a place (#738) - SlotGate: joinLine refreshes the entry's account on every ask, so a waiter whose thread moved to a full account no longer keeps a free machine slot from every other start. - A stored webhook delivery whose row cannot be read logs it and waits in its task's place (age: its first ask), not from the back of the line. - Tests pin the stored delivery's age, the waitingDeliveries guard and the fallback; README notes the account and the PR-watch choice. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/fork/agentSlotRoutes.test.ts | 130 +++++++++++++++++++ apps/server/src/fork/agentSlotRoutes.ts | 16 ++- apps/server/src/slots/SlotGate.test.ts | 29 +++++ apps/server/src/slots/SlotGate.ts | 4 +- docs/fork/README.md | 11 +- 5 files changed, 182 insertions(+), 8 deletions(-) diff --git a/apps/server/src/fork/agentSlotRoutes.test.ts b/apps/server/src/fork/agentSlotRoutes.test.ts index 00b46e727a5c0..b7d46e6fe9b40 100644 --- a/apps/server/src/fork/agentSlotRoutes.test.ts +++ b/apps/server/src/fork/agentSlotRoutes.test.ts @@ -23,6 +23,7 @@ import * as Metric from "effect/Metric"; import * as Option from "effect/Option"; import * as Schema from "effect/Schema"; import * as SqlClient from "effect/sql/SqlClient"; +import * as SqlError from "effect/sql/SqlError"; import * as TestClock from "effect/testing/TestClock"; import * as ThreadLaunchService from "../orchestration-v2/ThreadLaunchService.ts"; @@ -1525,6 +1526,135 @@ describe("a scheduled run or webhook delivery that waits keeps its place in the }, ); + it.effect( + "a stored delivery is ahead of a waiter that joined after it was stored, though the worker asks after it", + () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, started, webhookWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + const accepted = yield* service.triggerWebhook(webhookRequest(task)); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + yield* settle; + // A restart: the line is memory only; the stored row keeps its due time. + yield* Option.getOrThrow(gate).leave([`webhook-delivery:${accepted.deliveryId}`]); + yield* TestClock.adjust("1 second"); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* TestClock.adjust("1 second"); + yield* webhookWorker; + yield* settle; + assert.deepEqual(yield* lineOf(gate), [ + `webhook-delivery:${accepted.deliveryId}`, + "later-waiter", + ]); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* until(() => started.length === 1, "the delivery started"); + }), + { slots }, + ); + }, + ); + + it.effect("a direct delivery waiting in memory keeps its place through the 5 s sweep", () => { + const slots = slotWorld([BUSY]); + return withService( + ({ service, sql, started, notesWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + // A row with its id is there already, not pending: it is not stored, it waits in memory. + const deliveryId = "delivery:relay:held"; + yield* sql`INSERT INTO scheduled_task_webhook_pending + (delivery_id, task_id, coalesce_key, due_at, state, task_json, request_json, delivery_ids_json) + VALUES (${deliveryId}, ${task.id}, '', ${START}, 'running', '{}', '{}', '[]')`; + const accepted = yield* service.triggerWebhook({ + ...webhookRequest(task), + relayDeliveryId: "held", + }); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + assert.equal(accepted.deliveryId, deliveryId); + yield* settle; + yield* TestClock.adjust("1 second"); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + // The sweep finds no pending row for it, but it still waits: its place stays. + yield* notesWorker; + assert.deepEqual(yield* lineOf(gate), [`webhook-delivery:${deliveryId}`, "later-waiter"]); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* until(() => started.length === 1, "the delivery started"); + }), + { slots }, + ); + }); + + it.effect("a stored delivery whose row cannot be read is logged and still keeps a place", () => { + const slots = slotWorld([BUSY]); + const readFails = { value: false }; + const warnings: Array = []; + const logger = Logger.make(({ message, logLevel }) => { + if (logLevel === "Warn") warnings.push(JSON.stringify(message)); + }); + return boot( + ({ service, started, webhookWorker, gate }) => + Effect.gen(function* () { + yield* TestClock.setTime(START); + const { task } = yield* service.upsert( + yield* taskInput({ schedule: { type: "webhook" } }), + ); + const accepted = yield* service.triggerWebhook(webhookRequest(task)); + if (accepted._tag !== "accepted") return assert.fail(`not accepted: ${accepted._tag}`); + yield* settle; + // A restart, then the worker's read of the row's place fails: its task's place, now. + yield* Option.getOrThrow(gate).leave([`webhook-delivery:${accepted.deliveryId}`]); + readFails.value = true; + yield* TestClock.adjust("1 second"); + yield* webhookWorker; + yield* settle; + assert.lengthOf( + warnings.filter((warning) => warning.includes("place in line")), + 1, + ); + assert.deepEqual(yield* lineOf(gate), [`webhook-task:${task.id}`]); + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + slots.running = []; + assert.equal((yield* laterWaiter(gate)).type, "refuse"); + yield* until(() => started.length === 1, "the delivery started"); + assert.deepEqual(yield* lineOf(gate), ["later-waiter"]); + }), + { slots }, + ).pipe( + Effect.updateService( + SqlClient.SqlClient, + (sql) => + new Proxy(sql, { + apply: (target, self, args: Array) => + readFails.value && + Array.isArray(args[0]) && + args[0].join("?").includes("SELECT delivery_id, due_at FROM") + ? Effect.fail( + new SqlError.SqlError({ + reason: new SqlError.LockTimeoutError({ cause: null }), + }), + ) + : Reflect.apply(target, self, args), + }), + ), + Effect.provide( + Layer.mergeAll( + SqlitePersistence.layerMemory, + Logger.layer([logger], { mergeWithExisting: true }), + ), + ), + ); + }); + it.effect( "a waiting run's place goes when its task is deleted or paused (no slot kept for it)", () => { diff --git a/apps/server/src/fork/agentSlotRoutes.ts b/apps/server/src/fork/agentSlotRoutes.ts index 87d0a69cfd0fd..5ed9f53fb4ce9 100644 --- a/apps/server/src/fork/agentSlotRoutes.ts +++ b/apps/server/src/fork/agentSlotRoutes.ts @@ -391,6 +391,8 @@ const CurrentDirectDelivery = Context.Reference( */ const SCHEDULED_PLACE = "scheduled-run:"; const DELIVERY_PLACE = "webhook-delivery:"; +/** A stored delivery whose row cannot be read: its task's (one stored delivery runs at a time). */ +const DELIVERY_TASK_PLACE = "webhook-task:"; /** * What a run takes from its task's row, decoded as `ScheduledTaskService` decodes it (so a prompt @@ -511,7 +513,8 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( /** * The run's place in line: a run on its schedule at its due time, a webhook delivery at the * time it was stored. "Run now" asks once and has none. A stored delivery is the one its task's - * worker runs now (one at a time per task, in the worker's order); unreadable, it asks with none. + * worker runs now (one at a time per task, in the worker's order); unreadable, it is logged and + * keeps its task's place from this ask (`waitForWebhookSlot` asks in it, and leaves it). */ const placeOf = Effect.fnUntraced(function* ( task: ScheduledTask, @@ -531,9 +534,16 @@ export const makeScheduledRunSlots = Effect.fnUntraced(function* ( SELECT delivery_id, due_at FROM scheduled_task_webhook_pending WHERE task_id = ${task.id} AND state IN ('dispatching', 'running', 'completed', 'failed') ORDER BY due_at, rowid LIMIT 1 - `.pipe(Effect.orElseSucceed(() => [])); + `.pipe( + Effect.catchCause((cause) => + Effect.logWarning("Could not read a waiting webhook delivery's place in line", { + taskId: task.id, + cause, + }).pipe(Effect.as([])), + ), + ); return stored === undefined - ? undefined + ? ({ key: `${DELIVERY_TASK_PLACE}${task.id}` } satisfies SlotGate.SlotPlace) : ({ key: `${DELIVERY_PLACE}${stored.delivery_id}`, sinceMs: Number(stored.due_at), diff --git a/apps/server/src/slots/SlotGate.test.ts b/apps/server/src/slots/SlotGate.test.ts index c9bd67daffb19..e21ecb37206bf 100644 --- a/apps/server/src/slots/SlotGate.test.ts +++ b/apps/server/src/slots/SlotGate.test.ts @@ -2238,6 +2238,35 @@ describe("SlotGate: one line for every waiting start (issue 738)", () => { }), ); + it.effect( + "a waiter whose thread moved to a full account holds no slot from the others (each ask's account)", + () => + Effect.gen(function* () { + const world = newWorld(); + world.threads = [ + { id: "busy", instanceId: SECOND }, + { id: "other", instanceId: MAIN }, + ]; + yield* run( + world, + on({ maxPerMachine: 2, maxPerAccount: { [SECOND]: 1 } }), + Effect.gen(function* () { + assert.strictEqual((yield* wake("w", "w", MAIN)).type, "refuse"); + // A machine slot frees; the waiter's thread now runs on SECOND, which is full. + world.threads = [{ id: "busy", instanceId: SECOND }]; + const asks: Array = []; + for (let round = 0; round < 12; round += 1) { + asks.push((yield* wake("w", "w", SECOND)).type); + asks.push((yield* start("a")).type); + if (asks.at(-1) === "allow") break; + yield* TestClock.adjust("5 seconds"); + } + assert.deepStrictEqual(asks, ["refuse", "allow"]); + }), + ); + }), + ); + it.effect("a slot held for a slot wait is its thread's: a waiter behind does not take it", () => Effect.gen(function* () { const world = newWorld(); diff --git a/apps/server/src/slots/SlotGate.ts b/apps/server/src/slots/SlotGate.ts index babd76cc77343..db185164cb0b5 100644 --- a/apps/server/src/slots/SlotGate.ts +++ b/apps/server/src/slots/SlotGate.ts @@ -276,7 +276,8 @@ interface LineEntry { /** A route's wake-up, or a slot wait (granted only on a whole read of the held slots). */ readonly kind: "wake" | "slotWait"; readonly threadId: string; - readonly instanceId: string | null; + /** The account of its last ask: a thread moved to another account is modelled on that one. */ + instanceId: string | null; readonly sinceMs: number; /** Joins at the same time keep their order. */ readonly order: number; @@ -367,6 +368,7 @@ const make = Effect.gen(function* () { const seen = line.get(entry.key); if (seen !== undefined) { seen.lastAskMs = entry.nowMs; + seen.instanceId = entry.instanceId; seen.staleMs = entry.staleMs; seen.holdMs = entry.holdMs; // It came back: a slot kept for it is its own ask's now. diff --git a/docs/fork/README.md b/docs/fork/README.md index 25fc1d16778c3..e124d0bec010e 100644 --- a/docs/fork/README.md +++ b/docs/fork/README.md @@ -477,13 +477,16 @@ These routes start a new agent run and go through the same gate, with the same c goes to the oldest waiter that can use it: a start without a place (`delegate_task`, a send, "run now") and every younger waiter count the waiters ahead of them as starts. A thread with two places (a question wake and a continuation) counts from its oldest. A waiter whose own account is - full holds nothing from the others, a thread that shows a run only queues, and a thread with a - held slot starts on it. A free slot is kept for the oldest waiter for 15 s (a slot wait: 2 + full (the account of its last ask: its thread may have moved) holds nothing from the others, a + thread that shows a run only queues, and a thread with a held slot starts on it. A free slot is kept for the oldest waiter for 15 s (a slot wait: 2 sweeps) without it asking; if it does not come back by then, the waiters behind it go first until it asks again, still in its place. So a waiter that asks seldom (the PR-watch wake-up, at upstream's 2-minute sweep) or that died leaves a slot idle 15 s at most; the PR-watch wake-up - gets a slot only when one is free as it asks, as before the line. A waiter leaves the line when - it is let through, when its wake is no longer owed (its thread gone, its question answered, a + gets a slot only when one is free as it asks, as before the line. The fork could hold a refused + PR-watch wake-up and ask every 5 s itself (as it does for idle compaction); it does not, because + upstream's reactor waits for that dispatch, so holding it would delay the other watches' news, + and a retry of its own would race upstream's next sync for the thread. A waiter leaves the line + when it is let through, when its wake is no longer owed (its thread gone, its question answered, a handoff that is no longer due), or when it has not asked again for 30 s (a PR-watch wake-up: 5 minutes; a slot wait: 3 sweeps); places that stopped asking are dropped at the next ask, under any limit. A refusal names the waiters in line ahead apart from the running agents ("2 agents