From a9502ed47bd21fea8fabb79a34d01bcbe01d489f Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:20:45 +0000 Subject: [PATCH 1/3] slots observe: one count per agent, stored held slots, seams list, tests (#353) - Slots off: each allowed start gets a token of the capacity rule's own, so started/release reach it. A start counts until its dispatch ends, then for the start grace unless its child shows a run (then once, as that run). A dispatch with no child stops counting at once. - Held slots in the wait store count as the gate counts them, read in the observer's fiber with the read time limit, never on the decision path. The acquiring thread's own held slot is not counted twice. - The observer fiber starts after the decision returns. - Test for slotRule in wait_list (observe mode only). - fork-seams.json lists AgentScope.ts and agentScope.ts (ownScope). Co-Authored-By: Claude Opus 5.5 --- .github/fork-seams.json | 22 ++ apps/server/src/process/agentScope.ts | 1 + apps/server/src/slots/SlotGate.ts | 89 ++++-- apps/server/src/slots/SlotObserver.test.ts | 258 +++++++++++++++++- apps/server/src/slots/SlotObserver.ts | 88 ++++-- .../src/waits/WaitService.slots.test.ts | 18 ++ 6 files changed, 420 insertions(+), 56 deletions(-) diff --git a/.github/fork-seams.json b/.github/fork-seams.json index 0e860c1296057..645de3afcb38f 100644 --- a/.github/fork-seams.json +++ b/.github/fork-seams.json @@ -41,6 +41,28 @@ "Effect.flatMap(ForkSettingsWrite.withCheck(update, \"test settings\")), // Fork: save check", "yield* ForkSettingsWrite.check(current, updated, settingsPath); // Fork: save check" ] + }, + "packages/shared/src/AgentScope.ts": { + "why": "The slot observer (apps/server/src/slots/SlotObserver.ts) reads an agent's memory from its own systemd scope; one optional method on upstream's interface asks for it.", + "added": [ + "/** Fork: the thread's own systemd scope and slice, for reading its memory; none when shared. */", + "readonly ownScope?: (", + "threadId: string,", + ") => Effect.Effect<{ unit: string; slice: string } | undefined>;" + ] + }, + "apps/server/src/process/agentScope.ts": { + "why": "The scope map is private to upstream's make(); the fork's ownScope (see packages/shared/src/AgentScope.ts) is a 7-line lookup in it.", + "added": [ + "// Fork: the thread's own scope, for the slot observer (slots/SlotObserver.ts).", + "ownScope: (threadId) =>", + "Effect.sync(() => {", + "const tracked = scopeByThread.get(threadId);", + "return tracked === undefined || tracked.shared", + "? undefined", + ": { unit: tracked.unit, slice: ownedSlice };", + "})," + ] } } } diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 79408c8b7daf8..74f69a1d89f38 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -431,6 +431,7 @@ export const make = Effect.gen(function* () { stop, kill, clear, + // Fork: the thread's own scope, for the slot observer (slots/SlotObserver.ts). ownScope: (threadId) => Effect.sync(() => { const tracked = scopeByThread.get(threadId); diff --git a/apps/server/src/slots/SlotGate.ts b/apps/server/src/slots/SlotGate.ts index 244ee6149aed8..0dd55c83a4b66 100644 --- a/apps/server/src/slots/SlotGate.ts +++ b/apps/server/src/slots/SlotGate.ts @@ -41,7 +41,10 @@ import { makeSlotObserver } from "./SlotObserver.ts"; export type SlotToken = number; export type SlotAcquire = - /** `token` is `null` when the gate is off or has no limit (nothing to release). */ + /** + * `token` is `null` when the gate is off or has no limit (nothing to release), unless the + * capacity rule beside it keeps the start itself (`observe` mode). + */ | { readonly type: "allow"; readonly token: SlotToken | null } | { readonly type: "refuse"; readonly message: string }; @@ -200,23 +203,30 @@ const make = Effect.gen(function* () { /** * Every live held slot: granted in memory and kept in the wait store, one per wait (counted one * per thread, `onePerThread`). None while waits are off (nothing of the store is read then). A - * store that cannot be read holds none. + * store that cannot be read holds none. Taken now; the store's part is read when the returned + * effect runs, without the used and hidden keys of now (the capacity rule reads it later). */ - const liveHeld = Effect.fn("SlotGate.liveHeld")(function* (waitsOn: boolean, nowMs: number) { + const heldAt = Effect.fn("SlotGate.heldAt")(function* (waitsOn: boolean, nowMs: number) { for (const [key, untilMs] of usedKeys) if (untilMs <= nowMs) usedKeys.delete(key); // A grant never kept nor dropped (a cut-off call) hides the old holds only until they end. for (const [key, olds] of replacing) { if (olds.every((old) => old.untilMs <= nowMs)) replacing.delete(key); } const memory = yield* Ref.updateAndGet(held, (current) => liveReservations(current, nowMs)); - if (!waitsOn) return []; - const fromStore = stored === null ? [] : yield* stored.read; - const hidden = hiddenKeys(); - return liveReservations([...memory, ...fromStore], nowMs).filter( - (reservation) => !usedKeys.has(reservation.key) && !hidden.has(reservation.key), + if (!waitsOn) return Effect.succeed>([]); + const skipped = new Set([...usedKeys.keys(), ...hiddenKeys()]); + const source = stored; + return (source === null ? Effect.succeed([]) : source.read).pipe( + Effect.map((fromStore) => + liveReservations([...memory, ...fromStore], nowMs).filter( + (reservation) => !skipped.has(reservation.key), + ), + ), ); }); + const liveHeld = (waitsOn: boolean, nowMs: number) => Effect.flatten(heldAt(waitsOn, nowMs)); + /** * The live holds that re-asks not kept yet hide, of threads other than `threadId`. They still * count against their own account (never the machine, where the new hold counts), so a slot @@ -245,6 +255,7 @@ const make = Effect.gen(function* () { startGraceMs: settings.value.agentSlots.startGraceSeconds * 1000, // Off or without a limit, the gate keeps no records of new starts. gateRecords: effectiveSlotLimits(settings.value.agentSlots) !== null, + waitsOn: settings.value.waitFor.enabled, }) : Option.none(), ), @@ -253,26 +264,50 @@ const make = Effect.gen(function* () { /** * Runs a decision and, in `observe` mode, hands today's answers to the rule beside it with the * records as they were before the decision changed them. Forked there: never changed, never - * delayed. Inside the semaphore, so observations come in decision order. + * delayed (the wait store's held slots are read by the rule, off this path). Inside the + * semaphore, so observations come in decision order. */ const observeBeside = Effect.fn("SlotGate.observeBeside")(function* ( decision: Effect.Effect, answers: (result: A) => ReadonlyArray<"allow" | "refuse">, + /** + * `acquire` only. An allow the gate keeps no record of (`untracked`) gets a token of the + * rule's own, so `started` and `release` reach the rule's record of that start. + */ + caller?: { + readonly threadId: string; + readonly instanceId: string; + readonly untracked: (result: A) => boolean; + readonly withToken: (result: A, token: SlotToken) => A; + }, ) { const observe = yield* observing; if (Option.isNone(observe)) return yield* decision; + const { startGraceMs, gateRecords, waitsOn } = observe.value; const atMs = yield* Clock.currentTimeMillis; const starting = yield* Ref.get(entries); - const heldBefore = yield* Ref.get(held); + const heldBefore = yield* heldAt(waitsOn, atMs); const result = yield* decision; + // Only a start the gate did not record: never a token of the gate's own (settings changed + // between the two reads). + const ownToken = + !gateRecords && caller !== undefined && caller.untracked(result) + ? yield* Ref.getAndUpdate(nextToken, (value) => value + 1) + : null; yield* observer.observe({ today: answers(result), atMs, starting, held: heldBefore, - ...observe.value, + callerThreadId: caller?.threadId, + startGraceMs, + gateRecords, + ownStart: + ownToken === null || caller === undefined + ? undefined + : { token: ownToken, instanceId: caller.instanceId, atMs }, }); - return result; + return ownToken === null || caller === undefined ? result : caller.withToken(result, ownToken); }); /** Used up: they stop counting (memory ones go; stored ones are skipped until their end). */ @@ -319,6 +354,9 @@ const make = Effect.gen(function* () { ), ); + /** A decision and the caller's held slots it used up (typed whole, for `observeBeside`). */ + const asDecided = (outcome: SlotAcquire, used: ReadonlyArray) => ({ outcome, used }); + /** * The decision, under the semaphore, and the caller's held slot it used up (if any). The * caller's own held slots never count against it; they are not a pass past a limit either. @@ -333,10 +371,7 @@ const make = Effect.gen(function* () { ? null : effectiveSlotLimits(settings.value.agentSlots); if (Option.isNone(settings) || limits === null) { - return { - outcome: { type: "allow", token: null } satisfies SlotAcquire, - used: [] as ReadonlyArray, - }; + return asDecided({ type: "allow", token: null }, []); } const waitsOn = settings.value.waitFor.enabled; const nowMs = yield* Clock.currentTimeMillis; @@ -357,13 +392,7 @@ const make = Effect.gen(function* () { freeMemoryBytes: yield* freeMemoryBytes(limits), }); if (decision.type === "refuse") { - return { - outcome: { - type: "refuse", - message: slotRefusalText(decision, waitsOn), - } satisfies SlotAcquire, - used: [] as ReadonlyArray, - }; + return asDecided({ type: "refuse", message: slotRefusalText(decision, waitsOn) }, []); } const token = yield* Ref.getAndUpdate(nextToken, (value) => value + 1); yield* Ref.update(entries, (current) => [ @@ -372,9 +401,15 @@ const make = Effect.gen(function* () { ]); // The agent it held a slot for starts now: its held slot is used up, with every older copy. const used = yield* useUp(reservations, input.callerThreadId); - return { outcome: { type: "allow", token } satisfies SlotAcquire, used }; + return asDecided({ type: "allow", token }, used); }, - (decided) => observeBeside(decided, (result) => [result.outcome.type]), + (decided, input) => + observeBeside(decided, (result) => [result.outcome.type], { + threadId: input.callerThreadId, + instanceId: input.targetInstanceId, + untracked: (result) => result.outcome.type === "allow" && result.outcome.token === null, + withToken: (result, token) => asDecided({ type: "allow", token }, result.used), + }), // One at a time: the next caller sees the entry the last one added. semaphore.withPermits(1), ); @@ -575,12 +610,13 @@ const make = Effect.gen(function* () { for (const key of keys) replacing.delete(key); yield* Ref.update(held, (current) => current.filter((entry) => !keys.includes(entry.key))); }), + // A token is the gate's or the capacity rule's (`observeBeside`); each ignores the other's. started: Effect.fn("SlotGate.started")((token: SlotToken | null, childThreadId: ThreadId) => token === null ? Effect.void : Ref.update(entries, (current) => current.map((entry) => (entry.token === token ? { ...entry, childThreadId } : entry)), - ), + ).pipe(Effect.andThen(observer.started(token, childThreadId))), ), release: Effect.fn("SlotGate.release")(function* (token: SlotToken | null) { if (token === null) return; @@ -594,6 +630,7 @@ const make = Effect.gen(function* () { : [entry.endedAtMs === undefined ? { ...entry, endedAtMs: nowMs } : entry], ), ); + yield* observer.released(token, nowMs); }), }); }); diff --git a/apps/server/src/slots/SlotObserver.test.ts b/apps/server/src/slots/SlotObserver.test.ts index c778fa2ba0ec5..055f505aae452 100644 --- a/apps/server/src/slots/SlotObserver.test.ts +++ b/apps/server/src/slots/SlotObserver.test.ts @@ -161,6 +161,12 @@ const acquire = SlotGate.SlotGate.pipe( Effect.flatMap((gate) => gate.acquire({ callerThreadId: caller, targetInstanceId: MAIN })), ); +/** The dispatch ended without naming a child (it failed before one existed). */ +const release = (answer: SlotGate.SlotAcquire) => + SlotGate.SlotGate.pipe( + Effect.flatMap((gate) => gate.release(answer.type === "allow" ? answer.token : null)), + ); + /** Lets forked observations and the fake host's promises finish (scheduler turns, no real wait). */ const settle = Effect.forEach(Array.from({ length: 20 }), () => Effect.yieldNow, { discard: true }); @@ -222,7 +228,10 @@ describe("SlotGate observe mode", () => { world, {}, Effect.gen(function* () { - assert.deepStrictEqual(yield* acquire, { type: "allow", token: null }); + const answer = yield* acquire; + // A token for the rule's own record of the start (the gate keeps none while off). + assert.strictEqual(answer.type, "allow"); + assert.isNumber(answer.type === "allow" ? answer.token : null); return yield* statusAfter(1); }), ); @@ -232,6 +241,26 @@ describe("SlotGate observe mode", () => { }), ); + it.effect("the observation starts after the decision returns, never on the caller's stack", () => + Effect.gen(function* () { + const world = newWorld(); + // The fake projection read is synchronous: forked to start at once, it would run before + // the caller got its answer (inside the gate's lock). + const readsAtAnswer = yield* run( + world, + observe(), + Effect.gen(function* () { + assert.strictEqual((yield* acquire).type, "allow"); + const reads = world.snapshotReads; + yield* statusAfter(1); + return reads; + }), + ); + assert.strictEqual(readsAtAnswer, 0); + assert.strictEqual(world.snapshotReads, 1); + }), + ); + it.effect("the same decisions in fixed and observe mode, even with a hanging machine", () => Effect.gen(function* () { // Two slots: allow, allow, refuse; a release frees one; a slot wait is granted, then full. @@ -415,7 +444,7 @@ describe("SlotGate observe mode", () => { world, observe(), Effect.gen(function* () { - yield* acquire; + yield* release(yield* acquire); yield* TestClock.adjust(CALM_WAIT); yield* acquire; return yield* statusAfter(2); @@ -444,25 +473,232 @@ describe("SlotGate observe mode", () => { }), ); - it.effect("slots off: an allow counts as starting for the start grace, then goes", () => + it.effect("slots off: a start counts until its dispatch ends, then for the start grace", () => Effect.gen(function* () { const world = newWorld(); - const [soon, later] = yield* run( + const [during, grace, gone] = yield* run( world, observe({ startGraceSeconds: 30 }), Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + // A dispatch that has not named its child yet counts, however long it takes. + const first = yield* acquire; + yield* TestClock.adjust(Duration.seconds(45)); yield* acquire; + const during = yield* statusAfter(2); + // The child is named and the dispatch ends; it shows no run yet (a queued run). + const token = first.type === "allow" ? first.token : null; + yield* gate.started(token, ThreadId.make("child-queued")); + yield* gate.release(token); yield* TestClock.adjust(Duration.seconds(29)); + yield* release(yield* acquire); + const grace = yield* statusAfter(3); + yield* TestClock.adjust(Duration.seconds(2)); yield* acquire; - const soon = yield* statusAfter(2); - yield* TestClock.adjust(Duration.seconds(20)); + return [during, grace, yield* statusAfter(4)] as const; + }), + ); + assert.strictEqual(during.last?.pieces, 1); + // At 74 s: the queued child is 29 s past its dispatch's end, inside its 30 s grace; the + // second start (45 s) never ended. + assert.strictEqual(grace.last?.pieces, 2); + // At 76 s: the queued child's grace is over; the third start ended with no child at once. + assert.strictEqual(gone.last?.pieces, 1); + }), + ); + + it.effect("slots off: a start whose child shows its run counts once, as that run (#353)", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + observe({ startGraceSeconds: 30 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const first = yield* acquire; + const token = first.type === "allow" ? first.token : null; + yield* gate.started(token, ThreadId.make("child")); + // The child shows its run (300 MB in its own scope) inside the 30 s window. + world.threads = [{ id: "child", unit: "t3code-claude-cccc0000.scope" }]; + scopeFiles(world, "t3code-claude-cccc0000.scope", 300); + yield* TestClock.adjust(Duration.seconds(5)); + // A start whose dispatch ends with no child, so only the first agent is left. + yield* release(yield* acquire); + const running = yield* statusAfter(2); + // Its dispatch ends after the run showed: still counted once, as the run. + yield* gate.release(token); + yield* TestClock.adjust(Duration.seconds(5)); yield* acquire; - return [soon, yield* statusAfter(3)] as const; + return [running, yield* statusAfter(3)] as const; }), ); - assert.strictEqual(soon.last?.pieces, 1); - // At 49 s: the allow at 29 s still starts; the one at 0 s is past its 30 s grace. - assert.strictEqual(later.last?.pieces, 1); + // Not also as starting: that would be 2 pieces, owing 768 MB more. + for (const observed of status) assert.deepInclude(observed.last, { pieces: 1, owedMb: 468 }); + }), + ); + + it.effect("slots off: a start whose run showed and ended stops counting, grace or not", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + observe({ startGraceSeconds: 30 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const first = yield* acquire; + const token = first.type === "allow" ? first.token : null; + yield* gate.started(token, ThreadId.make("child")); + yield* gate.release(token); + // Its run shows, then ends, all inside the 30 s grace of its dispatch's end. + world.threads = [{ id: "child", unit: "t3code-claude-cccc0000.scope" }]; + scopeFiles(world, "t3code-claude-cccc0000.scope", 300); + yield* TestClock.adjust(Duration.seconds(5)); + yield* release(yield* acquire); + yield* statusAfter(2); + world.threads = []; + yield* TestClock.adjust(Duration.seconds(5)); + yield* release(yield* acquire); + return yield* statusAfter(3); + }), + ); + // A finished agent: not counted again as starting for the rest of its grace. + assert.strictEqual(status.last?.pieces, 0); + }), + ); + + it.effect("slots off: a child named while the observation waits is not counted twice", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + observe({ startGraceSeconds: 30 }), + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const first = yield* acquire; + yield* statusAfter(1); + yield* TestClock.adjust(Duration.seconds(5)); + // Decided while the first start has no child yet; observed after it shows its run. + yield* acquire; + yield* gate.started(first.type === "allow" ? first.token : null, ThreadId.make("child")); + world.threads = [{ id: "child", unit: "t3code-claude-cccc0000.scope" }]; + scopeFiles(world, "t3code-claude-cccc0000.scope", 300); + return yield* statusAfter(2); + }), + ); + assert.deepInclude(status.last, { pieces: 1, owedMb: 468 }); + }), + ); + + it.effect("slots off: a dispatch that ends with no child stops counting at once", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + observe({ startGraceSeconds: 30 }), + Effect.gen(function* () { + yield* release(yield* acquire); + yield* TestClock.adjust(Duration.seconds(1)); + yield* acquire; + return yield* statusAfter(2); + }), + ); + assert.deepInclude(status.last, { pieces: 0 }); + }), + ); + + it.effect("held slots in the wait store count as the gate counts them (a restart, #353)", () => + Effect.gen(function* () { + const HOLD = { key: "w_held", threadId: "waiter", untilMs: 60_000 }; + const cases = [ + // After a restart the store has the only copy of a delivered slot wait's hold. + { name: "slots off", agentSlots: {}, holds: [HOLD], caller, pieces: 1, reads: 1 }, + { + name: "slots on", + agentSlots: { enabled: true, maxPerMachine: 5 }, + holds: [HOLD], + caller, + pieces: 1, + // The gate's own decision reads it too. + reads: 2, + }, + // The holder's own delegation uses its hold up: counted once, as the new start. + { + name: "the holder", + agentSlots: {}, + holds: [HOLD], + caller: "waiter", + pieces: 0, + reads: 1, + }, + { + name: "an ended hold", + agentSlots: {}, + holds: [{ ...HOLD, untilMs: 0 }], + caller, + pieces: 0, + reads: 1, + }, + { name: "an empty store", agentSlots: {}, holds: [], caller, pieces: 0, reads: 1 }, + // Waits off: the gate counts no hold, and nothing of the store is read. + { + name: "waits off", + agentSlots: {}, + holds: [HOLD], + caller, + pieces: 0, + reads: 0, + waitsOff: true, + }, + ]; + for (const entry of cases) { + const world = newWorld(); + let reads = 0; + const status = yield* run( + world, + { + agentSlots: { mode: "observe", ...entry.agentSlots }, + waitFor: { enabled: entry.waitsOff !== true }, + }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* gate.attachReservations({ + read: Effect.sync(() => { + reads += 1; + return entry.holds; + }), + used: () => Effect.void, + }); + const answer = yield* gate.acquire({ + callerThreadId: ThreadId.make(entry.caller), + targetInstanceId: MAIN, + }); + assert.strictEqual(answer.type, "allow", entry.name); + return yield* statusAfter(1); + }), + ); + assert.strictEqual(status.last?.pieces, entry.pieces, entry.name); + assert.strictEqual(reads, entry.reads, entry.name); + } + }), + ); + + it.effect("a wait store read that hangs never delays the decision; the rule reads slow", () => + Effect.gen(function* () { + const world = newWorld(); + const status = yield* run( + world, + { ...observe(), waitFor: { enabled: true } }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* gate.attachReservations({ read: Effect.never, used: () => Effect.void }); + // No clock is moved before the answer: a decision that waited would never end here. + assert.strictEqual((yield* acquire).type, "allow"); + yield* settle; + yield* TestClock.adjust(OBSERVE_READ_TIMEOUT); + return yield* statusAfter(1); + }), + ); + assert.deepInclude(status.last, { rule: "refuse", reason: "unreadable", readings: "slow" }); }), ); @@ -474,7 +710,7 @@ describe("SlotGate observe mode", () => { observe(), Effect.gen(function* () { yield* TestClock.adjust(CALM_WAIT); - yield* acquire; + yield* release(yield* acquire); yield* TestClock.adjust(CALM_WAIT); const both = yield* Effect.all([acquire, acquire], { concurrency: "unbounded" }); assert.deepStrictEqual( diff --git a/apps/server/src/slots/SlotObserver.ts b/apps/server/src/slots/SlotObserver.ts index d1c6e55d4716d..b3642623b6351 100644 --- a/apps/server/src/slots/SlotObserver.ts +++ b/apps/server/src/slots/SlotObserver.ts @@ -38,6 +38,7 @@ import * as DateTime from "effect/DateTime"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as FiberSet from "effect/FiberSet"; +import * as Option from "effect/Option"; import * as Ref from "effect/Ref"; import * as Semaphore from "effect/Semaphore"; @@ -61,10 +62,21 @@ export interface SlotObserveInput { readonly today: ReadonlyArray; readonly atMs: number; readonly starting: ReadonlyArray; - readonly held: ReadonlyArray; + /** + * The live held slots as the gate counts them (memory and wait store), as of the decision. Read + * in the observer's fiber, so a slow wait store never delays the decision. + */ + readonly held: Effect.Effect>; + /** `acquire`'s caller: its own held slot is used up by the start it asks for (counted once). */ + readonly callerThreadId?: string | undefined; readonly startGraceMs: number; /** The gate keeps `starting` (slots on with a limit); else the rule keeps its own allows. */ readonly gateRecords: boolean; + /** + * While the gate keeps no records: the allowed start, under a token of its own that the gate's + * `started` and `release` hand on (`started`, `released` here). + */ + readonly ownStart?: StartingEntry | undefined; } /** systemd nests `a-b-c.slice` as `a.slice/a-b.slice/a-b-c.slice`. */ @@ -104,10 +116,12 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( let waiting = 0; let hanging = 0; /** - * While the gate keeps no records (slots off), the times of the agents allowed lately: each - * counts as starting for `startGraceMs`, the time a new agent takes to show a run. + * While the gate keeps no records (slots off), the agents allowed lately, kept as the gate keeps + * its own (`liveStarting`): each counts until its dispatch ends, then for `startGraceMs` unless + * its child shows a run (then it counts as that run, once). A dispatch that ends with no child + * stops counting at once. */ - let ownStarts: ReadonlyArray = []; + let ownStarts: ReadonlyArray = []; const readFacts = async (pieces: ReadonlyArray): Promise => { const machine = await readMachine(host); @@ -159,30 +173,50 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( const run = Effect.fn("SlotObserver.run")(function* ( input: SlotObserveInput, - ownStarting: number, + ownBefore: ReadonlyArray, ) { const running = yield* readRunning; + // Off the decision path; a store read past the time limit leaves the pieces unknown. + const heldRead = + running === undefined + ? Option.none() + : yield* Effect.timeoutOption(input.held, OBSERVE_READ_TIMEOUT); let pieces: ReadonlyArray | null = null; - if (running !== undefined) { + if (running !== undefined && Option.isSome(heldRead)) { + // The starts of before the decision as they are now, read with the runs of now: a child + // named or a dispatch ended while this waited is seen (else a child that shows its run + // meanwhile would count twice). + const before = new Set(ownBefore.map((entry) => entry.token)); + const own = ownStarts.filter((entry) => before.has(entry.token)); + // A child seen with a run counts as running from now on, as in the gate's own records. + ownStarts = ownStarts.filter( + (entry) => entry.childThreadId === undefined || !running.visible.has(entry.childThreadId), + ); const starting = liveStarting( - input.starting, + [...input.starting, ...own], running.visible, input.atMs, input.startGraceMs, ); - const held = onePerThread(liveReservations(input.held, input.atMs)); + const held = onePerThread( + liveReservations(heldRead.value, input.atMs).filter( + (reservation) => reservation.threadId !== input.callerThreadId, + ), + ); const scopes = yield* Effect.forEach([...running.visible], (threadId) => agentScope.ownScope === undefined ? Effect.undefined : agentScope.ownScope(threadId), ); pieces = [ ...scopes.map((scope) => ({ scope })), ...[...starting, ...held].map(() => ({ scope: undefined })), - ...Array.from({ length: ownStarting }, () => ({ scope: undefined })), ]; } const read = pieces === null - ? { facts: null, readings: "failed" as SlotReadings } + ? { + facts: null, + readings: (running === undefined ? "failed" : "slow") as SlotReadings, + } : yield* readings(pieces); const { decisions, pressure: next } = observeSlotAnswers(CAPACITY_RULE, { facts: read.facts, @@ -221,20 +255,17 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( // Kept at the call, before any skip, so a skipped allow still counts for the next ones. ownStarts = input.gateRecords ? [] - : ownStarts.filter((atMs) => atMs <= input.atMs && input.atMs - atMs < input.startGraceMs); - const ownStarting = ownStarts.length; - if (!input.gateRecords) { - ownStarts = [ - ...ownStarts, - ...input.today.filter((answer) => answer === "allow").map(() => input.atMs), - ]; + : liveStarting(ownStarts, new Set(), input.atMs, input.startGraceMs); + const ownBefore = ownStarts; + if (!input.gateRecords && input.ownStart !== undefined) { + ownStarts = [...ownStarts, input.ownStart]; } if (waiting >= OBSERVE_MAX_WAITING) { return Ref.update(status, (current) => ({ ...current, skipped: current.skipped + 1 })); } waiting += 1; return lock - .withPermits(1)(run(input, ownStarting)) + .withPermits(1)(run(input, ownBefore)) .pipe( Effect.catchCause((cause) => Effect.logWarning("agent slots: the capacity rule could not be observed", { cause }), @@ -244,13 +275,32 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( waiting -= 1; }), ), - FiberSet.run(fibers), + // Not on the caller's stack: it runs once the decision is answered, whatever it reads. + FiberSet.run(fibers, { startImmediately: false }), Effect.asVoid, ); }).pipe(Effect.ignoreCause); return { observe, + /** The gate's `started` with one of the rule's own tokens: the child that start names. */ + started: (token: number, childThreadId: string) => + Effect.sync(() => { + ownStarts = ownStarts.map((entry) => + entry.token === token ? { ...entry, childThreadId } : entry, + ); + }), + /** The gate's `release`: the dispatch ended (as the gate ends its own entries). */ + released: (token: number, atMs: number) => + Effect.sync(() => { + ownStarts = ownStarts.flatMap((entry) => + entry.token !== token + ? [entry] + : entry.childThreadId === undefined + ? [] + : [entry.endedAtMs === undefined ? { ...entry, endedAtMs: atMs } : entry], + ); + }), status: Ref.get(status), }; }); diff --git a/apps/server/src/waits/WaitService.slots.test.ts b/apps/server/src/waits/WaitService.slots.test.ts index 6db8d8abfd694..d9a3762a2eef7 100644 --- a/apps/server/src/waits/WaitService.slots.test.ts +++ b/apps/server/src/waits/WaitService.slots.test.ts @@ -578,6 +578,24 @@ describe("WaitService slot waits", () => { }), ); + it.effect("wait_list shows the capacity rule in observe mode only (#353)", () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW); + for (const mode of ["observe", "fixed"] as const) { + const world = newWorld(); + const store = disk(); + const send = fakeDelivery(); + const listed = yield* Effect.gen(function* () { + const { delegate, service } = yield* startService(store.fs, send.delivery); + assert.strictEqual(yield* delegate(T1), "allow"); + return yield* service.list({ threadId: T1, projectId: PROJECT }); + }).pipe(Effect.provide(serverLayer(world, slotsOn(2, { agentSlots: { mode } })))); + if (mode === "observe") assert.deepInclude(listed.slotRule, { mode: "observe" }); + else assert.isUndefined(listed.slotRule); + } + }), + ); + it.effect( "one thread holds at most one slot: repeated met slot waits do not hoard every free slot", () => From 485b92c0d2bd4e5ba4d5c32204b5c2cbb45b0e51 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 20:55:29 +0000 Subject: [PATCH 2/3] slots: drop the agentScope.ts marker line until its seam entry is on main Main's seam check reads the allow list from main, so a new line in an upstream file cannot be allowed by the same PR's list entry. The entry stays; the marker comes back in a follow-up once the entry is on main. Co-Authored-By: Claude Opus 5.5 --- apps/server/src/process/agentScope.ts | 1 - 1 file changed, 1 deletion(-) diff --git a/apps/server/src/process/agentScope.ts b/apps/server/src/process/agentScope.ts index 74f69a1d89f38..79408c8b7daf8 100644 --- a/apps/server/src/process/agentScope.ts +++ b/apps/server/src/process/agentScope.ts @@ -431,7 +431,6 @@ export const make = Effect.gen(function* () { stop, kill, clear, - // Fork: the thread's own scope, for the slot observer (slots/SlotObserver.ts). ownScope: (threadId) => Effect.sync(() => { const tracked = scopeByThread.get(threadId); From f57196fe8a03d90788014971f88037ad2ba79e56 Mon Sep 17 00:00:00 2001 From: scuttletoby <123843277+scuttletoby@users.noreply.github.com> Date: Tue, 17 Nov 2026 21:05:20 +0000 Subject: [PATCH 3/3] slots: drop the agentScope.ts marker line from its seam entry too The marker line left the file in 485b92c0; the list must match the file. It comes back with the line in #381. Co-Authored-By: Claude Opus 5.5 --- .github/fork-seams.json | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/fork-seams.json b/.github/fork-seams.json index 41c85b47b3730..62c7913a1e012 100644 --- a/.github/fork-seams.json +++ b/.github/fork-seams.json @@ -82,7 +82,6 @@ "apps/server/src/process/agentScope.ts": { "why": "The scope map is private to upstream's make(); the fork's ownScope (see packages/shared/src/AgentScope.ts) is a 7-line lookup in it.", "added": [ - "// Fork: the thread's own scope, for the slot observer (slots/SlotObserver.ts).", "ownScope: (threadId) =>", "Effect.sync(() => {", "const tracked = scopeByThread.get(threadId);",