diff --git a/apps/server/src/slots/SlotGate.ts b/apps/server/src/slots/SlotGate.ts index 0dd55c83a4b66..1e273f5f04465 100644 --- a/apps/server/src/slots/SlotGate.ts +++ b/apps/server/src/slots/SlotGate.ts @@ -4,7 +4,7 @@ * is memory only; slots held by delivered slot waits come from the wait store (`attachReservations`). */ import type { ProviderInstanceId, ThreadId } from "@t3tools/contracts"; -import type { SlotRuleStatus } from "@t3tools/contracts/waitFor"; +import type { SlotRuleStatus, WaitToolFailure } from "@t3tools/contracts/waitFor"; import { accountOnlyThreads, agentSlotsMode, @@ -50,12 +50,24 @@ export type SlotAcquire = /** Slots held by delivered slot waits, as the wait store keeps them (`reservedUntil`). */ export interface StoredReservations { - /** The live ones; read only while waits are on. */ - readonly read: Effect.Effect>; + /** + * The live ones; read only while waits are on. Fails when the store (or the settings) cannot be + * read: that is never an empty store. + */ + readonly read: Effect.Effect, WaitToolFailure>; /** The gate let the holder start an agent: clear the wait's `reservedUntil` (a write). */ readonly used: (key: string) => Effect.Effect; } +/** + * The live held slots, one read. `whole: false`: the wait store could not be read, so `held` is the + * gate's memory only (a lower count, never a good read). + */ +interface HeldRead { + readonly held: ReadonlyArray; + readonly whole: boolean; +} + /** One slot wait asking for a slot; `key` is its wait id. */ export interface SlotRequest { readonly key: string; @@ -74,7 +86,7 @@ export type SlotGrant = /** An earlier request of the same thread was granted in this call: asked again next time. */ | { readonly type: "sameThread" } | { readonly type: "full"; readonly decision: Extract } - /** The running agents cannot be counted now: never "free". */ + /** The running agents or the held slots cannot be counted now: never "free". */ | { readonly type: "unread"; readonly message: string }; /** What `wait_list` shows while slots are on (agents running, starting or held). */ @@ -126,7 +138,10 @@ export class SlotGate extends Context.Service< * the thread held before counts again. */ readonly drop: (keys: ReadonlyArray) => Effect.Effect; - /** For `wait_list`: null while slots are off, or when the running agents cannot be read. */ + /** + * For `wait_list`: null while slots are off, or when the running agents or the held slots + * cannot be read. + */ readonly usage: Effect.Effect; /** For `wait_list`: the capacity rule beside the gate; null unless the mode is `observe`. */ readonly observed: Effect.Effect; @@ -156,6 +171,12 @@ const make = Effect.gen(function* () { const hiddenKeys = () => new Set([...replacing.values()].flatMap((olds) => olds.map((old) => old.key))); let stored: StoredReservations | null = null; + /** + * Threads started while the wait store could not be read: the next good read uses their stored + * holds up and clears this. A counted grant needs a good read (or waits off, which clears this), + * so none comes in between; an uncounted one may, and counts against nothing. + */ + const unreadStarts = new Set(); /** * Threads with an active run, settled or not (settling a thread does not stop its run), nobody @@ -201,10 +222,9 @@ const make = Effect.gen(function* () { ); /** - * Every live held slot: granted in memory and kept in the wait store, one per wait (counted one - * per thread, `onePerThread`). None while waits are off (nothing of the store is read then). A - * store that cannot be read holds none. Taken now; the store's part is read when the returned - * effect runs, without the used and hidden keys of now (the capacity rule reads it later). + * Every live held slot (memory and wait store, one per wait); none while waits are off. A store + * that cannot be read gives `whole: false` (memory only). Taken now; the store's part is read when + * the returned effect runs, without the used and hidden keys of now (the rule reads it later). */ const heldAt = Effect.fn("SlotGate.heldAt")(function* (waitsOn: boolean, nowMs: number) { for (const [key, untilMs] of usedKeys) if (untilMs <= nowMs) usedKeys.delete(key); @@ -213,14 +233,30 @@ const make = Effect.gen(function* () { if (olds.every((old) => old.untilMs <= nowMs)) replacing.delete(key); } const memory = yield* Ref.updateAndGet(held, (current) => liveReservations(current, nowMs)); - if (!waitsOn) return Effect.succeed>([]); + if (!waitsOn) { + // Stored holds do not count while waits are off; a start then never uses them up either. + unreadStarts.clear(); + return Effect.succeed({ held: [], whole: true }); + } const skipped = new Set([...usedKeys.keys(), ...hiddenKeys()]); const source = stored; + const live = (reservations: ReadonlyArray) => + liveReservations(reservations, nowMs).filter((reservation) => !skipped.has(reservation.key)); return (source === null ? Effect.succeed([]) : source.read).pipe( - Effect.map((fromStore) => - liveReservations([...memory, ...fromStore], nowMs).filter( - (reservation) => !skipped.has(reservation.key), - ), + Effect.map((fromStore): HeldRead => { + // A start allowed while the store was unread uses its thread's stored holds up now. + for (const reservation of fromStore) { + if (!unreadStarts.has(reservation.threadId)) continue; + usedKeys.set(reservation.key, reservation.untilMs); + skipped.add(reservation.key); + } + unreadStarts.clear(); + return { held: live([...memory, ...fromStore]), whole: true }; + }), + Effect.catch((cause) => + Effect.logWarning("agent slots: the slots held in the wait store cannot be read", { + cause, + }).pipe(Effect.as({ held: live(memory), whole: false })), ), ); }); @@ -282,7 +318,11 @@ const make = Effect.gen(function* () { }, ) { const observe = yield* observing; - if (Option.isNone(observe)) return yield* decision; + if (Option.isNone(observe)) { + // A decision outside observe mode may grant a hold the rule's notes must never take. + yield* observer.outside; + return yield* decision; + } const { startGraceMs, gateRecords, waitsOn } = observe.value; const atMs = yield* Clock.currentTimeMillis; const starting = yield* Ref.get(entries); @@ -298,7 +338,7 @@ const make = Effect.gen(function* () { today: answers(result), atMs, starting, - held: heldBefore, + held: Effect.map(heldBefore, (read) => (read.whole ? read.held : undefined)), callerThreadId: caller?.threadId, startGraceMs, gateRecords, @@ -379,7 +419,10 @@ const make = Effect.gen(function* () { const read = counted ? yield* readRunning : undefined; const running = read?.running.filter((agent) => agent.threadId !== input.callerThreadId); const kept = yield* keepStarting(read?.visible, nowMs, limits); - const reservations = yield* liveHeld(waitsOn, nowMs); + // A store that cannot be read: its holds go uncounted (memory only), as unread counts never + // stop a delegation; the caller's stored holds are used up at the next good read. + const heldRead = yield* liveHeld(waitsOn, nowMs); + const reservations = heldRead.held; const decision = slotDecision({ limits: running === undefined ? { ...limits, machine: null, account: () => null } : limits, running: running ?? [], @@ -401,6 +444,7 @@ const make = Effect.gen(function* () { ]); // The agent it held a slot for starts now: its held slot is used up, with every older copy. const used = yield* useUp(reservations, input.callerThreadId); + if (!heldRead.whole) unreadStarts.add(input.callerThreadId); return asDecided({ type: "allow", token }, used); }, (decided, input) => @@ -447,7 +491,8 @@ const make = Effect.gen(function* () { const kept = yield* keepStarting(read?.visible, nowMs, limits); const memoryBytes = yield* freeMemoryBytes(limits); const reserveMs = settings.value.agentSlots.reserveSeconds * 1000; - let reservations = yield* liveHeld(settings.value.waitFor.enabled, nowMs); + const heldRead = yield* liveHeld(settings.value.waitFor.enabled, nowMs); + let reservations = heldRead.held; const grants: Array = []; const grantedThreads = new Set(); for (const request of requests) { @@ -457,8 +502,14 @@ const make = Effect.gen(function* () { continue; } const own = limitsFor(request); - if (read === undefined && limitsCount(own, request.account ?? "")) { - grants.push({ type: "unread", message: "The running agents cannot be counted now." }); + if ((read === undefined || !heldRead.whole) && limitsCount(own, request.account ?? "")) { + grants.push({ + type: "unread", + message: + read === undefined + ? "The running agents cannot be counted now." + : "The held slots cannot be counted now.", + }); continue; } const decision = slotDecision({ @@ -553,7 +604,9 @@ const make = Effect.gen(function* () { limits === null ? [] : liveStarting(yield* Ref.get(entries), read.visible, nowMs, limits.startGraceMs); - const reservations = onePerThread(yield* liveHeld(settings.value.waitFor.enabled, nowMs)); + const heldRead = yield* liveHeld(settings.value.waitFor.enabled, nowMs); + if (!heldRead.whole) return null; + const reservations = onePerThread(heldRead.held); // Holds hidden by a re-ask not kept yet: on their own account only, as `decide` counts them. const hidden = hiddenHeld(settings.value.waitFor.enabled, nowMs, ""); const accountMax = (id: string) => { diff --git a/apps/server/src/slots/SlotObserver.test.ts b/apps/server/src/slots/SlotObserver.test.ts index 0043a2e17fb63..f45d1b5b61dac 100644 --- a/apps/server/src/slots/SlotObserver.test.ts +++ b/apps/server/src/slots/SlotObserver.test.ts @@ -11,12 +11,12 @@ import { type ServerSettings as ServerSettingsValue, ThreadId, } from "@t3tools/contracts"; -import type { SlotRuleStatus } from "@t3tools/contracts/waitFor"; +import { type SlotRuleStatus, WaitToolFailure } from "@t3tools/contracts/waitFor"; import { AgentScope, type AgentScopeShape } from "@t3tools/shared/AgentScope"; import type { CapacityFileRead, CapacityHost } from "@t3tools/shared/capacityReadings"; import * as HostProcess from "@t3tools/shared/HostProcess"; import type { DeepPartial } from "@t3tools/shared/Struct"; -import { DISPATCH_SAFETY_MS } from "@t3tools/shared/agentSlots"; +import { DISPATCH_SAFETY_MS, type SlotReservation } from "@t3tools/shared/agentSlots"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -176,18 +176,25 @@ const acquireAndRelease = Effect.tap(acquire, release).pipe(Effect.map((answer) /** Lets forked observations and the fake host's promises finish (scheduler turns, no real wait). */ const settle = Effect.forEach(Array.from({ length: 20 }), () => Effect.yieldNow, { discard: true }); -/** The status once `observed + skipped` reached `count`. */ -const statusAfter = (count: number) => +/** `status` once `observed + skipped` reached `count` (scheduler turns only, at most 200). */ +const counted = ( + status: Effect.Effect, + count: number, + name = "", +) => Effect.gen(function* () { - const gate = yield* SlotGate.SlotGate; for (let turn = 0; turn < 200; turn += 1) { - const status = yield* gate.observed; - if (status !== null && status.observed + status.skipped >= count) return status; + const current = yield* status; + if (current !== null && current.observed + current.skipped >= count) return current; yield* settle; } - return yield* Effect.die(new Error(`fewer than ${count} observations`)); + return yield* Effect.die(new Error(`${name}fewer than ${count} observations`)); }); +/** The gate's status once `observed + skipped` reached `count`. */ +const statusAfter = (count: number) => + SlotGate.SlotGate.pipe(Effect.flatMap((gate) => counted(gate.observed, count))); + const observe = (patch: Record = {}) => ({ agentSlots: { mode: "observe", ...patch }, }); @@ -827,15 +834,7 @@ describe("SlotGate observe mode", () => { gateRecords: false, ...patch, }); - const done = (count: number) => - Effect.gen(function* () { - for (let turn = 0; turn < 200; turn += 1) { - const status = yield* observer.status; - if (status.observed + status.skipped >= count) return status; - yield* settle; - } - return yield* Effect.die(new Error(`${name}: fewer than ${count} observations`)); - }); + const done = (count: number) => counted(observer.status, count, `${name}: `); let decisions = 0; if (entry.own === "skipped") { // Other threads' observations fill the queue; the first one's store read takes 1 s. @@ -973,10 +972,7 @@ describe("SlotGate observe mode", () => { ...input, }); decided += 1; - for (let turn = 0; (yield* observer.status).observed < decided; turn += 1) { - if (turn === 200) return yield* Effect.die(new Error("not observed")); - yield* settle; - } + yield* counted(observer.status, decided); }); // Slots off: the holder starts (a dispatch that never ends: a lost release). yield* decide({ @@ -1038,10 +1034,7 @@ describe("SlotGate observe mode", () => { slowDecided += 1; yield* settle; yield* TestClock.adjust(OBSERVE_READ_TIMEOUT); - for (let turn = 0; (yield* slow.status).observed < slowDecided; turn += 1) { - if (turn === 200) return yield* Effect.die(new Error("not observed")); - yield* settle; - } + yield* counted(slow.status, slowDecided); }); const startAt = (atMs: number, token: number) => ({ atMs, @@ -1058,6 +1051,128 @@ describe("SlotGate observe mode", () => { }).pipe(Effect.scoped), ); + it.effect("a wait store that cannot be read is no good read: the start's note stays (#415)", () => + Effect.gen(function* () { + const world = newWorld(); + const HOLD = { key: "w_held", threadId: caller, untilMs: 100_000 }; + let failing = true; + const statuses = yield* run( + world, + { ...observe(), waitFor: { enabled: true } }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* gate.attachReservations({ + read: Effect.suspend(() => + failing + ? Effect.fail( + new WaitToolFailure({ code: "store_unavailable", message: "newer version" }), + ) + : Effect.succeed([HOLD]), + ), + used: () => Effect.void, + }); + // The holder starts while the store cannot be read; its dispatch keeps running. + assert.strictEqual((yield* acquire).type, "allow"); + const unread = yield* statusAfter(1); + failing = false; + // Another thread's decision: the start counts once, its hold is used up (not both). + yield* gate.acquire({ callerThreadId: ThreadId.make("other"), targetInstanceId: MAIN }); + return [unread, yield* statusAfter(2)] as const; + }), + ); + assert.deepInclude(statuses[0].last, { readings: "failed", pieces: null }); + assert.deepInclude(statuses[1].last, { readings: "ok", pieces: 1 }); + }), + ); + + it.effect( + "a decision outside observe mode drops the notes: a later hold still counts (#415)", + () => + Effect.gen(function* () { + const world = newWorld(); + let hang = true; + let inStore: ReadonlyArray = []; + const status = yield* run( + world, + { ...observe(), waitFor: { enabled: true } }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + const settings = yield* ServerSettings.ServerSettingsService; + yield* gate.attachReservations({ + read: Effect.suspend(() => (hang ? Effect.never : Effect.succeed(inStore))), + used: () => Effect.void, + }); + // Slots off: the caller starts (a note), its store read hangs so no read takes the + // note, and its dispatch ends with no child. + yield* release(yield* acquire); + yield* settle; + yield* TestClock.adjust(OBSERVE_READ_TIMEOUT); + yield* statusAfter(1); + // Fixed mode with a limit: the same thread is granted a new hold, which is written. + hang = false; + yield* settings.updateSettings({ + agentSlots: { mode: "fixed", enabled: true, maxPerMachine: 5 }, + }); + const grants = yield* gate.grant([{ key: "w_new", threadId: caller }]); + assert.deepStrictEqual( + grants.map((grant) => grant.type), + ["granted"], + ); + yield* gate.keep(["w_new"]); + inStore = [ + { + key: "w_new", + threadId: caller, + untilMs: (grants[0] as { untilMs: number }).untilMs, + }, + ]; + // Back to observe with slots off, well inside the note's 10 minutes. + yield* settings.updateSettings({ agentSlots: { mode: "observe", enabled: false } }); + yield* gate.acquire({ callerThreadId: ThreadId.make("other"), targetInstanceId: MAIN }); + return yield* statusAfter(2); + }), + ); + // The new hold was granted after the start: it counts. + assert.deepInclude(status.last, { readings: "ok", pieces: 1 }); + }), + ); + + it.effect("a second start of the same thread renews its note's time (#415)", () => + Effect.gen(function* () { + const world = newWorld(); + const slow = yield* observerFor(world); + let decided = 0; + const decideSlow = (input: Partial & { atMs: number }) => + Effect.gen(function* () { + yield* slow.observe({ + today: ["allow"], + starting: [], + held: Effect.never, + startGraceMs: 30_000, + gateRecords: false, + ...input, + }); + decided += 1; + yield* settle; + yield* TestClock.adjust(OBSERVE_READ_TIMEOUT); + yield* counted(slow.status, decided); + }); + const startAt = (atMs: number, token: number) => ({ + atMs, + callerThreadId: "waiter", + ownStart: { token, instanceId: MAIN, atMs }, + }); + // A hold may outlive the note's limit (`reserveSeconds` over 600): the later start's note + // must still take it. + yield* decideSlow(startAt(0, 1)); + yield* decideSlow(startAt(DISPATCH_SAFETY_MS / 2, 2)); + yield* decideSlow({ atMs: DISPATCH_SAFETY_MS }); + assert.strictEqual((yield* slow.kept).notes, 1, "renewed"); + yield* decideSlow({ atMs: DISPATCH_SAFETY_MS * 1.5 }); + assert.strictEqual((yield* slow.kept).notes, 0, "ended"); + }).pipe(Effect.scoped), + ); + it("nests a systemd slice the way systemd does", () => { assert.strictEqual( slicePath("app-t3code-agents-0123.slice"), diff --git a/apps/server/src/slots/SlotObserver.ts b/apps/server/src/slots/SlotObserver.ts index cce3db0e040e9..dbf5322045833 100644 --- a/apps/server/src/slots/SlotObserver.ts +++ b/apps/server/src/slots/SlotObserver.ts @@ -64,12 +64,12 @@ export interface SlotObserveInput { readonly atMs: number; readonly starting: ReadonlyArray; /** - * The live held slots as the gate counts them: memory as of the decision, the wait store as of - * when the observation runs (read in the observer's fiber, so a slow store never delays the - * decision). An observation that waits behind others can miss a hold a later decision used up - * meanwhile, or count one granted after its decision: one agent off, for a few seconds. + * The live held slots as the gate counts them: memory as of the decision, the wait store when the + * observation runs (off the decision path). A queued observation can be one agent off for a few + * seconds (a hold used up or granted meanwhile). `undefined`: the store could not be read, which + * is never a good read. */ - readonly held: Effect.Effect>; + readonly held: Effect.Effect | undefined>; /** `acquire`'s caller: its own held slot is used up by the start it asks for (counted once). */ readonly callerThreadId?: string | undefined; readonly startGraceMs: number; @@ -135,7 +135,8 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( /** * Slots off: threads allowed a start, noted at the decision (before any skip). The next good store * read uses their holds up, whatever became of the start meanwhile, and drops the notes. Slots on - * drops them all; a note no read took goes after `DISPATCH_SAFETY_MS`. + * drops them all, as does any decision outside observe mode (`outside`: it may grant a hold the + * note must not take); a note no read took goes `DISPATCH_SAFETY_MS` after the thread's last start. */ const startedThreads = new Map(); @@ -197,8 +198,10 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( running === undefined ? Option.none() : yield* Effect.timeoutOption(input.held, OBSERVE_READ_TIMEOUT); + // A store that could not be read is not an empty one: the pieces are unknown, the notes stay. + const storeFailed = Option.isSome(heldRead) && heldRead.value === undefined; let pieces: ReadonlyArray | null = null; - if (running !== undefined && Option.isSome(heldRead)) { + if (running !== undefined && Option.isSome(heldRead) && heldRead.value !== undefined) { // The starts of before the decision as they are now, read with the runs of now: a child // named or a dispatch ended while this waited is seen (else a child that shows its run // meanwhile would count twice). @@ -242,7 +245,7 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( pieces === null ? { facts: null, - readings: (running === undefined ? "failed" : "slow") as SlotReadings, + readings: (running === undefined || storeFailed ? "failed" : "slow") as SlotReadings, } : yield* readings(pieces); const { decisions, pressure: next } = observeSlotAnswers(CAPACITY_RULE, { @@ -339,6 +342,10 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( ); }), status: Ref.get(status), + /** The gate decided outside observe mode (fixed, or settings unread): the notes go. */ + outside: Effect.sync(() => { + startedThreads.clear(); + }), /** What the rule keeps in memory (for tests: nothing past the point it stops counting). */ kept: Effect.sync(() => ({ starts: ownStarts.length, diff --git a/apps/server/src/waits/WaitService.slots.test.ts b/apps/server/src/waits/WaitService.slots.test.ts index d9a3762a2eef7..ca762e3fe2790 100644 --- a/apps/server/src/waits/WaitService.slots.test.ts +++ b/apps/server/src/waits/WaitService.slots.test.ts @@ -498,6 +498,59 @@ describe("WaitService slot waits", () => { }), ); + it.effect( + "a store that cannot be loaded is not an empty one: no grant, used up later (#415)", + () => + Effect.gen(function* () { + yield* TestClock.setTime(NOW); + const world = newWorld(); + const store = disk(); + // A newer build's file: never read, so T1's held slot in it cannot be seen. + store.files.set(FILE, JSON.stringify({ version: 2, waits: [] })); + const send = fakeDelivery(); + yield* Effect.gen(function* () { + const { gate, delegate } = yield* startService(store.fs, send.delivery); + // Nothing runs, but the held slots cannot be counted: never "free", no usage shown. + const unread = yield* gate.grant([{ key: "w_cccccccccc", threadId: T2 }]); + assert.deepStrictEqual( + unread.map((grant) => grant.type), + ["unread"], + ); + assert.isNull(yield* gate.usage); + // T1 starts the agent its slot was held for, while the store still cannot be read. + const allowed = yield* gate.acquire({ callerThreadId: T1, targetInstanceId: MAIN }); + assert.strictEqual(allowed.type, "allow"); + // The store reads again, with T1's held slot in it; T1's dispatch ends with no child. + store.files.set( + FILE, + JSON.stringify({ + version: 1, + waits: [ + { + id: "w_aaaaaaaaaa", + threadId: T1, + projectId: PROJECT, + key: "000000000000000a", // gitleaks:allow (a fake test value, not a key) + createdAt: "2026-11-14T11:00:00.000Z", + deadlineAt: "2026-11-14T13:00:00.000Z", + state: "delivered", + endedAt: "2026-11-14T11:59:50.000Z", + attempt: 0, + kind: "slot", + spec: {}, + result: { outcome: "free", detail: "", at: "2026-11-14T11:59:50.000Z" }, + reservedUntil: "2026-11-14T12:00:50.000Z", + }, + ], + }), + ); + yield* gate.release(allowed.type === "allow" ? allowed.token : null); + // T1's start used its slot up: the slot is free for another thread, not held twice. + assert.strictEqual(yield* delegate(OUTSIDER), "allow"); + }).pipe(Effect.provide(serverLayer(world, slotsOn(1)))); + }), + ); + it.effect("waits off (the default): no store read or written, its held slots do not count", () => Effect.gen(function* () { yield* TestClock.setTime(NOW); diff --git a/apps/server/src/waits/WaitService.ts b/apps/server/src/waits/WaitService.ts index b1cc85c426786..d440e07f93312 100644 --- a/apps/server/src/waits/WaitService.ts +++ b/apps/server/src/waits/WaitService.ts @@ -672,15 +672,16 @@ export const make = Effect.fn("WaitService.make")(function* (input: { : slotJudgement(grant, settings, true); }); - /** The slots delivered slot waits still hold, for the gate (it reads them only while waits are on). */ + /** + * The slots delivered slot waits still hold, for the gate (it reads them only while waits are on). + * Fails when the store or the settings cannot be read: never an empty store then. + */ const heldInStore = lock.withPermits(1)( Effect.gen(function* () { - const loaded = yield* ensureLoaded(true); - if (!loaded.ok) return []; - const settings = yield* readSettingsOrNull; - if (settings === null) return []; + const contents = yield* loadedOrFail(true); + const settings = yield* readSettings; const now = yield* Clock.currentTimeMillis; - return loaded.contents.waits.flatMap((wait) => + return contents.waits.flatMap((wait) => storedReservation(wait, now, settings.agentSlots.reserveSeconds), ); }),