diff --git a/apps/server/src/slots/SlotObserver.test.ts b/apps/server/src/slots/SlotObserver.test.ts index 055f505aae452..1fec049554691 100644 --- a/apps/server/src/slots/SlotObserver.test.ts +++ b/apps/server/src/slots/SlotObserver.test.ts @@ -16,6 +16,7 @@ import { AgentScope, type AgentScopeShape } from "@t3tools/shared/AgentScope"; import type { CapacityFileRead, CapacityHost } from "@t3tools/shared/capacityReadings"; import * as HostProcess from "@t3tools/shared/HostProcess"; import type { DeepPartial } from "@t3tools/shared/Struct"; +import { DISPATCH_SAFETY_MS } from "@t3tools/shared/agentSlots"; import * as Duration from "effect/Duration"; import * as Effect from "effect/Effect"; import * as Layer from "effect/Layer"; @@ -26,10 +27,12 @@ import * as HostResources from "../resourceTelemetry/HostResources.ts"; import * as ServerSettings from "../serverSettings.ts"; import * as SlotGate from "./SlotGate.ts"; import { + makeSlotObserver, OBSERVE_MAX_HANGING, OBSERVE_MAX_WAITING, OBSERVE_READ_TIMEOUT, SlotCapacityHost, + type SlotObserveInput, slicePath, } from "./SlotObserver.ts"; @@ -167,6 +170,9 @@ const release = (answer: SlotGate.SlotAcquire) => Effect.flatMap((gate) => gate.release(answer.type === "allow" ? answer.token : null)), ); +/** A dispatch that ends with no child; its answer. */ +const acquireAndRelease = Effect.tap(acquire, release).pipe(Effect.map((answer) => answer.type)); + /** Lets forked observations and the fake host's promises finish (scheduler turns, no real wait). */ const settle = Effect.forEach(Array.from({ length: 20 }), () => Effect.yieldNow, { discard: true }); @@ -682,6 +688,89 @@ describe("SlotGate observe mode", () => { }), ); + it.effect("a held slot its holder's start used up counts once, slots off as on (#381)", () => + Effect.gen(function* () { + const HOLD = { key: "w_held", threadId: "waiter", untilMs: 100_000 }; + const waiter = ThreadId.make("waiter"); + const cases = [ + { name: "slots on", agentSlots: { enabled: true, maxPerMachine: 5 }, slowFirst: false }, + // Slots off: the gate never uses the hold up, so it stays in the store. + { name: "slots off", agentSlots: {}, slowFirst: false }, + // The holder's own observation reads the store too late: the next one still sees the start. + { name: "slots off, first read slow", agentSlots: {}, slowFirst: true }, + ]; + for (const entry of cases) { + const world = newWorld(); + let reads = 0; + const counts = yield* run( + world, + { + agentSlots: { mode: "observe", startGraceSeconds: 30, ...entry.agentSlots }, + waitFor: { enabled: true }, + }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* gate.attachReservations({ + read: Effect.suspend(() => { + reads += 1; + return entry.slowFirst && reads === 1 ? Effect.never : Effect.succeed([HOLD]); + }), + used: () => Effect.void, + }); + const answers: Array = []; + // The holder delegates; another thread delegates 5 s later. + const own = yield* gate.acquire({ callerThreadId: waiter, targetInstanceId: MAIN }); + answers.push(own.type); + if (entry.slowFirst) { + yield* settle; + yield* TestClock.adjust(OBSERVE_READ_TIMEOUT); + yield* statusAfter(1); + } + yield* TestClock.adjust(Duration.seconds(5)); + answers.push(yield* acquireAndRelease); + const started = (yield* statusAfter(2)).last?.pieces; + // Its child shows its run: counted as that run, the hold still used up. + const token = own.type === "allow" ? own.token : null; + yield* gate.started(token, ThreadId.make("child")); + world.threads = [{ id: "child" }]; + yield* TestClock.adjust(Duration.seconds(5)); + answers.push(yield* acquireAndRelease); + const running = (yield* statusAfter(3)).last?.pieces; + // The child's run ends: nothing of it counts, nor the hold it used. + yield* gate.release(token); + world.threads = []; + yield* TestClock.adjust(Duration.seconds(5)); + answers.push(yield* acquireAndRelease); + const ended = (yield* statusAfter(4)).last?.pieces; + // Observe data only: every answer stays today's allow. + assert.deepStrictEqual(answers, ["allow", "allow", "allow", "allow"], entry.name); + return [started, running, ended]; + }), + ); + assert.deepStrictEqual(counts, [1, 1, 0], entry.name); + } + // The holder's dispatch ends with no child before anyone else asks: nothing counts. + for (const agentSlots of [{ enabled: true, maxPerMachine: 5 }, {}]) { + const status = yield* run( + newWorld(), + { agentSlots: { mode: "observe", ...agentSlots }, waitFor: { enabled: true } }, + Effect.gen(function* () { + const gate = yield* SlotGate.SlotGate; + yield* gate.attachReservations({ + read: Effect.succeed([HOLD]), + used: () => Effect.void, + }); + yield* release(yield* gate.acquire({ callerThreadId: waiter, targetInstanceId: MAIN })); + yield* TestClock.adjust(Duration.seconds(5)); + yield* acquireAndRelease; + return yield* statusAfter(2); + }), + ); + assert.strictEqual(status.last?.pieces, 0, JSON.stringify(agentSlots)); + } + }), + ); + it.effect("a wait store read that hangs never delays the decision; the rule reads slow", () => Effect.gen(function* () { const world = newWorld(); @@ -755,6 +844,80 @@ describe("SlotGate observe mode", () => { }), ); + it.effect("the rule's memory keeps only what still counts (#381)", () => + Effect.gen(function* () { + const world = newWorld(); + const HOLD = { key: "w_held", threadId: "waiter", untilMs: 60_000 }; + const observer = yield* makeSlotObserver(Effect.succeed({ visible: new Set() })).pipe( + Effect.provide( + Layer.mergeAll( + Layer.succeed(SlotCapacityHost, hostOf(world)), + Layer.succeed(AgentScope, scopesOf(world)), + Layer.succeed(HostProcess.UserId, UID), + Layer.succeed(HostProcess.Platform, "linux"), + ), + ), + ); + let decided = 0; + /** One decision, observed to the end. */ + const decide = (input: Partial & { atMs: number }) => + Effect.gen(function* () { + yield* observer.observe({ + today: ["allow"], + starting: [], + held: Effect.succeed([HOLD]), + startGraceMs: 30_000, + gateRecords: false, + ...input, + }); + decided += 1; + for (let turn = 0; (yield* observer.status).observed < decided; turn += 1) { + if (turn === 200) return yield* Effect.die(new Error("not observed")); + yield* settle; + } + }); + // Slots off: the holder starts (a dispatch that never ends: a lost release). + yield* decide({ + atMs: 0, + callerThreadId: "waiter", + ownStart: { token: 1, instanceId: MAIN, atMs: 0 }, + }); + assert.deepStrictEqual(yield* observer.kept, { starts: 1, usedHolds: 1 }, "kept"); + // Past its hold's end and the dispatch's safety limit: the next decision lets both go. + yield* decide({ atMs: DISPATCH_SAFETY_MS, held: Effect.succeed([]) }); + assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "ended"); + // Slots switched on (the gate keeps the records): the rule's own go at once. + yield* decide({ + atMs: DISPATCH_SAFETY_MS + 1, + callerThreadId: "waiter", + ownStart: { token: 2, instanceId: MAIN, atMs: DISPATCH_SAFETY_MS + 1 }, + held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]), + }); + yield* decide({ atMs: DISPATCH_SAFETY_MS + 2, gateRecords: true }); + assert.deepStrictEqual(yield* observer.kept, { starts: 0, usedHolds: 0 }, "slots on"); + // Switched on while a slots-off observation still waits: the slots-on one counts the hold + // as the gate does (the gate never used it up). + const ON = DISPATCH_SAFETY_MS + 3; + yield* observer.observe({ + today: ["allow"], + atMs: ON, + starting: [], + held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]), + callerThreadId: "waiter", + startGraceMs: 30_000, + gateRecords: false, + ownStart: { token: 3, instanceId: MAIN, atMs: ON }, + }); + decided += 1; + yield* decide({ + atMs: ON, + gateRecords: true, + held: Effect.succeed([{ ...HOLD, untilMs: 2 * DISPATCH_SAFETY_MS }]), + }); + assert.strictEqual((yield* observer.status).last?.pieces, 1, "switched on"); + }).pipe(Effect.scoped), + ); + it("nests a systemd slice the way systemd does", () => { assert.strictEqual( slicePath("app-t3code-agents-0123.slice"), diff --git a/apps/server/src/slots/SlotObserver.ts b/apps/server/src/slots/SlotObserver.ts index b3642623b6351..d63f194eede27 100644 --- a/apps/server/src/slots/SlotObserver.ts +++ b/apps/server/src/slots/SlotObserver.ts @@ -63,8 +63,10 @@ export interface SlotObserveInput { readonly atMs: number; readonly starting: ReadonlyArray; /** - * The live held slots as the gate counts them (memory and wait store), as of the decision. Read - * in the observer's fiber, so a slow wait store never delays the decision. + * The live held slots as the gate counts them: memory as of the decision, the wait store as of + * when the observation runs (read in the observer's fiber, so a slow store never delays the + * decision). An observation that waits behind others can miss a hold a later decision used up + * meanwhile, or count one granted after its decision: one agent off, for a few seconds. */ readonly held: Effect.Effect>; /** `acquire`'s caller: its own held slot is used up by the start it asks for (counted once). */ @@ -85,6 +87,9 @@ export function slicePath(slice: string): string { return parts.map((_, index) => `/${parts.slice(0, index + 1).join("-")}.slice`).join(""); } +/** One of the rule's own starts, and the thread that asked for it (its holds are used up). */ +type OwnStart = StartingEntry & { readonly threadId?: string | undefined }; + interface OwnPiece { readonly scope: { readonly unit: string; readonly slice: string } | undefined; } @@ -121,7 +126,14 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( * its child shows a run (then it counts as that run, once). A dispatch that ends with no child * stops counting at once. */ - let ownStarts: ReadonlyArray = []; + let ownStarts: ReadonlyArray = []; + /** + * While the gate keeps no records: held slots of the threads those starts were for, by key, until + * they end. A start uses its thread's hold up, as the gate's `useUp` does with slots on (with slots + * off the gate leaves it in the store), so the start and the hold never both count, not even once + * the start stops counting (its child shows its run, or its dispatch ends). + */ + const usedHolds = new Map(); const readFacts = async (pieces: ReadonlyArray): Promise => { const machine = await readMachine(host); @@ -198,9 +210,23 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( input.atMs, input.startGraceMs, ); + const live = liveReservations(heldRead.value, input.atMs); + if (!input.gateRecords) { + // Also the starts this decision counts (a skipped or slow observation of a start's own + // decision leaves its hold to the next one). + const startedThreads = new Set(own.map((entry) => entry.threadId)); + if (input.ownStart !== undefined) startedThreads.add(input.callerThreadId); + for (const reservation of live) { + if (startedThreads.has(reservation.threadId)) { + usedHolds.set(reservation.key, reservation.untilMs); + } + } + } const held = onePerThread( - liveReservations(heldRead.value, input.atMs).filter( - (reservation) => reservation.threadId !== input.callerThreadId, + live.filter( + (reservation) => + reservation.threadId !== input.callerThreadId && + (input.gateRecords || !usedHolds.has(reservation.key)), ), ); const scopes = yield* Effect.forEach([...running.visible], (threadId) => @@ -256,9 +282,12 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( ownStarts = input.gateRecords ? [] : liveStarting(ownStarts, new Set(), input.atMs, input.startGraceMs); + for (const [key, untilMs] of usedHolds) { + if (input.gateRecords || untilMs <= input.atMs) usedHolds.delete(key); + } const ownBefore = ownStarts; if (!input.gateRecords && input.ownStart !== undefined) { - ownStarts = [...ownStarts, input.ownStart]; + ownStarts = [...ownStarts, { ...input.ownStart, threadId: input.callerThreadId }]; } if (waiting >= OBSERVE_MAX_WAITING) { return Ref.update(status, (current) => ({ ...current, skipped: current.skipped + 1 })); @@ -302,5 +331,7 @@ export const makeSlotObserver = Effect.fn("makeSlotObserver")(function* ( ); }), status: Ref.get(status), + /** What the rule keeps in memory (for tests: nothing past the point it stops counting). */ + kept: Effect.sync(() => ({ starts: ownStarts.length, usedHolds: usedHolds.size })), }; }); diff --git a/packages/shared/src/agentSlots.ts b/packages/shared/src/agentSlots.ts index 33d921931f9b1..8faa1b04da11b 100644 --- a/packages/shared/src/agentSlots.ts +++ b/packages/shared/src/agentSlots.ts @@ -119,12 +119,12 @@ export interface StartingEntry { * never get more slots than are free (a lost end stops after `DISPATCH_SAFETY_MS`). After the * dispatch, a child that shows no run yet (a queued run) counts for the start grace. */ -export function liveStarting( - entries: ReadonlyArray, +export function liveStarting( + entries: ReadonlyArray, visibleRunIds: ReadonlySet, nowMs: number, startGraceMs: number, -): ReadonlyArray { +): ReadonlyArray { return entries.filter((entry) => { if (entry.childThreadId !== undefined && visibleRunIds.has(entry.childThreadId)) return false; return entry.endedAtMs === undefined