Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
178 changes: 177 additions & 1 deletion apps/server/src/slots/SlotGate.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,12 +12,14 @@ import {
ServerSettingsError,
ThreadId,
} from "@t3tools/contracts";
import type { AgentSlotsSettings } from "@t3tools/contracts/waitFor";
import { type AgentSlotsSettings, WaitToolFailure } from "@t3tools/contracts/waitFor";
import type { DeepPartial } from "@t3tools/shared/Struct";
import { DISPATCH_SAFETY_MS, type SlotReservation } from "@t3tools/shared/agentSlots";
import * as Deferred from "effect/Deferred";
import * as Effect from "effect/Effect";
import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as Logger from "effect/Logger";
import { TestClock } from "effect/testing";

import * as OrchestratorV2 from "../orchestration-v2/Orchestrator.ts";
Expand Down Expand Up @@ -773,6 +775,180 @@ describe("SlotGate", () => {
}),
);

/** A store whose read fails with `problem` while it is set; `used` clears a hold in it. */
const flakyStore = (holds: ReadonlyArray<SlotReservation>) => {
const store = {
problem: null as string | null,
holds,
used: [] as Array<string>,
/** Set: the next `used` call completes it, then hangs (a call cut off half-way). */
hangNext: null as Deferred.Deferred<void> | null,
reads: 0,
};
const attach = (gate: SlotGate.SlotGate["Service"]) =>
gate.attachReservations({
read: Effect.suspend(() => {
store.reads += 1;
return store.problem === null
? Effect.succeed(store.holds)
: Effect.fail(
new WaitToolFailure({ code: "store_unavailable", message: store.problem }),
);
}),
used: (key) =>
Effect.suspend(() => {
const hang = store.hangNext;
if (hang !== null) {
store.hangNext = null;
return Deferred.succeed(hang, undefined).pipe(Effect.andThen(Effect.never));
}
store.used.push(key);
store.holds = store.holds.filter((entry) => entry.key !== key);
return Effect.void;
}),
});
return { store, attach };
};

it.effect(
"a start while the store cannot be read: the next good read clears its hold in the store (#574)",
() =>
Effect.gen(function* () {
// Fixed mode: no observation reads the store beside the decisions.
const settings = waitsOn({ mode: "fixed", maxPerMachine: 2 });
for (const path of ["acquire", "grant", "usage"] as const) {
const world = newWorld();
const { store, attach } = flakyStore([
{ key: "stored", threadId: holder, untilMs: 90_000 },
]);
yield* run(
world,
settings,
Effect.gen(function* () {
const gate = yield* SlotGate.SlotGate;
yield* attach(gate);
store.problem = "a newer file";
const start = yield* acquireAs(holder);
assert.strictEqual(start.type, "allow");
// Its dispatch ends with no child: nothing of it is left starting.
yield* gate.release(start.type === "allow" ? start.token : null);
assert.deepStrictEqual(store.used, []);
store.problem = null;
// The next good read, on any path, uses the hold up and clears it in the store.
if (path === "acquire") assert.strictEqual((yield* acquireAs(other)).type, "allow");
if (path === "grant") {
assert.deepStrictEqual(types(yield* grant({ key: "w-other", threadId: other })), [
"granted",
]);
}
if (path === "usage") {
assert.deepStrictEqual((yield* gate.usage)?.machine, { used: 0, max: 2 });
}
assert.deepStrictEqual(store.used, ["stored"], path);
// Runs again: it is not cleared twice (another start clears only its own hold).
yield* gate.usage;
yield* acquireAs(other);
assert.deepStrictEqual(
store.used.filter((key) => key === "stored"),
["stored"],
path,
);
}),
);
// A restart (a new gate on the same store): the used hold does not count again.
yield* run(
world,
settings,
Effect.gen(function* () {
const gate = yield* SlotGate.SlotGate;
yield* attach(gate);
assert.deepStrictEqual((yield* gate.usage)?.machine, { used: 0, max: 2 }, path);
}),
);
}
}),
);

it.effect("a clear cut off half-way is tried again by the next call (#574)", () =>
Effect.gen(function* () {
const world = newWorld();
const { store, attach } = flakyStore([
{ key: "stored", threadId: holder, untilMs: 90_000 },
]);
yield* run(
world,
waitsOn({ mode: "fixed", maxPerMachine: 2 }),
Effect.gen(function* () {
const gate = yield* SlotGate.SlotGate;
yield* attach(gate);
store.problem = "a newer file";
assert.strictEqual((yield* acquireAs(holder)).type, "allow");
store.problem = null;
const clearing = yield* Deferred.make<void>();
store.hangNext = clearing;
// The good read uses the hold up; its clear hangs and the call is cut off.
const fiber = yield* Effect.forkChild(gate.usage);
yield* Deferred.await(clearing);
yield* Fiber.interrupt(fiber);
assert.deepStrictEqual(store.used, []);
// Hidden in memory meanwhile, and cleared by the next call.
assert.deepStrictEqual((yield* gate.usage)?.machine, { used: 1, max: 2 });
assert.deepStrictEqual(store.used, ["stored"]);
assert.deepStrictEqual(store.holds, []);
}),
);
}),
);

it.effect("a store that cannot be read is warned about once per reason (#574)", () =>
Effect.gen(function* () {
const world = newWorld();
const warnings: Array<string> = [];
const logger = Logger.make(({ logLevel, message }) => {
const text = String(Array.isArray(message) ? message[0] : message);
if (logLevel === "Warn") warnings.push(text);
});
const storeWarnings = () =>
warnings.filter((text) => text.includes("held in the wait store cannot be read")).length;
const { store, attach } = flakyStore([]);
yield* run(
world,
waitsOn({ mode: "fixed", maxPerMachine: 2 }),
Effect.gen(function* () {
const gate = yield* SlotGate.SlotGate;
yield* attach(gate);
store.problem = "a newer file";
yield* acquireAs(other);
yield* grant({ key: "w1", threadId: holder });
assert.isNull(yield* gate.usage);
yield* acquireAs(other);
assert.strictEqual(storeWarnings(), 1);
// Another reason: warned again.
store.problem = "a damaged file";
yield* acquireAs(other);
yield* acquireAs(other);
assert.strictEqual(storeWarnings(), 2);
// Read again, then the same reason as before: a new spell of failures, warned again.
store.problem = null;
yield* gate.usage;
store.problem = "a damaged file";
yield* gate.usage;
yield* gate.usage;
assert.strictEqual(storeWarnings(), 3);
// Waits off, then on again with the same reason: a new spell, warned again.
const settings = yield* ServerSettings.ServerSettingsService;
yield* settings.updateSettings({ waitFor: { enabled: false } });
yield* acquireAs(other);
yield* settings.updateSettings({ waitFor: { enabled: true } });
yield* gate.usage;
yield* gate.usage;
assert.strictEqual(storeWarnings(), 4);
assert.isAbove(store.reads, 7);
}).pipe(Effect.provide(Logger.layer([logger], { mergeWithExisting: false }))),
);
}),
);

it.effect("usage: null while slots are off; counts running, starting and held", () =>
Effect.gen(function* () {
const world = newWorld();
Expand Down
52 changes: 47 additions & 5 deletions apps/server/src/slots/SlotGate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -177,6 +177,15 @@ const make = Effect.gen(function* () {
* so none comes in between; an uncounted one may, and counts against nothing.
*/
const unreadStarts = new Set<string>();
/**
* Stored holds that a good read used up for `unreadStarts`, not yet cleared in the store. Cleared
* outside the semaphore by the next acquire, grant or usage (`clearUsedLater`), as `acquire`
* clears its own; a key leaves only after its write was tried, so a call cut off half-way leaves
* it for the next one (clearing twice changes nothing). Lost on a restart, like `unreadStarts`.
*/
const usedLater = new Set<string>();
/** Why the store last could not be read; warned once until the reason changes or a read works. */
let storeProblem: string | null = null;

/**
* Threads with an active run, settled or not (settling a thread does not stop its run), nobody
Expand Down Expand Up @@ -236,6 +245,7 @@ const make = Effect.gen(function* () {
if (!waitsOn) {
// Stored holds do not count while waits are off; a start then never uses them up either.
unreadStarts.clear();
storeProblem = null;
return Effect.succeed<HeldRead>({ held: [], whole: true });
}
const skipped = new Set([...usedKeys.keys(), ...hiddenKeys()]);
Expand All @@ -248,16 +258,22 @@ const make = Effect.gen(function* () {
for (const reservation of fromStore) {
if (!unreadStarts.has(reservation.threadId)) continue;
usedKeys.set(reservation.key, reservation.untilMs);
usedLater.add(reservation.key);
skipped.add(reservation.key);
}
unreadStarts.clear();
storeProblem = null;
return { held: live([...memory, ...fromStore]), whole: true };
}),
Effect.catch((cause) =>
Effect.logWarning("agent slots: the slots held in the wait store cannot be read", {
Effect.catch((cause) => {
const unread = Effect.succeed<HeldRead>({ held: live(memory), whole: false });
// Every decision and observation reads the store: one warning per reason, not per read.
if (cause.message === storeProblem) return unread;
storeProblem = cause.message;
return Effect.logWarning("agent slots: the slots held in the wait store cannot be read", {
cause,
}).pipe(Effect.as<HeldRead>({ held: live(memory), whole: false })),
),
}).pipe(Effect.andThen(unread));
}),
);
});

Expand Down Expand Up @@ -394,6 +410,23 @@ const make = Effect.gen(function* () {
),
);

/**
* Clears the stored holds a later good read used up (`usedLater`), outside the semaphore. Each
* key leaves the set once its write was tried (a failed write is logged; its copy ends by itself).
* With no store attached nothing is written and the keys still leave: such a hold would count
* again after a restart until it ends, the safe direction (fewer starts, never more).
*/
const clearUsedLater = Effect.suspend(() => {
const keys = [...usedLater];
return clearStored(keys).pipe(
Effect.andThen(
Effect.sync(() => {
for (const key of keys) usedLater.delete(key);
}),
),
);
});

/** A decision and the caller's held slots it used up (typed whole, for `observeBeside`). */
const asDecided = (outcome: SlotAcquire, used: ReadonlyArray<string>) => ({ outcome, used });

Expand Down Expand Up @@ -466,10 +499,11 @@ const make = Effect.gen(function* () {
// Cleared in the store after the decision, outside the semaphore; a failed write only means
// the stored copy ends by itself (it no longer counts here either way).
yield* clearStored(used);
yield* clearUsedLater;
return outcome;
});

const grant = Effect.fn("SlotGate.grant")(
const grantLocked = Effect.fn("SlotGate.grantLocked")(
function* (requests: ReadonlyArray<SlotRequest>) {
const settings = yield* readSettings;
const limits = Option.isNone(settings)
Expand Down Expand Up @@ -581,6 +615,12 @@ const make = Effect.gen(function* () {
semaphore.withPermits(1),
);

const grant = Effect.fn("SlotGate.grant")(function* (requests: ReadonlyArray<SlotRequest>) {
const grants = yield* grantLocked(requests);
yield* clearUsedLater;
return grants;
});

const keepLocked = Effect.fn("SlotGate.keepLocked")(function* (keys: ReadonlyArray<string>) {
const replaced: Array<string> = [];
for (const key of keys) {
Expand All @@ -605,6 +645,8 @@ const make = Effect.gen(function* () {
? []
: liveStarting(yield* Ref.get(entries), read.visible, nowMs, limits.startGraceMs);
const heldRead = yield* liveHeld(settings.value.waitFor.enabled, nowMs);
// Not under the semaphore: a hold this read used up is cleared in the store here.
yield* clearUsedLater;
if (!heldRead.whole) return null;
const reservations = onePerThread(heldRead.held);
// Holds hidden by a re-ask not kept yet: on their own account only, as `decide` counts them.
Expand Down
11 changes: 10 additions & 1 deletion apps/server/src/waits/WaitService.slots.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -499,7 +499,7 @@ describe("WaitService slot waits", () => {
);

it.effect(
"a store that cannot be loaded is not an empty one: no grant, used up later (#415)",
"a store that cannot be loaded is not an empty one: no grant, used up later and cleared (#415, #574)",
() =>
Effect.gen(function* () {
yield* TestClock.setTime(NOW);
Expand Down Expand Up @@ -547,6 +547,15 @@ describe("WaitService slot waits", () => {
yield* gate.release(allowed.type === "allow" ? allowed.token : null);
// T1's start used its slot up: the slot is free for another thread, not held twice.
assert.strictEqual(yield* delegate(OUTSIDER), "allow");
// Cleared in the file too, not only hidden in this gate's memory.
const used = stored(store.files).find((wait) => wait.id === "w_aaaaaaaaaa");
assert.strictEqual(used?.state, "delivered");
assert.isUndefined(used?.reservedUntil);
}).pipe(Effect.provide(serverLayer(world, slotsOn(1))));
// A restart: a new gate on the same file does not count T1's used slot again.
yield* Effect.gen(function* () {
const { delegate } = yield* startService(store.fs, send.delivery);
assert.strictEqual(yield* delegate(T2), "allow");
}).pipe(Effect.provide(serverLayer(world, slotsOn(1))));
}),
);
Expand Down
Loading