diff --git a/.env.production.example b/.env.production.example index 73055a4..d90d304 100644 --- a/.env.production.example +++ b/.env.production.example @@ -16,11 +16,13 @@ ACME_EMAIL=you@yourcompany.com COOKIE_DOMAIN=.content.yourcompany.com COOKIE_SAMESITE=lax -# Encrypts the API keys stored in the settings table. +# Encrypts the API keys stored in the settings table. REQUIRED in production — +# the API refuses to start without it. # -# MOVE THE EXISTING VALUE from your local .env. A different secret does not -# fail loudly — it makes every key already saved unreadable, which reads as -# "the key vanished". +# MOVE THE EXISTING VALUE from your local .env. Leaving this blank is caught at +# boot, but setting the WRONG value is not: a different secret does not fail +# loudly, it makes every key already saved unreadable, which reads on the page +# as "the key vanished". CONTENTOS_SECRET= # How the model is reached on a server. diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml deleted file mode 100644 index ce345f4..0000000 --- a/.github/workflows/check.yml +++ /dev/null @@ -1,29 +0,0 @@ -# Typecheck, lint, tests, and a production build — the same four things -# `npm run check` and `npm run build:check` do on a laptop. -# -# No database and no model: nothing here talks to Postgres or spends a credit. -# The tests are the pure ones in tests/, and the build only has to compile. -name: check - -on: - push: - branches: ["**"] - pull_request: - -jobs: - check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-node@v4 - with: - node-version: 24 - cache: npm - - run: npm ci - - run: npm ci --prefix api - - run: npm run check - # Into .next-check, so it can never be confused with a dev server's - # output — see next.config.ts. - - run: npm run build:check - # The API compiles on its own; the image is what a server runs. - - run: npm run build --prefix api diff --git a/.gitignore b/.gitignore index 2f0ba7c..04ff2e0 100644 --- a/.gitignore +++ b/.gitignore @@ -61,3 +61,14 @@ next-env.d.ts # the NestJS API has its own install and build output api/node_modules api/dist + +# the Tauri desktop app: Rust build output, and the ACL schemas tauri-build +# writes from tauri.conf.json on every compile. Cargo.lock is NOT here — this +# is an application, so the exact versions it was built against are part of it. +src-tauri/target +src-tauri/gen/schemas + +# The Node the desktop app ships. ~90 MB, fetched and checksummed by +# src-tauri/scripts/fetch-node.mjs at build time — a runtime is not a thing to +# keep in a repository. +src-tauri/binaries diff --git a/.vscode/settings.json b/.vscode/settings.json new file mode 100644 index 0000000..434e3cb --- /dev/null +++ b/.vscode/settings.json @@ -0,0 +1,9 @@ +{ + // Use the TypeScript in node_modules (5.9.3) for IntelliSense, not the one + // bundled with the editor. A newer bundled compiler reports options this + // repo's compiler still accepts — `moduleResolution: node` and `baseUrl` in + // api/tsconfig.json are deprecated in TypeScript 7 — as errors in files that + // build and typecheck clean. Editor and CI now agree on one compiler. + "typescript.tsdk": "node_modules/typescript/lib", + "typescript.enablePromptUseWorkspaceTsdk": true +} diff --git a/AGENTS.md b/AGENTS.md index d9092aa..9ba9d78 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -10,11 +10,17 @@ This block is written and re-added by `next dev` — verify at `node_modules/nex # Read these first -Four short files, written so a session does not have to rediscover the same -things by grepping and clicking. Read the one that matches the job. +Written so a session does not have to rediscover the same things by grepping +and clicking. Read the one that matches the job. -- **`docs/ARCHITECTURE.md`** — what lives where, the routes, the data flow, and - the vocabulary (the UI says "Template" where the code says `pack`). +- **`docs/ARCHITECTURE.md`** — what lives where, the routes, the two data flows, + and the vocabulary (the UI says "Template" where the code says `pack`, and + "Machine" where it says `worker`). +- **`docs/WORKER.md`** — why writing a section and fetching a reel became jobs + on a queue that teammates' laptops claim: the `workers` and `jobs` tables, the + claim query, the leases, and what a machine's bearer token may reach. Read it + before touching `worker/`, `api/src/workers/`, `lib/server/repos/jobs.ts` or + `advance()` in `lib/server/services/runs.ts`. - **`docs/DEVELOPING.md`** — how to start it (port 3333), what to run to check work, why `next build` must not touch the dev server, and what to do when another session is editing the same files. @@ -42,3 +48,25 @@ Compose from the UI kit in `components/ui` — `Page`, `PageHeader`, `Toolbar`, listed in `components/ui/README.md` — read that file before adding anything. - Every icon-only control takes a `label`, which becomes its tooltip and its accessible name. + +# Code that runs on somebody else's machine + +`worker/` is a sidecar on a teammate's laptop. It claims jobs over HTTPS, spawns +the CLIs, and posts the results back. It has no database and must never look +like it wants one. + +- From `lib/server` it may import `tools.ts`, `brain-defs.ts` and + `brain-transports.ts`, and **nothing else**. Everything else there reaches a + repo, and an import of a repo puts `pg` on somebody's desktop. That is how the + worker ended up carrying a hand-copied brain table once already — and the copy + drifted, on the half of the system that does the work. +- Use relative specifiers in anything the worker imports, not `@/`. It runs + under Node's own type stripping through `worker/resolve-ts.mjs`, which knows + nothing about tsconfig `paths`: an `@/` import typechecks, passes review, and + is a module-not-found the first time somebody starts the sidecar. +- The server decides and the worker executes. A payload carries finished text + and a command — never a pack, a rule, a dependency graph or the brand voice — + and a worker that cannot honour what it was told fails the job rather than + quietly answering some other way. +- `npm run worker` starts it; there is no build step. `docs/WORKER.md` lists the + environment it takes and why two of the variables have no default. diff --git a/README.md b/README.md index 6bcabbe..4733553 100644 --- a/README.md +++ b/README.md @@ -259,16 +259,28 @@ gone, so an old run can still be rewritten. ### Running one -**No API key needed.** The default brain shells out to the `claude` CLI already -installed on this machine, which is what the old system called its local brain. -Set `ANTHROPIC_API_KEY` and it uses the Anthropic SDK instead. - -Sections run **one request each**, driven from the browser in dependency order. -That is deliberate: you watch it happen, and a failure costs one section rather -than the run. Rewriting a section is the same request again. The server checks -the dependencies rather than trusting the caller — asking for the Bangla script -first would otherwise hand the model an empty "already generated" block, and it -would invent the English script it was meant to be translating. +**No API key needed.** The brain shells out to the `claude` CLI already signed +in on a machine, billed to that subscription rather than per token. Which model +writes is a property of the **workspace**, not of the console. + +Pressing Run does not write anything itself. `advance()` looks at the template's +dependency waves, and for every section whose inputs are written it queues a +**job**; a worker — a process on somebody's own computer, with the CLIs and the +GPU on it — claims the job, writes the section, and posts it back, at which +point `advance()` runs again. So a run survives the tab closing, the API +restarting, and the machine that started it going to sleep: another one picks +it up. `docs/WORKER.md` is the whole design. + +A failure still costs one section rather than the run, and rewriting one is +still a single request. The server checks the dependencies rather than trusting +the caller — asking for the Bangla script first would otherwise hand the model +an empty "already generated" block, and it would invent the English script it +was meant to be translating. + +`ANTHROPIC_API_KEY` still works and is **off by default**. A workspace can turn +on API fallback, which lets the server write a section when no machine is +awake; it is opt-in because a fallback that fired by itself would spend money +at the moment nobody was watching. ### First run diff --git a/api/Dockerfile b/api/Dockerfile index 5577c39..53254ee 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -14,10 +14,21 @@ WORKDIR /app COPY --from=deps /app/api/node_modules ./api/node_modules # The shared code, and the root packages it imports (pg, the Anthropic SDK). COPY package.json package-lock.json* ./ -RUN npm ci --omit=dev +# Everything, including devDependencies — `tsc` needs them. +# +# This was `--omit=dev`, which looks right for an image and is not: the type +# packages for what lib/server imports live in the root's devDependencies, so +# `@types/pg` was absent and the build below failed with a screenful of +# implicit-any on code that compiles fine on a laptop. The dev dependencies +# are needed to BUILD and not to RUN, which is what `npm prune` is for. +RUN npm ci COPY lib ./lib COPY api ./api RUN cd api && npm run build +# Now that tsc is done, take them back out. The runtime stage copies this +# node_modules wholesale, so pruning here is what keeps the shipped image to +# what a request actually reaches. +RUN npm prune --omit=dev FROM node:24-bookworm-slim AS runtime WORKDIR /app @@ -28,12 +39,22 @@ ENV TS_NODE_BASEURL=./dist # Downloads and frames live here; mount a volume on it. ENV CONTENTOS_DATA_DIR=/app/.data -# The tools the ingest runs. Without them the API still starts; source-video -# features report the binary as missing, which is the truth. +# ffmpeg, and only ffmpeg. +# +# This image used to carry python3 and a downloaded yt-dlp as well, because +# fetching a link ran here. It does not any more: an `ingest_source` job goes +# to a machine that has yt-dlp, the download happens there, and what comes +# back is the stills and a 16 kHz WAV. Nothing on this side touches a URL's +# media. +# +# ffmpeg stays for the one ingest that is still local — a BROWSER UPLOAD, +# whose bytes already arrived here, and where posting them back out to a +# laptop to cut eight stills would double the transfer to learn nothing. +# `ffprobe` ships in the same package. Whisper is deliberately absent: +# transcription is a job over that WAV, aimed at a desktop with a GPU, and a +# CPU transcriber on a 2GB VPS is the slowest way to get the worst answer. RUN apt-get update \ - && apt-get install -y --no-install-recommends ffmpeg python3 ca-certificates curl \ - && curl -fsSL https://github.com/yt-dlp/yt-dlp/releases/latest/download/yt-dlp -o /usr/local/bin/yt-dlp \ - && chmod a+rx /usr/local/bin/yt-dlp \ + && apt-get install -y --no-install-recommends ffmpeg ca-certificates \ && rm -rf /var/lib/apt/lists/* COPY --from=build /app/node_modules ./node_modules diff --git a/api/src/app.module.ts b/api/src/app.module.ts index e647f3f..03becb5 100644 --- a/api/src/app.module.ts +++ b/api/src/app.module.ts @@ -15,6 +15,9 @@ import { IntegrationsController } from "./integrations/integrations.controller"; import { RunsController } from "./runs/runs.controller"; import { SourcesController } from "./sources/sources.controller"; import { ResearchController } from "./research/research.controller"; +import { WorkersController } from "./workers/workers.controller"; +import { MachinesController } from "./machines/machines.controller"; +import { Reaper } from "./workers/reaper"; /** * One module, because there is one database and one set of rules. @@ -22,8 +25,8 @@ import { ResearchController } from "./research/research.controller"; * Nest's usual shape is a module per feature, each with its own providers. * Here the "providers" are the functions in `lib/server` — repos and services * shared with the repo root — and a controller is a thin translation from HTTP - * to those. Ten controllers in one module is honest about that; ten modules - * each importing nothing would be ceremony. + * to those. Thirteen controllers in one module is honest about that; + * thirteen modules each importing nothing would be ceremony. * * The session guard is global. Every route is behind it unless the handler * says otherwise with `@Public()` — the auth gap that the audit found was @@ -43,7 +46,22 @@ import { ResearchController } from "./research/research.controller"; RunsController, SourcesController, ResearchController, + WorkersController, + /* + * The console's view of the machines, not the machines' own protocol. + * `MachinesController` is behind the global session guard like every other + * console route; `WorkersController` above is bearer-token only. Two + * controllers because they are two principals — see docs/WORKER.md. + */ + MachinesController, ], - providers: [{ provide: APP_GUARD, useClass: SessionGuard }], + /* + * The reaper is a provider rather than something started in `main`, so that + * its interval is owned by the application: Nest constructs it once and, + * with shutdown hooks on, clears it on the way out. A `setInterval` in + * `main` would survive `app.close()` and hold a database pool open forever + * in tests and on a dev reload. + */ + providers: [{ provide: APP_GUARD, useClass: SessionGuard }, Reaper], }) export class AppModule {} diff --git a/api/src/auth/auth.controller.ts b/api/src/auth/auth.controller.ts index f94c1a7..94b1558 100644 --- a/api/src/auth/auth.controller.ts +++ b/api/src/auth/auth.controller.ts @@ -1,4 +1,5 @@ import { + BadRequestException, Body, Controller, Delete, @@ -8,12 +9,15 @@ import { NotFoundException, Param, Post, + Query, Res, } from "@nestjs/common"; import type { Response } from "express"; import { + adoptTarget, authenticate, + createHandoff, createInvite, createUser, endOtherSessions, @@ -24,38 +28,82 @@ import { removeUser, revokeInvite, signupOpen, + spendHandoff, startSession, + sweepHandoffs, sweepSessions, updatePassword, type User, } from "@/lib/server/auth"; import { clearSession, setSession } from "../common/cookie"; -import { Public } from "../common/session.guard"; +import { Public, SESSION_SCHEME } from "../common/session.guard"; import { CurrentUser, SessionId } from "../common/user.decorator"; /** * Signing in, signing up, and the people who can. * - * The four routes a visitor reaches before they have a session are `@Public()`; + * The routes a visitor reaches before they have a session are `@Public()`; * everything else is behind the global guard. The cookie is set by this * process, on this origin — see common/cookie.ts for what that means when the * web app is somewhere else. + * + * `auth/adopt` is the newest of the public ones and the only one that is not a + * form: it takes a one-time code minted by `auth/handoff` over an existing + * session and turns it into that same session's cookie, so the desktop app's + * webview arrives signed in instead of at a second login form. */ @Controller() export class AuthController { + /** + * Sign in. The cookie always; the session id only when it is asked for. + * + * A browser needs nothing but the cookie and has never read this body for a + * credential. The desktop app cannot use the cookie at all — its webview is + * on a local origin, so the API's cookie is third-party — so it holds the + * session id itself and sends it back as `Authorization: Session `; see + * `common/session.guard.ts` for why that is the same session and not a new + * principal. + * + * Behind a header rather than given to everybody, because a credential in a + * response body is a credential in whatever logs that body. Nothing on the + * console asks for this one, so nothing on the console can start recording + * it: a request-logging middleware, a browser devtools export somebody + * pastes into a bug report, a proxy keeping response samples. A client that + * sends `X-Session-Return: 1` has said it intends to store the thing, and + * has somewhere to store it — the OS credential store, in this app's case. + * + * The header is the request's, not a response flag, so the web app's login + * is byte-for-byte what it was. And it is a header rather than a body field + * so that the shape of what a client *posts* stays the same for both, which + * keeps `authenticate` the one thing this route is about. + */ @Public() @Post("auth/login") @HttpCode(200) async login( @Body() body: { email?: string; password?: string }, @Headers("user-agent") agent: string | undefined, + @Headers("x-session-return") wantsSession: string | undefined, @Res({ passthrough: true }) res: Response, ) { const user = await authenticate(body.email ?? "", body.password ?? ""); void sweepSessions(); - setSession(res, await startSession(user.id, agent ?? "")); - return user; + const sid = await startSession(user.id, agent ?? ""); + setSession(res, sid); + // Any value that is not an explicit "no" counts as asking: a client that + // sets this header at all is a non-browser client that will not get the + // cookie home, and failing it over the spelling of "true" would produce a + // sign-in that looks like it worked and a machine that never enrols. + if (!asked(wantsSession)) return user; + /* + * The scheme travels with the credential. The app has to send it back as + * `Authorization: Session ` and there is no second place it could read + * that word from — it is not in a header of this response, it is not in + * the cookie — so saying it here is what stops a client inventing `Bearer` + * and colliding with the worker token. + */ + return { ...user, session: sid, scheme: SESSION_SCHEME }; } @Public() @@ -115,6 +163,85 @@ export class AuthController { return { ok: true }; } + /** + * A one-time code that carries this session into a webview. + * + * Behind the global guard, and that is the whole authorisation: the session + * it hands out is `@SessionId()`, which is the one the caller just + * authenticated with. There is no parameter naming a session or a user, so + * there is nothing here that could be pointed at somebody else's — a caller + * can only ever ask for a code for the session it already has, which means + * this route hands out no access that the request did not arrive holding. + * + * The desktop app is what calls it, over its `Authorization: Session …`, + * immediately before opening the console window. See "The hand-off" in + * lib/server/auth.ts for why the code is single-use, short-lived, hashed at + * rest, and buys this session rather than minting another. + * + * The code is in the response body because the caller is a Rust process with + * somewhere to put it, exactly as `POST /auth/login` returns a session id to + * a client that asked for one — and nothing on the console calls this, so + * nothing on the console can start logging it. + */ + @Post("auth/handoff") + @HttpCode(200) + handoff(@SessionId() sid: string) { + // Spent and expired rows, on the way past. Same shape as `sweepSessions` + // on login: the tidying rides on the traffic rather than on a timer. + void sweepHandoffs(); + return createHandoff(sid); + } + + /** + * Spend a hand-off code, set the session cookie, and go to the console. + * + * `@Public()` because the point of it is to be reachable by a webview that + * has no cookie yet — that is the state it exists to fix. The code is the + * credential, and it is checked by `spendHandoff`, which refuses a spent, an + * expired, an unknown and a since-revoked code identically. + * + * **`next` is validated before the code is spent**, and a `next` that is not + * the console is refused outright rather than replaced with something safe. + * Two reasons in that order: a redirect that also sets a session cookie is + * the pairing an open redirect is actually worth something for, and a typo + * in a deployment's `WEB_ORIGIN` should fail loudly on the first try rather + * than work-ish forever. Spending the code only after that check means a + * refusal does not also burn the app's one credential, so it can simply open + * the console the ordinary way and let the person sign in. + * + * **A bad code still redirects, with no cookie set.** This is a window + * somebody is looking at, mid-navigation, and a JSON error in it is a dead + * end with nothing to click. Sending them to the console signed out gets + * them the console's own login page, which is precisely the second sign-in + * this feature removes when it works and is a perfectly good outcome when it + * does not. It is also one response for all four failures, so it says + * nothing about which one happened. + */ + @Public() + @Get("auth/adopt") + async adopt( + @Query("code") code: string | undefined, + @Query("next") next: string | undefined, + @Res() res: Response, + ) { + const target = adoptTarget(String(next ?? "")); + if (!target) { + throw new BadRequestException( + "That is not this console's address, so this app will not sign a browser in and send it there.", + ); + } + + const sid = await spendHandoff(String(code ?? "")); + if (sid) setSession(res, sid); + /* + * Never stored anywhere on the way. A 302 carrying a `Set-Cookie` is the + * one response in this API that a shared proxy caching it would hand one + * person's session to the next person down the wire. + */ + res.setHeader("Cache-Control", "no-store"); + res.redirect(302, target); + } + @Get("auth/sessions") sessions(@CurrentUser() user: User, @SessionId() sid: string) { return listSessions(user.id, sid); @@ -162,3 +289,18 @@ export class AuthController { return { ok: true }; } } + +/** + * Whether a client asked for its session id back. + * + * Generous in one direction only. Absent is no — that is the browser, and the + * whole reason this is opt-in — but anything present counts as yes unless it + * is one of the words that plainly means no. A desktop app that sent `true` + * where this wanted `1` would otherwise sign in successfully, be handed + * nothing to keep, and fail at enrolment with a sentence about a server that + * is working perfectly. + */ +const asked = (header: string | undefined) => { + const value = String(header ?? "").trim().toLowerCase(); + return value !== "" && value !== "0" && value !== "false" && value !== "no"; +}; diff --git a/api/src/common/session.guard.ts b/api/src/common/session.guard.ts index eadda0b..e2633e1 100644 --- a/api/src/common/session.guard.ts +++ b/api/src/common/session.guard.ts @@ -14,20 +14,101 @@ import { SESSION_COOKIE, sessionUser, type User } from "@/lib/server/auth"; export const PUBLIC = "contentos:public"; export const Public = () => SetMetadata(PUBLIC, true); +/** + * The scheme a client that is not a browser sends its session under: + * `Authorization: Session `. + * + * **Not `Bearer`.** `WorkerGuard` owns that word, and the two credentials it + * would then share a spelling with are the two that must never be confused: + * a session is a person and reaches the whole console, a worker token is a + * machine and reaches nine queue routes. Two credentials that look alike at a + * glance is how one ends up pasted where the other belongs — into a support + * message, a `.env`, or the wrong field of a setup window — and the paste that + * matters is the one that hands somebody's whole console to a machine's + * credential store. A different scheme name costs one word and makes the two + * impossible to mistake in a log line, a proxy rule or a curl command. + * + * `Session` is not on the IANA scheme registry and does not need to be: it is + * read by this guard and sent by this repository's own desktop app, and a + * registered name would only invite something else to assume it means what it + * means somewhere else. + */ +export const SESSION_SCHEME = "Session"; + +const SESSION_HEADER = /^Session\s+(.+)$/i; + +/** + * The session id an `Authorization` header carries, or an empty string. + * + * Split out so it can be tested without a database, a request or a window + * system, and so there is exactly one place that decides what counts: the + * important half of this function is what it *refuses*. A `Bearer` header is + * a worker token and must read as no session at all, or the desktop app's two + * credentials would each be accepted in the other's place — precisely the + * merge `docs/WORKER.md` argues against in "The desktop app holds two + * credentials, on purpose". + */ +export function sessionFromHeader(header: string | undefined): string { + return SESSION_HEADER.exec(String(header ?? "").trim())?.[1]?.trim() ?? ""; +} + /** The request, once the guard has run: the user rides on it. */ export type SessionRequest = Request & { user?: User; sessionId: string }; /** * The lock on every route. * - * Reads the session cookie off the request, looks the session up, and puts - * the user on the request for `@CurrentUser()`. Global, so a new controller - * is closed by default: the audit found fifteen Next route handlers that - * each had to remember to call `requireUser()`, and did not. + * Reads the session off the request, looks it up, and puts the user on the + * request for `@CurrentUser()`. Global, so a new controller is closed by + * default: the audit found fifteen Next route handlers that each had to + * remember to call `requireUser()`, and did not. * * A route that must be open — login, signup, the "is signup open" probe — * says so with `@Public()`. The session id is put on the request either way, * because logout needs it whether or not the session is still valid. + * + * ## Two transports, one session + * + * The cookie is how a browser carries it, and nothing about that path changed: + * same name, same options in `common/cookie.ts`, same behaviour for the web + * app on `WEB_ORIGIN`. + * + * The header exists because the desktop app is not a browser. Its webview is + * on a local origin (`tauri://localhost`) and the API is on somebody's server, + * which makes the session cookie a *third-party* cookie — exactly the thing + * webviews and browsers are progressively refusing, and a thing this app would + * have to keep working across four Chromium releases it does not control. The + * app therefore holds the session id itself, in the OS credential store beside + * the worker token, and attaches it deliberately to each request. The team's + * plan is that only this API is deployed, with no hosted console to visit, so + * "mint a token in Settings → Machines and paste it" stopped being an + * instruction anybody could follow; the app signs in with an email and a + * password instead and enrols itself through `POST /api/machines`. + * + * **This is not a new principal, and it cannot reach anything a signed-in + * browser could not.** There is one lookup, `sessionUser`, and it is the same + * call the cookie path makes: the same `sessions` row, the same `expires_at`, + * the same user. Everything that ends a session ends this one too — Settings → + * Team deleting a person's rows, "sign out everywhere" through + * `endOtherSessions`, a password change deleting all of them, `endSession` on + * logout, and the expiry sweep. Nothing here mints anything, nothing here + * extends a lifetime, and there is no branch anywhere that asks which + * transport a session arrived on — which is the point: a second code path that + * decided *authorisation* differently would be a second thing to audit, and + * would be the one that drifts. + * + * What it genuinely does widen, said plainly: a session id now lives somewhere + * other than a cookie jar. A cookie is `httpOnly`, so script on the console + * cannot read it; a header value is held by whatever client sends it. That is + * why the desktop app keeps it in Windows Credential Manager rather than in + * `settings.json` next to the addresses, and why it never puts it in the + * webview's DOM. The console itself must keep using the cookie and does. + * + * The other direction is closed by construction. A worker token in a `Session` + * header is not a session id — `sessions` has no such row and `sessionUser` + * answers null. A session id in a `Bearer` header is not a worker — `WorkerGuard` + * hashes it and finds nothing in `workers`. Neither credential becomes the + * other by being sent under the wrong scheme; both simply fail. */ @Injectable() export class SessionGuard implements CanActivate { @@ -35,7 +116,26 @@ export class SessionGuard implements CanActivate { async canActivate(context: ExecutionContext) { const req = context.switchToHttp().getRequest(); - req.sessionId = String(req.cookies?.[SESSION_COOKIE] ?? ""); + + /* + * The header first, and the cookie only when there is no `Session` header + * at all. + * + * The explicit credential beats the ambient one. A cookie is sent by the + * browser whether or not this request meant to use it; a header is what + * this client chose to authenticate as, and a client holding both — a + * webview that once signed in on the API's own origin, an app being + * debugged against a console in the same browser profile — means the one + * it attached. + * + * `||` falls through only when the header is absent or is somebody else's + * scheme, never when it is a `Session` header carrying a value that turns + * out to be dead. A bad header is a failed request, not a quiet promotion + * to whichever session the cookie jar happens to hold. + */ + req.sessionId = + sessionFromHeader(req.headers?.authorization) || + String(req.cookies?.[SESSION_COOKIE] ?? ""); const open = this.reflector.getAllAndOverride(PUBLIC, [ context.getHandler(), diff --git a/api/src/common/user.decorator.ts b/api/src/common/user.decorator.ts index e6c639a..efb1855 100644 --- a/api/src/common/user.decorator.ts +++ b/api/src/common/user.decorator.ts @@ -14,7 +14,16 @@ export const CurrentUser = createParamDecorator( ctx.switchToHttp().getRequest().user ?? null, ); -/** The session id off the cookie, valid or not — logout needs it either way. */ +/** + * The session id, valid or not — logout needs it either way. + * + * From the cookie or from an `Authorization: Session …` header, whichever the + * caller used: the guard resolves that before anything reads this, so a + * handler never has to know which transport carried it. A browser sends the + * cookie; the desktop app, whose webview is on a local origin against a remote + * API, sends the header, because a cookie in that position is third-party and + * is what webviews are progressively refusing. + */ export const SessionId = createParamDecorator( (_data: unknown, ctx: ExecutionContext) => ctx.switchToHttp().getRequest().sessionId, diff --git a/api/src/common/worker.guard.ts b/api/src/common/worker.guard.ts new file mode 100644 index 0000000..75a156f --- /dev/null +++ b/api/src/common/worker.guard.ts @@ -0,0 +1,96 @@ +import { + CanActivate, + ExecutionContext, + Injectable, + SetMetadata, + UnauthorizedException, + UseGuards, + applyDecorators, +} from "@nestjs/common"; +import type { Request } from "express"; + +import { PUBLIC } from "./session.guard"; +import { touchWorker, workerByToken, type Worker } from "@/lib/server/repos/workers"; + +/** Marks a handler as belonging to the worker protocol rather than the console. */ +export const WORKER_ROUTE = "contentos:worker"; + +/** The request, once the guard has run: the machine rides on it. */ +export type WorkerRequest = Request & { + worker?: Worker; + /** The raw bearer token, kept only for register, which re-hashes it. */ + workerToken?: string; +}; + +/** + * The lock on the six worker endpoints. + * + * A bearer token, not the session cookie, and the reasons are in + * `docs/WORKER.md`: a cookie is a browser mechanism the worker would have to + * scrape out of a webview; sessions are deleted by Settings → Team, which is + * the wrong lever for a machine; a session can read the entire console while a + * worker needs six routes; and a headless process cannot be sent to + * `/login?next=…`, which is what every 401 does to a browser. + * + * Hashing goes through `hashToken` inside `workerByToken` — the only + * `createHash` in the codebase — so no controller can put a raw token in a + * column, or hash it with a different encoding and produce a machine that + * enrols and then can never claim anything. + * + * `last_seen_at` is touched here rather than in each handler because liveness + * is a fact about the connection, not about which endpoint was called: a + * machine that is heartbeating is awake whether or not it is holding work. The + * write is deliberately not awaited. Liveness is a display notion, the claim + * long-poll holds for twenty-five seconds either way, and an UPDATE on the + * hot path of every request should not add a round trip to it. + */ +@Injectable() +export class WorkerGuard implements CanActivate { + async canActivate(context: ExecutionContext) { + const req = context.switchToHttp().getRequest(); + + const header = String(req.headers.authorization ?? ""); + const token = /^Bearer\s+(.+)$/i.exec(header.trim())?.[1]?.trim() ?? ""; + // A session cookie must never satisfy a worker route. There is no fallback + // here on purpose: the two credentials are different principals with + // different lifetimes and different blast radii, and a guard that accepted + // either would quietly merge them. + if (!token) throw new UnauthorizedException("Worker token required"); + + const worker = await workerByToken(token); + if (!worker) throw new UnauthorizedException("Unknown worker token"); + + req.worker = worker; + req.workerToken = token; + void touchWorker(worker.id).catch(() => {}); + return true; + } +} + +/** + * The third decorator, beside `@Public()`. + * + * It does two things at once, and both are needed: it hangs `WorkerGuard` on + * the handler, and it marks the route `PUBLIC` so the global `SessionGuard` + * stands aside instead of refusing a request that carries no cookie. Standing + * aside is all that means — `SessionGuard` still puts a session on the request + * if one happens to be there, and `WorkerGuard` still demands the bearer + * token, so a signed-in browser cannot reach these routes and a worker token + * cannot reach anything else. + * + * Written as one decorator rather than the pair, because the pair is a thing + * somebody would half-apply: `@Public()` without the guard is an open route, + * and that is exactly the class of hole the global guard exists to close. + */ +export const WorkerRoute = () => + applyDecorators( + SetMetadata(WORKER_ROUTE, true), + SetMetadata(PUBLIC, true), + UseGuards(WorkerGuard), + ); + +/** The machine on the request. Never undefined inside a `@WorkerRoute()`. */ +export const workerOf = (req: WorkerRequest): Worker => { + if (!req.worker) throw new UnauthorizedException("Worker token required"); + return req.worker; +}; diff --git a/api/src/integrations/integrations.controller.ts b/api/src/integrations/integrations.controller.ts index 1bc2a69..87ad65b 100644 --- a/api/src/integrations/integrations.controller.ts +++ b/api/src/integrations/integrations.controller.ts @@ -2,119 +2,455 @@ import { BadRequestException, Body, Controller, - ConflictException, Get, HttpCode, + NotFoundException, + Param, Patch, Post, Query, } from "@nestjs/common"; -import { brainStatuses, findBrain, write } from "@/lib/server/brains"; -import { getSettings, listSecrets, setSettings } from "@/lib/server/repos/settings"; -import { toolStatuses } from "@/lib/server/tools"; +import { + BRAINS, + findBrain, + type BrainDef, + type Transport, +} from "@/lib/server/brain-defs"; +import { enqueue, getJob } from "@/lib/server/repos/jobs"; +import { getSettings, listSecrets } from "@/lib/server/repos/settings"; +import { + getWorker, + listWorkers, + updateWorker, + type Worker, +} from "@/lib/server/repos/workers"; +import { getWorkspace, updateWorkspace } from "@/lib/server/repos/workspaces"; const MEDIA = ["yt-dlp", "ffmpeg", "ffprobe", "whisper"]; /** - * What this machine can actually reach, probed by running it. + * What a MACHINE can reach — read, not probed. + * + * This controller used to call `toolStatuses()` and `brainStatuses()`, which + * spawn `claude --version` and `yt-dlp --version` in the API process. That was + * exactly right while the API process was also the machine with the binaries + * on it. On a server it is a question asked of the wrong box: the answer is + * "nothing installed", forever, and the Integrations page becomes a list of + * red rows about a machine nobody uses — while the four laptops that really do + * have the tools are invisible. + * + * So the page stops asking "what is on this machine" and starts asking "what + * is on WHICH machine". Every row here comes out of `workers.tools`, which the + * worker probed on its own hardware and reported when it registered — the same + * `ToolStatus` shape, so the component barely changes. "Re-check" stops + * meaning "spawn five processes here" and starts meaning "ask that machine to + * probe again", which is what re-registration already is. * - * The models and the local binaries, each checked by invoking it rather than - * by reading a flag somebody set. `?recheck=1` runs the probes again instead - * of answering from the cache. + * The distinction the page has to be able to draw, and could not before: + * `whisper` is not installed on that machine, versus that machine has not been + * seen since Tuesday. One is an install command; the other is "open your + * laptop". */ @Controller("integrations") export class IntegrationsController { + /** + * The wiring, as of one machine. + * + * `machine` names a worker; without one the most recently seen live machine + * is used, because "where can this run right now" is the question the page + * is actually asking and a laptop shut since Tuesday is not an answer to it. + * `workspace` names whose brain and API-fallback switch to report — both of + * those moved off the global settings in migration 0016, and neither has a + * console-wide value any more. + */ @Get() - async wiring(@Query("recheck") recheck?: string) { - const again = recheck === "1"; - const [brains, tools, settings, keys] = await Promise.all([ - again ? toolStatuses(true).then(brainStatuses) : brainStatuses(), - toolStatuses(again), + async wiring( + @Query("machine") machineId?: string, + @Query("workspace") workspaceId?: string, + ) { + const [machines, settings, keys] = await Promise.all([ + listWorkers(), getSettings(), listSecrets(), ]); + + const machine = + (machineId ? machines.find((m) => m.id === machineId) : undefined) ?? + machines.find((m) => m.live) ?? + machines[0] ?? + null; + + const workspace = workspaceId ? await getWorkspace(workspaceId) : null; + return { - brains, - tools: tools.filter((t) => MEDIA.includes(t.id)), + /* + * Everything the picker needs, and nothing a token could be rebuilt + * from. `token_hash` never leaves the repo; the rest of a worker row is + * what the page draws. + */ + machines: machines.map((m) => ({ + id: m.id, + name: m.name, + platform: m.platform, + version: m.version, + lastSeenAt: m.lastSeenAt, + live: m.live, + })), + machine: machine + ? { + id: machine.id, + name: machine.name, + live: machine.live, + lastSeenAt: machine.lastSeenAt, + } + : null, + brains: brainsOn(machine, Object.keys(keys)), + // Read off the row rather than probed. No machine enrolled yet means an + // empty list, which the page must render as "no machines" rather than as + // "nothing installed" — those are opposite sentences. + tools: (machine?.tools ?? []).filter((t) => MEDIA.includes(t.id)), + /** The subset the machine's owner has switched on. `workers.enabled`. */ + enabled: machine?.enabled ?? [], + /** `workers.can_read_frames`, which defaults OFF. Per machine, on purpose. */ + canReadFrames: machine?.canReadFrames ?? false, + /** Per workspace now, not per console. Null when none was named. */ + brain: workspace?.brain ?? null, + apiFallback: workspace?.apiFallback ?? null, settings, keys: Object.keys(keys), }; } - /** Pick the brain, or switch local tools on and off. */ + /** + * Set one of the things this page can set — each on the thing it belongs to. + * + * There is no `setSettings({ brain, enabled })` any more, and that is the + * point of migration 0016: `brain` is an editorial choice about a + * workspace's content, `enabled` is a fact about one person's machine, and a + * single PATCH that wrote both into a console-wide table let a teammate + * retune every brand in the database and switch off a binary on somebody + * else's desktop with the same click. + * + * `enabled` is also settable on `PATCH /api/machines/:id`, which is the + * canonical place for a machine's switches; it is accepted here because it + * is what the toggle on this page does, and it goes through the same repo + * call rather than a second copy of the rule. + */ @Patch() - async update(@Body() body: { brain?: string; enabled?: string[] }) { - if (body.brain) { - const status = (await brainStatuses()).find((b) => b.id === body.brain); - if (!status) throw new BadRequestException(`No model called "${body.brain}"`); - if (!status.available) throw new ConflictException(status.reason); + async update( + @Body() + body: { + workspace?: string; + brain?: string; + apiFallback?: boolean; + machine?: string; + enabled?: string[]; + }, + ) { + if (body.brain !== undefined || body.apiFallback !== undefined) { + if (!body.workspace) { + throw new BadRequestException( + "Which workspace? The model and the API-key fallback are per workspace now, not per console.", + ); + } + if (body.brain !== undefined && !findBrain(body.brain)) { + throw new BadRequestException(`No model called "${body.brain}"`); + } + /* + * Not checked against a machine before saving, on purpose. + * + * The old code refused a brain whose CLI this box did not have. A + * workspace's model is an editorial decision that outlives whichever + * laptops happen to be awake when it is made, and a machine with the CLI + * may be enrolled an hour later. A run that cannot be routed says so at + * enqueue, by name, with the install command — which is a better place + * to find out than a picker that would not let you choose. + */ + const updated = await updateWorkspace(body.workspace, { + ...(body.brain !== undefined ? { brain: body.brain } : {}), + ...(body.apiFallback !== undefined ? { apiFallback: body.apiFallback } : {}), + }); + if (!updated) throw new NotFoundException("No such workspace"); + return { brain: updated.brain, apiFallback: updated.apiFallback }; } - return setSettings(body); + + if (body.enabled !== undefined) { + if (!body.machine) { + throw new BadRequestException( + "Which machine? A tool being switched off is a fact about one desktop, not about the console.", + ); + } + const updated = await updateWorker(body.machine, { enabled: body.enabled }); + if (!updated) throw new NotFoundException("No such machine"); + return { machine: updated.id, enabled: updated.enabled }; + } + + /* + * Nothing else on this page is a setting any more. + * + * `quality`, `autoApprove` and `reduceMotion` are genuinely console-wide + * and are edited through `PATCH /api/settings`, which is where they always + * were. Refusing here rather than quietly succeeding matters because an + * old client still sends `{ brain }` or `{ enabled }` with no scope, and a + * 200 with nothing written would look exactly like a saved switch that + * silently is not. + */ + throw new BadRequestException( + "Nothing to set. `brain` and `apiFallback` need a workspace; `enabled` needs a machine.", + ); } - /** Ask a model for two words, and report what happened. */ + /** + * Ask a model for two words — on a named machine. + * + * It cannot be answered here any more. `write()` on the server would spawn a + * CLI that is not installed, or quietly spend an API key to answer a + * question about somebody's laptop; neither tells you the thing the button + * exists to tell you, which is whether THAT machine can write. So it becomes + * a `test_brain` job, and the answer arrives by polling it. + * + * A model that works on one teammate's laptop and not on another's is now + * something the page can express, and before this it could not even ask. + */ @Post("test") @HttpCode(200) - async test(@Body() body: { id?: string }) { + async test(@Body() body: { id?: string; machine?: string }) { const def = findBrain(body.id ?? ""); if (!def) throw new BadRequestException("No such model"); + if (!body.machine) throw new BadRequestException("Which machine should try it?"); + + const machine = await getWorker(body.machine); + if (!machine) throw new NotFoundException("No such machine"); - const status = (await brainStatuses()).find((b) => b.id === def.id)!; - if (!status.available) { - return { ok: false, ms: 0, reply: "", error: status.reason, fix: "" }; + /* + * Refuse before queueing when the machine plainly cannot do it. + * + * A job pinned to a worker that does not have the CLI would sit queued + * until `wait_until` and then fail on a deadline, which reads as "it was + * slow" rather than "it is not installed there". Answering now is the same + * information, two minutes earlier and in the right words. + */ + const tool = def.command + ? machine.tools.find((t) => t.id === def.command) + : undefined; + if (def.command && !tool?.present) { + return { + ok: false, + ms: 0, + reply: "", + error: `${def.command} is not installed on ${machine.name}`, + fix: tool?.install + ? `On ${machine.name}: ${tool.install}` + : `Install the ${def.command} CLI on ${machine.name}`, + machine: machine.name, + }; } - const started = Date.now(); - try { - const reply = await write(def.id, { + const { job } = await enqueue({ + kind: "test_brain", + wantsWorker: machine.id, + needs: def.command ? [def.command] : [], + // One attempt. A retry would ask the same machine the same question and, + // if it is asleep, take twice as long to say so. + maxAttempts: 1, + // Short, because somebody is watching this button. Past it the reaper + // fails the job naming the machine — which is the honest answer when a + // laptop is shut, and the one this page could not give before. + waitUntil: new Date(Date.now() + 120_000), + error: machine.live + ? "" + : `${machine.name} has not been seen recently, so nothing picked this up.`, + payload: { + brain: def.id, + transport: (def.command ? "cli" : "api") as Transport, system: "You are a terse assistant. Answer in one short line, no preamble.", user: "Reply with exactly: BRAIN OK", tier: "cheap", timeoutMs: 150_000, - }); - return { - ok: Boolean(reply.trim()), - ms: Date.now() - started, - reply: reply.slice(0, 200), - error: reply.trim() ? "" : "It answered with nothing", - fix: "", - transport: status.transport, - }; - } catch (e) { - return { - ok: false, - ms: Date.now() - started, - reply: "", - error: e instanceof Error ? e.message.slice(0, 400) : "It did not answer", - fix: e instanceof Error ? fixFor(def.id, e.message) : "", - transport: status.transport, - }; + }, + }); + + return { jobId: job.id, machine: machine.name, state: job.state }; + } + + /** + * How the test went, once a machine has answered. + * + * Polled by the page, because the result no longer comes back from the POST + * that started it: the work happens on a laptop the request has no way to + * wait for. `fixFor` runs here rather than on the worker for the same reason + * it always did — it is a lookup from an error string to a sentence, and the + * sentences belong with the console that shows them. + */ + @Get("test/:jobId") + async testResult(@Param("jobId") jobId: string) { + const job = await getJob(jobId); + if (!job || job.kind !== "test_brain") { + throw new NotFoundException("No such test"); } + + const machine = job.workerId ? await getWorker(job.workerId) : null; + const where = machine?.name ?? ""; + const brainId = String(job.payload.brain ?? ""); + + if (job.state === "queued" || job.state === "claimed") { + // `job.error` carries the sentence written at enqueue when the machine + // was not live, so "waiting for a laptop that is shut" is visible while + // it is still true rather than only once the deadline passes. + return { state: job.state, running: true, machine: where, note: job.error }; + } + + const result = job.result as { + ok?: boolean; + ms?: number; + reply?: string; + error?: string; + transport?: Transport; + }; + const reply = String(result.reply ?? ""); + const error = String(result.error || job.error || ""); + + return { + state: job.state, + running: false, + // Answering with nothing is not a pass. The button exists to prove the + // model writes, and an empty string proves the opposite. + ok: job.state === "done" && result.ok !== false && Boolean(reply.trim()), + ms: Number(result.ms ?? 0), + reply: reply.slice(0, 200), + error, + fix: error ? fixFor(brainId, error, where) : "", + transport: result.transport ?? "", + machine: where, + }; } } -/** The one-line remedy for the failures people actually hit. */ -function fixFor(id: string, message: string) { +/** + * Which models this machine could write with, derived from what it reported. + * + * The same shape `brainStatuses()` produced, built from a table instead of + * from five child processes. `cli` is present AND switched on, because the + * claim query intersects a job's `needs` against `workers.enabled` — a CLI + * that is installed but off is a model no job will ever reach that machine + * for, and a page calling it available would be describing a route that does + * not exist. + * + * `api` stays a fact about the SERVER rather than about the machine: the keys + * live here, encrypted, and the API path runs here as the workspace's + * fallback. That is why the two halves of one row can be true of two different + * computers, and why the reason strings have to say which. + */ +function brainsOn(machine: Worker | null, storedKeys: string[]) { + return BRAINS.map((def) => { + const tool = def.command + ? machine?.tools.find((t) => t.id === def.command) + : undefined; + const cli = Boolean( + def.command && tool?.present && machine?.enabled.includes(def.command), + ); + const { api, apiFrom } = keyState(def, storedKeys); + const available = cli || api; + + return { + ...def, + cli, + cliVersion: tool?.version ?? "", + api, + apiFrom, + available, + transport: (available ? (cli ? "cli" : "api") : "") as Transport | "", + // A CLI with no override uses whatever the signed-in account defaults + // to, and that account is on the machine, not here. + using: + available && cli ? (def.cliModel ?? "the account's default model") : def.model, + reason: reasonFor(def, machine, cli, api, tool?.present === true), + }; + }); +} + +/** A key is set, in this process's environment or in the settings table. */ +function keyState(def: BrainDef, storedKeys: string[]) { + if (def.env && (process.env[def.env] ?? "").trim()) { + return { api: true, apiFrom: "env" as const }; + } + if (def.secret && storedKeys.includes(def.secret)) { + return { api: true, apiFrom: "settings" as const }; + } + return { api: false, apiFrom: "" as const }; +} + +/** + * Why it cannot be used — and on which computer. + * + * "Install the claude CLI" was a complete sentence when the reader was sitting + * at the machine that would run it. On a team it is an instruction with no + * address on it, and the commonest case is not a missing binary at all: it is + * a machine that is simply not switched on, which no install command fixes. + */ +function reasonFor( + def: BrainDef, + machine: Worker | null, + cli: boolean, + api: boolean, + installed: boolean, +) { + if (cli || api) return ""; + if (!machine) { + return "No machine has been enrolled yet — install the desktop app on one and add it in Settings → Machines"; + } + if (def.id === "ollama") { + // Ollama is an HTTP server on the machine that runs it, so there is no + // binary in `tools` to look for and nothing here can reach it. Only that + // machine can answer this, which is what Test now asks it to do. + return `Ollama runs on the machine itself — press Test to ask ${machine.name}`; + } + if (installed) { + return `${def.command} is installed on ${machine.name} but switched off for it`; + } + return `Install the ${def.command} CLI on ${machine.name}, or paste a key in Settings → API keys`; +} + +/** + * The one-line remedy for the failures people actually hit. + * + * Kept, because it is a lookup from an error string to a sentence and that + * does not move anywhere. What changed is who is being told: every one of + * these is now advice about a terminal on a machine the reader may not be + * sitting at, so the machine's name goes into the sentence. "Run `codex + * login` in a terminal" is still the right advice, and it is useless run on + * the wrong computer. + */ +function fixFor(id: string, message: string, machine: string) { const m = message.toLowerCase(); + // "On Shakhawat's desktop: " — dropped when nothing ever claimed the job, + // because an instruction addressed to nowhere is worse than one addressed + // to you. + const on = machine ? `On ${machine}: ` : ""; if (m.includes("unknown variant") || m.includes("failed to refresh available models")) { return id === "codex-cli" - ? "That CLI is out of date — run: npm i -g @openai/codex@latest" - : "That CLI is out of date — reinstall it at the latest version"; + ? `${on}that CLI is out of date — run: npm i -g @openai/codex@latest` + : `${on}that CLI is out of date — reinstall it at the latest version`; } if (m.includes("auth method") || m.includes("not logged in") || m.includes("login")) { if (id === "gemini-cli") { - return "Run `gemini` once in a terminal to sign in, or paste a Google AI key in Settings → API keys"; + return `${on}run \`gemini\` once in a terminal to sign in, or paste a Google AI key in Settings → API keys`; } - if (id === "codex-cli") return "Run `codex login` in a terminal"; - if (id === "claude-cli") return "Run `claude` once in a terminal to sign in"; + if (id === "codex-cli") return `${on}run \`codex login\` in a terminal`; + if (id === "claude-cli") return `${on}run \`claude\` once in a terminal to sign in`; } if (m.includes("api key") || m.includes("401") || m.includes("unauthorized")) { + // The only one with no machine in it, and correctly so: the keys are on + // the server, so this is the same sentence wherever the job ran. return "The key was refused — check it in Settings → API keys"; } if (m.includes("timed out") || m.includes("timeout")) { - return "It did not answer in time. Try again, or pick another model."; + return `${on}it did not answer in time. Try again, or pick another model.`; + } + if (m.includes("not been seen") || m.includes("waiting for")) { + return machine ? `Open ${machine} and try again` : "Open that machine and try again"; } return ""; } diff --git a/api/src/machines/machines.controller.ts b/api/src/machines/machines.controller.ts new file mode 100644 index 0000000..750e507 --- /dev/null +++ b/api/src/machines/machines.controller.ts @@ -0,0 +1,230 @@ +import { + BadRequestException, + Body, + Controller, + Delete, + Get, + NotFoundException, + Param, + Patch, + Post, +} from "@nestjs/common"; +import { randomBytes } from "node:crypto"; + +import { listUsers, type User } from "@/lib/server/auth"; +import { + deleteWorker, + getWorker, + listWorkers, + registerWorker, + updateWorker, + type Worker, +} from "@/lib/server/repos/workers"; + +import { CurrentUser } from "../common/user.decorator"; + +/** + * The console's half of the worker split. + * + * Everything here is behind the global `SessionGuard` — a person, on the + * Settings screen or in the desktop app's setup window. It is emphatically NOT + * a worker route: + * `api/src/workers/workers.controller.ts` is the machine's six endpoints and + * every one of them is `@WorkerRoute()` with a bearer token. The two are + * different principals on purpose (`docs/WORKER.md`, "The desktop app holds + * two credentials"), so they are different controllers on different paths and + * neither decorator appears on the other's file. + * + * What is here is the half a machine is deliberately not allowed to do for + * itself: mint its credential, and set the switches its owner owns. + */ +@Controller("machines") +export class MachinesController { + /** + * Every machine, with whose it is. + * + * The owner's name is joined in here rather than being looked up by the + * browser, because the picker's whole job is to let somebody choose a + * machine and "Rakib's desktop" is the answer to "whose GPU am I about to + * spend". `Worker.live` is already derived in the repo from `last_seen_at`, + * so the page never has to know what the liveness window is. + */ + @Get() + async list(): Promise { + const [workers, users] = await Promise.all([listWorkers(), listUsers()]); + const byId = new Map(users.map((u) => [u.id, u])); + return workers.map((worker) => decorate(worker, byId.get(worker.userId))); + } + + /** + * Enrol a machine: mint the token, hand it back exactly once. + * + * 256 bits from `node:crypto` — the same randomness the session ids use, and + * not `Math.random`, because this is a credential that grants the queue. It + * is returned in this one response body and never again: `registerWorker` + * stores only its sha256, so a database dump is a list of machines, not a + * list of live logins. Lose it and the answer is to revoke the row and mint + * another, which costs one paste in the desktop app. + * + * The row is created by the same `registerWorker` the machine itself calls, + * which is what makes the first real registration an UPDATE on this row + * rather than a second one: enrolment and re-registration are the same + * insert keyed on the same token hash. The machine has not spoken yet, so + * `platform`, `version` and `tools` stay empty until it does — that emptiness + * is what the panel reads as "waiting for this machine to check in". + * + * ## The desktop app calls this too, and that is the ordinary path now + * + * There is no hosted console to visit in the deployment this is heading for + * — only this API — so "mint a token in Settings → Machines and paste it + * into the app" stopped being an instruction anybody could follow. The + * desktop app signs in with an email and a password, keeps the session, and + * posts here with the computer's hostname as the name. Nothing about that is + * a wider door: the session arrives as `Authorization: Session ` rather + * than as a cookie (`common/session.guard.ts` has the whole argument), it is + * the same person's same session either way, and the machine is bound to + * `user.id` off the guard — never to anything in the body — so this route + * cannot mint a machine for somebody else however it is called. + * + * The name is the hostname because that is the app naming a machine nobody + * has named yet, and it is what the person will recognise in the picker + * before they think of something better. It is not a rename: `registerWorker` + * deliberately does not take `name` on conflict, so a machine somebody called + * "Shakhawat's desktop" here keeps that name through every re-registration. + * + * ## Why there is no find-or-create, and who is responsible instead + * + * A row's identity IS its token hash, and the token exists in one response + * and nowhere afterwards. So a "give me this machine's row again" route could + * not hand back the credential that row is keyed on; it could only mint a + * second token and orphan the first, which is a duplicate row wearing a + * helpful name. The client is therefore the one that must not ask twice: the + * desktop app records the machine id it was given — in its settings file, + * beside the addresses, while the token itself goes to the OS credential + * store — and re-enrols only when one of the two is gone, or when the id is + * no longer listed by `GET /machines` against the person who has just signed + * in. Anybody tempted to add a server-side + * dedupe here should read that paragraph again first — the thing that would + * have to be returned is the thing this route exists never to keep. + */ + @Post() + async mint( + @CurrentUser() user: User, + @Body() body: { name?: string }, + ): Promise<{ machine: Machine; token: string }> { + const name = (body.name ?? "").trim(); + if (!name) throw new BadRequestException("Give the machine a name"); + + const token = `wrk_${randomBytes(32).toString("base64url")}`; + const worker = await registerWorker({ userId: user.id, name, token }); + return { machine: decorate(worker, user), token }; + } + + /** + * The owner's switches. + * + * `registerWorker` refuses to overwrite these when a machine re-registers, + * on purpose — a worker restarting with its defaults must not hand itself + * back a permission somebody switched off — which makes this route the only + * place they are ever set. `tools` is not patchable for the mirror-image + * reason: nobody types a binary into existence from a browser. + * + * `enabled` is intersected with what the machine actually reported, so a + * stale form cannot leave a tool switched on that the machine has since + * stopped advertising. The claim query would then hand it work it cannot + * run, and the job would fail on somebody's laptop rather than here. + */ + @Patch(":id") + async patch( + @Param("id") id: string, + @Body() + body: { + name?: string; + enabled?: string[]; + canReadFrames?: boolean; + maxConcurrency?: number; + }, + ): Promise { + const patch: Parameters[1] = {}; + + if (body.name !== undefined) { + const name = body.name.trim(); + if (!name) throw new BadRequestException("A machine needs a name"); + patch.name = name; + } + if (body.enabled !== undefined) { + if (!Array.isArray(body.enabled)) + throw new BadRequestException("enabled must be a list of tool ids"); + const worker = await requireWorker(id); + const advertised = new Set(worker.tools.map((tool) => tool.id as string)); + patch.enabled = body.enabled.map(String).filter((t) => advertised.has(t)); + } + if (body.canReadFrames !== undefined) + patch.canReadFrames = body.canReadFrames === true; + if (body.maxConcurrency !== undefined) { + const n = Number(body.maxConcurrency); + if (!Number.isFinite(n)) throw new BadRequestException("A number, please"); + // Clamped in `updateWorker` as well; rejecting here would turn a stepper + // held down into an error toast for something the repo already handles. + patch.maxConcurrency = Math.round(n); + } + + const saved = await updateWorker(id, patch); + if (!saved) throw new NotFoundException("No such machine"); + + const users = await listUsers(); + return decorate(saved, users.find((u) => u.id === saved.userId)); + } + + /** + * Revoke: the row goes, and with it the only copy of the token's hash. + * + * There is no separate revoked flag because a hash that is not in the table + * cannot authenticate, and a row that exists but is refused is a second + * state to keep in sync with the first. Work the machine was holding is not + * lost: `jobs.worker_id` is ON DELETE SET NULL, so a claimed job simply + * loses its lease and the reaper puts it back in the queue for whoever is + * next. What it will not do is come back on this machine. + */ + @Delete(":id") + async revoke(@Param("id") id: string) { + const gone = await deleteWorker(id); + if (!gone) throw new NotFoundException("That machine is already gone"); + return { ok: true }; + } +} + +/** + * A worker as the console reads it: the row, plus whose machine it is. + * + * The extra fields are display only. They are added here rather than in the + * repo because `listWorkers` is also read by the claim path, which has no + * interest in anybody's email address. + */ +export type Machine = Worker & { + ownerName: string; + ownerEmail: string; + /** True until the machine itself has registered — nothing reported yet. */ + awaitingFirstContact: boolean; +}; + +const decorate = (worker: Worker, owner: User | undefined): Machine => ({ + ...worker, + ownerName: owner?.name || owner?.email || "Unknown", + ownerEmail: owner?.email ?? "", + /* + * A machine that has never spoken has no platform, no version and no tools, + * because only `POST /api/workers/register` sets those. `last_seen_at` is + * stamped by the mint itself — the enrolment row is written by the same + * insert — so liveness alone would call a brand new row "live" for two + * minutes and the page would be waiting for tools from something that has + * not been started yet. + */ + awaitingFirstContact: !worker.platform && !worker.version && !worker.tools.length, +}); + +async function requireWorker(id: string): Promise { + const worker = await getWorker(id); + if (!worker) throw new NotFoundException("No such machine"); + return worker; +} diff --git a/api/src/main.ts b/api/src/main.ts index b7ea029..a9b33c7 100644 --- a/api/src/main.ts +++ b/api/src/main.ts @@ -5,7 +5,6 @@ import type { NestExpressApplication } from "@nestjs/platform-express"; import cookieParser from "cookie-parser"; import { ready } from "@/lib/server/http"; -import { resumeOrphans } from "@/lib/server/services/runs"; import { AppModule } from "./app.module"; import { ErrorsFilter } from "./common/errors.filter"; @@ -56,11 +55,24 @@ async function main() { */ await ready(); - // Whatever the last process was writing when it stopped, this one finishes. - const resumed = await resumeOrphans(); - if (resumed.length) { - console.log(`Resuming ${resumed.length} run(s) left mid-write: ${resumed.join(", ")}`); - } + /* + * Nothing at boot touches a section's state. This is where `resumeOrphans()` + * used to run, requeueing every `writing` row on the reasoning that its + * driver died with the last process. That was true while there was one + * process and the model was its own child; once the writing happens on + * somebody's laptop it is false, and a deploy would requeue a section a + * machine was four minutes into — written twice, paid for twice, the second + * answer overwriting the first. With two API replicas every boot would steal + * the other's in-flight work. + * + * Migration 0015 swept the genuine orphans of the old world once, and that + * was the last time. From here `jobs.lease_until` decides, per job, on + * evidence — a lease that ran out — and the reaper acts on it. + */ + + // So the reaper's interval is cleared on SIGTERM rather than left holding a + // database pool open while the container waits to be killed. + app.enableShutdownHooks(); const port = Number(process.env.PORT ?? 4000); await app.listen(port); diff --git a/api/src/runs/runs.controller.ts b/api/src/runs/runs.controller.ts index 683162c..f4e7d36 100644 --- a/api/src/runs/runs.controller.ts +++ b/api/src/runs/runs.controller.ts @@ -13,28 +13,77 @@ import { } from "@nestjs/common"; import { - deleteRun, getRun, listRuns, requeueFailed, setRunDecision, } from "@/lib/server/repos/runs"; +import { jobsForRun, type JobWithWorker } from "@/lib/server/repos/jobs"; import { - driveRun, + advance, editSection, - isDriving, + isRunning, + queueSection, + removeRun, startRun, - writeSection, + stopRun, } from "@/lib/server/services/runs"; +import type { User } from "@/lib/server/auth"; +import { CurrentUser } from "../common/user.decorator"; + +/** + * What a job looks like from the browser: an allowlist, not a row. + * + * A `Pick` of the repo's own type rather than a fresh interface, for the same + * reason lib/runs-client.ts imports `Run` instead of redeclaring it — two ends + * of a JSON boundary that describe a field separately are two things that can + * disagree, and the disagreement shows up as a blank chip rather than an + * error. The client picks the same keys off the same type, so renaming one in + * repos/jobs.ts breaks both compiles instead of quietly emptying the page. + * + * What is deliberately NOT here: + * + * - `payload`. A `write_section` job carries six to eight thousand characters + * of system prompt. The browser has no use for a single one of them, and a + * twelve-section run polled every second and a half would ship more prompt + * per minute than the run has content in it. + * - `result`. Its `text` IS the section, and `run_sections` already holds the + * copy every screen reads; sending it again doubles the page for nothing. + * - the ids of machines and workspaces. The page names a machine, it never + * addresses one. + * + * An allowlist rather than an omit-these because a column added to `jobs` next + * month must not start reaching browsers because nobody remembered this route. + */ +export type RunJob = Pick< + JobWithWorker, + | "id" + | "kind" + | "state" + | "sectionId" + | "workerName" + | "workerLive" + | "error" + | "attempts" + | "maxAttempts" + | "waitUntil" + | "createdAt" +>; /** * Runs, and the sections written into them. * - * The driver is `driveRun` in lib/server/services — a loop inside THIS - * process, one section at a time, carrying on whether or not anybody is - * watching. That is the reason the API is its own long-lived service and not - * a serverless function: a run takes minutes and shells out to a local - * model, and both need somewhere to stand. + * Every route here returns as soon as the database says so. The driver used to + * be `driveRun` — a loop inside THIS process holding one run for its whole + * length, one section at a time, because the model was a child process of the + * loop. It is `advance` now: it puts jobs on the queue for whatever is ready + * and stops, and the machines that hold the CLIs pick them up. So a run is not + * something a request or a process owns, and restarting the API mid-run loses + * nothing. + * + * These are the person-facing routes, authenticated by session. What a worker + * calls lives under /api/workers with its own credential — see docs/WORKER.md + * for why the two are deliberately different principals. */ @Controller("runs") export class RunsController { @@ -43,9 +92,17 @@ export class RunsController { return listRuns(workspaceId || undefined); } - /** Create a run with every section queued. It does not start writing. */ + /** + * Create a run with every section queued. It does not start writing. + * + * The author comes off the session, never out of the body: the guard has + * already established who this is, and a `createdBy` a caller could type + * would let anyone charge a run to a colleague. Same rule as `brandVoice`, + * which the service reads from the workspace for the same reason. + */ @Post() create( + @CurrentUser() user: User, @Body() body: { workspaceId?: string; @@ -63,6 +120,7 @@ export class RunsController { packSlug: body.packSlug ?? "enbn-website-package", inputs: body.inputs ?? {}, title: body.title, + userId: user.id, }); } @@ -83,36 +141,110 @@ export class RunsController { return { id, decision: d }; } + /** + * Delete a run, having first stopped whatever is writing it. + * + * `removeRun` cancels the jobs before the row goes. Deleting first would + * leave a machine minutes into a section for a run that no longer exists. + */ @Delete(":id") async remove(@Param("id") id: string) { - return { ok: await deleteRun(id) }; + return { ok: await removeRun(id) }; } /** - * Write everything that is left, on the server, and return at once. + * Queue everything that is ready, and return at once. * - * Pressing this on a run that failed is a retry: the failures go back in - * the queue first, so the loop tries them again rather than skipping them. + * Pressing this on a run that failed is a retry: the failures go back to + * queued first, so `advance` finds them eligible again rather than skipping + * them. Pressing it twice is harmless — the queue's one-live-job-per-section + * index makes both presses converge on the same job instead of paying two + * subscriptions to write one section. */ @Post(":id/start") @HttpCode(200) async start(@Param("id") id: string) { if (!(await getRun(id))) throw new NotFoundException("No such run"); const retried = await requeueFailed(id); - return { ...driveRun(id), retried, running: true }; + const queued = await advance(id); + return { + started: queued.enqueued.length > 0, + retried, + running: await isRunning(id), + ...queued, + }; } - /** Is this run being written right now, by anyone? */ - @Get(":id/start") - running(@Param("id") id: string) { - return { running: isDriving(id) }; + /** + * The queue's own record of this run, one entry per job, with the machine. + * + * Three things the page had to fake because none of this was reachable: + * + * - "no machine here has the tool" was a regex against the section's error + * text, matching a sentence `advance()` writes. Rewording that sentence + * silently downgraded a NO MACHINE chip to a plain FAILED one and nothing + * failed anywhere. `state` says `unroutable` here, as a state. + * - the name of a machine that is asleep survived only in the answer to + * `POST :id/start`, so it appeared only in the tab that pressed the + * button, never for a colleague and never for a single-section write. + * - "being written on Shakhawat's desktop" could not be said at all: + * `run_sections.wrote_with` is set on success, so while a section is in + * flight there is no name anywhere the browser can see. `worker_id` is + * joined to a name here from the moment the job is claimed. + * + * Session-authenticated, like every other route on this controller — the + * global `SessionGuard` covers it. It is NOT a worker route: a worker talks + * to /api/workers with its own credential, and deliberately cannot read the + * content library. Nothing here could reconstruct that credential either; + * the token is a sha256 in `workers` and never leaves the database at all. + * + * This is also what `GET :id/start` above answers, in full rather than + * reduced to a boolean: `isRunning` IS `jobsForRun(...).some(live)`. The + * page's poll therefore swaps one call for the other rather than adding a + * second timer to the tick. + */ + @Get(":id/jobs") + async jobs(@Param("id") id: string): Promise { + if (!(await getRun(id))) throw new NotFoundException("No such run"); + return (await jobsForRun(id)).map((j) => ({ + id: j.id, + kind: j.kind, + state: j.state, + sectionId: j.sectionId, + workerName: j.workerName, + workerLive: j.workerLive, + error: j.error, + attempts: j.attempts, + maxAttempts: j.maxAttempts, + waitUntil: j.waitUntil, + createdAt: j.createdAt, + })); } - /** Write one section, now, and answer with the whole run. */ + /** + * Stop a run that is writing. + * + * The jobs are cancelled here; a machine already running one finds out at its + * next heartbeat, within fifteen seconds, and kills the child process. + */ + @Post(":id/stop") + @HttpCode(200) + async stop(@Param("id") id: string) { + if (!(await getRun(id))) throw new NotFoundException("No such run"); + return { ok: true, ...(await stopRun(id)) }; + } + + /** + * Queue one section, and answer with the whole run. + * + * It no longer writes it here and waits: the request would be holding open + * for however long a laptop takes. The row comes back saying `writing`, and + * the page reads the result from the run like every other section. + */ @Post(":id/sections/:sectionId") @HttpCode(200) write(@Param("id") id: string, @Param("sectionId") sectionId: string) { - return writeSection(id, sectionId); + return queueSection(id, sectionId); } /** Replace a section's text by hand. It counts as finished from then on. */ diff --git a/api/src/sources/sources.controller.ts b/api/src/sources/sources.controller.ts index ed67a4f..bb41425 100644 --- a/api/src/sources/sources.controller.ts +++ b/api/src/sources/sources.controller.ts @@ -29,11 +29,25 @@ import { /** * Source videos: fetched from a link, or uploaded from a machine. * - * Slow on purpose. A fetch holds while yt-dlp downloads, ffmpeg cuts stills - * and a transcriber runs — minutes, not seconds — and the browser waits with - * a spinner on it. Express has no request timeout of its own, so there is - * nothing here to raise; the Next handlers needed `maxDuration` and this does - * not. + * The two halves are no longer the same shape, and the difference is the whole + * of `docs/WORKER.md`'s "Uploads stay on the server" decision: + * + * - **A link returns immediately.** `ingest()` writes an `ingest_source` job + * and hands back a source in `fetching`; yt-dlp and ffmpeg run on whichever + * machine has them. This route used to hold for minutes with a spinner on + * the other end, which only ever worked because the API was somebody's + * laptop — on a server it is a request held open across a download that + * machine cannot do at all. + * - **An upload still holds.** The bytes are already arriving here, and + * sending a four hundred megabyte recording back out to a laptop to cut + * eight stills would double the transfer to learn nothing. The ffmpeg half + * runs in this process; only the transcription leaves, as its own job. + * + * So a caller polls `GET /sources/:id` after a fetch and watches `state` move + * from `fetching` to `ready` or `failed`. It must not treat the POST's answer + * as finished work: `frames` and `transcript` are empty in it by design, and + * a screen that renders that response as the result shows an empty picker for + * every link. */ @Controller("sources") export class SourcesController { @@ -42,7 +56,15 @@ export class SourcesController { return listSources(workspaceId || undefined); } - /** Fetch a reel by link, or hand back the one already fetched. */ + /** + * Queue a fetch for a link, or hand back the one already fetched. + * + * Returns a source in `fetching` with a job behind it — or, when nothing on + * this estate has yt-dlp, one already `failed` with a sentence naming what + * to install. Both are terminal-or-moving states a person can act on, which + * is the rule: a source must never sit on `fetching` with an empty queue + * behind it. + */ @Post() fetch( @Body() body: { url?: string; workspaceId?: string | null; refresh?: boolean }, @@ -62,6 +84,11 @@ export class SourcesController { * parser: a parser buffers, and the whole point of the ingest's streaming * path is that a 400MB recording never sits in memory. The declared size is * refused up front as a courtesy; the ingest counts the bytes itself. + * + * Still the slow route, and still on purpose — the frames are cut here. It + * comes back `ready` with the stills on it, and the transcript arrives later + * on whichever machine has whisper, so `transcript` may fill in after this + * response the same way a link's does. */ @Post("upload") upload(@Req() req: Request) { @@ -117,7 +144,14 @@ export class SourcesController { return { ok: await removeSource(id) }; } - /** One more still, at a moment somebody asked for. */ + /** + * One more still, at a moment somebody asked for. + * + * Uploads only. A fetched link's video is downloaded on the machine that + * ingests it and never comes here — only the stills and the audio do — so + * this answers 409 with a sentence saying so rather than pretending a file + * is missing. + */ @Post(":id/frames") @HttpCode(201) addFrame(@Param("id") id: string, @Body() body: { at?: number }) { diff --git a/api/src/workers/reaper.ts b/api/src/workers/reaper.ts new file mode 100644 index 0000000..0b63a5a --- /dev/null +++ b/api/src/workers/reaper.ts @@ -0,0 +1,152 @@ +import { + Injectable, + type OnApplicationShutdown, + type OnModuleInit, +} from "@nestjs/common"; + +import { getJob, reap, sweepPayloads } from "@/lib/server/repos/jobs"; +import { sourceFailed } from "@/lib/server/services/ingest"; +import { sectionFailed } from "@/lib/server/services/runs"; + +/** + * How often the queue is looked at, and how often the prompts are thrown away. + * + * Fifteen seconds is a quarter of the ninety-second lease, so a job whose + * machine has gone quiet is back in the queue within six missed heartbeats + * plus a tick — noticed before anybody has reloaded the page twice. + * + * The payload sweep runs on its own, much slower clock. It only ever matches + * jobs that finished a day ago, so running it every fifteen seconds would be + * the same UPDATE finding nothing hundreds of times an hour; five minutes is + * often enough that a row is never carrying a prompt for long after its day, + * and rare enough to stay invisible. + */ +const REAP_MS = 15_000; +const SWEEP_EVERY = 20; // ticks — five minutes + +/** + * The one thing that moves a job out of `claimed` without the worker's say-so. + * + * It does it on evidence — a lease that ran out — rather than on the + * assumption `resumeOrphans()` used to make at boot, that a `writing` row must + * be dead because the process that started it was. That reasoning held while + * there was one process and the model was its own child; the moment the work + * runs on somebody's laptop it is false, and a deploy would requeue a section + * a machine was four minutes into writing, pay for it twice, and let the + * second answer overwrite the first. + * + * `reap` and `sweepPayloads` are both single UPDATEs with their conditions in + * the WHERE clause, so two API replicas running this interval at the same + * millisecond is uninteresting: the second one matches no rows. That is the + * property that lets this be an unguarded `setInterval` rather than a leader + * election. + * + * Three things the lifecycle has to get right, and each has a failure behind + * it: + * + * - Registered once. `onModuleInit` runs once per application, and the handle + * is kept so a second call cannot stack a second interval on top of the + * first — two reapers is not a correctness problem but it is a doubled load + * on the one query that must stay fast. + * - Cleared on shutdown. Without `onApplicationShutdown` a test or a dev + * reload leaves a timer holding a database pool open, and the process never + * exits. + * - A tick that throws must not take the process with it. `setInterval` with + * an async callback produces an unhandled rejection when the database blinks + * — a restart, a failover, `docker compose down` — and an unhandled + * rejection ends Node. Catching inside the tick means a blink costs one + * pass, not the API. `running` is the same guard applied to time: a pass + * that is somehow still going when the next tick fires is skipped rather + * than overlapped. + */ +@Injectable() +export class Reaper implements OnModuleInit, OnApplicationShutdown { + private timer: NodeJS.Timeout | null = null; + private running = false; + private ticks = 0; + + onModuleInit() { + if (this.timer) return; + this.timer = setInterval(() => void this.tick(), REAP_MS); + // The interval is maintenance, not work anybody is waiting on: it must + // never be the reason the process stays alive. + this.timer.unref?.(); + } + + onApplicationShutdown() { + if (!this.timer) return; + clearInterval(this.timer); + this.timer = null; + } + + private async tick() { + if (this.running) return; + this.running = true; + try { + const { requeued, failed } = await reap(); + // Said out loud because these are the two events nobody witnessed: a + // machine that went quiet mid-section, and a job that ran out of + // attempts or of patience. Silence otherwise — a pass that moves nothing + // is what almost every pass does. + if (requeued.length) { + console.log(`Reaper: ${requeued.length} job(s) back in the queue after a lost lease`); + } + if (failed.length) { + console.log(`Reaper: ${failed.length} job(s) failed — out of attempts or past their wait`); + } + + /* + * Tell the run about it, or the reaping is invisible. + * + * `reap()` ends a job on evidence the worker never reports: a lease that + * ran out because a laptop was shut, attempts spent, or a wait that + * nobody came back for. Nothing else is going to arrive and say so — the + * machine that had the job is by definition not talking — so a section + * left at `writing` or `queued` would stay there with an empty queue + * behind it, which is exactly the "Writing for five days" row this + * design is built to prevent. + * + * Each is done on its own rather than in one pass: a run whose topic + * cannot be updated must not stop the next job's section being marked. + */ + for (const jid of failed) { + try { + const job = await getJob(jid); + if (!job) continue; + const why = job.error || "No machine finished this."; + /* + * Every kind, not just the one. A reaped `ingest_source` used to end + * here silently, leaving its source on `fetching` with an empty queue + * behind it — the same dead row as a section stuck on `writing`, and + * the one a person is most likely to meet, because "nobody opened the + * laptop that has yt-dlp" is an ordinary Tuesday. + * + * `sourceFailed` is a no-op for `test_brain` and for anything with no + * source, so the branch does not have to enumerate the kinds that + * have nothing to tell. + */ + if (job.kind === "write_section") await sectionFailed(job, why); + else await sourceFailed(job, why); + } catch (e) { + console.error( + `Reaper: job ${jid} ended but its section could not be marked:`, + e instanceof Error ? e.message : e, + ); + } + } + + this.ticks += 1; + if (this.ticks % SWEEP_EVERY === 0) { + const swept = await sweepPayloads(); + if (swept) console.log(`Reaper: cleared the prompts on ${swept} finished job(s)`); + } + } catch (e) { + // One pass, not the API. The next tick tries again in fifteen seconds, + // and both statements are idempotent, so nothing is owed from the pass + // that failed. + console.error("Reaper pass failed:", e instanceof Error ? e.message : e); + } finally { + this.running = false; + } + } +} diff --git a/api/src/workers/workers.controller.ts b/api/src/workers/workers.controller.ts new file mode 100644 index 0000000..b21eb2c --- /dev/null +++ b/api/src/workers/workers.controller.ts @@ -0,0 +1,591 @@ +import { + BadRequestException, + Body, + ConflictException, + Controller, + Get, + NotFoundException, + Param, + Post, + Req, + Res, +} from "@nestjs/common"; +import busboy from "busboy"; +import type { Response } from "express"; +import { createReadStream, createWriteStream } from "node:fs"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { Transform } from "node:stream"; +import { pipeline } from "node:stream/promises"; + +import { + LEASE_SECONDS, + claim, + fail, + heartbeat, + heldBy, + succeed, +} from "@/lib/server/repos/jobs"; +import { registerWorker } from "@/lib/server/repos/workers"; +import { getSource } from "@/lib/server/repos/sources"; +import { + AUDIO_FILE, + FRAME_FILE, + MAX_UPLOAD_BYTES, + audioPath, + ensureSourceDir, + frameFile, + sourceFailed, + sourceResult, + transcriptResult, +} from "@/lib/server/services/ingest"; +import { sectionFailed, sectionResult } from "@/lib/server/services/runs"; +import type { ToolStatus } from "@/lib/server/tools"; + +import { WorkerRoute, workerOf, type WorkerRequest } from "../common/worker.guard"; + +/** + * How long a claim holds when there is nothing to hand out, and how often the + * worker is told to speak. + * + * Twenty-five seconds is under every proxy's thirty-second idle default, which + * is the only number that actually constrains it: a long-poll that outlives an + * intermediary comes back as a dropped connection the worker cannot tell from + * a server that has gone away. A second between polls keeps latency to about a + * second without anything stateful in between — the reason this is a long-poll + * and not a socket. + */ +const CLAIM_HOLD_MS = 25_000; +const CLAIM_POLL_MS = 1_000; +const HEARTBEAT_MS = 15_000; + +/** + * The nine endpoints a machine speaks, and nothing else. + * + * Every route here is `@WorkerRoute()`: a bearer token, no session, no cookie. + * The server decides and the worker executes — a job carries finished text and + * a command, never a pack, a template, a rule or a dependency graph — so there + * is deliberately nothing on this controller that reads the content library. + * A stolen worker token buys the queue and nothing more. + */ +@Controller("workers") +export class WorkersController { + /** + * Enrol a machine, or re-report the one already holding this token. + * + * Idempotent on the token because `registerWorker` makes it so: the desktop + * app registers at every start and after every update, and a newly installed + * `whisper` becomes visible by the machine saying so again. This is what + * replaces `toolStatuses(force)` and the 60s probe cache, both of which + * belonged to a server that could probe itself. + * + * The owner's switches — `canReadFrames`, `maxConcurrency`, `workspaceIds` — + * are not accepted here even though the protocol table lists them, because + * `registerWorker` refuses to overwrite them and it is right to: a worker + * restarting with its defaults must not hand itself back file-read + * permission somebody switched off in the console. They are the console's + * writes; this is the machine's report. + */ + @Post("register") + @WorkerRoute() + async register( + @Req() req: WorkerRequest, + @Body() + body: { + name?: string; + platform?: string; + version?: string; + tools?: ToolStatus[]; + enabled?: string[]; + }, + ) { + const worker = workerOf(req); + const saved = await registerWorker({ + userId: worker.userId, + name: body.name?.trim() || worker.name, + // The raw token off the header, hashed again inside the repo. The row is + // found by its hash, so this is the same machine by definition. + token: req.workerToken ?? "", + platform: body.platform ?? "", + version: body.version ?? "", + tools: Array.isArray(body.tools) ? body.tools : [], + enabled: Array.isArray(body.enabled) ? body.enabled.map(String) : [], + }); + return { + workerId: saved.id, + /* + * The name the CONSOLE has for this machine, handed back so the worker + * can say it. + * + * The worker knows only what it was configured with, which is usually a + * hostname; the console knows what a person typed in Settings → Machines, + * and that is the name every screen shows. Without this the sidecar's own + * log announced itself as "Shifa" while the machine picker called it + * "Shakhawat's PC" — two names for one computer, in front of somebody + * trying to work out which one is not claiming jobs. + */ + name: saved.name, + heartbeatMs: HEARTBEAT_MS, + claimMs: CLAIM_HOLD_MS, + }; + } + + /** + * Hand out work, waiting up to twenty-five seconds for some to appear. + * + * An empty answer is the normal answer. Most of the day there is nothing + * queued, and `{ jobs: [] }` after a quiet twenty-five seconds is a machine + * doing exactly what it should — so nothing here logs, throws or counts it + * as a failure. The alternative, a 500 or a warn per worker per half minute, + * would bury the one line that matters on the day something is actually + * wrong. + * + * The loop is written so it cannot outlive its own deadline or the + * connection. It re-checks the clock and the socket before every wait and + * again after it, and the timer it waits on is `unref`ed, so a worker that + * disconnects mid-poll leaves nothing behind holding the process open. There + * is no shared state between requests, which is what lets two API replicas + * serve claims for the same estate without knowing about each other — the + * queue's `FOR UPDATE SKIP LOCKED` is the only coordination there is. + */ + @Post("claim") + @WorkerRoute() + async claim(@Req() req: WorkerRequest, @Body() body: { max?: number }) { + const worker = workerOf(req); + const asked = Math.max(1, Math.min(8, Number(body?.max ?? 1) || 1)); + + let open = true; + const closed = () => { + open = false; + }; + req.on("close", closed); + + try { + const deadline = Date.now() + CLAIM_HOLD_MS; + for (;;) { + /* + * Recomputed every pass, not once. A worker holding one job can post + * its result on another connection while this one is waiting, and the + * next second of the same poll should then be allowed to hand it more + * work. `heldBy` is one indexed count; the claim itself is where the + * decision is actually made atomic. + */ + const room = worker.maxConcurrency - (await heldBy(worker.id)); + const jobs = room > 0 ? await claim(worker.id, worker.enabled, Math.min(asked, room)) : []; + if (jobs.length) { + return { + jobs: jobs.map((j) => ({ + id: j.id, + kind: j.kind, + payload: j.payload, + leaseUntil: j.leaseUntil, + })), + }; + } + if (!open || Date.now() + CLAIM_POLL_MS >= deadline) return { jobs: [] }; + await sleep(CLAIM_POLL_MS); + if (!open) return { jobs: [] }; + } + } finally { + req.off("close", closed); + } + } + + /** + * Extend the leases a machine still holds, and tell it what it has lost. + * + * `drop` is the whole reason this returns anything: it is how a worker + * learns a job was cancelled by a Stop button, or reaped out from under it + * while its laptop was asleep. The answer on the worker's side is to kill + * the child process rather than spend another four minutes producing + * something the server will refuse. Everything not kept is dropped, ids this + * server has never heard of included, because "I do not know that job" and + * "that job is not yours any more" call for the same action. + */ + @Post("heartbeat") + @WorkerRoute() + async heartbeat( + @Req() req: WorkerRequest, + @Body() body: { jobIds?: string[]; progress?: string }, + ) { + const worker = workerOf(req); + const ids = Array.isArray(body?.jobIds) ? body.jobIds.map(String) : []; + return heartbeat(worker.id, ids, LEASE_SECONDS); + } + + /** + * A finished job, from the machine that still holds it. + * + * `jobs.succeed` returns null for exactly one situation and it is not an + * error: the job is not this worker's, or its lease ran out. A laptop that + * wakes from sleep and posts a section that was reassigned twenty minutes + * ago must not overwrite the one that actually landed, so the write is + * refused, the result already in `run_sections` stands, and the worker is + * told 409 so that it stops rather than retrying into the same wall. + * + * Every kind is handed to the service that owns the row it writes, and + * nothing here writes `run_sections` or `sources` itself: the ordering, the + * dependency waves, what "written" means to a run and what a missing + * transcript means to a source all belong on that side, and a second + * implementation of them living in a controller is how the two quietly stop + * agreeing. + * + * Until now only `write_section` was applied and the other three kinds were + * stored on the job and nowhere else — which is the same bug in three + * places: a machine finished the work, said so, and the row a person is + * looking at never moved. An ingest that landed left its source on + * "Fetching…" with an empty queue behind it, and the frames and the WAV it + * had already uploaded sat on disk with nothing in the database pointing at + * them. + */ + @Post("jobs/:id/result") + @WorkerRoute() + async result( + @Req() req: WorkerRequest, + @Param("id") jobId: string, + @Body() body: { result?: Record }, + ) { + const worker = workerOf(req); + const result = body?.result ?? {}; + const job = await succeed(jobId, worker.id, result); + if (!job) throw new ConflictException(staleJob(jobId)); + + switch (job.kind) { + case "write_section": + await sectionResult(job, result, worker.name); + break; + /* + * The frames and the WAV are already here — they went up through the two + * upload endpoints below before this was posted — so the result only + * names files the server holds, and `sourceResult` re-checks every one + * of those names before it puts them on the row. + */ + case "ingest_source": + await sourceResult(job, result); + break; + case "transcribe_audio": + await transcriptResult(job, result); + break; + /* + * `test_brain` has nothing to apply anywhere else, and that is the whole + * of its handling — but it is written down rather than left to fall off + * the end, because the thing that matters about it is that `succeed()` + * above has already put the job at `done` with the reply on it. That is + * what the Integrations panel polls for; a job that stayed `claimed` + * because nobody moved it would leave the button spinning until the + * lease expired and the reaper handed the test to a second machine. + */ + case "test_brain": + break; + } + return { ok: true }; + } + + /** + * A failure, from the machine that still holds the job. + * + * Same ownership and lease test, same 409, for the same reason. `retryable: + * false` is the worker saying "this will fail identically on any machine" — + * a refusal, a bad payload, a transport it cannot honour — and the server + * does not spend another attempt on those. A worker that cannot do what the + * payload asked must fail rather than quietly answer some other way; a + * silent downgrade is the outcome `docs/DECISIONS.md` calls worse than an + * error. + */ + @Post("jobs/:id/fail") + @WorkerRoute() + async failJob( + @Req() req: WorkerRequest, + @Param("id") jobId: string, + @Body() body: { error?: string; retryable?: boolean }, + ) { + const worker = workerOf(req); + const message = String(body?.error ?? "").trim() || "The machine did not say why."; + const job = await fail(jobId, worker.id, message, body?.retryable !== false); + if (!job) throw new ConflictException(staleJob(jobId)); + + /* + * A job that is finished failing has to say so on the section. + * + * `fail()` either puts the job back in the queue for another attempt or + * ends it, and only the ending is news to the run: a retryable failure is + * about to be tried on another machine, and marking the section failed in + * between would flash "Failed" on a row that is still being written. + * + * Without this the row keeps whatever `advance` set — `writing` — with + * nothing left in the queue that will ever move it. That is the run that + * sat on "Writing" for five days, which `docs/DECISIONS.md` records and + * this whole design exists to make impossible. + */ + if (job.state === "failed") { + if (job.kind === "write_section") { + await sectionFailed(job, message); + } else { + /* + * The same rule for a source, and `sourceFailed` draws the one + * distinction that matters: a failed `ingest_source` fetched nothing, + * so the source fails and names the machine; a failed + * `transcribe_audio` cost the source its words and nothing else, so + * the source stays ready and says which step fell short. A missing or + * broken transcriber degrades a source; it does not fail it. A + * `test_brain` has no source and falls through as a no-op — its + * failure is the answer, and it is already on the job. + */ + await sourceFailed(job, message, worker.name); + } + } + return { ok: true }; + } + + /** + * Frames from an ingesting machine, streamed to the server's disk. + * + * The server is the single source of truth for frames: the browser renders + * them, the worker that ingests and the worker that writes are usually + * different machines, and desktops are not backed up. So ingest uploads and + * write downloads, and this is the uploading half. + * + * Every filename is re-checked against `FRAME_FILE`. That regex has until + * now been guarding a query string; these names arrive over the network from + * a process the server does not control, and it is now guarding a write — + * `frame-01.jpg` is a file in this source's directory, `../../../id_rsa` is + * not. Nothing else about the name is trusted: the part's own path + * components are discarded before the test, not after. + * + * Read straight off the request with busboy and counted as it passes, the + * way `writeCapped` does, because a parser buffers and a cap taken from the + * content-length header is a number the client chose. + */ + @Post("sources/:id/frames") + @WorkerRoute() + async frames(@Req() req: WorkerRequest, @Param("id") sourceId: string) { + const source = await getSource(sourceId); + if (!source) throw new NotFoundException("No such source"); + + // Through the ingest service rather than by rebuilding the path here: + // there is one place that knows where a source's files live, and a second + // copy of `.data/sources/` in a controller is a layout that drifts the + // first time CONTENTOS_DATA_DIR means something new. + const dir = await ensureSourceDir(sourceId); + + return new Promise<{ files: string[] }>((resolve, reject) => { + const bb = busboy({ headers: req.headers }); + const files: string[] = []; + const writes: Promise[] = []; + let bad: Error | null = null; + + bb.on("file", (field, file, info) => { + if (field !== "file") { + file.resume(); + return; + } + const name = path.basename(String(info.filename ?? "")); + if (!FRAME_FILE.test(name)) { + bad ??= new BadRequestException(`Not a frame filename: ${name}`); + file.resume(); + return; + } + files.push(name); + writes.push(writeCappedFile(file, path.join(dir, name))); + }); + + bb.on("error", reject); + bb.on("finish", () => { + Promise.all(writes) + .then(() => { + if (bad) throw bad; + if (!files.length) throw new BadRequestException("No frame in that request"); + resolve({ files }); + }) + .catch(reject); + }); + + req.pipe(bb); + }); + } + + /** + * The audio, from the machine that fetched the video. + * + * The other half of "ingest uploads, write downloads", and the reason + * `whisper` never became a dependency of this server. Audio is roughly a + * thirtieth of the video, so the ingesting worker sends a 16 kHz mono WAV — + * about a megabyte a minute rather than four hundred — and the machine with + * a GPU downloads only that when it claims the `transcribe_audio` job. + * + * One fixed filename rather than one the sender chooses. A frame's name + * carries information — which second it was cut at — and so has to be + * checked against `FRAME_FILE`; a source has exactly one audio track, so the + * safest name is the one the server picks and the sender cannot influence at + * all. Multipart, and counted as it passes, for the same reason the frames + * are: a parser buffers, and a cap read off content-length is a number the + * client chose. + */ + @Post("sources/:id/audio") + @WorkerRoute() + async audioUp(@Req() req: WorkerRequest, @Param("id") sourceId: string) { + const source = await getSource(sourceId); + if (!source) throw new NotFoundException("No such source"); + + await ensureSourceDir(sourceId); + const dest = audioPath(sourceId); + + return new Promise<{ ok: true; file: string }>((resolve, reject) => { + const bb = busboy({ headers: req.headers, limits: { files: 1 } }); + let handled = false; + + bb.on("file", (field, file) => { + if (field !== "file") { + file.resume(); + return; + } + handled = true; + writeCappedFile(file, dest) + .then(() => resolve({ ok: true, file: AUDIO_FILE })) + .catch(reject); + }); + + bb.on("error", reject); + bb.on("finish", () => { + if (!handled) reject(new BadRequestException("No audio in that request")); + }); + + req.pipe(bb); + }); + } + + /** + * The audio, back down to the machine about to transcribe it. + * + * Streamed rather than read into memory the way a frame is: a JPEG is tens + * of kilobytes and an hour of speech is sixty megabytes, and buffering that + * per concurrent transcription is the API process falling over rather than + * the request. + * + * A 404 here is a real answer, not a fault — an `ingest_source` that could + * not download the video uploaded no audio, and the job for it should not + * have been enqueued, but a source deleted mid-transcription reaches exactly + * this line. The worker fails the job, `sourceFailed` writes the sentence, + * and the source stays readable. + */ + @Get("sources/:id/audio") + @WorkerRoute() + async audioDown(@Param("id") sourceId: string, @Res() res: Response) { + const file = audioPath(sourceId); + const size = await fs + .stat(file) + .then((s) => s.size) + .catch(() => 0); + if (!size) throw new NotFoundException("No audio for that source"); + + res.set({ + "content-type": "audio/wav", + "content-length": String(size), + "cache-control": "private, max-age=31536000, immutable", + }); + createReadStream(file).pipe(res); + } + + /* + * There is deliberately no PUT for a transcript here. + * + * One existed, and nothing ever called it: a transcript comes back inside + * the `transcribe_audio` result, where `succeed()` has already checked that + * this worker holds that job and its lease is live. The PUT had no such + * check — it took a source id and a string, so ANY valid worker token could + * overwrite ANY source's transcript without holding a job for it. Dead code + * would be reason enough; dead code that is also the one write on this + * controller with no ownership test is reason twice over. + */ + + /** + * A frame, back down to the machine that is about to look at it. + * + * The browser's copy of this lives on `sources.controller.ts` behind a + * session, and it stays there. This is a second door rather than a widened + * one: making the existing route accept either credential would mean a + * stolen worker token could walk the content library through the same + * handler, and the whole argument for a separate token is that it reaches + * nine endpoints and no more. + * + * `write_section` payloads name frames by this path. None of the three CLIs + * takes bytes and each takes a file differently, so the worker writes them + * to a temp directory of its own and hands over local paths — which is why + * the server can stay the only place the frames actually live. + */ + @Get("sources/:id/frames/:file") + @WorkerRoute() + async frame( + @Param("id") sourceId: string, + @Param("file") file: string, + @Res() res: Response, + ) { + // The same name check the browser's route makes, for a stronger reason: + // this one is reached by a process, not a person, and `..` in a path + // segment is what it would be reached with. + const bytes = await frameFile(sourceId, path.basename(file)); + if (!bytes) throw new NotFoundException("No such frame"); + res + .set({ + "content-type": "image/jpeg", + "cache-control": "private, max-age=31536000, immutable", + }) + .send(bytes); + } +} + +/** + * The sentence a worker gets with its 409. + * + * Written as prose rather than a code, because the machine reading it logs it + * and a person reads that log: "409" on a laptop at midnight says nothing, + * and this says the one thing that is true and reassuring — the work was not + * lost, it was somebody else's by then. + */ +function staleJob(jobId: string) { + return `Job ${jobId} is not yours any more — its lease ran out or it was reassigned. Nothing was written.`; +} + +/** + * A wait that cannot outlive the process. + * + * `unref` so a claim held mid-poll never keeps Node alive through a shutdown, + * and a plain resolving timer rather than an abortable one because the caller + * re-checks the socket on the other side of it; there is nothing to cancel. + */ +function sleep(ms: number) { + return new Promise((resolve) => { + const timer = setTimeout(resolve, ms); + timer.unref?.(); + }); +} + +/** + * One part, to one file, counted as it goes. + * + * The same shape as `writeCapped` in services/ingest.ts — a counting Transform + * between the stream and the file, and the partial file removed on the way out + * — because the failure it prevents is the same one: a sender that declares + * ten megabytes and sends ten gigabytes, and a half-written JPEG left on disk + * that the picker will happily try to render. + */ +async function writeCappedFile(file: NodeJS.ReadableStream, dest: string) { + let seen = 0; + const cap = new Transform({ + transform(chunk: Buffer, _enc, done) { + seen += chunk.length; + if (seen > MAX_UPLOAD_BYTES) { + done(new BadRequestException("That upload is over the size cap.")); + return; + } + done(null, chunk); + }, + }); + + try { + await pipeline(file, cap, createWriteStream(dest)); + } catch (e) { + await fs.rm(dest, { force: true }).catch(() => {}); + throw e; + } +} diff --git a/components/overlays/RunSetupSheet.tsx b/components/overlays/RunSetupSheet.tsx index 3c3eed2..fc846ca 100644 --- a/components/overlays/RunSetupSheet.tsx +++ b/components/overlays/RunSetupSheet.tsx @@ -5,12 +5,22 @@ import { useEffect, useRef, useState } from "react"; import { QUALITIES } from "@/lib/data"; import { PACKS } from "@/lib/packs/enbn-website"; import { startTopicRun } from "@/lib/start-run"; -import { fetchSource, frameUrl, type Source } from "@/lib/sources-client"; +import { + fetchSourceToEnd, + fetchingLine, + frameUrl, + isStopped, + sourceProblem, + transcriptNote, + transcriptState, + type Source, +} from "@/lib/sources-client"; import { useStore } from "@/lib/store"; import { themesOf } from "@/lib/topics"; import { font, ghost, ghostHover, layer, primary, primaryActive, primaryHover, t, w } from "@/lib/theme"; import { Hov } from "@/components/ui/Hov"; import { Popover } from "@/components/ui/Popover"; +import { Button, Chip } from "@/components/ui"; import { ChevronDown, PlayIcon } from "@/components/ui/Icons"; export function RunSetupSheet() { @@ -129,6 +139,19 @@ export function RunSetupSheet() { const [source, setSource] = useState(null); const [fetching, setFetching] = useState(false); const [sourceError, setSourceError] = useState(""); + /** + * The row of a link still being fetched, as the server last told it. + * + * Held apart from `source` on purpose: a row in `fetching` has no frames and + * no transcript on it, so putting it in `source` would draw the card that + * says what the run has been handed over a video that has not been fetched — + * and worse, hand its id to `startTopicRun`. + */ + const [pending, setPending] = useState(null); + /** Said quietly when a wait was let go of, which is not an error. */ + const [sourceNote, setSourceNote] = useState(""); + /** Calls off the poll: Stop waiting, or the sheet going away. */ + const waiting = useRef(null); /* * Nothing is written against a workspace the server has not confirmed. @@ -140,6 +163,23 @@ export function RunSetupSheet() { */ const ready = projectsLoaded && Boolean(project.id); + /* + * The sheet going away stops the poll. + * + * Cleanup only. Without it a fetch left behind by a closed sheet keeps asking + * the API every few seconds until the give-up window runs out. The job itself + * is on the server and carries on; pressing Fetch again picks it up. + */ + useEffect(() => () => waiting.current?.abort(), []); + + /** Let go of a fetch without cancelling it. */ + const stopWaiting = () => { + waiting.current?.abort(); + setSourceNote( + "Stopped waiting. The fetch is still on the queue — press Fetch again to pick it up, or run without it.", + ); + }; + const pullSource = async () => { const url = sourceUrl.trim(); if (!url || fetching) return; @@ -147,17 +187,47 @@ export function RunSetupSheet() { setSourceError("Still loading the workspace — try again in a second."); return; } + const stop = new AbortController(); + waiting.current = stop; setFetching(true); setSourceError(""); + setSourceNote(""); + setSource(null); try { - setSource(await fetchSource({ url, workspaceId: project.id })); + /* + * Follow the job; do not believe the POST. + * + * `POST /api/sources` writes an `ingest_source` job and answers at once + * with a row in `fetching` — empty frames, empty transcript. This sheet + * used to show that as the fetched video, so the run was handed a source + * with nothing on it and the first two sections were asked to describe + * stills that did not exist. `fetchSourceToEnd` waits for the real thing. + */ + const settled = await fetchSourceToEnd( + { url, workspaceId: project.id }, + setPending, + stop.signal, + ); + /* + * Failed is an answer, and its sentence is the useful part — which tool + * no machine here has and what installs it, or which machine it waited + * for. It goes in the red block; the run can still start without a reel. + */ + if (settled.state === "failed") { + setSourceError(settled.error || "Could not fetch that link"); + return; + } + setSource(settled); } catch (e) { - setSource(null); + // Letting go of the wait is something the person did, not a failure. + if (isStopped(e)) return; setSourceError( e instanceof Error ? e.message : "Could not fetch that link", ); } finally { setFetching(false); + setPending(null); + waiting.current = null; } }; @@ -174,6 +244,20 @@ export function RunSetupSheet() { setStartError("Still loading the workspace — try again in a second."); return; } + /* + * Do not start a run that quietly loses the reel. + * + * `sourceId` is `source?.id`, and `source` is only set once the fetch has + * landed — so pressing Run mid-fetch would start the whole template against + * no video at all, minutes of model time after you pasted the link that was + * the entire reason for this run. Say so, and offer the two ways out. + */ + if (fetching) { + setStartError( + "The reel is still coming. Wait for it, or press Stop waiting to run without it.", + ); + return; + } setStarting(true); setStartError(""); try { @@ -571,10 +655,70 @@ export function RunSetupSheet() { - {fetching ? ( + {/* + What the wait is doing, in the row's own words. + + A link is a job on somebody else's machine now, and "a machine is + downloading it" and "waiting for a laptop that is switched off" are + different evenings. `ingest()` writes the sentence for each onto the + source; a spinner would hide both. + */} + {pending ? ( +
+
+ + FETCHING + + + {pending.title || pending.url} + + +
+
+ {fetchingLine(pending)} +
+
+ ) : fetching ? ( +
+ Asking the server to queue it… +
+ ) : null} + + {/* Letting go of a wait is not a failure, so it is not in red. */} + {sourceNote && !fetching ? (
- Downloading, cutting stills and transcribing. This takes a minute - or two — leave the sheet open. + {sourceNote}
) : null} @@ -620,7 +764,19 @@ export function RunSetupSheet() { source.frames.length ? `${source.frames.length} stills` : "no stills", - source.transcript ? "transcript" : "no transcript", + /* + Three answers, not two. The words are transcribed by a + separate job on a separate machine, so a reel is routinely + fetched, framed and ready with its transcript still queued — + and "no transcript" for that reports a loss that has not + happened. The run can start either way; a section that reads + the stills has everything it needs. + */ + transcriptState(source) === "here" + ? "transcript" + : transcriptState(source) === "coming" + ? "transcript on the way" + : "no transcript", ] .filter(Boolean) .join(" · ")} @@ -652,9 +808,20 @@ export function RunSetupSheet() { ))} ) : null} - {source.error ? ( + {/* The video's own bad news — "Metadata only — this video is + private" — which is worth seeing before spending a run on it. */} + {sourceProblem(source) ? (
- {source.error} + {sourceProblem(source)} +
+ ) : null} + + {/* The transcript's news, quieter and separate. Both share one + `error` column, and amber over a card with eight stills on it + says the fetch went wrong when it went fine. */} + {transcriptNote(source) ? ( +
+ {transcriptNote(source)}
) : null} @@ -760,7 +927,10 @@ export function RunSetupSheet() { (null); + const [name, setName] = useState(""); + const [busy, setBusy] = useState(false); + const [error, setError] = useState(""); + /** The one moment the token exists in a browser. Cleared, never re-fetched. */ + const [minted, setMinted] = useState<{ name: string; token: string } | null>( + null, + ); + const [copied, setCopied] = useState(false); + const [editing, setEditing] = useState(null); + const [revoking, setRevoking] = useState(null); + + const load = useCallback(() => { + void listMachines() + .then(setMachines) + .catch((e: unknown) => { + setMachines([]); + setError( + e instanceof Error ? e.message : "Could not read the machines", + ); + }); + }, []); + + useEffect(load, [load]); + + const enrol = async () => { + if (busy || !name.trim()) return; + setBusy(true); + setError(""); + try { + const made = await mintMachine(name.trim()); + setName(""); + setMachines((prev) => [made.machine, ...(prev ?? [])]); + setMinted({ name: made.machine.name, token: made.token }); + // Straight to the clipboard, as an invite link is: the only thing to do + // with a fresh token is paste it into the desktop app, and it is 50 + // characters nobody should be reading off a screen. + await navigator.clipboard?.writeText(made.token).catch(() => {}); + setCopied(true); + } catch (e) { + setError(e instanceof Error ? e.message : "Could not enrol a machine"); + } finally { + setBusy(false); + } + }; + + const save = async (id: string, patch: Parameters[1]) => { + const saved = await patchMachine(id, patch).catch((e: unknown) => { + setError(e instanceof Error ? e.message : "That did not save"); + return null; + }); + if (!saved) return; + setMachines((prev) => (prev ?? []).map((m) => (m.id === saved.id ? saved : m))); + setEditing((prev) => (prev && prev.id === saved.id ? saved : prev)); + }; + + const revoke = async (machine: Machine) => { + setRevoking(null); + setMachines((prev) => (prev ?? []).filter((m) => m.id !== machine.id)); + await revokeMachine(machine.id).catch(() => {}); + load(); + }; + + return ( + <> +
+
Enrol a machine
+

+ A machine is one desktop running the Content OS worker, under one + person’s login, with their own model CLI and their own GPU. Name + it here, paste the token it gives you into the desktop app on that + computer, and it starts asking for work. Nothing runs on the server + itself. +

+ +
+ + setName(e.target.value)} + onKeyDown={(e) => { + if (e.key === "Enter") void enrol(); + }} + placeholder="e.g. Rakib’s desktop" + /> + +
+ +
+
+ + {error ? ( +
{error}
+ ) : null} + + {/* + Shown once, and this is the once. + The server keeps only a sha256 of it, for the same reason API keys + are encrypted and invite tokens are hashed: a database dump must not + be a list of live credentials. There is no route that can show it + again — if it is lost, revoke the machine and enrol it afresh. + */} + {minted ? ( +
+
+ + The token for {minted.name} +
+

+ Copied to your clipboard. This is the only time it is shown — + only a hash of it is stored, so nothing here can print it again. + Paste it into the desktop app on that machine; if you lose it, + revoke the machine below and enrol it again. +

+
+ + {minted.token} + + + +
+
+ ) : null} +
+ + + + + {machines === null ? ( + + ) : machines.length === 0 ? ( + } + title="No machines yet" + line="Until one is enrolled there is nowhere for a run to happen." + /> + ) : ( + machines.map((machine) => ( + setEditing(machine)} + > + + + + } + title={machine.name} + subtitle={[ + machine.ownerName, + machine.platform, + machine.version, + ] + .filter(Boolean) + .join(" · ")} + /> + + + + + + + {machine.lastSeenAt ? ago(machine.lastSeenAt) : "never"} + + + + setEditing(machine)} + > + + + setRevoking(machine)} + > + + + + + )) + )} + + +

+ A grey tool is one that machine does not have — an install away, on that + computer. A machine that is offline is a different thing entirely: its + tools are still listed because they are still installed; it is simply + not there to run them, and the fix is somebody opening their laptop. +

+ + {editing ? ( + setEditing(null)} + onSave={save} + onRevoke={() => { + const m = editing; + setEditing(null); + setRevoking(m); + }} + /> + ) : null} + + {revoking ? ( + } + onClose={() => setRevoking(null)} + footer={ + <> + + + + } + > +

+ Anything it was part way through is not lost and is not finished + either: the job goes back in the queue within a minute or two and + the next capable machine picks it up from the start. A section is + written whole or not at all, so nothing half-done is saved. +

+

+ The token cannot be brought back — only its hash was ever stored. + Putting this machine back means enrolling it again and pasting a new + one into the desktop app. +

+ + ) : null} + + ); +} + +/** + * Three states, and the difference between them is the point of the column. + * + * "Waiting" is a row that exists because somebody made a token and has not + * pasted it anywhere yet — it has never reported a thing about itself. + * "Offline" is a machine that has told us all about itself and is currently + * shut. Collapsing the two would make a token nobody used look like a laptop + * somebody closed. + */ +function Status({ machine }: { machine: Machine }) { + if (machine.awaitingFirstContact) + return ( + + Waiting + + ); + if (machine.live) + return ( + + Online + + ); + return ( + + Offline + + ); +} + +/** What the machine reported, and which of it its owner has switched on. */ +function Tools({ machine }: { machine: Machine }) { + if (machine.awaitingFirstContact) + return ( + nothing reported yet + ); + + const present = machine.tools.filter((tool) => tool.present); + if (!present.length) + return ( + no tools installed + ); + + return ( + + {present.map((tool) => { + const on = machine.enabled.includes(tool.id); + return ( + + {tool.id} + + ); + })} + + ); +} + +/** + * The switches the machine is not allowed to set for itself. + * + * `registerWorker` refuses to overwrite any of these when a worker + * re-registers, which is the whole reason this dialog exists: a machine + * restarting with its defaults must not hand itself back a permission somebody + * switched off here. + */ +function MachineSettings({ + machine, + onClose, + onSave, + onRevoke, +}: { + machine: Machine; + onClose: () => void; + onSave: ( + id: string, + patch: { + name?: string; + enabled?: string[]; + canReadFrames?: boolean; + maxConcurrency?: number; + }, + ) => Promise; + onRevoke: () => void; +}) { + const [name, setName] = useState(machine.name); + const [enabled, setEnabled] = useState(machine.enabled); + const [canReadFrames, setCanReadFrames] = useState(machine.canReadFrames); + const [maxConcurrency, setMaxConcurrency] = useState(machine.maxConcurrency); + const [busy, setBusy] = useState(false); + + const toggle = (id: string) => + setEnabled((prev) => + prev.includes(id) ? prev.filter((x) => x !== id) : [...prev, id], + ); + + const apply = async () => { + setBusy(true); + try { + await onSave(machine.id, { + name: name.trim() || machine.name, + enabled, + canReadFrames, + maxConcurrency, + }); + onClose(); + } finally { + setBusy(false); + } + }; + + return ( + } + footer={ + <> + + + + + } + > + + setName(e.target.value)} /> + + + + {null} + + + {machine.tools.map((tool) => ( + + toggle(tool.id)} + label={`Let this machine use ${tool.id}`} + /> + + ))} + + + setCanReadFrames((v) => !v)} + label="Let the Claude CLI open frame files" + /> + + + + ({ value: v, label: v }))} + value={String(maxConcurrency)} + onChange={(v) => setMaxConcurrency(Number(v))} + /> + + + ); +} + +/** One labelled control in the settings dialog. */ +function Row({ + label, + hint, + mono = false, + children, +}: { + label: string; + hint: string; + mono?: boolean; + children: ReactNode; +}) { + return ( +
+
+
+ {label} +
+
+ {hint} +
+
+ {children ?
{children}
: null} +
+ ); +} diff --git a/components/tools/FramePicker.tsx b/components/tools/FramePicker.tsx index 5e37f16..54b6bcc 100644 --- a/components/tools/FramePicker.tsx +++ b/components/tools/FramePicker.tsx @@ -324,8 +324,10 @@ export function FramePicker({ ) : (

- No frames were cut — the video itself was not downloaded, or - ffmpeg is off in Integrations. It can still read the transcript. + No frames were cut — the video itself was not downloaded, or the + machine that fetched it has ffmpeg switched off. That switch is per + machine now, on Settings → Machines. It can still read the + transcript.

)} diff --git a/components/views/ContentResearcherView.tsx b/components/views/ContentResearcherView.tsx index 37d7d1b..ac954ec 100644 --- a/components/views/ContentResearcherView.tsx +++ b/components/views/ContentResearcherView.tsx @@ -5,10 +5,15 @@ import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { findTool } from "@/lib/tools"; import { ago } from "@/lib/packs-client"; import { - fetchSource, + fetchSourceToEnd, + fetchingLine, frameUrl, + isStopped, listSources, removeSource, + sourceProblem, + transcriptNote, + transcriptState, type Source, } from "@/lib/sources-client"; import { @@ -117,12 +122,30 @@ export function ContentResearcherView() { const [at, setAt] = useState(""); const [fetching, setFetching] = useState(false); + /** + * The row of a link that is still being fetched, as the server last told it. + * + * Deliberately not `source`: a source in `fetching` has no frames on it, and + * putting it there would draw step 2 with an empty picker — the very bug this + * screen had when it treated the POST's answer as the finished video. This is + * only what the wait is allowed to say about itself, and it says the source's + * own sentence: "Waiting for Shakhawat's desktop…". + */ + const [pending, setPending] = useState(null); + /** Calls off the poll: Stop waiting, or leaving the tool. */ + const waiting = useRef(null); const [reading, setReading] = useState(false); const [error, setError] = useState(""); const [result, setResult] = useState(null); /** Said out loud when something worked but not the way you asked. */ const [notice, setNotice] = useState(""); + /** + * The same, for the fetch box — which needs its own line because `notice` is + * rendered inside step 2, and step 2 does not exist until a video has landed. + * "Stopped waiting" is said precisely when one has not. + */ + const [fetchNote, setFetchNote] = useState(""); const [series, setSeries] = useState(NO_SERIES); const [saved, setSaved] = useState([]); const fileBox = useRef(null); @@ -138,6 +161,14 @@ export function ContentResearcherView() { const frames = source?.frames ?? []; const chosen = picked.length ? picked : frames.map((f) => f.file); + /** + * Where this video's words are — here, still coming, or not coming at all. + * + * Read from the row rather than from `transcript` being empty, because those + * are different facts: transcription is a separate job on a separate machine, + * and an empty string means "not yet" at least as often as it means "never". + */ + const words = source ? transcriptState(source) : "none"; const seriesOptions = useMemo( () => [ @@ -236,20 +267,70 @@ export function ContentResearcherView() { */ const workspaceReady = projectsLoaded && Boolean(project?.id); + /* + * Leaving the tool stops the poll. + * + * Cleanup only — a fetch that is still queued would otherwise go on asking + * the API every few seconds from a screen nobody is looking at, for up to the + * give-up window. The job is unaffected: it is on the server, and pressing + * Fetch again picks it up where it got to. + */ + useEffect(() => () => waiting.current?.abort(), []); + + /** Let go of a fetch without cancelling it. */ + const stopWaiting = () => { + waiting.current?.abort(); + setFetchNote( + "Stopped waiting. The fetch is still on the queue — press Fetch again to pick it up.", + ); + }; + const fetchUrl = async () => { if (!url.trim() || fetching) return; if (!workspaceReady) { setError("Still loading the workspace — try again in a second."); return; } + const stop = new AbortController(); + waiting.current = stop; setFetching(true); setError(""); + setNotice(""); + setFetchNote(""); try { - start(await fetchSource({ url: url.trim(), workspaceId: project?.id })); + /* + * Follow the job; do not believe the POST. + * + * `POST /api/sources` only writes an `ingest_source` job now and answers + * with a row in `fetching` — no frames, no transcript. Handing that + * straight to `start()` is what drew an empty frame picker for every + * link, every time. `fetchSourceToEnd` polls until the row is ready or + * failed, and `setPending` shows what it is doing meanwhile. + */ + const settled = await fetchSourceToEnd( + { url: url.trim(), workspaceId: project?.id }, + setPending, + stop.signal, + ); + /* + * Failed is an answer, not an exception — and its sentence is the useful + * part: which tool no machine here has and what installs it, or which + * machine it waited for and never got. + */ + if (settled.state === "failed") { + setError(settled.error || "That link could not be fetched"); + return; + } + start(settled); } catch (e) { + // Stopping the wait is something the person did; a banner about it would + // be the screen complaining about being obeyed. + if (isStopped(e)) return; setError(e instanceof Error ? e.message : "That link could not be fetched"); } finally { setFetching(false); + setPending(null); + waiting.current = null; } }; @@ -260,7 +341,16 @@ export function ContentResearcherView() { } setFetching(true); setError(""); + setFetchNote(""); try { + /* + * Uploads are not a job, so there is nothing to follow. + * + * The bytes arrive here and ffmpeg cuts the stills in the API process, so + * this answer really is the finished video — polling it would be a round + * trip to be told what we were just handed. Only the transcript leaves, + * as its own job, and the row says so itself. + */ const { uploadSource } = await import("@/lib/tools-client"); start(await uploadSource(file, project?.id)); } catch (e) { @@ -451,14 +541,86 @@ export function ContentResearcherView() { }} /> - {fetching ? ( + {/* + What the wait is actually doing, in the row's own words. + + A link is a job on somebody else's machine now, and the three things + that can be true of it — running, queued behind a laptop that is shut, + or already failed because nothing here has yt-dlp — read completely + differently to the person waiting. `ingest()` writes the sentence for + each onto the source; a spinner would throw all three away. + + An upload has no such row to show: the bytes are still going up and + the stills are cut on the server, so it gets its own line. + */} + {pending ? ( +
+
+ + FETCHING + + + {nameOf(pending)} + + +
+
+ {fetchingLine(pending)} +
+
+ ) : fetching ? (
- Downloading, cutting stills and transcribing. Minutes, not seconds — - most of it is the download. + Uploading, and cutting the stills here on the server. The transcript + follows on whichever machine has whisper. +
+ ) : null} + + {/* Letting go of a wait is not a failure, so it is not in the red + banner at the top of the screen. */} + {fetchNote && !fetching ? ( +
+ {fetchNote}
) : null} - {source ? ( + {/* + The video this screen is on — hidden while another one is being + fetched, because the two lines sit one under the other and "0:41 · + transcript · 8 frames" directly beneath "FETCHING " reads + as a description of the thing that is still coming. The steps below + keep the old video, which stays perfectly readable until the new one + lands and `start()` replaces it. + */} + {source && !pending ? (
{source.duration ? clock(source.duration) : ""} - {source.transcript ? " · transcript" : " · no transcript"} + {/* + Three words, not two. `transcribe_audio` is its own job on its + own machine, so a video is routinely ready and worth reading + with the words still on their way — and "no transcript" for + that is a failure being reported before it has happened. + */} + {words === "here" + ? " · transcript" + : words === "coming" + ? " · transcript on the way" + : " · no transcript"} {` · ${frames.length} frames`}
) : null} - {source?.error ? ( + {/* The source's own bad news: amber, because it is about the video. */} + {source && !pending && sourceProblem(source) ? (
- {source.error} + {sourceProblem(source)} +
+ ) : null} + + {/* + The transcript's news, kept quieter and kept separate. One `error` + column carries both, and painting "a machine is working on it" in + warning amber next to eight perfectly good frames says the video is + broken when nothing is. + */} + {source && !pending && transcriptNote(source) ? ( +
+ {transcriptNote(source)}
) : null} @@ -541,6 +726,21 @@ export function ContentResearcherView() { {reading ? "Watching…" : "Watch it and give me ideas"} + + {/* + Said, not enforced. The frames are the evidence this tool is built + on — an address bar in frame four names the site that a transcript + only gestures at — so a video whose words are still queued behind + somebody's GPU is entirely readable now. Blocking the button until + whisper landed would stall the tool on an estate that may have no + whisper at all. + */} + {words === "coming" ? ( +
+ The transcript is still coming. This reads the frames either way — + or wait for it and press again. +
+ ) : null} ) : null} diff --git a/components/views/DocumentView.tsx b/components/views/DocumentView.tsx index 8743437..ba88d1c 100644 --- a/components/views/DocumentView.tsx +++ b/components/views/DocumentView.tsx @@ -22,12 +22,17 @@ import { import { useStore } from "@/lib/store"; import { latestRunFor, topicSlug } from "@/lib/run-doc"; import { packForSeries } from "@/lib/packs"; -import { editSection, startWriting, writeSection } from "@/lib/runs-client"; -import { useRunDocument } from "@/lib/use-run-document"; +import { + editSection, + queueSection, + startWriting, + stopRun, +} from "@/lib/runs-client"; +import { useRunDocument, type Placement } from "@/lib/use-run-document"; import { frameUrl, type Source } from "@/lib/sources-client"; import { font, panel, primary, rise, t, w } from "@/lib/theme"; import { Hov } from "@/components/ui/Hov"; -import { Button, Rail, TextArea } from "@/components/ui"; +import { Button, Chip, Rail, TextArea, type Tone } from "@/components/ui"; import { PencilIcon } from "@/components/ui/Icons"; import { UnwrittenTopic } from "@/components/views/UnwrittenTopic"; import { ReadingMode } from "@/components/overlays/ReadingMode"; @@ -203,6 +208,84 @@ const STATUS_TONE = { }, } as const; +/** + * A place, as one word and one colour. + * + * `blocked` and `idle` are grey because neither is a problem: one is waiting + * its turn and the other is waiting for you. `waiting` is amber because it is + * an errand — somebody has to switch a machine on. `unroutable` is red and says + * NO MACHINE rather than FAILED, because Retry cannot fix it: the tool is not + * installed anywhere on the estate, and pressing Retry until it is would be + * the only thing the old wording invited. + */ +const PLACE_LABEL: Record = { + done: "READY", + idle: "NOT STARTED", + blocked: "BLOCKED", + queued: "QUEUED", + waiting: "WAITING", + writing: "WRITING", + unroutable: "NO MACHINE", + failed: "FAILED", +}; + +const PLACE_TONE: Record = { + done: "good", + idle: "mute", + blocked: "mute", + queued: "accent", + waiting: "warn", + writing: "accent", + unroutable: "bad", + failed: "bad", +}; + +/** + * The sentence under the row, when there is something worth saying. + * + * Empty for the ordinary states — a row that is written, being written, or + * plainly not started yet explains itself, and a line of prose under every one + * of twelve sections is noise that hides the two that matter. + * + * The two that matter read as opposite instructions on purpose. "whisper is + * not installed on any machine" is a command to run; "that machine has not + * been seen recently" is a laptop to open. They were the same grey QUEUED chip + * before, which is how a run could sit all afternoon while everybody assumed + * the other thing. + */ +function placementNote(at: Placement | undefined): string { + if (!at) return ""; + switch (at.kind) { + case "blocked": + return `Needs ${at.on.join(" and ")} first.`; + case "writing": + /* + * Whose machine is typing it. The page could not say this at all until + * the queue became readable: `run_sections` records `wroteWith` on + * success, so mid-flight there was no name anywhere, and a run spread + * across four laptops looked exactly like a run on one. + */ + return at.machine ? `Being written on ${at.machine}.` : ""; + case "waiting": + /* + * The server's own sentence first, because it names every machine the + * job could go to and the browser has no list of machines to rebuild + * that from. The fallbacks cover a job pinned to one machine — a name + * and no sentence — and a section the poll has not caught up with yet. + */ + return at.message + ? `${at.message} Opening it starts this.` + : at.machine + ? `Waiting for ${at.machine}, which has not been seen recently — opening it starts this.` + : "On the queue. No machine has picked it up yet."; + case "unroutable": + case "failed": + return at.message; + default: + return ""; + } +} + export function DocumentView({ slug, runId, @@ -229,13 +312,27 @@ export function DocumentView({ setFetched, serverWriting, setServerWriting, + waitingForMachine, + placements, } = useRunDocument({ slug, runId }); const [openSections, setOpenSections] = useState>( new Set(["03"]), ); const [filter, setFilter] = useState(""); - const [busy, setBusy] = useState>(new Set()); + /* + * Sections whose request is in the air, by id. + * + * It used to be "sections this tab is writing", held for the length of the + * model call, because the tab WAS the driver: it asked for a section and + * waited minutes for the text. The request is a database write now and comes + * back in milliseconds with the row already saying `writing`, so this covers + * only the gap between the click and that answer — it stops a second click + * queueing the same section and gives the button something to say meanwhile. + */ + const [queuing, setQueuing] = useState>(new Set()); + /** True between pressing Stop and the server confirming the jobs are gone. */ + const [stopping, setStopping] = useState(false); const [copied, setCopied] = useState(null); /* @@ -264,70 +361,126 @@ export function DocumentView({ const [reading, setReading] = useState(false); /** - * Is it writing, waiting, or stopped? + * Is it writing, waiting on a machine, unrunnable, or stopped? * * The page showed "0/12 written" and twelve QUEUED chips whether the run was * mid-flight, finished with nothing, or had never been started — three very - * different situations that all look identical while you watch them. With - * auto-approve off a new run does not start on its own, so the honest answer - * for most of them is "waiting for you", and that was the one thing nothing - * on the page said. + * different situations that all look identical while you watch them. The + * split into workers added two more that look identical to those: a run + * whose sections are on the queue with every machine switched off, and a run + * that can never move because nothing on the estate has the tool. Those two + * have opposite answers — open a laptop, or install something — so the line + * says which. */ const status = useMemo(() => { if (!run) return null; + const placed = run.sections.map((x) => ({ + row: x, + at: placements.get(x.id), + })); + /* - * In flight here, or in flight anywhere. + * Something is actually being written. * - * `busy` is this tab's own loop; the row's state is what the server - * recorded. A run driven from another tab moves the second and not the - * first, and reading only the local set would report "not running" while - * the machine was busy writing. + * The row's own state, not this tab's: the writing happens on somebody + * else's machine now, so there is no local fact to prefer. */ - const serverBusy = run.sections.find((x) => x.state === "writing"); - if (busy.size || serverBusy || serverWriting) { - const now = - doc?.sections.find((x) => busy.has(x.n)) ?? - doc?.sections.find((x) => x.id === serverBusy?.id); - /* - * Between two sections there is nothing being written, and the honest - * word for that is not "Writing…" with no object. The one about to go is - * the next queued, which is what the server picks the moment the last one - * lands — so the pill names it rather than going vague for a second and a - * half every time a section finishes. - */ + const writing = placed.find((x) => x.at?.kind === "writing"); + if (writing || queuing.size) { + const now = doc?.sections.find((x) => x.id === writing?.row.id); const next = doc?.sections.find((x) => x.state === "queued"); + // Which machine, once one has claimed it. This said "a worker machine" + // to everybody, always, because nothing named one until the section had + // already landed and `wroteWith` was written. + const on = writing?.at?.kind === "writing" ? writing.at.machine : ""; return { tone: "busy" as const, + pulse: true, text: now ? `Writing ${now.n} · ${now.name}…` : next ? `Starting ${next.n} · ${next.name}…` : "Writing…", - // It runs from whichever tab started it: the browser asks for one - // section, waits, then asks for the next. Closing that tab stops it - // between sections. - note: busy.size ? "Keep this tab open" : "Running on the server", + note: on + ? `On ${on} — you can close this tab` + : "On a worker machine — you can close this tab", }; } - const failed = run.sections.filter((x) => x.state === "failed"); + /* + * On the queue, and nothing has picked it up. + * + * The state that did not exist before and that the old page had no word + * for: it read "Writing…" and pulsed at a run where nothing was happening + * at all, forever, because a laptop was shut. Naming the machine when the + * server told us which turns it into an errand somebody can run. + */ + const asleep = placed.find( + (x) => x.at?.kind === "waiting" && (x.at.machine || x.at.message), + )?.at; + if (waitingForMachine || asleep) { + return { + tone: "idle" as const, + pulse: false, + text: "Queued — waiting for a machine", + // The row's own sentence rather than a second wording of it: two + // copies of one fact, written in two places, drift apart — and this + // one already names the machines and says what opening one does. + note: placementNote(asleep) || "No machine has picked this up yet.", + }; + } + + /* + * Nothing here can ever run it. Not a failure to retry — an install. + * + * `advance` fails the section with the sentence naming the tool and the + * command, so the note is the server's own words rather than a second copy + * of them written here and gone stale. + */ + const unroutable = placed.filter((x) => x.at?.kind === "unroutable"); + if (unroutable.length) { + const first = unroutable[0]?.at; + return { + tone: "bad" as const, + pulse: false, + text: `${unroutable.length} section${unroutable.length === 1 ? "" : "s"} no machine can write`, + note: + first?.kind === "unroutable" + ? first.message + : "Nothing here has the tool it needs.", + }; + } + + const failed = placed.filter((x) => x.at?.kind === "failed"); if (failed.length) { + const first = failed[0]?.at; return { tone: "bad" as const, + pulse: false, text: `Stopped — ${failed.length} section${failed.length === 1 ? "" : "s"} failed`, - note: failed[0]?.error?.slice(0, 90) || "Open the section to see why", + note: + (first?.kind === "failed" ? first.message.slice(0, 90) : "") || + "Open the section to see why", }; } - if (!unwritten) return { tone: "good" as const, text: "All written", note: "" }; + if (!unwritten) { + return { + tone: "good" as const, + pulse: false, + text: "All written", + note: "", + }; + } return { tone: "idle" as const, + pulse: false, text: `Not running — ${unwritten} queued`, note: 'Press "Write the rest"', }; - }, [run, busy, doc, unwritten, serverWriting]); + }, [run, queuing, doc, unwritten, placements, waitingForMachine]); /* * Whether it carries on by itself. @@ -346,60 +499,115 @@ export function DocumentView({ }, [settings, unwritten]); /** - * Write these sections, by id, in order. + * Ask for these sections, by id. + * + * It used to be a driver: one request per section, awaited, in order, + * because the model was a child process of the request and the answer WAS + * the text. `POST /runs/:id/sections/:id` enqueues now and comes straight + * back with the run, so awaiting it in a loop asked for the second section + * while the first had not been written — and the page believed text had + * landed when nothing had. * - * One request each, because the model behind them is a single local process - * and parallel requests raced it into returning an error page instead of - * JSON. Slower, and it finishes. + * So: ask, take the run each answer carries, and let the watcher follow the + * rest. The server cascades on its own — a section landing runs `advance`, + * which queues whatever that unblocked — so this is only ever "please start + * this one". */ - const writeIds = useCallback(async (ids: string[]) => { - if (!run || !ids.length) return; - const ns = ids - .map((id) => doc?.sections.find((x) => x.id === id)?.n) - .filter((n): n is string => Boolean(n)); - - setBusy((prev) => new Set([...prev, ...ns])); - for (const id of ids) { - try { - const after = await writeSection(run.id, id); - // The server's copy, straight away: the driver reads `run` to decide - // what is ready next, and waiting for a list reload to tell it would - // stall between every section. - if (runId) setFetched(after); - } catch { - // The row shows what the server recorded on the next read; a failed - // one must not stop the sections queued behind it. + const writeIds = useCallback( + async (ids: string[]) => { + if (!run || !ids.length) return; + setQueuing((prev) => new Set([...prev, ...ids])); + /* + * Poll fast from this instant rather than from the first answer: the run + * IS on the queue the moment the request is accepted, and a page sitting + * on "Not running" for six seconds after a click reads as a click that + * missed. + */ + setServerWriting(true); + for (const id of ids) { + try { + const after = await queueSection(run.id, id); + // The server's copy, straight away — it already says `writing`, and + // waiting for a list reload to say so is a second of dead page. + setFetched(after); + } catch { + // Whatever the server recorded shows on the next poll; one section + // being refused must not stop the ones asked for behind it. + } + setQueuing((prev) => { + const next = new Set(prev); + next.delete(id); + return next; + }); } - setBusy((prev) => { - const next = new Set(prev); - const n = doc?.sections.find((x) => x.id === id)?.n; - if (n) next.delete(n); - return next; - }); - } - await reloadRuns(); - }, [run, doc, runId, reloadRuns, setFetched]); - - // The local driver, for "Write next" and for the auto-approve preference. - // The whole-run case is the server's job now — see `serverWriting` below. - useEffect(() => { - if (!auto || !run || busy.size || !pending.length) return; - const timer = setTimeout(() => void writeIds([pending[0]]), 150); - return () => clearTimeout(timer); - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [auto, run, pending, busy.size]); + await reloadRuns(); + }, + [run, reloadRuns, setFetched, setServerWriting], + ); - /** Write the lot, on the server, and start watching straight away. */ + /** + * Write the lot. + * + * The answer used to be held onto: its `waiting` list carried the only + * machine names the browser ever saw, so it was kept in a hook and drawn + * under sections it may already have stopped describing — and only ever in + * the tab that pressed this button, never for a colleague opening the same + * run and never for a single-section write. `GET /runs/:id/jobs` says all of + * that for every section, to everybody, however the run was started, so this + * is back to what it looks like: start it, and let the watcher follow. + */ const writeEverything = useCallback(async () => { if (!run) return; setServerWriting(true); try { - await startWriting(run.id); + const answer = await startWriting(run.id); + setServerWriting(answer.running); } catch { setServerWriting(false); } }, [run, setServerWriting]); + /** + * Stop a run that is writing. + * + * Queued jobs stop being work at once; a machine already part way through a + * section finds out at its next heartbeat, within about fifteen seconds, so + * a row can still say `writing` for a moment after this returns. Which is + * why the run is re-read afterwards rather than assumed idle. + */ + const stop = useCallback(async () => { + if (!run || stopping) return; + setStopping(true); + try { + await stopRun(run.id); + setServerWriting(false); + await reloadRuns(); + } catch { + // It is still running, and the next poll will say so. + } finally { + setStopping(false); + } + }, [run, stopping, setServerWriting, reloadRuns]); + + /* + * Auto-approve: carry on without being asked. + * + * One press of Start rather than a section at a time. The old loop drove the + * run from this tab — write one, wait, pick the next — which is exactly the + * job the server took over: `advance` queues everything ready, and every + * section that lands queues whatever it unblocked. Driving it from here as + * well would only race the server to enqueue the same section, and would + * stop the moment the tab was closed. + */ + useEffect(() => { + if (!auto || !run || serverWriting || queuing.size || !pending.length) { + return; + } + const timer = setTimeout(() => void writeEverything(), 150); + return () => clearTimeout(timer); + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [auto, run, pending, serverWriting, queuing.size]); + /* * Two ways to have nothing to show, and they are different problems. * @@ -504,10 +712,28 @@ export function DocumentView({ const written = writtenCount(doc); const stateOf = (s: DocSection): SectionState => - busy.has(s.n) ? "writing" : s.state; + s.id && queuing.has(s.id) ? "writing" : s.state; + + /** + * Where this row's work is, as opposed to what state it is in. + * + * `state` is one of four words the database keeps; this is the sentence a + * person can act on — which machine, awake or not, or nothing at all. See + * lib/use-run-document.ts for how the two are told apart. + */ + const placeOf = (s: DocSection): Placement | undefined => + s.id ? placements.get(s.id) : undefined; /** The one being written this second, if any, and how many gave up. */ const nowWriting = doc.sections.find((s) => stateOf(s) === "writing") ?? null; + /* + * And on which machine, once one has claimed it. Empty for the second + * between the row being marked and a worker's long poll taking it, and for + * a section the server is writing itself on the workspace's API key. + */ + const nowWritingAt = nowWriting ? placeOf(nowWriting) : undefined; + const nowWritingOn = + nowWritingAt?.kind === "writing" ? nowWritingAt.machine : ""; const failedCount = doc.sections.filter((s) => stateOf(s) === "failed").length; const rollUp = (sections: DocSection[]): SectionState => { @@ -674,12 +900,33 @@ export function DocumentView({
+ {/* + Stop, whenever there is anything on the queue for this run. + + It is offered for a run that is merely queued as well as one being + written, and that is the point: a run parked behind a machine + nobody is going to switch on is the commonest reason to want out, + and before this there was no way to cancel it — the only control + was the one that started it. + */} + {serverWriting ? ( + + ) : null} {/* While a run still has sections to write, the header is about finishing it, not about redoing what is already there. Redo all - comes back once it is done. + comes back once it is done. Nothing that starts work is offered + while work is already queued: pressing it would be asking for the + same sections a second time. */} - {unwritten ? ( + {serverWriting ? null : unwritten ? ( auto ? ( setAuto(false)} @@ -793,10 +1040,12 @@ export function DocumentView({ height: 7, borderRadius: "50%", background: "currentColor", - animation: - status.tone === "busy" - ? "os-pulse 1.1s ease-in-out infinite" - : undefined, + // A run waiting on a shut laptop is amber and still; only + // work actually moving is allowed to pulse, or the page + // animates at you about something that is not happening. + animation: status.pulse + ? "os-pulse 1.1s ease-in-out infinite" + : undefined, }} /> {status.text} @@ -917,10 +1166,22 @@ export function DocumentView({ {nowWriting ? ( writing {nowWriting.n} · {nowWriting.name} + {nowWritingOn ? ` on ${nowWritingOn}` : ""} - ) : serverWriting || busy.size ? ( + ) : queuing.size ? ( - picking the next section… + queueing… + + ) : waitingForMachine ? ( + /* + * Queued, and nothing has taken it. Not "picking the next + * section" — that sentence was written when picking took a + * second and a half and happened in the API process. It can now + * be until somebody gets back to their desk, and saying so is + * the difference between waiting and wondering. + */ + + on the queue — no machine has taken it yet ) : failedCount ? ( @@ -1121,6 +1382,8 @@ export function DocumentView({ {g.sections.map((s) => { const sState = stateOf(s); const sSt = STATE_STYLE[sState]; + const at = placeOf(s); + const said = placementNote(at); const open = openSections.has(s.n) && s.body.length > 0; return (
@@ -1204,24 +1467,38 @@ export function DocumentView({ {s.lang} ) : null} + {/* + One word for where the work is, not for what the + row's column says. + + Four situations shared the word QUEUED: nobody + has asked; a machine is about to take it; a + machine that is switched off is meant to take it; + nothing anywhere can take it. The first two are + fine, the third is an errand and the fourth is an + install — so they get different words and + different colours. + */} {sState !== "written" ? ( - - {sSt.label} - + at ? ( + + {PLACE_LABEL[at.kind]} + + ) : ( + + {sSt.label} + + ) ) : null} {/* Quiet, because it is a note about provenance rather than a state you have to act on. */} @@ -1254,6 +1531,36 @@ export function DocumentView({ > {s.purpose}
+ {/* + Why it is not written, in the server's own words. + + The failure text was carried all the way from + `run_sections.error` into the document and then + drawn nowhere, so a section that could not run said + FAILED and nothing else — and the sentence that says + WHICH tool to install was sitting in the object the + row was rendered from. It wraps, because an install + command truncated at one line is not an install + command. + */} + {said ? ( +
+ {said} +
+ ) : null}
diff --git a/components/views/IntegrationsView.tsx b/components/views/IntegrationsView.tsx index 3f7bdf8..f831c75 100644 --- a/components/views/IntegrationsView.tsx +++ b/components/views/IntegrationsView.tsx @@ -1,41 +1,69 @@ "use client"; -import { useState, type ReactNode } from "react"; +import { Fragment, useState, type ReactNode } from "react"; +import { ago } from "@/lib/packs-client"; import { - patchWiring, - testBrain, + patchMachineTools, + patchWorkspaceWiring, + useBrainTests, useWiringState, type BrainStatus, - type BrainTest, + type ChosenMachine, + type TestView, type ToolStatus, } from "@/lib/integrations-client"; import { useStore } from "@/lib/store"; -import { font, panel, rise, t, w } from "@/lib/theme"; -import { Button, Toggle } from "@/components/ui"; +import { color, font, panel, t, w } from "@/lib/theme"; +import { + Button, + Chip, + EmptyState, + ListCell, + ListHeader, + ListPanel, + ListRow, + Page, + PageHeader, + RowTile, + Select, + Toggle, + Toolbar, +} from "@/components/ui"; import { BrainGlyph, - CheckGlyph, - CloseGlyph, CpuGlyph, + KeyGlyph, WrenchGlyph, } from "@/components/ui/DocIcons"; +import { RefreshIcon } from "@/components/ui/Icons"; /** - * What Content OS can reach, checked rather than claimed. + * What can run where. + * + * This page used to ask "what is on this machine", and answered by spawning + * `claude --version` and `yt-dlp --version` inside the API process. That was + * the right question exactly as long as the API process and the machine with + * the binaries on it were the same computer. They are not: work happens on + * enrolled desktops now, and on a server the old question has one permanent + * answer — nothing installed — while the four laptops that really do have the + * tools stay invisible. * - * Every row here is the result of running the thing: `claude --version`, - * `yt-dlp --version`, a request to localhost:11434. A tool that is not - * installed says so and shows the one command that installs it, because the - * previous version of this page displayed a version number for software that - * might never have been on the machine. + * So it asks "what is on WHICH machine" instead, and the whole page is scoped + * by two pickers it did not have: the machine at the top, and the workspace + * the console is already switched to. Those are the two things migration 0016 + * separated — a fact about somebody's hardware and an editorial choice about a + * brand — and one flat settings object could not tell them apart. + * + * Settings → Machines enrols them and owns their permissions; this page + * answers a different question with the same vocabulary, deliberately, so the + * two read as one console rather than as two lists of the same computers. */ -const GOOD = "#4bb07a"; -const WARN = "#e0a83c"; -const BAD = "#d1656b"; +const MODELS = "1.5fr 1.25fr 168px"; +const TOOLS = "1.4fr 1fr 118px 62px"; -/** What each local tool is actually for, in this app's own terms. */ +/** What each tool is actually for, in this app's own terms. */ const TOOL_ROLE: Record = { "yt-dlp": "Pulls a source reel down so a template can be told what it shows", ffmpeg: "Cuts stills out of that video for the identify step to read", @@ -45,270 +73,389 @@ const TOOL_ROLE: Record = { }; export function IntegrationsView() { - const { go } = useStore(); - const { wiring, setWiring, loading, checking, error, reload } = useWiringState(); - const [saving, setSaving] = useState(""); - const [refused, setRefused] = useState(""); + const { go, project } = useStore(); /* - * Test results, per model, kept until the page is left. + * Empty means "whichever machine the server thinks is the live one". * - * A version number proves a command exists and nothing more: codex can be - * installed AND logged in and still fail because the CLI is a release - * behind. Only sending a prompt settles it, so the page offers to. + * Not defaulted to the first row here, because the server already answers + * that better than a browser can: it picks the most recently seen live + * machine, which is the honest answer to "where can this run right now". */ - const [tests, setTests] = useState>({}); - - async function test(id: string) { - setTests((prev) => ({ ...prev, [id]: "running" })); - try { - const result = await testBrain(id); - setTests((prev) => ({ ...prev, [id]: result })); - } catch (e) { - setTests((prev) => ({ - ...prev, - [id]: { - ok: false, - ms: 0, - reply: "", - error: e instanceof Error ? e.message : "The test could not run", - fix: "", - }, - })); - } - } + const [machineId, setMachineId] = useState(""); + const workspaceId = project?.id ?? ""; + const { wiring, setWiring, loading, checking, error, reload } = useWiringState( + machineId, + workspaceId, + ); + const { tests, test } = useBrainTests(); + const [saving, setSaving] = useState(""); + const [refused, setRefused] = useState(""); + const machines = wiring?.machines ?? []; + const machine = wiring?.machine ?? null; const brains = wiring?.brains ?? []; const tools = wiring?.tools ?? []; - const settings = wiring?.settings; - - /** Missing pieces, counted for the banner at the top. */ - const missingTools = tools.filter((x) => !x.present); - const readyBrains = brains.filter((b) => b.available); - const brain = brains.find((b) => b.id === settings?.brain); + const enabled = wiring?.enabled ?? []; + const ready = brains.filter((b) => b.available); + const installed = tools.filter((x) => x.present); + const chosenBrain = brains.find((b) => b.id === wiring?.brain); + const noMachines = !loading && machines.length === 0; async function chooseBrain(id: string) { - if (!wiring) return; + if (!wiring || !workspaceId) return; setSaving(id); setRefused(""); try { - const next = await patchWiring({ brain: id }); - setWiring({ ...wiring, settings: next }); + const next = await patchWorkspaceWiring(workspaceId, { brain: id }); + setWiring({ ...wiring, brain: next.brain, apiFallback: next.apiFallback }); } catch (e) { - setRefused(e instanceof Error ? e.message : "That model could not be set"); + setRefused( + e instanceof Error ? e.message : "That model could not be set for this workspace", + ); } finally { setSaving(""); } } + async function toggleFallback() { + if (!wiring || !workspaceId) return; + const apiFallback = !wiring.apiFallback; + setWiring({ ...wiring, apiFallback }); + setRefused(""); + await patchWorkspaceWiring(workspaceId, { apiFallback }).catch((e: unknown) => { + setWiring({ ...wiring, apiFallback: !apiFallback }); + setRefused(e instanceof Error ? e.message : "That switch did not save"); + }); + } + async function toggleTool(id: string) { - if (!wiring || !settings) return; - const enabled = settings.enabled.includes(id) - ? settings.enabled.filter((x) => x !== id) - : [...settings.enabled, id]; - setWiring({ ...wiring, settings: { ...settings, enabled } }); - await patchWiring({ enabled }).catch(() => {}); + if (!wiring || !machine) return; + const next = enabled.includes(id) + ? enabled.filter((x) => x !== id) + : [...enabled, id]; + setWiring({ ...wiring, enabled: next }); + await patchMachineTools(machine.id, next).catch((e: unknown) => { + setWiring({ ...wiring, enabled }); + setRefused(e instanceof Error ? e.message : "That switch did not save"); + }); } return ( -
-
-
-

- Integrations -

-

- {loading - ? "Checking this machine…" - : `${readyBrains.length} of ${brains.length} models usable · ${ - tools.filter((x) => x.present).length - } of ${tools.length} tools installed`} -

-
+ + + {/* + The two places the things on this page are actually owned: a + machine's permissions belong to whoever owns the machine, and the + keys belong to the server. Offered once here rather than as a + button in every row that lacks one. + */} + + + + } + /> + + + {machines.length ? ( + <> + + + +
+ +
+
+ +

+

+
+ + +
+ This machine's setup and worker + +
+
Starting…
+

+
+ What the worker is saying + 

+      

+    

+
+    

+      
+      

+        

+          
Console address — the pages this app opens

+          

+          

+        

+        

+          
API address — what the worker calls for jobs

+          

+          

+        

+      

+
+      

+        Use different addresses on this machine
+
+        
+        
+        

+        
+
+        
+        
+        

+        
+      

+
+      
+      

+        Set this machine up with a token instead
+        
+        
+        

+          Minted on the console under Settings → Machines and shown once. It is
+          kept in Windows Credential Manager, not in a file.
+        

+      

+
+      

+        
+        
+        

+        
+      

+      

+    

+    

+    
+
+    
+  
+
diff --git a/start.sh b/start.sh
index d2e1691..a7925b6 100644
--- a/start.sh
+++ b/start.sh
@@ -1,111 +1,144 @@
-#!/usr/bin/env bash
-# Start Content OS on port 3333.
-#
-#   ./start.sh          dev server with hot reload
-#   ./start.sh prod     production build, then serve it
-#   PORT=4000 ./start.sh    use a different port
-
-set -euo pipefail
-
-cd "$(dirname "$0")"
-
-PORT="${PORT:-3333}"
-MODE="${1:-dev}"
-
-if [ ! -d node_modules ]; then
-  echo "→ installing dependencies"
-  npm install
-fi
-
-# The database, before the app that needs it.
-#
-# Every screen reads through Postgres, so without it the app starts, serves a
-# blank shell and 500s in a log nobody has open. Bringing it up here — or
-# saying plainly that it cannot be — is the difference between a fix and ten
-# minutes wondering what broke.
-if ! docker compose ps --status running 2>/dev/null | grep -q contentos-postgres; then
-  echo "→ starting Postgres"
-  if ! docker compose up -d --wait 2>/dev/null; then
-    echo
-    echo "  Postgres could not be started."
-    echo "  Start Docker Desktop, then run:  npm run db:up"
-    echo
-    echo "  Content OS will run without it, but every screen will be empty."
-    echo
-  fi
-fi
-
-# Whisper, if this machine has not got it.
-#
-# The other tools are single binaries a person already has or does not need;
-# whisper is a Python package, and it is the one thing standing between a
-# source video and a transcript. Installing it here means a fresh clone can
-# read a reel without first finding the Integrations page and being told to
-# run something.
-#
-# It is a big download the first time — torch is most of a gigabyte — so it
-# says so, and SKIP_WHISPER=1 turns it off for anyone who does not want it.
-if [ "${SKIP_WHISPER:-0}" != "1" ] && ! command -v whisper >/dev/null 2>&1; then
-  # The first Python that can actually install something.
-  #
-  # Not simply the first one on PATH: `python` is often a virtualenv with no
-  # pip in it — this machine's is — and picking that one fails with "No module
-  # named pip" while a perfectly good `py` sits next to it. `pip` itself is
-  # frequently not on PATH at all, so the module form is what gets asked.
-  PY=""
-  for candidate in python python3 py; do
-    if command -v "$candidate" >/dev/null 2>&1 &&
-       "$candidate" -m pip --version >/dev/null 2>&1; then
-      PY="$candidate"
-      break
-    fi
-  done
-
-  if [ -z "$PY" ]; then
-    echo "  whisper needs Python with pip, which is not on PATH. Transcripts will be skipped."
-  elif "$PY" -m whisper --help >/dev/null 2>&1; then
-    : # Installed, just not as a command on PATH. The app finds it either way.
-  else
-    echo "→ installing whisper (one time, ~1GB — SKIP_WHISPER=1 to skip)"
-    if ! "$PY" -m pip install -q -U openai-whisper; then
-      echo "  whisper did not install. Transcripts will be skipped until it does:"
-      echo "    $PY -m pip install -U openai-whisper"
-    fi
-  fi
-fi
-
-# The API is its own process now, on API_PORT (4000). It is started here so
-# that one command still brings the whole thing up; stopping this script
-# stops both.
-API_PORT="${API_PORT:-4000}"
-if [ ! -d api/node_modules ]; then
-  echo "→ installing the API's dependencies"
-  (cd api && npm install)
-fi
-
-case "$MODE" in
-  dev)
-    echo "→ Content OS API (dev) on http://localhost:$API_PORT/api"
-    (cd api && PORT="$API_PORT" WEB_ORIGIN="http://localhost:$PORT" npm run dev) &
-    API_PID=$!
-    trap 'kill $API_PID 2>/dev/null' EXIT
-    echo "→ Content OS (dev) on http://localhost:$PORT"
-    NEXT_PUBLIC_API_URL="http://localhost:$API_PORT" npm run dev -- --port "$PORT"
-    ;;
-  prod | start)
-    echo "→ building the API"
-    (cd api && npm run build)
-    echo "→ building Content OS"
-    NEXT_PUBLIC_API_URL="http://localhost:$API_PORT" npm run build
-    echo "→ Content OS API on http://localhost:$API_PORT/api"
-    (cd api && PORT="$API_PORT" WEB_ORIGIN="http://localhost:$PORT" CONTENTOS_DATA_DIR=../.data npm run start) &
-    API_PID=$!
-    trap 'kill $API_PID 2>/dev/null' EXIT
-    echo "→ Content OS (production) on http://localhost:$PORT"
-    npm run start -- --port "$PORT"
-    ;;
-  *)
-    echo "usage: ./start.sh [dev|prod]   (PORT overrides the default 3333)" >&2
-    exit 1
-    ;;
-esac
+#!/usr/bin/env bash
+# Start Content OS on port 3333.
+#
+#   ./start.sh          dev server with hot reload
+#   ./start.sh prod     production build, then serve it
+#   PORT=4000 ./start.sh    use a different port
+
+set -euo pipefail
+
+cd "$(dirname "$0")"
+
+PORT="${PORT:-3333}"
+MODE="${1:-dev}"
+
+if [ ! -d node_modules ]; then
+  echo "→ installing dependencies"
+  npm install
+fi
+
+# The database, before the app that needs it.
+#
+# Every screen reads through Postgres, so without it the app starts, serves a
+# blank shell and 500s in a log nobody has open. Bringing it up here — or
+# saying plainly that it cannot be — is the difference between a fix and ten
+# minutes wondering what broke.
+if ! docker compose ps --status running 2>/dev/null | grep -q contentos-postgres; then
+  echo "→ starting Postgres"
+  if ! docker compose up -d --wait 2>/dev/null; then
+    echo
+    echo "  Postgres could not be started."
+    echo "  Start Docker Desktop, then run:  npm run db:up"
+    echo
+    echo "  Content OS will run without it, but every screen will be empty."
+    echo
+  fi
+fi
+
+# No tools are installed here.
+#
+# This used to pip-install openai-whisper, because whisper was the one thing
+# standing between a source video and a transcript AND it ran in this process.
+# Neither half is true now: transcription is a job, it runs on a worker's
+# machine, and the server never needs whisper at all. Installing a gigabyte of
+# torch to start a web app would be a gigabyte spent on nothing.
+#
+# The tools a WORKER needs — yt-dlp, ffmpeg, whisper.cpp and a model — come
+# from `npm run worker:setup`, pinned and checksummed, into .data/tools.
+
+# Anything this machine has been told, the same way every other script here
+# learns it. The worker's token is the only value start.sh needs that it
+# cannot work out for itself.
+if [ -f .env ]; then
+  set -a
+  # shellcheck disable=SC1091
+  . ./.env
+  set +a
+fi
+
+# The API is its own process now, on API_PORT (4000). It is started here so
+# that one command still brings the whole thing up; stopping this script
+# stops both.
+API_PORT="${API_PORT:-4000}"

+

+# The origin the browser will be on, worked out BEFORE it is handed to

+# anything.

+#

+# It used to be written inline as `PORT="$API_PORT" WEB_ORIGIN="http://localhost:$PORT"`,

+# and bash performs command-prefix assignments left to right: PORT became

+# 4000, and only then was $PORT expanded. So the API was told to allow its

+# OWN origin, every browser call from :3333 was refused by CORS, and the page

+# said "The API is not running" about an API that was running and answering

+# curl perfectly. A variable that cannot be shadowed is the whole fix.

+WEB_APP_ORIGIN="http://localhost:$PORT"
+

+# The worker, which is the thing that actually writes.

+#

+# Without one, pressing Run queues a job and nothing ever claims it. The page

+# says so — it names the machine it is waiting for — but a dev server that

+# cannot write a section is not a dev server anybody can work against, so it

+# starts here with the rest.

+#

+# It needs a token, and a token cannot be invented: it is minted once in

+# Settings → Machines and only its sha256 is kept, so nothing here can look it

+# up. Put it in .env as CONTENTOS_WORKER_TOKEN. Without one this says how to

+# get one and carries on, because a missing worker must not stop the console

+# coming up — signing in is where you go to fix it.

+#

+# WORKER=0 ./start.sh skips it, for working on a screen that has no runs in it.

+start_worker() {

+  if [ "${WORKER:-1}" = "0" ]; then

+    return

+  fi

+  if [ -z "${CONTENTOS_WORKER_TOKEN:-}" ]; then

+    echo "→ no worker: set CONTENTOS_WORKER_TOKEN in .env (Settings → Machines mints one)"

+    echo "  Runs will queue and wait. Everything else works."

+    return

+  fi

+  echo "→ worker on this machine, claiming from http://localhost:$API_PORT"

+  # No name is passed: the machine is named in Settings -> Machines when its
+  # token is minted, and the worker is not allowed to rename it.
+  CONTENTOS_API_URL="http://localhost:$API_PORT" npm run worker &
+  WORKER_PID=$!

+}

+

+# Everything this script started dies with it. A worker left behind would keep

+# claiming jobs against a console that is no longer there, and the section it

+# was three minutes into would be written by a process nobody can see.

+stop_all() {

+  kill ${API_PID:-} ${WORKER_PID:-} 2>/dev/null || true

+}

+
+if [ ! -d api/node_modules ]; then
+  echo "→ installing the API's dependencies"
+  (cd api && npm install)
+fi
+
+case "$MODE" in
+  dev)
+    echo "→ Content OS API (dev) on http://localhost:$API_PORT/api"
+    (cd api && PORT="$API_PORT" WEB_ORIGIN="$WEB_APP_ORIGIN" npm run dev) &
+    API_PID=$!
+    trap stop_all EXIT
+    start_worker
+    echo "→ Content OS (dev) on http://localhost:$PORT"
+    NEXT_PUBLIC_API_URL="http://localhost:$API_PORT" npm run dev -- --port "$PORT"
+    ;;
+  prod | start)
+    echo "→ building the API"
+    (cd api && npm run build)
+    echo "→ building Content OS"
+    NEXT_PUBLIC_API_URL="http://localhost:$API_PORT" npm run build
+    echo "→ Content OS API on http://localhost:$API_PORT/api"
+    (cd api && PORT="$API_PORT" WEB_ORIGIN="$WEB_APP_ORIGIN" CONTENTOS_DATA_DIR=../.data npm run start) &
+    API_PID=$!
+    trap stop_all EXIT
+    start_worker
+    echo "→ Content OS (production) on http://localhost:$PORT"
+    npm run start -- --port "$PORT"
+    ;;
+  *)
+    echo "usage: ./start.sh [dev|prod]   (PORT overrides the default 3333)" >&2
+    exit 1
+    ;;
+esac
diff --git a/tests/handoff.test.ts b/tests/handoff.test.ts
new file mode 100644
index 0000000..f8026a8
--- /dev/null
+++ b/tests/handoff.test.ts
@@ -0,0 +1,100 @@
+import { afterEach, beforeEach, describe, expect, it } from "vitest";
+
+import { adoptTarget } from "@/lib/server/auth";
+
+/**
+ * Where a hand-off is allowed to put somebody down.
+ *
+ * `GET /api/auth/adopt` sets a session cookie and then redirects, and those
+ * two halves together are worth more to an attacker than either on its own: a
+ * link that lands a browser on your page *already signed in as its owner* is a
+ * different thing from an open redirect. So the one rule that keeps it honest
+ * — the redirect target must be the console — is the thing worth pinning.
+ *
+ * Pure, and therefore here rather than in a request test. Nothing about a
+ * loosened check is visible to `tsc`, nothing about it shows in a screenshot,
+ * and the failure it prevents is not a broken page but a working one on
+ * somebody else's domain.
+ *
+ * The single-use and expiry rules live in Postgres — one `UPDATE ... WHERE
+ * used_at IS NULL` — and are proved against a real API rather than mocked
+ * here; see docs/WORKER.md, "The hand-off".
+ */
+
+const ORIGINAL = process.env.WEB_ORIGIN;
+
+beforeEach(() => {
+  process.env.WEB_ORIGIN = "https://console.example.com,https://old.example.com";
+});
+
+afterEach(() => {
+  if (ORIGINAL === undefined) delete process.env.WEB_ORIGIN;
+  else process.env.WEB_ORIGIN = ORIGINAL;
+});
+
+describe("where a hand-off may redirect to", () => {
+  it("allows the console it was told about", () => {
+    expect(adoptTarget("https://console.example.com/content")).toBe(
+      "https://console.example.com/content",
+    );
+  });
+
+  it("allows every origin WEB_ORIGIN names, because CORS already does", () => {
+    expect(adoptTarget("https://old.example.com/")).toBe("https://old.example.com/");
+  });
+
+  /* A path is what the app has when it wants to land somebody on one screen. */
+  it("resolves a path against the console", () => {
+    expect(adoptTarget("/runs/run_x")).toBe("https://console.example.com/runs/run_x");
+  });
+
+  it("sends an empty next to the console's front door", () => {
+    expect(adoptTarget("")).toBe("https://console.example.com");
+    expect(adoptTarget("   ")).toBe("https://console.example.com");
+  });
+
+  /*
+   * The four that matter, and they are one rule: whatever the string parses
+   * to, its ORIGIN has to be on the list. A scheme, a host and a port is all
+   * an origin is, so a protocol-relative host, a `javascript:` URL and the
+   * console's name smuggled in as a username all miss it without any of them
+   * needing to be recognised on its own.
+   */
+  it("refuses another host outright", () => {
+    expect(adoptTarget("https://evil.example.com/")).toBeNull();
+  });
+
+  it("refuses a protocol-relative host, which is a host and not a path", () => {
+    expect(adoptTarget("//evil.example.com/")).toBeNull();
+  });
+
+  it("refuses a scheme that is not the web", () => {
+    expect(adoptTarget("javascript:alert(1)")).toBeNull();
+    expect(adoptTarget("data:text/html,")).toBeNull();
+  });
+
+  it("refuses the console's name used as a username on somebody else's host", () => {
+    expect(adoptTarget("https://console.example.com@evil.example.com/")).toBeNull();
+  });
+
+  /* Same host, wrong scheme or wrong port, is a different origin and is not it. */
+  it("refuses the same host on another scheme or port", () => {
+    expect(adoptTarget("http://console.example.com/")).toBeNull();
+    expect(adoptTarget("https://console.example.com:8443/")).toBeNull();
+  });
+
+  it("refuses a subdomain of the console, which is not the console", () => {
+    expect(adoptTarget("https://uploads.console.example.com/")).toBeNull();
+  });
+
+  /*
+   * A deployment that names no console has nowhere legitimate to send anyone,
+   * and must refuse rather than fall back to a default host that would then be
+   * the one origin an attacker knows is always allowed.
+   */
+  it("refuses everything when WEB_ORIGIN is unusable", () => {
+    process.env.WEB_ORIGIN = "not a url";
+    expect(adoptTarget("https://console.example.com/")).toBeNull();
+    expect(adoptTarget("")).toBeNull();
+  });
+});
diff --git a/tests/queue.test.ts b/tests/queue.test.ts
new file mode 100644
index 0000000..cb25f9f
--- /dev/null
+++ b/tests/queue.test.ts
@@ -0,0 +1,294 @@
+import { Client } from "pg";
+import { afterAll, beforeAll, describe, expect, it } from "vitest";
+
+/**
+ * The queue, against a real Postgres.
+ *
+ * The only test here that touches a database, and the exception is argued
+ * rather than assumed. Everything else in `tests/` is a pure function, because
+ * that is what fails silently days later on one row — but the claim query is
+ * not a pure function and it carries more risk than any of them. It decides,
+ * across machines that cannot see each other, who writes which section; the
+ * failure modes are two laptops writing the same thing and paying twice, or a
+ * job no machine can see sitting queued forever. Neither is visible to
+ * `tsc`, neither shows up in a screenshot, and both cost money the first time
+ * they happen in front of the team.
+ *
+ * Three bugs found while writing it are the argument for keeping it:
+ * `bool_and` over zero rows is NULL rather than true, so a job needing nothing
+ * was invisible to every worker; the reaper failed jobs a machine was still
+ * running; and the payload sweep rewrote every historical row on every tick.
+ *
+ * It builds and drops its OWN database. Nothing here can reach the workspace
+ * the dev server is using, and it skips itself entirely when there is no
+ * Postgres to talk to, so `npm run check` on a laptop with the container down
+ * is still green.
+ */
+
+const BASE =
+  process.env.DATABASE_URL ??
+  "postgres://contentos:contentos@localhost:5437/contentos";
+const TEST_DB = "contentos_queue_test";
+const TEST_URL = BASE.replace(/\/[^/?]+(\?|$)/, `/${TEST_DB}$1`);
+const ADMIN_URL = BASE.replace(/\/[^/?]+(\?|$)/, "/postgres$1");
+
+/** Whether there is a Postgres to talk to at all, decided once, quickly. */
+async function reachable() {
+  const c = new Client({ connectionString: ADMIN_URL, connectionTimeoutMillis: 1500 });
+  try {
+    await c.connect();
+    await c.end();
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+const up = await reachable();
+
+async function admin(sql: string) {
+  const c = new Client({ connectionString: ADMIN_URL });
+  await c.connect();
+  try {
+    await c.query(sql);
+  } finally {
+    await c.end();
+  }
+}
+
+// Bound before the repos are imported: `lib/server/db/client.ts` reads
+// DATABASE_URL once, at module load, so the import has to come after this.
+process.env.DATABASE_URL = TEST_URL;
+
+const dbMod = () => import("@/lib/server/db/client");
+const jobsMod = () => import("@/lib/server/repos/jobs");
+const workersMod = () => import("@/lib/server/repos/workers");
+
+let uid = "";
+let n = 0;
+/** A token nothing else in this file will collide with. */
+const token = () => `tok-${uid}-${(n += 1)}`;
+
+describe.skipIf(!up)("the queue", () => {
+  beforeAll(async () => {
+    await admin(`DROP DATABASE IF EXISTS ${TEST_DB}`);
+    await admin(`CREATE DATABASE ${TEST_DB}`);
+
+    const { ready, q, id } = await dbMod();
+    // The real migration list, in order, against an empty database — which is
+    // also the only place the 0015 SQL is ever executed by a test.
+    await ready();
+
+    uid = id("usr");
+    await q(
+      "INSERT INTO users (id, email, name, password_hash, owner) VALUES ($1, $2, 'Test', 'x:y', true)",
+      [uid, `${uid}@test.invalid`],
+    );
+  }, 60_000);
+
+  afterAll(async () => {
+    if (!up) return;
+    const { pool } = await dbMod();
+    await pool().end();
+    await admin(`DROP DATABASE IF EXISTS ${TEST_DB}`);
+  }, 30_000);
+
+  it("applies every migration, and the worker split's columns land", async () => {
+    const { q } = await dbMod();
+    const { MIGRATIONS } = await import("@/lib/server/db/schema");
+    const names = await q<{ name: string }>("SELECT name FROM migrations ORDER BY name");
+    // Compared against the list itself rather than a name written here: a
+    // hardcoded "the last one is 0015" fails the day somebody appends 0016,
+    // which is a passing test breaking on correct work.
+    expect(names.map((r) => r.name)).toEqual(MIGRATIONS.map((m) => m.name).sort());
+
+    const cols = await q<{ t: string; c: string }>(
+      `SELECT table_name AS t, column_name AS c FROM information_schema.columns
+        WHERE table_schema = 'public'`,
+    );
+    const have = new Set(cols.map((r) => `${r.t}.${r.c}`));
+    for (const want of [
+      "workers.token_hash",
+      "workers.workspace_ids",
+      "workers.can_read_frames",
+      "jobs.needs",
+      "jobs.lease_until",
+      "jobs.wait_until",
+      "jobs.swept_at",
+      "run_sections.wrote_with",
+      "workspaces.api_fallback",
+      "runs.created_by",
+      "workspaces.brain",
+    ]) {
+      expect(have.has(want), `migration did not create ${want}`).toBe(true);
+    }
+  });
+
+  it("hands out a job that needs nothing", async () => {
+    // `bool_and` over zero rows is NULL, not true. Without the COALESCE in the
+    // claim query this job is invisible to every worker for ever, and a brain
+    // test simply never answers.
+    const { enqueue, claim } = await jobsMod();
+    const { registerWorker } = await workersMod();
+    const w = await registerWorker({ userId: uid, name: "any", token: token(), enabled: [] });
+    const { job } = await enqueue({ kind: "test_brain", needs: [] });
+    const got = await claim(w.id, [], 5);
+    expect(got.map((j) => j.id)).toContain(job.id);
+  });
+
+  it("shows a job only to a machine that has the tool switched on", async () => {
+    const { enqueue, claim } = await jobsMod();
+    const { registerWorker } = await workersMod();
+    const without = await registerWorker({
+      userId: uid, name: "no whisper", token: token(), enabled: ["claude"],
+    });
+    const { job } = await enqueue({ kind: "transcribe_audio", needs: ["whisper"] });
+    expect((await claim(without.id, ["claude"], 5)).map((j) => j.id)).not.toContain(job.id);
+
+    const withIt = await registerWorker({
+      userId: uid, name: "has whisper", token: token(), enabled: ["whisper"],
+    });
+    expect((await claim(withIt.id, ["whisper"], 5)).map((j) => j.id)).toContain(job.id);
+  });
+
+  it("never hands the same job to two machines at once", async () => {
+    // The whole reason the claim is `FOR UPDATE SKIP LOCKED`. Two workers
+    // asking in the same millisecond must come away with disjoint sets, or a
+    // section is written twice and two subscriptions are spent on it.
+    const { enqueue, claim } = await jobsMod();
+    const { registerWorker } = await workersMod();
+    const a = await registerWorker({ userId: uid, name: "A", token: token(), enabled: ["claude"] });
+    const b = await registerWorker({ userId: uid, name: "B", token: token(), enabled: ["claude"] });
+    for (let i = 0; i < 8; i += 1) {
+      await enqueue({ kind: "write_section", needs: ["claude"] });
+    }
+    const [ga, gb] = await Promise.all([
+      claim(a.id, ["claude"], 8),
+      claim(b.id, ["claude"], 8),
+    ]);
+    const ids = [...ga, ...gb].map((j) => j.id);
+    expect(ids.length).toBeGreaterThan(0);
+    expect(new Set(ids).size).toBe(ids.length);
+  });
+
+  it("keeps an owner's switch off when a machine reports itself again", async () => {
+    /*
+     * A worker re-registers every few minutes. The switch in Settings →
+     * Machines has to survive that, or it is a lie: turn ffmpeg off and the
+     * machine turns it back on by itself within the tick. A tool installed
+     * AFTER enrolment still has to arrive switched on, or it is reported,
+     * listed, and never used.
+     */
+    const { registerWorker, updateWorker, getWorker } = await workersMod();
+    const tool = (tid: string) => ({
+      id: tid, command: tid, env: {}, lead: [], present: true,
+      version: "1", error: "", install: "",
+    });
+    const tok = token();
+
+    // A probed-but-absent tool is reported (the page has to be able to say
+    // "not installed on that computer") and must NOT be switched on: `enabled`
+    // answers "may be used", and the claim query matches `needs` against it
+    // alone.
+    const missing = (tid: string) => ({ ...tool(tid), present: false });
+
+    const first = await registerWorker({
+      userId: uid, name: "re-reg", token: tok,
+      tools: [tool("claude"), tool("ffmpeg"), missing("codex")] as never,
+    });
+    expect([...first.enabled].sort()).toEqual(["claude", "ffmpeg"]);
+
+    // The owner switches ffmpeg off.
+    await updateWorker(first.id, { enabled: ["claude"] });
+
+    // The machine reports again, unchanged, and must not undo that.
+    await registerWorker({
+      userId: uid, name: "re-reg", token: tok,
+      tools: [tool("claude"), tool("ffmpeg")] as never,
+    });
+    expect((await getWorker(first.id))!.enabled).toEqual(["claude"]);
+
+    // The machine may not rename itself either. The name is what a person
+    // reads in the picker when choosing where a run happens, and it is typed
+    // in Settings -> Machines; a check-in that carried a hostname replaced it.
+    await registerWorker({
+      userId: uid, name: "shifa-hostname", token: tok,
+      tools: [tool("claude"), tool("ffmpeg")] as never,
+    });
+    expect((await getWorker(first.id))!.name).toBe("re-reg");
+
+    // Now whisper is installed on that machine. It is new, so it arrives on —
+    // and ffmpeg is still off, because that was a decision.
+    await registerWorker({
+      userId: uid, name: "re-reg", token: tok,
+      tools: [tool("claude"), tool("ffmpeg"), tool("whisper")] as never,
+    });
+    const after = (await getWorker(first.id))!.enabled;
+    expect([...after].sort()).toEqual(["claude", "whisper"]);
+  });
+
+  it("refuses a result from a machine that no longer holds the job", async () => {
+    // The laptop that wakes from sleep and posts a section reassigned twenty
+    // minutes ago. It must not overwrite the one that actually landed.
+    const { enqueue, claim, succeed, fail } = await jobsMod();
+    const { registerWorker } = await workersMod();
+    const mine = await registerWorker({ userId: uid, name: "mine", token: token(), enabled: ["claude"] });
+    const other = await registerWorker({ userId: uid, name: "other", token: token(), enabled: ["claude"] });
+    const { job } = await enqueue({ kind: "write_section", needs: ["claude"] });
+    expect((await claim(mine.id, ["claude"], 1))[0].id).toBe(job.id);
+
+    expect(await succeed(job.id, other.id, { text: "stolen" })).toBeNull();
+    expect(await fail(job.id, other.id, "not mine")).toBeNull();
+    expect(await succeed(job.id, mine.id, { text: "ok" })).not.toBeNull();
+  });
+
+  it("does not fail a job a machine is still running on its last attempt", async () => {
+    // The reaper's passes are order-dependent: expired leases go back first,
+    // and only then are spent jobs failed. Done in one pass it would end a
+    // section three minutes into the attempt that was going to finish it.
+    const { q } = await dbMod();
+    const { enqueue, claim, reap, getJob } = await jobsMod();
+    const { registerWorker } = await workersMod();
+    const w = await registerWorker({ userId: uid, name: "slow", token: token(), enabled: ["claude"] });
+    const { job } = await enqueue({ kind: "write_section", needs: ["claude"], maxAttempts: 1 });
+    const [held] = await claim(w.id, ["claude"], 1);
+    expect(held.attempts).toBe(1); // already at max_attempts, and still running
+
+    await reap();
+    expect((await getJob(job.id))!.state).toBe("claimed");
+
+    await q("UPDATE jobs SET lease_until = now() - interval '1 minute' WHERE id = $1", [job.id]);
+    await reap();
+    expect((await getJob(job.id))!.state).toBe("failed");
+  });
+
+  it("clears a finished job's prompt once, not on every pass", async () => {
+    const { q } = await dbMod();
+    const { enqueue, sweepPayloads } = await jobsMod();
+    const { job } = await enqueue({
+      kind: "write_section",
+      needs: ["claude"],
+      payload: { system: "x".repeat(4000), user: "y" },
+    });
+    await q(
+      `UPDATE jobs SET state = 'done', finished_at = now() - interval '2 days',
+              result = '{"text":"written","ms":1200}'::jsonb
+        WHERE id = $1`,
+      [job.id],
+    );
+
+    expect(await sweepPayloads()).toBeGreaterThan(0);
+    // The second pass must match nothing: without `swept_at` this UPDATE
+    // rewrote every historical row every fifteen seconds, for ever.
+    expect(await sweepPayloads()).toBe(0);
+
+    const [row] = await q<{ payload: Record; result: Record }>(
+      "SELECT payload, result FROM jobs WHERE id = $1",
+      [job.id],
+    );
+    expect(row.payload).toEqual({});
+    // The operational record survives — it is the only thing that says where
+    // the work ran and what it cost.
+    expect(row.result.ms).toBe(1200);
+  });
+});
diff --git a/tests/session-credentials.test.ts b/tests/session-credentials.test.ts
new file mode 100644
index 0000000..27a3389
--- /dev/null
+++ b/tests/session-credentials.test.ts
@@ -0,0 +1,54 @@
+import { describe, expect, it } from "vitest";
+
+import { sessionFromHeader } from "@/api/src/common/session.guard";
+
+/**
+ * Which credential is which.
+ *
+ * The API takes three now — the session cookie a browser sends, the session id
+ * the desktop app sends as `Authorization: Session ` because a webview on
+ * a local origin cannot keep a third-party cookie, and the worker token a
+ * machine sends as `Authorization: Bearer `. Two of those three arrive
+ * in the same header, and telling them apart is one regex in `SessionGuard`.
+ *
+ * That regex is worth a test for one specific failure: a `Bearer` header
+ * reading as a session. A worker token is a machine's credential over nine
+ * queue routes; a session is a person's credential over the whole console —
+ * every run, every template, and the ability to spend model budget. A guard
+ * that accepted either word would let the first stand in for the second, which
+ * is the merge `docs/WORKER.md` argues against under "The desktop app holds
+ * two credentials, on purpose". Nothing about that is visible in a screenshot
+ * and nothing in `tsc` would notice a loosened pattern.
+ *
+ * The lookup itself is not tested here on purpose: it is `sessionUser`, the
+ * same call the cookie path has always made, and a second copy of that
+ * assertion would only be testing Postgres.
+ */
+describe("what an Authorization header counts as", () => {
+  it("reads a session sent under its own scheme", () => {
+    expect(sessionFromHeader("Session abc123")).toBe("abc123");
+  });
+
+  it("is not fussy about the case or the spacing a client uses", () => {
+    expect(sessionFromHeader("session   abc123")).toBe("abc123");
+    expect(sessionFromHeader("  SESSION abc123  ")).toBe("abc123");
+  });
+
+  /*
+   * The one that matters. `Bearer` belongs to `WorkerGuard`, and a worker
+   * token that satisfied the console's guard would be a stolen machine
+   * credential reading the content library.
+   */
+  it("refuses a worker's bearer token", () => {
+    expect(sessionFromHeader("Bearer wrk_aaaabbbbcccc")).toBe("");
+  });
+
+  it("refuses anything else, including a scheme that only starts the same", () => {
+    expect(sessionFromHeader("Basic dXNlcjpwYXNz")).toBe("");
+    expect(sessionFromHeader("Sessionabc123")).toBe("");
+    expect(sessionFromHeader("Session")).toBe("");
+    expect(sessionFromHeader("Session   ")).toBe("");
+    expect(sessionFromHeader("")).toBe("");
+    expect(sessionFromHeader(undefined)).toBe("");
+  });
+});
diff --git a/vitest.config.mts b/vitest.config.mts
index 6cdb821..5ec3f68 100644
--- a/vitest.config.mts
+++ b/vitest.config.mts
@@ -3,11 +3,17 @@ import { defineConfig } from "vitest/config";
 /**
  * Unit tests, for the logic a screenshot cannot check.
  *
- * Deliberately narrow. No component tests and no database: what these cover is
- * the pure functions everything else stands on — the status a row shows, the
- * slug a URL resolves by, what an imported template turns into, what the
- * researcher makes of a model's answer. Those are the things that fail
- * silently, days later, on one row.
+ * Deliberately narrow. No component tests: what these cover is the pure
+ * functions everything else stands on — the status a row shows, the slug a URL
+ * resolves by, what an imported template turns into, what the researcher makes
+ * of a model's answer. Those are the things that fail silently, days later, on
+ * one row.
+ *
+ * `tests/queue.test.ts` is the one exception, and it argues itself in its own
+ * header. The claim query decides, across machines that cannot see each other,
+ * who writes which section; it is not a pure function and no screenshot shows
+ * it working. It builds and drops its own database and skips itself when there
+ * is no Postgres, so `npm run check` with the container down is still green.
  *
  * `.mts` because the package is CommonJS and this file is ESM; `tsconfigPaths`
  * is what makes `@/lib/...` resolve the way it does in the app, so a test
diff --git a/worker/api.ts b/worker/api.ts
new file mode 100644
index 0000000..cd515d0
--- /dev/null
+++ b/worker/api.ts
@@ -0,0 +1,231 @@
+import { openAsBlob } from "node:fs";
+import path from "node:path";
+
+import type { ToolStatus } from "../lib/server/tools";
+import type { Config } from "./config";
+
+/**
+ * The six endpoints a machine speaks, and nothing else.
+ *
+ * Every call carries the bearer token and never a cookie: the worker is not a
+ * browser and must not be able to reach the console with a machine credential.
+ * `docs/WORKER.md` has the argument; the shape of this file is the argument
+ * made in code — there is deliberately no method here that reads a run, a
+ * pack or a template, because the token is not allowed to.
+ */
+
+/**
+ * An answer the server refused, kept with its status so 409 can be told apart.
+ *
+ * The field is assigned in the body rather than declared as a constructor
+ * parameter property: Node strips types out of these files rather than
+ * compiling them, and a parameter property is the one piece of TypeScript that
+ * emits code rather than deleting it — so it is a syntax error at run time
+ * even though `tsc` is perfectly happy with it.
+ */
+export class HttpError extends Error {
+  readonly status: number;
+
+  constructor(message: string, status: number) {
+    super(message);
+    this.status = status;
+  }
+}
+
+/** A job that is no longer this machine's. Not a failure, and not lost work. */
+export const isStale = (e: unknown) => e instanceof HttpError && e.status === 409;
+
+export type ClaimedJob = {
+  id: string;
+  kind: string;
+  payload: Record;
+  leaseUntil: string | null;
+};
+
+export class Api {
+  private readonly cfg: Config;
+
+  // Assigned here rather than as a parameter property, for the reason on
+  // `HttpError` above: Node strips these files, it does not compile them.
+  constructor(cfg: Config) {
+    this.cfg = cfg;
+  }
+
+  private url(pathname: string) {
+    // Absolute paths arrive in payloads — `/api/workers/sources/x/frames/y` —
+    // so the base is joined here rather than concatenated at each call site,
+    // where a doubled or missing slash becomes a 404 that reads like a missing
+    // frame.
+    return new URL(pathname, `${this.cfg.base}/`).toString();
+  }
+
+  private async send(
+    method: string,
+    pathname: string,
+    body?: unknown,
+    signal?: AbortSignal,
+  ): Promise {
+    const res = await fetch(this.url(pathname), {
+      method,
+      headers: {
+        authorization: `Bearer ${this.cfg.token}`,
+        ...(body === undefined ? {} : { "content-type": "application/json" }),
+      },
+      ...(body === undefined ? {} : { body: JSON.stringify(body) }),
+      signal,
+    });
+
+    if (!res.ok) {
+      /*
+       * The server's own sentence, when it wrote one.
+       *
+       * The 409 on a stale job is prose on purpose — "its lease ran out, and
+       * nothing was written" — and that sentence is the whole of what a person
+       * reading this laptop's log at midnight needs. Reducing it to the status
+       * code would throw away the reassuring half.
+       */
+      const text = await res.text().catch(() => "");
+      let said = text.slice(0, 400);
+      try {
+        const parsed = JSON.parse(text) as { message?: string; error?: string };
+        said = String(parsed.message ?? parsed.error ?? said);
+      } catch {
+        // Not JSON — a proxy's HTML error page, most likely. Its first 400
+        // characters still say more than the number does.
+      }
+      throw new HttpError(said || `${method} ${pathname} answered ${res.status}`, res.status);
+    }
+
+    return (await res.json().catch(() => ({}))) as T;
+  }
+
+  /**
+   * Say what this machine is and what it can run.
+   *
+   * On start and again on a timer, because re-registering is what replaces
+   * `toolStatuses(force)`: the server has no machine to probe, so a newly
+   * installed whisper becomes visible by this machine saying so again. It is
+   * idempotent on the token — the same machine gets the same row.
+   */
+  register(body: {
+    name: string;
+    platform: string;
+    version: string;
+    tools: ToolStatus[];
+    enabled: string[];
+  }) {
+    return this.send<{
+      workerId: string;
+      /** What the console calls this machine — not what we called ourselves. */
+      name?: string;
+      heartbeatMs: number;
+      claimMs: number;
+    }>(
+      "POST",
+      "/api/workers/register",
+      body,
+    );
+  }
+
+  /**
+   * Ask for work, and wait.
+   *
+   * The long poll is held on the server for about twenty-five seconds and an
+   * empty answer is the normal answer. The signal is how a shutdown stops
+   * waiting immediately instead of holding the process open for the rest of
+   * the poll.
+   */
+  claim(max: number, signal?: AbortSignal) {
+    return this.send<{ jobs: ClaimedJob[] }>("POST", "/api/workers/claim", { max }, signal);
+  }
+
+  /** Extend the leases still held, and learn which have been taken away. */
+  heartbeat(jobIds: string[], progress?: string) {
+    return this.send<{ keep: string[]; drop: string[] }>("POST", "/api/workers/heartbeat", {
+      jobIds,
+      ...(progress ? { progress } : {}),
+    });
+  }
+
+  result(jobId: string, result: Record) {
+    return this.send<{ ok: true }>("POST", `/api/workers/jobs/${jobId}/result`, { result });
+  }
+
+  /**
+   * A failure, and whether anyone should try again.
+   *
+   * `retryable: false` means "this will fail identically on any machine" — a
+   * refusal, a payload this worker cannot honour, a tool that is not here.
+   * Spending the second attempt on those buys nothing and costs another
+   * lease's worth of somebody waiting.
+   */
+  fail(jobId: string, error: string, retryable: boolean) {
+    return this.send<{ ok: true }>("POST", `/api/workers/jobs/${jobId}/fail`, {
+      error: error.slice(0, 2000),
+      retryable,
+    });
+  }
+
+  /** Bytes from the console — a frame to look at, or a WAV to transcribe. */
+  async download(pathname: string, dest: string, signal?: AbortSignal) {
+    const res = await fetch(this.url(pathname), {
+      headers: { authorization: `Bearer ${this.cfg.token}` },
+      signal,
+    });
+    if (!res.ok) {
+      throw new HttpError(`${pathname} answered ${res.status}`, res.status);
+    }
+    const { writeFile } = await import("node:fs/promises");
+    await writeFile(dest, Buffer.from(await res.arrayBuffer()));
+    return dest;
+  }
+
+  /**
+   * Frames up to the console, in one request.
+   *
+   * The server is the single source of truth for frames — the browser renders
+   * them, the machine that ingests and the machine that writes are usually
+   * different, and desktops are not backed up — so an ingest uploads and a
+   * write downloads. Every part is named `file`, which is the field the
+   * controller reads, and each filename is checked against `FRAME_FILE` on the
+   * far side before a byte of it is written.
+   */
+  async uploadFrames(pathname: string, files: string[]) {
+    const form = new FormData();
+    for (const file of files) {
+      // `openAsBlob` rather than reading each JPEG into a Buffer: eight stills
+      // is nothing, but this is the same code path a hundred would take.
+      form.append("file", await openAsBlob(file, { type: "image/jpeg" }), path.basename(file));
+    }
+    const res = await fetch(this.url(pathname), {
+      method: "POST",
+      headers: { authorization: `Bearer ${this.cfg.token}` },
+      body: form,
+    });
+    if (!res.ok) {
+      throw new HttpError(`Uploading frames answered ${res.status}`, res.status);
+    }
+  }
+
+  /**
+   * The 16 kHz WAV up to the console.
+   *
+   * Streamed off disk rather than buffered. Audio is roughly a thirtieth of
+   * the video, which is the whole reason transcription can afford to be a
+   * separate job on a different desktop — but an hour of speech is still sixty
+   * megabytes, and holding that in the heap of a sidecar that also has ffmpeg
+   * running is how a laptop's worker dies rather than its download.
+   */
+  async uploadAudio(pathname: string, file: string) {
+    const form = new FormData();
+    form.append("file", await openAsBlob(file, { type: "audio/wav" }), path.basename(file));
+    const res = await fetch(this.url(pathname), {
+      method: "POST",
+      headers: { authorization: `Bearer ${this.cfg.token}` },
+      body: form,
+    });
+    if (!res.ok) {
+      throw new HttpError(`Uploading the audio answered ${res.status}`, res.status);
+    }
+  }
+}
diff --git a/worker/brains.ts b/worker/brains.ts
new file mode 100644
index 0000000..3c9ab4e
--- /dev/null
+++ b/worker/brains.ts
@@ -0,0 +1,202 @@
+import {
+  findBrain,
+  type BrainDef,
+  type Transport,
+  type WriteRequest,
+} from "../lib/server/brain-defs";
+import {
+  viaClaudeCli,
+  viaCodexCli,
+  viaGeminiCli,
+  viaOllama,
+} from "../lib/server/brain-transports";
+import { toolStatuses } from "../lib/server/tools";
+import type { Config } from "./config";
+import { Refusal } from "./job-types";
+
+/**
+ * Reaching a model from this machine — the rules, not the flags.
+ *
+ * This file used to be a verbatim copy of the four transports in
+ * `lib/server/brains.ts`, held under protest, because `write()` cannot be
+ * called from here: it re-derives the transport through `brainStatuses()` →
+ * `getSecret()` → Postgres, and a sidecar on somebody's desktop has neither a
+ * database nor any business having one. The transports were private to that
+ * module, so there was nothing to import either.
+ *
+ * Both halves are fixed. The brain table is `lib/server/brain-defs.ts`, which
+ * imports nothing; the four transports are `lib/server/brain-transports.ts`,
+ * which imports only `tools.ts` — no repo, no client, no `pg` — and every
+ * hard-won flag in them now exists once. What is left here is the part that
+ * was never a copy of anything: the decisions a worker makes, and the ones it
+ * refuses to make.
+ *
+ * The rule that matters is negative. **There is no fallback.** A worker told
+ * `cli` that has no CLI fails; it does not reach for an API key, and there is
+ * no key on this machine to reach for. The API transports were deliberately
+ * left behind in `brains.ts` with the keys, so this file cannot perform a
+ * silent CLI-to-API downgrade even by accident — which is the outcome
+ * `docs/DECISIONS.md` calls worse than an error, and importing the shared
+ * transports must never become the day a worker acquires the ability.
+ */
+
+/**
+ * A job's ask, in the shape the transports already take.
+ *
+ * `WriteRequest` minus `images`: bytes are the one field a worker never
+ * carries. The frames arrive as URLs it downloads to its own disk and hands
+ * over as paths, and the only transport that takes base64 is the Anthropic
+ * API, which lives on the server. Naming the omission keeps a future payload
+ * from quietly adding an image path through here.
+ */
+export type BrainCall = Omit & {
+  brain: string;
+  transport: Transport;
+};
+
+/**
+ * Ask the model, exactly the way the payload said to.
+ *
+ * Everything editorial was settled on the server — which brain, which
+ * transport, what to say — so nothing here chooses. It checks that this
+ * machine can honour what it was told, and refuses by name when it cannot: a
+ * refusal is an answer somebody can act on, and an answer produced some other
+ * way than the run asked for, presented as though it had been, is the failure
+ * this whole split was careful to make impossible.
+ */
+export async function callBrain(
+  cfg: Config,
+  req: BrainCall,
+): Promise<{ text: string; using: string }> {
+  const def = findBrain(req.brain);
+  if (!def) {
+    /*
+     * The table is the server's own now, so an unknown id means this machine
+     * is running an older worker than the console that queued the job. Say
+     * that, rather than picking the nearest row: guessing means running a
+     * different model than the run was costed for, and saying afterwards that
+     * it was the one asked for.
+     */
+    throw new Refusal(
+      `This machine's worker does not know a model called "${req.brain}". Update the worker on this machine.`,
+    );
+  }
+
+  const frames = req.imageFiles ?? [];
+
+  /*
+   * Ollama is a server on this machine, not a command on it.
+   *
+   * So its transport is `api` and always was — that is `transportFor()` on the
+   * server saying "no command to install", not a downgrade — and it is the one
+   * `api` a worker may honour, because the address is local and nothing is
+   * spent per token. It is also the one brain that cannot be shown a picture:
+   * it is handed a prompt and nothing else, and answering a question about
+   * frames from the words alone is the outcome worse than an error.
+   */
+  if (def.id === "ollama") {
+    if (req.transport !== "api") {
+      throw new Refusal(
+        `The job asked for Ollama over "${req.transport}", and Ollama is only ever reached over its own HTTP API.`,
+      );
+    }
+    if (frames.length) {
+      throw new Refusal(
+        "Ollama cannot be shown frames, so it must not answer a question about them.",
+      );
+    }
+    /*
+     * This machine's address, not the console's. `ollama-url` is a setting
+     * about one host, and the host that matters is whichever one this laptop
+     * can reach.
+     */
+    return { text: await viaOllama(def, req, cfg.ollamaUrl), using: def.model };
+  }
+
+  if (req.transport !== "cli") {
+    /*
+     * A key would work here, and that is precisely why it is refused.
+     *
+     * The API transports stay on the server, deliberately: this split exists
+     * so a run costs a subscription somebody already pays for, and a worker
+     * that quietly spent money per token when its CLI was missing would do it
+     * on the evening nobody was watching.
+     */
+    throw new Refusal(
+      `This machine only writes through a signed-in CLI, and the job asked for ${def.name} over an API key. API keys stay on the server.`,
+    );
+  }
+
+  const tool = await toolFor(cfg, def);
+
+  if (frames.length && def.id === "claude-cli") {
+    /*
+     * Two checks, and this is the second one.
+     *
+     * The server sent `allowFrameRead` from `workers.can_read_frames`; this
+     * machine checks its own switch again before granting Claude the `Read`
+     * tool, because the grant lands on this filesystem and the person who owns
+     * this filesystem gets the last word. Codex and Gemini are handed the
+     * picture and granted nothing, so neither comes through here.
+     */
+    if (req.allowFrameRead !== true) {
+      throw new Refusal(
+        "Claude can read the frames, but the console has not allowed this machine to open frame files. Turn it on for this machine in Settings → Machines.",
+      );
+    }
+    if (!cfg.allowFrameRead) {
+      throw new Refusal(
+        "Claude can read the frames, but this worker is not allowed to open files: set CONTENTOS_WORKER_ALLOW_FRAME_READ=1 on this machine and restart it.",
+      );
+    }
+  }
+
+  /*
+   * What was actually asked for, which is not always `model`: a CLI with no
+   * override uses whatever the signed-in account defaults to, and reporting a
+   * model name it was never told is how "which model wrote this" stops being
+   * answerable.
+   */
+  const using = def.cliModel ?? "the account's default model";
+
+  // Named one by one rather than falling through to a last branch. A chain
+  // whose `else` was gemini would answer a fifth brand as Gemini the day one
+  // is added to the table — a substitution nobody asked for and nobody would
+  // see, which is the exact failure this file exists to prevent.
+  if (def.id === "claude-cli") return { text: await viaClaudeCli(def, req, tool), using };
+  if (def.id === "codex-cli") return { text: await viaCodexCli(def, req, tool), using };
+  if (def.id === "gemini-cli") return { text: await viaGeminiCli(def, req, tool), using };
+
+  throw new Refusal(
+    `This machine has no way to write with ${def.name}, and it must not answer as some other model instead.`,
+  );
+}
+
+/**
+ * The invocation that answered the probe, or the reason there was none.
+ *
+ * `tool.command` and `tool.lead` rather than the brand's own command string,
+ * for the reason `probe()` records them at all: an install the probe reached a
+ * second way — `python -m whisper` is the usual one — is an install this can
+ * use, and hard-coding the first name means telling somebody to install what
+ * they already have. The whole `ToolStatus` goes to the transport, which takes
+ * it as its `CliSpawn`; the server, which has no probe at the call site,
+ * passes nothing and gets the brand's own command name.
+ */
+async function toolFor(cfg: Config, def: BrainDef) {
+  // Every brand that reaches this line has a command — Ollama returned above,
+  // and it is the only row without one.
+  const id = def.command ?? def.id;
+  const status = (await toolStatuses()).find((t) => t.id === id);
+  if (!status?.present) {
+    throw new Refusal(
+      `${def.name}'s CLI is not installed on this machine. Install it with: ${status?.install ?? id}`,
+    );
+  }
+  if (cfg.toolsOff.has(id)) {
+    throw new Refusal(
+      `${id} is switched off on this machine (CONTENTOS_WORKER_TOOLS_OFF), so it must not be used to write.`,
+    );
+  }
+  return status;
+}
diff --git a/worker/config.ts b/worker/config.ts
new file mode 100644
index 0000000..31f3ad1
--- /dev/null
+++ b/worker/config.ts
@@ -0,0 +1,166 @@
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+import { fileURLToPath } from "node:url";
+
+/**
+ * Everything this machine needs to be told, and where it is told it.
+ *
+ * Environment rather than a config file, because the same code has to start
+ * three ways: `npm run worker` in a checkout today, a service on somebody's
+ * desktop, and a Tauri sidecar tomorrow — and only one of those has a
+ * directory anybody would think to put a config file in. The desktop app will
+ * hand the token over from the OS keychain as an env var, which is the whole
+ * reason it is read from here and never from a file next to the binary.
+ *
+ * Two of these are required and there is no default that could be right for
+ * either: an API URL guessed as localhost would silently make a worker that
+ * serves nobody, and a blank token authenticates as nothing. A missing one is
+ * reported as the sentence naming what to set, not as a stack trace — the
+ * person reading it is looking at a console window on their own laptop, not at
+ * a server log.
+ */
+export type Config = {
+  /** The console's origin, no trailing slash. */
+  base: string;
+  /** The bearer token minted in Settings → Machines, shown once. */
+  token: string;
+  /** What the machine picker calls this machine. */
+  name: string;
+  /** Sidecar version, reported so a stale desktop app is visible. */
+  version: string;
+  /** How many jobs to ask for at once. */
+  maxJobs: number;
+  /**
+   * Whether Claude's CLI may be granted `Read` on THIS filesystem.
+   *
+   * Checked again here even though the payload already carries the server's
+   * answer, because the grant lands on this disk and whoever owns the disk
+   * gets the last word. Defaults off: starting a worker must not silently
+   * carry over a permission somebody granted on a different machine.
+   */
+  allowFrameRead: boolean;
+  /** Tool ids switched off on this machine, whatever the probe found. */
+  toolsOff: Set;
+  /** Where Ollama is, for the one brain that is an HTTP server rather than a CLI. */
+  ollamaUrl: string;
+  /** Where `npm run worker:setup` put yt-dlp, ffmpeg and whisper.cpp. */
+  toolsDir: string;
+  /** How often to re-report the tool probe. */
+  registerEveryMs: number;
+};
+
+class ConfigError extends Error {}
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+/**
+ * Where the installed tools live, decided once and put in the environment.
+ *
+ * Resolved from THIS FILE's location rather than from the working directory,
+ * because a sidecar is started by whatever launched it and a desktop app
+ * launches it from wherever Windows felt like — while the tools sit in the app
+ * folder, next to `worker/`, always.
+ *
+ * It goes into the environment because `lib/server/tools.ts` cannot work this
+ * out for itself: that file is also type-checked by `api/tsconfig.json` as
+ * CommonJS, where `import.meta.url` is a compile error (TS1343), so it reads
+ * `CONTENTOS_TOOLS_DIR` and this is what sets it. An explicit value always
+ * wins, which is how somebody puts a three-gigabyte whisper model on the drive
+ * that has room for it.
+ *
+ * `.data/tools`, not `.tools`, and for one flat reason: `.gitignore` already
+ * ignores `.data/`. A tools folder anywhere else would put two hundred
+ * megabytes of downloaded executables into `git status` for every teammate who
+ * ran the installer. It is deliberately NOT under `CONTENTOS_DATA_DIR` — that
+ * variable belongs to `cliHome()` and to nothing else, and quietly widening it
+ * would make the sentence in docs/WORKER.md about it false.
+ *
+ * Called by the executor as well as by the supervisor, because each job runs
+ * in a process of its own and an environment variable does not survive a
+ * `spawn` of a fresh node.
+ */
+export function toolsRoot() {
+  const set = (process.env.CONTENTOS_TOOLS_DIR ?? "").trim();
+  const dir = set ? path.resolve(set) : path.resolve(here, "..", ".data", "tools");
+  process.env.CONTENTOS_TOOLS_DIR = dir;
+  return dir;
+}
+
+/**
+ * The version, off the repo's package.json.
+ *
+ * Read rather than imported: an import assertion would tie the worker to a
+ * JSON module flag, and a sidecar that cannot find its own package.json should
+ * report an honest "0.0.0-dev" rather than refusing to start over a version
+ * string nothing depends on.
+ */
+function version() {
+  try {
+    const raw = fs.readFileSync(path.join(here, "..", "package.json"), "utf8");
+    return String((JSON.parse(raw) as { version?: string }).version ?? "") || "0.0.0-dev";
+  } catch {
+    return "0.0.0-dev";
+  }
+}
+
+const flag = (value: string | undefined, fallback = false) =>
+  value === undefined || value.trim() === ""
+    ? fallback
+    : /^(1|true|yes|on)$/i.test(value.trim());
+
+export function loadConfig(): Config {
+  const base = (process.env.CONTENTOS_API_URL ?? "").trim().replace(/\/+$/, "");
+  const token = (process.env.CONTENTOS_WORKER_TOKEN ?? "").trim();
+
+  if (!base) {
+    throw new ConfigError(
+      "Set CONTENTOS_API_URL to the console this machine works for, for example http://localhost:4000.",
+    );
+  }
+  if (!/^https?:\/\//i.test(base)) {
+    throw new ConfigError(
+      `CONTENTOS_API_URL is "${base}", which is not a URL — it has to start with http:// or https://.`,
+    );
+  }
+  if (!token) {
+    throw new ConfigError(
+      "Set CONTENTOS_WORKER_TOKEN to the token from Settings → Machines. It is shown once, when the machine is added.",
+    );
+  }
+
+  const asked = Number(process.env.CONTENTOS_WORKER_MAX_JOBS ?? "1");
+  return {
+    base,
+    token,
+    name: (process.env.CONTENTOS_WORKER_NAME ?? "").trim() || os.hostname(),
+    version: version(),
+    /*
+     * One at a time unless the owner says otherwise, and capped low.
+     *
+     * One machine is one CLI login, so two sections in parallel are two
+     * processes competing for the same subscription and the same GPU. The
+     * parallelism this design is after comes from more laptops; the server
+     * caps it as well, at `workers.max_concurrency`, and the smaller of the
+     * two wins.
+     */
+    maxJobs: Number.isFinite(asked) ? Math.max(1, Math.min(4, Math.trunc(asked))) : 1,
+    allowFrameRead: flag(process.env.CONTENTOS_WORKER_ALLOW_FRAME_READ, false),
+    toolsOff: new Set(
+      (process.env.CONTENTOS_WORKER_TOOLS_OFF ?? "")
+        .split(",")
+        .map((id) => id.trim())
+        .filter(Boolean),
+    ),
+    ollamaUrl:
+      (process.env.CONTENTOS_OLLAMA_URL ?? "").trim().replace(/\/+$/, "") ||
+      "http://localhost:11434",
+    // Called for its side effect as much as for its value: from here on,
+    // every probe in this process — and in the executor processes it spawns —
+    // looks in the app's own tools folder before it looks at PATH.
+    toolsDir: toolsRoot(),
+    registerEveryMs: 5 * 60_000,
+  };
+}
+
+export const isConfigError = (e: unknown) => e instanceof ConfigError;
diff --git a/worker/executor.ts b/worker/executor.ts
new file mode 100644
index 0000000..b189be9
--- /dev/null
+++ b/worker/executor.ts
@@ -0,0 +1,100 @@
+import { loadConfig } from "./config";
+import { Api } from "./api";
+import { ingestSource } from "./ingest-source";
+import { RESULT_MARK, Refusal } from "./job-types";
+import type {
+  IngestPayload,
+  TestPayload,
+  TranscribePayload,
+  WritePayload,
+} from "./job-types";
+import { testBrain } from "./test-brain";
+import { transcribeAudio } from "./transcribe-audio";
+import { writeSection } from "./write-section";
+
+/**
+ * One job, in a process of its own.
+ *
+ * The supervisor could call these functions directly and the code would be
+ * shorter. It runs them here instead for one reason, and it is the reason the
+ * protocol has a `drop` list at all: when the server takes a job back — a Stop
+ * button, a lease reaped while a laptop slept — the worker is supposed to
+ * **kill the child process** rather than spend four more minutes producing
+ * something the server will refuse. `run()` in `lib/server/tools.ts` returns a
+ * promise and keeps its `ChildProcess` to itself, so there is nothing to kill
+ * from the outside; a whole process that can be killed as a tree is.
+ *
+ * It buys two more things worth having on somebody's own desktop. A CLI that
+ * wedges, leaks or dies takes only its own job down, and the supervisor keeps
+ * heartbeating for the others. And the temp directory of a killed job is
+ * cleaned by the next start's sweep rather than being lost with the code that
+ * would have removed it.
+ *
+ * The cost is a serialisation boundary: the job arrives on stdin as JSON and
+ * the answer leaves on stdout behind `RESULT_MARK`, because a CLI or a library
+ * can print to stdout at any time and the last line is not reliably ours.
+ */
+
+type Incoming = { id: string; kind: string; payload: Record };
+
+async function main() {
+  const job = JSON.parse(await readAll()) as Incoming;
+  const cfg = loadConfig();
+  const api = new Api(cfg);
+
+  switch (job.kind) {
+    case "write_section":
+      return writeSection(cfg, api, job.id, job.payload as unknown as WritePayload);
+    case "ingest_source":
+      return ingestSource(cfg, api, job.id, job.payload as unknown as IngestPayload);
+    case "transcribe_audio":
+      return transcribeAudio(cfg, api, job.id, job.payload as unknown as TranscribePayload);
+    case "test_brain":
+      return testBrain(cfg, job.payload as unknown as TestPayload);
+    default:
+      /*
+       * A kind this worker has never heard of.
+       *
+       * Not retryable: every machine on the estate is running some version of
+       * this file, and a newer server that invented a kind will get the same
+       * answer from all of them. The sentence names the machine's own
+       * staleness, because that is the thing somebody can act on.
+       */
+      throw new Refusal(
+        `This machine's worker does not know how to run a "${job.kind}" job. Update the worker on this machine.`,
+      );
+  }
+}
+
+main().then(
+  (result) => done({ ok: true, result }),
+  (e: unknown) =>
+    done({
+      ok: false,
+      error: e instanceof Error ? e.message : String(e),
+      // A `Refusal` is this machine saying "the next one will fail the same
+      // way": a bad payload, a transport it cannot honour, a tool that is not
+      // here. Everything else gets the one retry, because the failures that
+      // actually recur are the closed laptop and the dropped connection.
+      retryable: !(e instanceof Refusal),
+    }),
+);
+
+function done(answer: unknown) {
+  process.stdout.write(`\n${RESULT_MARK}${JSON.stringify(answer)}\n`, () => {
+    // Written, then out — `process.exit` on Windows can truncate a pipe that
+    // has not flushed, and the truncated line is the one carrying the answer.
+    process.exit(0);
+  });
+}
+
+/** The whole job off stdin, because a payload is thousands of characters. */
+function readAll() {
+  return new Promise((resolve, reject) => {
+    let text = "";
+    process.stdin.setEncoding("utf8");
+    process.stdin.on("data", (chunk) => (text += chunk));
+    process.stdin.on("end", () => resolve(text));
+    process.stdin.on("error", reject);
+  });
+}
diff --git a/worker/index.ts b/worker/index.ts
new file mode 100644
index 0000000..7f65acc
--- /dev/null
+++ b/worker/index.ts
@@ -0,0 +1,425 @@
+import { spawn, type ChildProcess } from "node:child_process";
+import { rm } from "node:fs/promises";
+import path from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+
+import { toolStatuses, type ToolStatus } from "../lib/server/tools";
+import { Api, HttpError, isStale, type ClaimedJob } from "./api";
+import { isConfigError, loadConfig, type Config } from "./config";
+import { RESULT_MARK } from "./job-types";
+import { TEMP_ROOT, sweepTemp } from "./temp";
+
+/**
+ * The worker: register, claim, run, report. Forever, until somebody stops it.
+ *
+ * It opens outbound connections only — no inbound port, no static IP, no hole
+ * in anybody's router — which is what makes it something a teammate can run on
+ * their own Windows laptop without asking anyone for anything. The server
+ * decides what happens and this executes it: a job carries finished text and a
+ * command, never a pack, a template, a rule or a dependency graph.
+ *
+ * Nothing here is stateful across a restart, and that is deliberate. A job
+ * held by a process that dies is not lost — its lease expires, the reaper puts
+ * it back, another machine takes it — so this process has nothing to persist
+ * and no file to be careful about.
+ */
+
+const here = path.dirname(fileURLToPath(import.meta.url));
+
+/** A job this machine is holding, and the process actually doing it. */
+type Running = {
+  job: ClaimedJob;
+  child: ChildProcess;
+  /** Killed on purpose — a `drop` from the server, or a shutdown. */
+  killed: boolean;
+};
+
+async function main() {
+  const cfg = loadConfig();
+  const api = new Api(cfg);
+
+  // Whatever an earlier run left behind. A killed process cleans up nothing,
+  // and a laptop that has been fetching videos for a month should not be
+  // storing a month of them in its temp directory.
+  await sweepTemp();
+
+  const running = new Map();
+  let stopping = false;
+  let claiming: AbortController | null = null;
+
+  /**
+   * Stop claiming, and mean it.
+   *
+   * Installed before anything that waits — the first registration can wait for
+   * minutes — because a Ctrl-C that does nothing is a machine you have to kill
+   * to stop, and a killed machine hands nothing back.
+   */
+  function stop(signal: string) {
+    if (stopping) {
+      // A second Ctrl-C is somebody who has decided not to wait. Honour it.
+      log("Stopping now.");
+      process.exit(1);
+    }
+    stopping = true;
+    log(`${signal} — finishing up. Not claiming any more work.`);
+    // The claim is a long poll; aborting it is the difference between stopping
+    // now and stopping in twenty-five seconds.
+    claiming?.abort();
+  }
+
+  process.on("SIGINT", () => stop("SIGINT"));
+  process.on("SIGTERM", () => stop("SIGTERM"));
+
+  /*
+   * Keep trying to enrol.
+   *
+   * A worker that exited because the console was not up yet would be a
+   * desktop app that has to be started in the right order, on a laptop that
+   * opens before its wifi does — every morning. A bad token is different and
+   * does exit: no amount of waiting turns a 401 into a machine.
+   */
+  let enrolled: Awaited> | null = null;
+  while (!enrolled && !stopping) {
+    try {
+      enrolled = await register(cfg, api);
+    } catch (e) {
+      if (e instanceof HttpError && e.status === 401) throw e;
+      log(`Cannot reach ${cfg.base} yet: ${say(e)}. Trying again in 10s.`);
+      await sleep(10_000);
+    }
+  }
+  if (!enrolled) return;
+
+  log(
+    // The console's name for this machine, not the one we guessed. A machine is
+    // named by a person in Settings -> Machines and that is what every screen
+    // shows; announcing a hostname here gave one computer two names in front of
+    // somebody working out which of them had stopped claiming.
+    `Registered as ${enrolled.workerId} (${enrolled.name || cfg.name}). Claiming for ${enrolled.claimMs}ms at a time, heartbeat every ${enrolled.heartbeatMs}ms.`,
+  );
+
+  /*
+   * Re-register on a timer.
+   *
+   * This is what replaces `toolStatuses(force)` and the sixty-second probe
+   * cache: the server has no machine to probe, so a whisper installed this
+   * afternoon becomes visible by this machine saying so again. It is
+   * idempotent on the token — the same machine, the same row, a fresher tool
+   * report.
+   */
+  const registerTimer = setInterval(() => {
+    void register(cfg, api).catch((e: unknown) => {
+      // Not fatal. The claim loop is still talking to the same server, and the
+      // next tick reports again; a worker that exited over a failed re-report
+      // would stop working over something that changed nothing.
+      log(`Could not re-report this machine's tools: ${say(e)}`);
+    });
+  }, cfg.registerEveryMs);
+
+  /*
+   * Heartbeat, and act on what comes back.
+   *
+   * `drop` is the whole reason this returns anything: it is how this machine
+   * learns a job was cancelled by a Stop button or reaped while the lid was
+   * shut. The answer is to kill the child, not to finish work nobody wants —
+   * four more minutes of a subscription spent on something the server will
+   * refuse with a 409 anyway.
+   */
+  const heartbeatTimer = setInterval(() => {
+    if (stopping || running.size === 0) return;
+    void api
+      .heartbeat([...running.keys()])
+      .then(({ drop }) => {
+        for (const id of drop ?? []) {
+          const held = running.get(id);
+          if (!held) continue;
+          log(`Job ${id} was taken back by the server — stopping it here.`);
+          held.killed = true;
+          killTree(held.child);
+        }
+      })
+      .catch((e: unknown) => log(`Heartbeat failed: ${say(e)}`));
+  }, enrolled.heartbeatMs);
+
+  /**
+   * Run a job, then say what happened.
+   *
+   * A 409 on the way out is not a failure and is not logged as one: it means
+   * the job stopped being this machine's — a lease that ran out while the
+   * laptop slept, and somebody else has already written it. Nothing was lost,
+   * and there is nothing to retry.
+   */
+  const start = (job: ClaimedJob) => {
+    const child = spawnExecutor(job);
+    const held: Running = { job, child, killed: false };
+    running.set(job.id, held);
+    log(`Claimed ${job.kind} ${job.id}.`);
+
+    void collect(child)
+      .then(async (answer) => {
+        if (held.killed) {
+          // The server already took it back. Posting anything now would be
+          // answering a question nobody is still asking.
+          log(`Job ${job.id} was stopped before it finished.`);
+          return;
+        }
+        if (answer.ok) {
+          await api.result(job.id, answer.result);
+          log(`Finished ${job.kind} ${job.id}.`);
+        } else {
+          await api.fail(job.id, answer.error, answer.retryable);
+          log(
+            `Failed ${job.kind} ${job.id}${answer.retryable ? "" : " (for good)"}: ${answer.error}`,
+          );
+        }
+      })
+      .catch((e: unknown) => {
+        if (isStale(e)) {
+          log(`Job ${job.id}: ${say(e)}`);
+          return;
+        }
+        log(`Could not report job ${job.id}: ${say(e)}`);
+      })
+      .finally(() => {
+        running.delete(job.id);
+      });
+  };
+
+  /*
+   * The loop.
+   *
+   * A long poll rather than a socket: one code path, survives every proxy, and
+   * about a second of latency without anything stateful in between. An empty
+   * answer after twenty-five quiet seconds is the normal answer and says
+   * nothing — a line per worker per half minute would bury the one line that
+   * matters on the day something is actually wrong.
+   */
+  while (!stopping) {
+    if (running.size >= cfg.maxJobs) {
+      await sleep(1_000);
+      continue;
+    }
+
+    claiming = new AbortController();
+    try {
+      const { jobs } = await api.claim(cfg.maxJobs - running.size, claiming.signal);
+      for (const job of jobs ?? []) start(job);
+    } catch (e) {
+      if (stopping) break;
+      /*
+       * The console is not answering.
+       *
+       * Said once and then waited out, because the ordinary cause is a laptop
+       * that woke before its wifi did, and a machine that logged a line per
+       * failed poll would fill a terminal with the same sentence. Nothing is
+       * lost while this is happening: any job this worker held has a lease
+       * that will expire, and the reaper hands it to somebody awake.
+       */
+      log(`No answer from ${cfg.base}: ${say(e)}. Trying again in 5s.`);
+      await sleep(5_000);
+    } finally {
+      claiming = null;
+    }
+  }
+
+  clearInterval(registerTimer);
+  clearInterval(heartbeatTimer);
+
+  /*
+   * Shutting down: nothing is left half-said.
+   *
+   * Every job still running is killed and then FAILED explicitly, as
+   * retryable, rather than being abandoned to its lease. Both end with the job
+   * back in the queue, but ninety seconds apart — and those ninety seconds are
+   * somebody watching a row that says "Writing" on a machine that is already
+   * closed.
+   */
+  for (const held of running.values()) {
+    held.killed = true;
+    killTree(held.child);
+    await api
+      .fail(
+        held.job.id,
+        `${cfg.name} was shut down while this job was running. Nothing was written.`,
+        true,
+      )
+      .catch((e: unknown) => {
+        if (!isStale(e)) log(`Could not hand back job ${held.job.id}: ${say(e)}`);
+      });
+    log(`Handed back ${held.job.kind} ${held.job.id}.`);
+  }
+
+  await rm(TEMP_ROOT, { recursive: true, force: true }).catch(() => {});
+  log("Stopped.");
+}
+
+/**
+ * Say what this machine is and what it can run.
+ *
+ * `tools` is the whole `ToolStatus` array, probed here, because probing is the
+ * worker's job now — the Integrations page renders exactly these rows and the
+ * server has no machine of its own to ask. `enabled` is the subset this
+ * machine's owner has left switched on, which is a fact about a machine and
+ * never about a console: "ffmpeg is off" was one global row for the whole
+ * install, and on an estate of four laptops that is three separate wrong
+ * things.
+ */
+async function register(cfg: Config, api: Api) {
+  // Forced past the sixty-second cache: this call exists precisely to notice
+  // an install that happened since the last one.
+  const tools: ToolStatus[] = await toolStatuses(true);
+  return api.register({
+    name: cfg.name,
+    platform: `${process.platform} ${process.arch}`,
+    version: cfg.version,
+    tools,
+    enabled: tools.filter((t) => t.present && !cfg.toolsOff.has(t.id)).map((t) => t.id),
+  });
+}
+
+/**
+ * A job's own process, started the same way this one was.
+ *
+ * The same `--import` hook, because the executor imports the app's TypeScript
+ * by plain path exactly as this file does. On anything but Windows it is
+ * `detached` so that it leads its own process group and a kill can take the
+ * shell and the CLI under it; on Windows there are no process groups to lead
+ * and `taskkill /T` walks the tree instead.
+ */
+function spawnExecutor(job: ClaimedJob) {
+  const child = spawn(
+    process.execPath,
+    [
+      "--disable-warning=MODULE_TYPELESS_PACKAGE_JSON",
+      "--import",
+      pathToFileURL(path.join(here, "register.mjs")).href,
+      path.join(here, "executor.ts"),
+    ],
+    {
+      windowsHide: true,
+      stdio: ["pipe", "pipe", "pipe"],
+      ...(process.platform === "win32" ? {} : { detached: true }),
+    },
+  );
+
+  // The payload is thousands of characters of system prompt, so it goes down
+  // stdin rather than into an argument — the same reason the CLIs are fed on
+  // stdin, and the same Windows command-line limit behind it.
+  child.stdin?.write(JSON.stringify({ id: job.id, kind: job.kind, payload: job.payload }));
+  child.stdin?.end();
+  return child;
+}
+
+type Answer =
+  | { ok: true; result: Record }
+  | { ok: false; error: string; retryable: boolean };
+
+/**
+ * Wait for the job's process, and read its answer out of the noise.
+ *
+ * Behind a sentinel rather than "the last line of stdout", because a CLI or
+ * one of its libraries can print at any moment and being wrong about which
+ * line is the answer means posting a progress bar as somebody's section.
+ */
+function collect(child: ChildProcess): Promise {
+  return new Promise((resolve) => {
+    let out = "";
+    let err = "";
+    child.stdout?.on("data", (d: Buffer) => (out += String(d)));
+    child.stderr?.on("data", (d: Buffer) => (err += String(d)));
+
+    child.on("error", (e) =>
+      resolve({ ok: false, error: `The job process would not start: ${e.message}`, retryable: true }),
+    );
+
+    child.on("close", (code) => {
+      const at = out.lastIndexOf(RESULT_MARK);
+      if (at >= 0) {
+        try {
+          return resolve(JSON.parse(out.slice(at + RESULT_MARK.length).trim()) as Answer);
+        } catch {
+          // Fall through: a mangled answer is a crashed job, not a silent
+          // success, and it is reported as one below.
+        }
+      }
+      /*
+       * No answer at all: killed, out of memory, or a crash before it could
+       * write one. Retryable, because that is the shape of every one of those
+       * — another machine, or this one after a restart, may well manage it.
+       */
+      resolve({
+        ok: false,
+        error:
+          `The job stopped without answering (exit ${code}).` +
+          (err.trim() ? ` ${err.trim().split("\n").slice(-3).join(" ").slice(0, 400)}` : ""),
+        retryable: true,
+      });
+    });
+  });
+}
+
+/**
+ * Kill a job and everything it started.
+ *
+ * The CLIs are spawned through a shell — they are .cmd shims on Windows and
+ * spawning one directly fails with ENOENT — so the model process is a
+ * grandchild, and killing only the child would leave `claude` running with
+ * nobody waiting for it. `taskkill /T` walks the tree; a process group does
+ * the same everywhere else.
+ */
+function killTree(child: ChildProcess) {
+  if (!child.pid || child.exitCode !== null) return;
+  try {
+    if (process.platform === "win32") {
+      spawn("taskkill", ["/pid", String(child.pid), "/t", "/f"], { windowsHide: true });
+    } else {
+      process.kill(-child.pid, "SIGKILL");
+    }
+  } catch {
+    // Already gone, most likely — which is the outcome this was after.
+  }
+}
+
+/**
+ * A wait that keeps the process alive while it waits.
+ *
+ * This timer used to be `unref`ed, and that is the difference between a worker
+ * that retries and one that quietly gives up. Before the first successful
+ * registration nothing else is holding Node's event loop open — the heartbeat
+ * and re-register intervals do not exist yet — so an unref'd timer left
+ * NOTHING ref'd, and the process printed "Trying again in 10s" and exited 0 on
+ * the spot. The one case it broke is the ordinary one: a laptop opened before
+ * its wifi is up.
+ *
+ * It stays ref'd on purpose. Shutdown is not this function's job: the signal
+ * handlers stop the loop and clear the intervals, which is what actually ends
+ * the process.
+ */
+const sleep = (ms: number) => new Promise((resolve) => setTimeout(resolve, ms));
+
+const say = (e: unknown) => (e instanceof Error ? e.message : String(e));
+
+function log(line: string) {
+  // A person reads this in a console window on their own laptop, so it says
+  // the time rather than a level nobody filters on.
+  console.log(`[${new Date().toLocaleTimeString()}] ${line}`);
+}
+
+main().catch((e: unknown) => {
+  /*
+   * The one thing this must not do on somebody's desktop is print a stack
+   * trace at them. A missing setting is a sentence naming what to set.
+   */
+  if (isConfigError(e)) {
+    console.error(say(e));
+    process.exit(1);
+  }
+  if (e instanceof HttpError && e.status === 401) {
+    console.error(
+      `${say(e)}\nThe console did not recognise CONTENTOS_WORKER_TOKEN. Add this machine again in Settings → Machines and paste the new token.`,
+    );
+    process.exit(1);
+  }
+  console.error(`The worker stopped: ${say(e)}`);
+  process.exit(1);
+});
diff --git a/worker/ingest-source.ts b/worker/ingest-source.ts
new file mode 100644
index 0000000..7bbb85f
--- /dev/null
+++ b/worker/ingest-source.ts
@@ -0,0 +1,393 @@
+import fs from "node:fs/promises";
+import path from "node:path";
+
+import { run, toolStatuses, type ToolId } from "../lib/server/tools";
+import type { Api } from "./api";
+import type { Config } from "./config";
+import { Refusal, type IngestPayload, type JobResult } from "./job-types";
+import { discard, jobDir } from "./temp";
+
+/**
+ * Turning a link into frames, a caption and a WAV — on the machine that has
+ * the tools, the cookies and the bandwidth.
+ *
+ * This is the old inline pipeline out of `lib/server/services/ingest.ts`,
+ * moved rather than rewritten: the same yt-dlp ladder, the same evenly spaced
+ * stills, the same 16 kHz mono WAV. What changed is where it runs and what it
+ * does with the result. The server keeps every decision — how many frames, how
+ * tall, how long this may take, where the bytes go — and sends them in the
+ * payload; this machine only executes, and then uploads.
+ *
+ * **Degrading is the design, not a compromise.** A missing ffmpeg costs the
+ * frames and nothing else; a video the platform will not serve still yields a
+ * title, a caption and a duration, which is far more than nothing. So almost
+ * nothing in here throws: what fell short comes back in `error`, in the tool's
+ * own words, and the source is still `ready`. `whisper` is not in this file at
+ * all — transcription is its own job on whatever machine has a GPU, which is
+ * why an estate with no whisper anywhere can still fetch a link.
+ */
+export async function ingestSource(
+  cfg: Config,
+  api: Api,
+  jobId: string,
+  payload: IngestPayload,
+): Promise {
+  if (!payload.sourceId || !payload.url) {
+    throw new Refusal("The ingest job arrived without a source or a link.");
+  }
+
+  const dir = await jobDir(jobId, "ingest");
+  const deadline = Date.now() + (payload.timeoutMs ?? 900_000);
+
+  try {
+    /*
+     * Metadata first, and this is the one step allowed to fail the job.
+     *
+     * If yt-dlp cannot even read the link there is nothing to degrade to — no
+     * title, no caption, no duration — and the source would be `ready` with
+     * every field empty, which reads as a bug rather than as a refusal. It is
+     * left retryable because the same link often works from another machine:
+     * a different network, a different residential IP, a browser signed in to
+     * the platform.
+     */
+    const meta = await fetchMeta(cfg, payload.url, left(deadline, 90_000));
+
+    const video = await download(cfg, payload, dir, deadline);
+    const frames = video.file
+      ? await cutFrames(cfg, video.file, dir, payload, deadline)
+      : [];
+    // Only when there is a file. With nothing downloaded there is no audio,
+    // and a second sentence about a step that was never going to run buries
+    // the first one that matters.
+    const audio = video.file ? await extractAudio(cfg, video.file, dir, payload) : "";
+
+    /*
+     * The bytes go up BEFORE the result is posted.
+     *
+     * The result only names files, and `sourceResult` puts those names on the
+     * row for the browser to render — so a result that arrived before its
+     * frames would give the picker eight broken pictures for as long as the
+     * upload took. Uploading first means the row is never ahead of the disk.
+     */
+    if (frames.length) await api.uploadFrames(payload.uploadFrames, frames.map((f) => f.path));
+    if (audio) await api.uploadAudio(payload.uploadAudio, audio);
+
+    return {
+      title: str(meta.title),
+      uploader: str(meta.uploader) || str(meta.channel),
+      description: str(meta.description).slice(0, 8000),
+      duration: meta.duration == null ? 0 : Math.round(Number(meta.duration)) || 0,
+      thumbnail: str(meta.thumbnail),
+      meta,
+      frames: frames.map((f) => ({ at: f.at, file: f.file })),
+      // Whether there is a WAV on the server to transcribe. The server reads
+      // this to decide whether to ask for a `transcribe_audio` at all, so it
+      // has to be the truth about the upload rather than about ffmpeg.
+      audio: Boolean(audio),
+      /*
+       * Whichever step fell short, in its own words.
+       *
+       * The download's reason wins when there is one: it is the larger loss.
+       * "Could not be downloaded" is the same sentence for a private video, a
+       * dead link and a platform refusing anonymous requests, and only one of
+       * those is worth trying again — the tool already says which.
+       */
+      error: video.file
+        ? frames.length
+          ? ""
+          : `No frames — ${await frameReason(cfg)}`
+        : `Metadata only — ${video.error}. The caption and title are still usable.`,
+    };
+  } finally {
+    // The video is the reason this matters: a few hundred megabytes per link,
+    // on somebody's own laptop, and nothing on the server will ever come back
+    // to clean it up.
+    await discard(dir);
+  }
+}
+
+/** What is left of the server's budget, capped at what this step is worth. */
+const left = (deadline: number, most: number) =>
+  Math.max(5_000, Math.min(most, deadline - Date.now()));
+
+const str = (v: unknown) => (typeof v === "string" ? v : "");
+
+/**
+ * Whether a tool may run here: installed AND switched on, on this machine.
+ *
+ * `allowed()` used to AND the probe with a console-wide `settings.enabled`,
+ * which was one row for the whole install — right while the install was one
+ * laptop, wrong the moment "ffmpeg is off" became a fact about one machine
+ * among several. The switch is this machine's own, and so is the probe.
+ */
+async function allowed(cfg: Config, id: ToolId) {
+  if (cfg.toolsOff.has(id)) return null;
+  const status = (await toolStatuses()).find((t) => t.id === id);
+  return status?.present ? status : null;
+}
+
+/** What the platform says about the video, without downloading it. */
+async function fetchMeta(
+  cfg: Config,
+  url: string,
+  timeout: number,
+): Promise> {
+  const tool = await allowed(cfg, "yt-dlp");
+  if (!tool) {
+    // The claim query should never have handed this job to a machine without
+    // yt-dlp switched on, so if it did, no other machine's answer will be
+    // different for the same reason — this one is about the queue, not the
+    // link.
+    throw new Refusal("yt-dlp is not available on this machine, so it cannot fetch links.");
+  }
+
+  const { code, out, err } = await run(
+    tool.command,
+    [...tool.lead, "--dump-single-json", "--no-warnings", "--no-playlist", url],
+    { env: tool.env, timeout },
+  );
+  if (code !== 0 || !out.trim()) {
+    throw new Error(firstUseful(err) || "yt-dlp could not read that link");
+  }
+  try {
+    return JSON.parse(out) as Record;
+  } catch {
+    throw new Error("yt-dlp returned something that was not JSON");
+  }
+}
+
+/**
+ * The video file itself.
+ *
+ * Tried more than once, because "download this video" is not one operation any
+ * more. YouTube in particular refuses anonymous requests for the media even
+ * when it happily hands over the metadata, so the ladder goes: a single
+ * progressive file (smallest, most often allowed), then separate video+audio
+ * streams muxed by ffmpeg, then the same again with the cookies from a signed
+ * in browser.
+ *
+ * That last rung is the reason this job belongs on a desktop rather than on
+ * the server: the cookies it reads are a person's own browser session, on a
+ * machine they are signed in to, and there was never going to be one of those
+ * on a VPS.
+ *
+ * Capped at the height the server asked for throughout: the frames are read
+ * for on-screen text and a browser address bar, and 4K costs bandwidth and
+ * disk to answer the same question.
+ *
+ * Returns the path, or "" and the reason it could not — metadata alone is
+ * still a useful source, so this degrades rather than throwing.
+ */
+async function download(
+  cfg: Config,
+  payload: IngestPayload,
+  dir: string,
+  deadline: number,
+): Promise<{ file: string; error: string }> {
+  const tool = await allowed(cfg, "yt-dlp");
+  if (!tool) {
+    return { file: "", error: "yt-dlp is switched off on this machine" };
+  }
+
+  const height = payload.maxHeight || 720;
+  const attempts: { label: string; args: string[] }[] = [
+    { label: "progressive", args: ["-f", `b[height<=${height}]/b`] },
+    { label: "split streams", args: ["-f", `bv*[height<=${height}]+ba/bv*+ba`] },
+    // Last, and only if the others failed: reading the browser's cookies is
+    // how you get at something the platform will only serve to a signed-in
+    // session. It fails harmlessly when the browser is running and holding its
+    // cookie database open.
+    {
+      label: "browser cookies",
+      args: ["--cookies-from-browser", "chrome", "-f", `b[height<=${height}]/b`],
+    },
+  ];
+
+  const failures: string[] = [];
+
+  for (const attempt of attempts) {
+    if (Date.now() >= deadline) break;
+    const { code, err } = await run(
+      tool.command,
+      [
+        ...tool.lead,
+        ...attempt.args,
+        "--no-playlist",
+        "--no-warnings",
+        "-o",
+        path.join(dir, "video.%(ext)s"),
+        payload.url,
+      ],
+      { env: tool.env, timeout: left(deadline, 300_000) },
+    );
+
+    const files = await fs.readdir(dir).catch(() => []);
+    const file = files.find((f) => f.startsWith("video."));
+    if (code === 0 && file) {
+      return { file: path.join(dir, file), error: "" };
+    }
+
+    failures.push(`${attempt.label}: ${firstUseful(err) || `exit ${code}`}`);
+  }
+
+  return { file: "", error: failures[0] ?? "yt-dlp could not fetch the media" };
+}
+
+/**
+ * Why a downloaded video produced no stills.
+ *
+ * Four different situations used to arrive as the same empty frame list —
+ * ffmpeg switched off, ffmpeg missing, ffprobe missing so the duration is
+ * unknown, and ffmpeg present and failing — and only one of those is fixed by
+ * an install. The person reading this sentence is deciding what to do next, so
+ * it has to say which.
+ */
+async function frameReason(cfg: Config) {
+  for (const id of ["ffmpeg", "ffprobe"] as const) {
+    if (cfg.toolsOff.has(id)) return `${id} is switched off on this machine.`;
+    const status = (await toolStatuses()).find((t) => t.id === id);
+    if (!status?.present) {
+      return `${id} is not installed on this machine. ${status?.install ?? ""}`.trim();
+    }
+  }
+  return "ffmpeg could not cut any stills out of that video.";
+}
+
+/**
+ * Evenly spaced stills, as JPEGs in this job's temp directory.
+ *
+ * They are named `frame-NN.jpg` because that is the shape `FRAME_FILE` on the
+ * server allows, and the server re-checks every one of these names before it
+ * writes a byte — the regex that used to guard a query string is now guarding
+ * a write from a machine the server does not control. A name invented here
+ * that does not match is a frame silently refused, so the shape is not a
+ * convention, it is the contract.
+ */
+async function cutFrames(
+  cfg: Config,
+  video: string,
+  dir: string,
+  payload: IngestPayload,
+  deadline: number,
+): Promise<{ at: number; file: string; path: string }[]> {
+  const tool = await allowed(cfg, "ffmpeg");
+  if (!tool) return [];
+
+  const seconds = await durationOf(cfg, video, deadline);
+  if (!seconds) return [];
+
+  const count = payload.frameCount || 8;
+  const height = payload.maxHeight || 720;
+  const frames: { at: number; file: string; path: string }[] = [];
+
+  for (let i = 0; i < count; i += 1) {
+    if (Date.now() >= deadline) break;
+    // Offset by half a step so the first frame is not the black one platforms
+    // so often start on, and the last is not past the end.
+    const at = Math.max(0, ((i + 0.5) * seconds) / count);
+    const file = `frame-${String(i + 1).padStart(2, "0")}.jpg`;
+    const out = path.join(dir, file);
+    const { code } = await run(
+      tool.command,
+      [
+        ...tool.lead,
+        "-y",
+        "-ss",
+        at.toFixed(2),
+        "-i",
+        video,
+        "-frames:v",
+        "1",
+        "-vf",
+        `scale=-2:${height}`,
+        "-q:v",
+        "4",
+        out,
+      ],
+      { env: tool.env, timeout: left(deadline, 60_000) },
+    );
+    if (code === 0) frames.push({ at: Number(at.toFixed(2)), file, path: out });
+  }
+  return frames;
+}
+
+/**
+ * The audio, on its own, in the only format the transcriber takes.
+ *
+ * `-vn` drops the video, `-ac 1` mixes to mono and `-ar 16000` resamples,
+ * which together are whisper.cpp's required input and, not by coincidence,
+ * about a thirtieth of the bytes. That ratio is the whole reason transcription
+ * can be a separate job on a different desktop: a megabyte a minute crosses
+ * the network, not four hundred.
+ *
+ * Returns the path, or "" — a silent screen recording with no audio track at
+ * all is a real thing people post, and it is not a failure. It is a source
+ * with no words in it, which the server says out loud rather than leaving as
+ * an empty transcript nobody can explain.
+ */
+async function extractAudio(
+  cfg: Config,
+  video: string,
+  dir: string,
+  payload: IngestPayload,
+): Promise {
+  const tool = await allowed(cfg, "ffmpeg");
+  if (!tool) return "";
+
+  const out = path.join(dir, payload.audioFile || "audio.wav");
+  const { code } = await run(
+    tool.command,
+    [
+      ...tool.lead,
+      "-y",
+      "-i",
+      video,
+      "-vn",
+      "-ac",
+      "1",
+      "-ar",
+      String(payload.audioRate || 16_000),
+      out,
+    ],
+    { env: tool.env, timeout: 300_000 },
+  );
+  if (code !== 0) return "";
+  // ffmpeg can exit 0 having written a header and nothing else, and a WAV with
+  // no samples sent to a GPU is a minute of somebody's machine spent to
+  // transcribe silence.
+  const size = await fs
+    .stat(out)
+    .then((s) => s.size)
+    .catch(() => 0);
+  return size > 1024 ? out : "";
+}
+
+async function durationOf(cfg: Config, video: string, deadline: number) {
+  const tool = await allowed(cfg, "ffprobe");
+  if (!tool) return 0;
+  const { out } = await run(
+    tool.command,
+    [
+      ...tool.lead,
+      "-v",
+      "error",
+      "-show_entries",
+      "format=duration",
+      "-of",
+      "default=noprint_wrappers=1:nokey=1",
+      video,
+    ],
+    { env: tool.env, timeout: left(deadline, 30_000) },
+  );
+  const seconds = Number(out.trim());
+  return Number.isFinite(seconds) && seconds > 0 ? seconds : 0;
+}
+
+/** The first line of yt-dlp's stderr that is a reason rather than a warning. */
+function firstUseful(err: string) {
+  const line = err
+    .split("\n")
+    .map((l) => l.trim())
+    .find((l) => l && !l.startsWith("WARNING"));
+  return (line ?? "").replace(/^ERROR:\s*/, "").slice(0, 300);
+}
diff --git a/worker/job-types.ts b/worker/job-types.ts
new file mode 100644
index 0000000..a7e1025
--- /dev/null
+++ b/worker/job-types.ts
@@ -0,0 +1,88 @@
+/**
+ * The shapes the server sends and the shapes it expects back.
+ *
+ * Written out here rather than imported from `lib/server/services/*` on
+ * purpose: those modules reach the database on import, and a sidecar on
+ * somebody's desktop has no database and must never look like it wants one.
+ * These are the payloads documented in `docs/WORKER.md` and built by
+ * `payloadFor()` in services/runs.ts and `ingest()` in services/ingest.ts; if
+ * one of those changes shape, this is the file that has to move with it.
+ *
+ * The worker is opaque to everything else in a payload. There is no pack here,
+ * no rule, no dependency graph and no brand voice, because the server has
+ * already resolved all of it — a job carries finished text and a command.
+ */
+
+export type Transport = "cli" | "api";
+
+/**
+ * How a job's own process hands its answer back.
+ *
+ * It lives here, in a module with nothing but types and constants in it,
+ * because the supervisor needs the same string the executor writes — and
+ * importing it from `executor.ts` would run that file's `main()` inside the
+ * supervisor, which would sit there reading a stdin that never ends.
+ *
+ * Long enough that no CLI, progress bar or stack trace says it by accident:
+ * whatever follows the LAST one of these on stdout is the answer, and being
+ * wrong about that means posting a progress bar as somebody's section.
+ */
+export const RESULT_MARK = "<>";
+
+export type FrameRef = { url: string; at: number };
+
+export type WritePayload = {
+  brain: string;
+  transport: Transport;
+  system: string;
+  user: string;
+  tier: string;
+  timeoutMs?: number;
+  frames?: FrameRef[];
+  allowFrameRead?: boolean;
+};
+
+export type TestPayload = {
+  brain: string;
+  transport: Transport;
+  system: string;
+  user: string;
+  tier: string;
+  timeoutMs?: number;
+};
+
+export type IngestPayload = {
+  sourceId: string;
+  url: string;
+  refresh?: boolean;
+  frameCount: number;
+  maxHeight: number;
+  audioRate: number;
+  audioFile: string;
+  timeoutMs?: number;
+  uploadFrames: string;
+  uploadAudio: string;
+};
+
+export type TranscribePayload = {
+  sourceId: string;
+  audioUrl: string;
+  model: string;
+  timeoutMs?: number;
+};
+
+/**
+ * A failure this machine is sure about.
+ *
+ * `retryable: false` is a claim, not a mood: it says the next machine will
+ * fail identically — a refusal, a payload naming a transport this worker
+ * cannot honour, a tool that is not installed. Everything else is thrown as an
+ * ordinary Error and retried once, because the failures that actually recur
+ * are the closed laptop and the dropped connection.
+ */
+export class Refusal extends Error {
+  readonly retryable = false;
+}
+
+/** What comes back from a handler: the result body the server stores. */
+export type JobResult = Record;
diff --git a/worker/register.mjs b/worker/register.mjs
new file mode 100644
index 0000000..6b7b4b1
--- /dev/null
+++ b/worker/register.mjs
@@ -0,0 +1,9 @@
+/**
+ * What `node --import` loads before the worker starts.
+ *
+ * Separate from the hook itself because a resolve hook runs on its own thread
+ * and cannot register itself; this is the two lines that live on the main one.
+ */
+import { register } from "node:module";
+
+register("./resolve-ts.mjs", import.meta.url);
diff --git a/worker/resolve-ts.mjs b/worker/resolve-ts.mjs
new file mode 100644
index 0000000..150a649
--- /dev/null
+++ b/worker/resolve-ts.mjs
@@ -0,0 +1,28 @@
+/**
+ * Let the worker import the app's TypeScript by its plain path.
+ *
+ * Node 22.18+ strips types out of a .ts file on its own, so this sidecar needs
+ * no bundler and no ts-node — but its ESM resolver still refuses an
+ * extensionless specifier, so `import "../lib/server/tools"` fails at run time
+ * even though every editor and `tsc` resolve it happily. Writing the extension
+ * instead (`"../lib/server/tools.ts"`) runs, and then `tsc --noEmit` on the
+ * repo fails with TS5097, because `allowImportingTsExtensions` is off in the
+ * root tsconfig and this worker has no business turning it on for the whole
+ * app.
+ *
+ * So the resolution gap is closed here, in eleven lines, rather than by
+ * changing a compiler option that every other file in the repo lives under.
+ * Only relative specifiers are touched, and only when they carry no extension:
+ * a bare package name still resolves exactly as node_modules says it should.
+ */
+export async function resolve(specifier, context, nextResolve) {
+  if (/^\.{1,2}\//.test(specifier) && !/\.[cm]?[jt]s$/.test(specifier)) {
+    try {
+      return await nextResolve(`${specifier}.ts`, context);
+    } catch {
+      // Not a .ts after all — fall through and let the default resolver give
+      // its own error, which names the specifier the author actually wrote.
+    }
+  }
+  return nextResolve(specifier, context);
+}
diff --git a/worker/setup.ts b/worker/setup.ts
new file mode 100644
index 0000000..54df375
--- /dev/null
+++ b/worker/setup.ts
@@ -0,0 +1,907 @@
+import { createHash } from "node:crypto";
+import { createWriteStream } from "node:fs";
+import fs from "node:fs/promises";
+import path from "node:path";
+import { createInterface } from "node:readline/promises";
+import { Readable, Transform } from "node:stream";
+import { pipeline } from "node:stream/promises";
+import type { ReadableStream as WebReadableStream } from "node:stream/web";
+
+import { run, toolStatuses, toolsBin, whisperModels } from "../lib/server/tools";
+import { toolsRoot } from "./config";
+
+/**
+ * `npm run worker:setup` — the tools, on a machine belonging to somebody who
+ * does not have a compiler.
+ *
+ * The team this worker is for are not developers and they are on Windows.
+ * That single fact decides everything in this file. `pip install openai-whisper`
+ * is not an instruction you can give them: it needs a Python, then PyTorch,
+ * then gigabytes, and then usually a conversation about CUDA. `winget install
+ * Gyan.FFmpeg` needs an admin prompt and a PATH they will never look at again.
+ * So this fetches four things into one folder inside the app and teaches the
+ * worker to look there — and that is the whole mechanism. There is no
+ * installer, no service, no registry key and nothing to uninstall.
+ *
+ * Five rules, and each of them is here because the alternative is somebody's
+ * own computer:
+ *
+ * - **Pinned, and said out loud.** Every version is a constant in this file
+ *   with the URL beside it, and the plan is printed before anything is
+ *   fetched. Nothing here ever resolves "latest": a script that silently pulls
+ *   whatever a server feels like handing over today is not something to run on
+ *   a colleague's laptop, and a pinned version is also the only way "it worked
+ *   last week" is a question with an answer.
+ * - **Verified.** Every download is checked against a sha256 published by the
+ *   project that made it, and against the exact byte count where the publisher
+ *   states one. Bytes land in `downloads/` under a `.part` name and are only
+ *   moved into place after they pass — because a half-downloaded `yt-dlp.exe`
+ *   left in `bin/` by a dropped wifi connection is a file the probe will
+ *   happily report as an installed tool, and the failure would surface a week
+ *   later as an ingest that cannot read any link.
+ * - **Inside its own folder, always.** No PATH edit, no registry, no admin, no
+ *   `%APPDATA%`. Removing this app means deleting a folder, and that promise
+ *   is worth keeping literally.
+ * - **Windows first, and honest elsewhere.** ffmpeg and whisper.cpp are
+ *   fetched as Windows x64 builds. On macOS and Linux this says so and prints
+ *   the platform's own one-liner rather than downloading a binary that cannot
+ *   run.
+ * - **Re-runnable.** What is already here is reported and skipped. Running it
+ *   twice is how somebody adds a bigger model later, and it must not mean
+ *   re-downloading three gigabytes.
+ *
+ * What it deliberately does NOT install: `claude`, `codex` and `gemini`, which
+ * need that person's own login and are the app's job to detect rather than to
+ * provide; and Ollama, which is a 700 MB download with its own installer and
+ * is pointed at rather than pulled down uninvited. Both are reported at the
+ * end, so one command still answers "is this machine ready".
+ */
+
+/* ── What gets downloaded, pinned ───────────────────────────────────────── */
+
+/**
+ * yt-dlp, 2026-08-19.
+ *
+ * The one tool here that goes stale on a schedule nobody controls: it breaks
+ * when a platform changes its player, which happens without warning and
+ * roughly monthly. It also updates itself — `yt-dlp -U` replaces the very file
+ * this puts in `bin/`, needs no admin because the file is ours, and is the
+ * right answer when a link that used to work stops working.
+ */
+const YT_DLP = "2026.08.19";
+
+/** FFmpeg 9.0.1, gyan.dev's "essentials" build — the one ffmpeg.org links to
+ *  for Windows. Static, so ffmpeg.exe and ffprobe.exe carry no DLLs with them. */
+const FFMPEG = "9.0.1";
+
+/** whisper.cpp release b4938 (v1.9.3), 2026-08-20. */
+const WHISPER = "b4938";
+
+/** A file to fetch: where from, and what it must turn out to be. */
+type Asset = {
+  url: string;
+  /** Lower-case hex, as the project that built the file publishes it. */
+  sha256: string;
+  /** Exact size, or 0 when the publisher does not state one. */
+  bytes: number;
+  /** Roughly how big, for the line printed before anything is downloaded. */
+  about: string;
+};
+
+const YT_DLP_ASSETS: Record = {
+  win32: {
+    url: `https://github.com/yt-dlp/yt-dlp/releases/download/${YT_DLP}/yt-dlp.exe`,
+    sha256: "66674953fe251b89f4d08c5f0e35e0728679bd67ab3d7d05c0562af101dd3e7a",
+    bytes: 17_840_399,
+    about: "17 MB",
+  },
+  linux: {
+    url: `https://github.com/yt-dlp/yt-dlp/releases/download/${YT_DLP}/yt-dlp_linux`,
+    sha256: "b16e4dab368a816cd05d477d698a605a6ae87ccee1c8ffd38fa21d7254141fcc",
+    bytes: 40_446_224,
+    about: "40 MB",
+  },
+  darwin: {
+    url: `https://github.com/yt-dlp/yt-dlp/releases/download/${YT_DLP}/yt-dlp_macos`,
+    sha256: "0f192b7ec147ab6288885d6351d9ab67367640029b4377576ef46dd79cf7b202",
+    bytes: 37_146_048,
+    about: "37 MB",
+  },
+};
+
+/**
+ * gyan.dev states a sha256 next to each package and does not state a byte
+ * count, so `bytes` is 0 and the digest carries the whole verification. Taken
+ * from https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-9.0.1-essentials_build.zip.sha256
+ */
+const FFMPEG_ASSET: Asset = {
+  url: `https://www.gyan.dev/ffmpeg/builds/packages/ffmpeg-${FFMPEG}-essentials_build.zip`,
+  sha256: "fec81ae03971d9dd4be3ebe02e263bd2ec1d789483f931bdba5f5715e65da2e9",
+  bytes: 0,
+  about: "~106 MB",
+};
+
+/**
+ * Two whisper.cpp builds, and the second one is a decision worth making
+ * deliberately.
+ *
+ * `cpu` is eight megabytes and runs anywhere. `cublas` is the same program
+ * built against CUDA 12.4 — the GPU path, and the reason transcription is
+ * worth sending to a desktop at all — but it is 671 MB and it does nothing at
+ * all without an NVIDIA card and a current driver. Defaulting to it would mean
+ * two thirds of a gigabyte downloaded onto a laptop that cannot use a byte of
+ * it, so it is asked for by name.
+ */
+const WHISPER_ASSETS: Record = {
+  cpu: {
+    url: `https://github.com/ggml-org/whisper.cpp/releases/download/${WHISPER}/whisper-bin-x64.zip`,
+    sha256: "c2a4b60edb11f7e11a9191ffb50929535527d4d91c9903dbe3e554583bbbc63d",
+    bytes: 8_361_840,
+    about: "8 MB",
+  },
+  cublas: {
+    url: `https://github.com/ggml-org/whisper.cpp/releases/download/${WHISPER}/whisper-cublas-12.4.0-bin-x64.zip`,
+    sha256: "c1b17166e1e31a91cc8e9c1f910d3785e3ce757bb2958bf9dce13fdb4880005f",
+    bytes: 671_045_732,
+    about: "671 MB — needs an NVIDIA GPU and a CUDA 12 driver",
+  },
+};
+
+/**
+ * The models, and what each one costs.
+ *
+ * This table is the point of the `--models` flag: choosing a whisper model is
+ * a trade between disk, time and how many words come back wrong, and nobody
+ * can make it from a list of names. Speeds are for a CPU; a GPU build moves
+ * every row up by roughly an order of magnitude.
+ *
+ * sha256 and byte counts come from the Hugging Face API for
+ * `ggerganov/whisper.cpp`, which stores them as the LFS object id — so these
+ * are the digests the file is addressed by, not ones anybody typed.
+ */
+const MODELS: Record = {
+  tiny: {
+    url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-tiny.bin",
+    sha256: "be07e048e1e599ad46341c8d2a135645097a538221678b7acdd1b1919c6e1b21",
+    bytes: 77_691_713,
+    about: "74 MB",
+    accuracy: "roughest. Clear English only; mangles names and accents. Several times faster than real time.",
+  },
+  base: {
+    url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.bin",
+    sha256: "60ed5bc3dd14eea856493d334349b405782ddcaf0028d4b5df4088345fba2efe",
+    bytes: 147_951_465,
+    about: "141 MB",
+    accuracy: "the default. Fine on clean speech, roughly real time on a CPU, wrong often enough on strong accents and music beds to notice.",
+  },
+  small: {
+    url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small.bin",
+    sha256: "1be3a9b2063867b937e64e2ec7483364a79917e157fa98c5d94b5c1fffea987b",
+    bytes: 487_601_967,
+    about: "465 MB",
+    accuracy: "clearly better on accents, noise and non-English. About three times slower than base.",
+  },
+  medium: {
+    url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-medium.bin",
+    sha256: "6c14d5adee5f86394037b4e4e8b59f1673b6cee10e3cf0b11bbdbee79c156208",
+    bytes: 1_533_763_059,
+    about: "1.4 GB",
+    accuracy: "close to the best. Slow enough on a CPU that a ten-minute reel can outlast the job's timeout.",
+  },
+  "large-v3-turbo": {
+    url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin",
+    sha256: "1fc70f774d38eb169993ac391eea357ef47c88757ef72ee5943879b7e8e2bc69",
+    bytes: 1_624_555_275,
+    about: "1.5 GB",
+    accuracy: "near-large accuracy at roughly medium speed. The best trade on a machine with a GPU.",
+  },
+  "large-v3": {
+    url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3.bin",
+    sha256: "64d182b440b98d5203c4f9bd541544d84c605196c4f7b845dfa11fb23594d1e2",
+    bytes: 3_095_033_483,
+    about: "2.9 GB",
+    accuracy: "the best there is. Hours per hour of audio without a GPU — do not pick this for a laptop.",
+  },
+};
+
+/** Named so somebody reading the folder knows what put it there. */
+const MANIFEST = "installed.json";
+
+type Manifest = {
+  updatedAt?: string;
+  tools?: Record;
+};
+
+/* ── Running it ─────────────────────────────────────────────────────────── */
+
+type Args = {
+  yes: boolean;
+  force: boolean;
+  help: boolean;
+  listModels: boolean;
+  dryRun: boolean;
+  only: Set;
+  model: string;
+  whisperBuild: string;
+  dir: string;
+};
+
+const KNOWN = ["yt-dlp", "ffmpeg", "whisper"];
+
+function parseArgs(argv: string[]): Args {
+  const args: Args = {
+    yes: false,
+    force: false,
+    help: false,
+    listModels: false,
+    dryRun: false,
+    only: new Set(),
+    model: "base",
+    whisperBuild: "cpu",
+    dir: "",
+  };
+  for (let i = 0; i < argv.length; i += 1) {
+    const a = argv[i];
+    const next = () => argv[++i] ?? "";
+    if (a === "-y" || a === "--yes") args.yes = true;
+    else if (a === "--force") args.force = true;
+    else if (a === "-h" || a === "--help") args.help = true;
+    else if (a === "--models") args.listModels = true;
+    else if (a === "--dry-run") args.dryRun = true;
+    else if (a === "--only") for (const id of next().split(",")) args.only.add(id.trim());
+    else if (a === "--model") args.model = next().trim();
+    else if (a === "--whisper") args.whisperBuild = next().trim();
+    else if (a === "--dir") args.dir = next().trim();
+    else throw new Error(`I do not know the option "${a}". Run with --help to see the ones I do.`);
+  }
+  const unknown = [...args.only].filter((id) => !KNOWN.includes(id));
+  if (unknown.length) {
+    throw new Error(`--only takes ${KNOWN.join(", ")}; "${unknown.join(", ")}" is none of them.`);
+  }
+  if (!MODELS[args.model]) {
+    throw new Error(
+      `There is no whisper model called "${args.model}". Run with --models to see the choices.`,
+    );
+  }
+  if (!WHISPER_ASSETS[args.whisperBuild]) {
+    throw new Error(`--whisper takes cpu or cublas, not "${args.whisperBuild}".`);
+  }
+  return args;
+}
+
+function help() {
+  say(`npm run worker:setup -- [options]
+
+Downloads the tools this machine is missing into the app's own folder. Nothing
+is installed system-wide: no PATH edit, no registry, no admin. Undo it by
+deleting that folder.
+
+  -y, --yes            Do not ask before downloading.
+      --force          Fetch again even if the tool is already here.
+      --only a,b       Only these: ${KNOWN.join(", ")}.
+      --model NAME     Which whisper model (default: base). --models lists them.
+      --whisper cpu    Which whisper.cpp build. "cublas" is the GPU one:
+                       671 MB, and useless without an NVIDIA card.
+      --dir PATH       Put the tools somewhere else. Also settable as
+                       CONTENTOS_TOOLS_DIR, which the worker reads too.
+      --dry-run        Print the plan and stop.
+      --models         Print the model sizes and what they cost in accuracy.
+  -h, --help           This.`);
+}
+
+function listModels() {
+  say("whisper models — disk, and what the size buys:\n");
+  for (const [name, m] of Object.entries(MODELS)) {
+    say(`  ${name.padEnd(16)} ${m.about.padEnd(8)} ${m.accuracy}`);
+  }
+  say(`
+Every one of them is multilingual; the ".en" variants are English-only and are
+deliberately not offered, because this console is used for Bangla as well.
+Pick one with:  npm run worker:setup -- --model small`);
+}
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (args.help) return help();
+  if (args.listModels) return listModels();
+
+  // Before anything reads the tools directory: --dir wins, then whatever is
+  // already in the environment, then the folder beside worker/.
+  if (args.dir) process.env.CONTENTOS_TOOLS_DIR = path.resolve(args.dir);
+  const root = toolsRoot();
+  const bin = toolsBin();
+  const models = whisperModels();
+
+  say(`Content OS worker setup`);
+  say(`Tools folder: ${root}`);
+  say(`Platform:     ${process.platform} ${process.arch}\n`);
+
+  const steps = await plan(args, bin, models);
+
+  for (const step of steps.filter((s) => s.state !== "fetch")) {
+    // The path matters when the answer is "it is already there" and is noise
+    // when the answer is "you did not ask for it".
+    say(`  ${mark(step.state)} ${step.title}${step.state === "skip" ? "" : ` — ${step.why}`}`);
+  }
+
+  const fetching = steps.filter((s) => s.state === "fetch");
+  if (!fetching.length) {
+    say("\nNothing to download.\n");
+    await report(root);
+    return;
+  }
+
+  /*
+   * Everything, out loud, before a byte moves.
+   *
+   * This is the part that is not negotiable. Somebody is being asked to let a
+   * script pull two hundred megabytes of executables onto their own computer,
+   * and the least they are owed is the list: what, from whose server, how big,
+   * and pinned to which version.
+   */
+  say("\nAbout to download:\n");
+  for (const step of fetching) {
+    say(`  ${step.title}  (${step.asset.about})`);
+    say(`      from ${step.asset.url}`);
+    say(`      sha256 ${step.asset.sha256}`);
+  }
+  say(`\n  into ${root}`);
+  say("  Nothing outside that folder is written, changed or added to PATH.\n");
+
+  if (args.dryRun) return say("--dry-run: stopping here.");
+  if (!(await confirm(args.yes))) return say("Nothing was downloaded.");
+
+  say("");
+  const manifest = await readManifest(root);
+  manifest.tools ??= {};
+
+  for (const step of fetching) {
+    await step.install();
+    manifest.tools[step.id] = {
+      version: step.version,
+      from: step.asset.url,
+      sha256: step.asset.sha256,
+      at: new Date().toISOString(),
+    };
+    await writeManifest(root, manifest);
+  }
+
+  // The staging area is scratch, and a gigabyte of it. Nothing here is needed
+  // once the files have been verified and moved.
+  await fs.rm(path.join(root, "downloads"), { recursive: true, force: true }).catch(() => {});
+
+  say("");
+  await report(root);
+}
+
+/* ── The plan ───────────────────────────────────────────────────────────── */
+
+type State = "fetch" | "have" | "skip" | "unsupported";
+
+type Step = {
+  id: string;
+  title: string;
+  version: string;
+  asset: Asset;
+  state: State;
+  why: string;
+  install: () => Promise;
+};
+
+const mark = (state: State) =>
+  (state === "have" ? "already here:" : state === "unsupported" ? "not here:" : "not asked for:").padEnd(15);
+
+/**
+ * What each tool's state is, and why, before anything is downloaded.
+ *
+ * Four states rather than two, because "I am not going to download this" has
+ * three different meanings and a person deciding what to do next needs to know
+ * which: it is already here, you did not ask for it, or this platform is not
+ * one I have a binary for. Collapsing them into a silent skip is how somebody
+ * ends up waiting for an ffmpeg that was never going to arrive.
+ */
+async function plan(args: Args, bin: string, models: string): Promise {
+  const wanted = (id: string) => args.only.size === 0 || args.only.has(id);
+  const win = process.platform === "win32";
+  const steps: Step[] = [];
+
+  const decide = async (id: string, supported: boolean, already: Promise | boolean) =>
+    !wanted(id) ? "skip" : !supported ? "unsupported" : (await already) ? "have" : ("fetch" as State);
+
+  /* yt-dlp — one file, one download, and the project builds one per platform. */
+  const ytAsset = YT_DLP_ASSETS[process.platform] ?? YT_DLP_ASSETS.linux;
+  const ytName = win ? "yt-dlp.exe" : "yt-dlp";
+  const ytHere = path.join(bin, ytName);
+  const ytState = await decide(
+    "yt-dlp",
+    Boolean(YT_DLP_ASSETS[process.platform]),
+    settled(ytHere, ytAsset.bytes, args.force),
+  );
+  steps.push({
+    id: "yt-dlp",
+    title: `yt-dlp ${YT_DLP}`,
+    version: YT_DLP,
+    asset: ytAsset,
+    state: ytState,
+    why:
+      ytState === "unsupported"
+        ? `yt-dlp publishes no single-file build for ${process.platform}; install it the way this machine installs things.`
+        : ytHere,
+    install: async () => {
+      const file = await fetchVerified(ytAsset, "yt-dlp", bin, ytName);
+      // Downloaded files are not executable on macOS or Linux, and the failure
+      // is EACCES from a shell rather than anything that names the cause.
+      if (!win) await fs.chmod(file, 0o755).catch(() => {});
+    },
+  });
+
+  /* ffmpeg + ffprobe — one zip, two executables out of it. */
+  const ffHere = path.join(bin, win ? "ffmpeg.exe" : "ffmpeg");
+  const ffState = await decide(
+    "ffmpeg",
+    win,
+    (async () =>
+      (await exists(ffHere)) && (await exists(path.join(bin, win ? "ffprobe.exe" : "ffprobe"))) &&
+      !args.force)(),
+  );
+  steps.push({
+    id: "ffmpeg",
+    title: `ffmpeg + ffprobe ${FFMPEG}`,
+    version: FFMPEG,
+    asset: FFMPEG_ASSET,
+    state: ffState,
+    why: ffState === "unsupported" ? nativeAdvice("ffmpeg") : ffHere,
+    install: async () => {
+      const zip = await fetchVerified(FFMPEG_ASSET, "ffmpeg", "", "");
+      const stage = await unzip(zip, "ffmpeg");
+      // Found by walking rather than by the folder name inside the archive:
+      // that name carries the version, so hard-coding it would mean this
+      // function needs editing every time the pin moves.
+      for (const want of ["ffmpeg.exe", "ffprobe.exe"]) {
+        const found = await findFile(stage, want);
+        if (!found) throw new Error(`The FFmpeg archive did not contain ${want}.`);
+        await place(found, path.join(bin, want));
+      }
+      await fs.rm(stage, { recursive: true, force: true }).catch(() => {});
+    },
+  });
+
+  /* whisper.cpp — the executable, its DLLs, and separately a model. */
+  const whisperAsset = WHISPER_ASSETS[args.whisperBuild];
+  const whisperHere = path.join(bin, "whisper-cli.exe");
+  const whisperState = await decide(
+    "whisper",
+    win,
+    (async () => (await exists(whisperHere)) && !args.force)(),
+  );
+  steps.push({
+    id: "whisper",
+    title: `whisper.cpp ${WHISPER} (${args.whisperBuild})`,
+    version: `${WHISPER} ${args.whisperBuild}`,
+    asset: whisperAsset,
+    state: whisperState,
+    why: whisperState === "unsupported" ? nativeAdvice("whisper.cpp") : whisperHere,
+    install: async () => {
+      const zip = await fetchVerified(whisperAsset, "whisper", "", "");
+      const stage = await unzip(zip, "whisper");
+      const exe = (await findFile(stage, "whisper-cli.exe")) ?? (await findFile(stage, "main.exe"));
+      if (!exe) throw new Error("The whisper.cpp archive did not contain whisper-cli.exe.");
+      await place(exe, path.join(bin, "whisper-cli.exe"));
+      /*
+       * The DLLs beside it, all of them.
+       *
+       * whisper.cpp's Windows build is not one static file: whisper.dll,
+       * ggml.dll and a handful of ggml-cpu-* variants sit next to the exe and
+       * are loaded from the exe's own directory. Copying only the executable
+       * produces a program that exists, is found by the probe, and dies on
+       * launch with a dialog about a missing DLL — which reaches the console
+       * as a whisper that "crashed" for no stated reason.
+       */
+      const beside = path.dirname(exe);
+      for (const file of await fs.readdir(beside)) {
+        if (file.toLowerCase().endsWith(".dll")) {
+          await place(path.join(beside, file), path.join(bin, file));
+        }
+      }
+      await fs.rm(stage, { recursive: true, force: true }).catch(() => {});
+    },
+  });
+
+  /*
+   * The model, which is a separate step because it is a separate failure.
+   *
+   * A machine with whisper-cli.exe and no `.bin` cannot transcribe anything,
+   * and the probe in lib/server/tools.ts reports exactly that rather than
+   * calling whisper present — so this row going missing is visible on the
+   * Integrations page as its own sentence with its own fix.
+   */
+  const model = MODELS[args.model];
+  const modelFile = path.join(models, `ggml-${args.model}.bin`);
+  steps.push({
+    id: "whisper-model",
+    title: `whisper model "${args.model}"`,
+    version: args.model,
+    asset: model,
+    // Always supported: a model is a data file, and somebody who installed
+    // whisper.cpp with brew still needs one.
+    state: await decide("whisper", true, settled(modelFile, model.bytes, args.force)),
+    why: `${modelFile} — ${model.accuracy}`,
+    install: async () => {
+      await fetchVerified(model, "whisper-model", models, `ggml-${args.model}.bin`);
+    },
+  });
+
+  return steps;
+}
+
+/** What to type on a platform this script does not fetch binaries for. */
+function nativeAdvice(what: string) {
+  const mac = what === "ffmpeg" ? "brew install ffmpeg" : "brew install whisper-cpp";
+  const linux = what === "ffmpeg" ? "sudo apt install ffmpeg" : "build it from ggml-org/whisper.cpp";
+  return process.platform === "darwin"
+    ? `only fetched for Windows here. On macOS: ${mac}`
+    : `only fetched for Windows here. On Linux: ${linux}`;
+}
+
+/* ── Fetching, and being sure about what arrived ────────────────────────── */
+
+/**
+ * Download one file, prove it is the right one, then put it where it goes.
+ *
+ * The order is the whole point. Bytes go to `downloads/.part`, the hash
+ * and the length are computed while they stream past, and only a file that
+ * matches both is moved into `bin/` or `models/`. A dropped connection, a
+ * captive-portal login page served with a 200, a proxy that truncates at 2 GB
+ * — every one of those leaves a `.part` behind and nothing that the probe will
+ * later mistake for an installed tool.
+ *
+ * `dest`/`name` empty means "leave it in downloads": the archives are unpacked
+ * from there and never belong in bin/.
+ */
+async function fetchVerified(asset: Asset, label: string, dest: string, name: string) {
+  const staging = path.join(toolsRoot(), "downloads");
+  await fs.mkdir(staging, { recursive: true });
+  const part = path.join(staging, `${label}.part`);
+  /*
+   * An archive keeps the extension it was published with.
+   *
+   * `Expand-Archive` refuses, by name, any path that does not end in `.zip` —
+   * it is a check on the string, not on the bytes — so a staged file called
+   * `ffmpeg` would unpack fine through `tar` and fail on exactly the machines
+   * old enough to need the PowerShell fallback.
+   */
+  const final = dest
+    ? path.join(dest, name)
+    : path.join(staging, label + path.extname(new URL(asset.url).pathname));
+
+  await fs.rm(part, { force: true }).catch(() => {});
+
+  const res = await fetch(asset.url, { redirect: "follow" });
+  if (!res.ok || !res.body) {
+    throw new Error(`${asset.url} answered ${res.status} ${res.statusText}. Nothing was written.`);
+  }
+
+  /*
+   * Content-Length, checked before the download rather than after it.
+   *
+   * When the server states a length and it is not the length we pinned, the
+   * file at that URL is not the file this script was written against — a
+   * re-rolled release, a mirror, or an error page with a 200 on it — and there
+   * is no point spending somebody's bandwidth to find that out at the end.
+   */
+  const declared = Number(res.headers.get("content-length") ?? 0);
+  if (asset.bytes && declared && declared !== asset.bytes) {
+    throw new Error(
+      `${asset.url} is ${declared} bytes and this expects ${asset.bytes}. ` +
+        `That is not the file this version was pinned to; nothing was written.`,
+    );
+  }
+
+  const hash = createHash("sha256");
+  let got = 0;
+  let shown = 0;
+  const watch = new Transform({
+    transform(chunk: Buffer, _enc, done) {
+      hash.update(chunk);
+      got += chunk.length;
+      const total = asset.bytes || declared;
+      if (total && got - shown > total / 20) {
+        shown = got;
+        progress(label, got, total);
+      }
+      done(null, chunk);
+    },
+  });
+
+  say(`  ${label}: downloading ${asset.about}…`);
+  await pipeline(
+    Readable.fromWeb(res.body as unknown as WebReadableStream),
+    watch,
+    createWriteStream(part),
+  );
+  progress(label, got, asset.bytes || declared || got);
+  if (process.stdout.isTTY) process.stdout.write("\n");
+
+  const digest = hash.digest("hex");
+  if (digest !== asset.sha256) {
+    await fs.rm(part, { force: true }).catch(() => {});
+    throw new Error(
+      `${label} downloaded, but its sha256 is ${digest} and this expects ${asset.sha256}. ` +
+        `The file was deleted. Either the release was re-published or something is between ` +
+        `this machine and ${new URL(asset.url).host}.`,
+    );
+  }
+  if (asset.bytes && got !== asset.bytes) {
+    await fs.rm(part, { force: true }).catch(() => {});
+    throw new Error(`${label} arrived as ${got} bytes, not ${asset.bytes}. The file was deleted.`);
+  }
+  if (!got) {
+    await fs.rm(part, { force: true }).catch(() => {});
+    throw new Error(`${label} arrived empty. The file was deleted.`);
+  }
+
+  await fs.mkdir(path.dirname(final), { recursive: true });
+  // rename, not copy: it is one filesystem operation, it cannot half-happen,
+  // and there is never a moment when a partly-written file sits at the name
+  // the probe looks for.
+  await fs.rename(part, final);
+  say(`  ${label}: verified (sha256 ok, ${got.toLocaleString()} bytes) → ${final}`);
+  return final;
+}
+
+/** Whether a file is here AND the size it should be. */
+async function settled(file: string, bytes: number, force: boolean) {
+  if (force) return false;
+  try {
+    const stat = await fs.stat(file);
+    if (!stat.isFile()) return false;
+    /*
+     * A model half-copied off a USB stick, or left by a version of this script
+     * that did not stage its downloads, is a file that exists and does not
+     * work. Skipping it because the name is right is how somebody spends a
+     * week being told whisper crashed.
+     */
+    if (bytes && stat.size !== bytes) return false;
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+const exists = (file: string) =>
+  fs
+    .stat(file)
+    .then((s) => s.isFile())
+    .catch(() => false);
+
+/**
+ * Unpack a zip with what Windows already has.
+ *
+ * `tar` has shipped in Windows since 1803 and reads zips, which makes it the
+ * cheapest option and the one that does not load a PowerShell profile. If it
+ * is not there, `Expand-Archive` is. Neither needs a dependency in
+ * package.json, which is deliberate: a setup script whose job is to avoid
+ * making people install things should not itself need an install.
+ */
+async function unzip(zip: string, label: string) {
+  const stage = path.join(toolsRoot(), "downloads", `${label}-unpacked`);
+  await fs.rm(stage, { recursive: true, force: true }).catch(() => {});
+  await fs.mkdir(stage, { recursive: true });
+
+  const viaTar = await run("tar", ["-xf", zip, "-C", stage], { timeout: 600_000 });
+  if (viaTar.code === 0) return stage;
+
+  if (process.platform === "win32") {
+    // Single quotes inside the -Command string, so nothing has to survive
+    // cmd.exe's idea of a backslash-escaped double quote on the way in.
+    const ps = (v: string) => `'${v.replace(/'/g, "''")}'`;
+    const viaPs = await run(
+      "powershell",
+      [
+        "-NoProfile",
+        "-NonInteractive",
+        "-Command",
+        `Expand-Archive -LiteralPath ${ps(zip)} -DestinationPath ${ps(stage)} -Force`,
+      ],
+      { timeout: 600_000 },
+    );
+    if (viaPs.code === 0) return stage;
+    throw new Error(
+      `Could not unpack ${path.basename(zip)}. tar said "${firstLine(viaTar.err)}" and ` +
+        `Expand-Archive said "${firstLine(viaPs.err)}".`,
+    );
+  }
+
+  throw new Error(
+    `Could not unpack ${path.basename(zip)}: ${firstLine(viaTar.err) || `tar exited ${viaTar.code}`}`,
+  );
+}
+
+/** The first file with this name, anywhere under a directory. */
+async function findFile(root: string, name: string): Promise {
+  const want = name.toLowerCase();
+  for (const entry of await fs.readdir(root, { withFileTypes: true })) {
+    const full = path.join(root, entry.name);
+    if (entry.isDirectory()) {
+      const deeper = await findFile(full, name);
+      if (deeper) return deeper;
+    } else if (entry.name.toLowerCase() === want) {
+      return full;
+    }
+  }
+  return "";
+}
+
+/** Copy one file into place, replacing whatever was there. */
+async function place(from: string, to: string) {
+  await fs.mkdir(path.dirname(to), { recursive: true });
+  try {
+    await fs.copyFile(from, to);
+  } catch (e) {
+    /*
+     * Windows will not overwrite a file something has open, and the something
+     * is almost always the worker itself — an ffmpeg mid-ingest, or a
+     * whisper-cli holding its own DLLs. "EBUSY: resource busy or locked" names
+     * the syscall and not the situation, and the fix is one sentence.
+     */
+    const code = (e as NodeJS.ErrnoException).code;
+    if (code === "EBUSY" || code === "EPERM" || code === "ETXTBSY") {
+      throw new Error(
+        `${path.basename(to)} is in use, so it cannot be replaced. Stop the worker (Ctrl-C in its window) and run this again.`,
+      );
+    }
+    throw e;
+  }
+}
+
+/* ── Saying what happened ───────────────────────────────────────────────── */
+
+/**
+ * What this machine can actually do now, probed rather than assumed.
+ *
+ * The same `toolStatuses()` the worker reports at registration, forced past
+ * its cache — so this is not a summary of what was downloaded, it is the
+ * worker's own answer to "what is on this machine", asked immediately. A tool
+ * that downloaded perfectly and cannot run — a missing Visual C++ runtime, an
+ * antivirus that quarantined the exe — shows up here rather than three days
+ * later as a job that failed on somebody's laptop.
+ */
+async function report(root: string) {
+  say("This machine, as the worker sees it:\n");
+  const tools = await toolStatuses(true);
+  for (const tool of tools) {
+    const what = tool.present
+      ? `${tool.version || "installed"}${tool.flavor ? ` (${tool.flavor})` : ""}`
+      : tool.error;
+    say(`  ${tool.present ? "yes" : "no "}  ${tool.id.padEnd(10)} ${what}`);
+  }
+
+  const missing = tools.filter((t) => !t.present);
+  const clis = missing.filter((t) => ["claude", "codex", "gemini"].includes(t.id));
+  if (clis.length) {
+    say(`
+The model CLIs are not installed by this script on purpose: each one signs in
+as you, with your own subscription, and the app's job is to detect them rather
+than to own them. Install and sign in to whichever you use:`);
+    for (const cli of clis) say(`  ${cli.id.padEnd(8)} ${cli.install}`);
+  }
+
+  await ollama();
+
+  say(`
+Everything this script installed lives in ${root}. Nothing else on this
+computer was changed — no PATH, no registry, nothing installed for all users.
+To undo it, delete that folder.
+
+yt-dlp goes stale when a platform changes its player, which is why an ingest
+that worked last month can stop. It updates itself:
+  ${path.join(toolsBin(), process.platform === "win32" ? "yt-dlp.exe" : "yt-dlp")} -U
+
+Now start the worker:  npm run worker`);
+}
+
+/**
+ * Ollama, detected and pointed at rather than downloaded.
+ *
+ * It is roughly 700 MB with its own installer and its own model downloads on
+ * top, and it is optional — one brain out of four, and the only one that is an
+ * HTTP server rather than a CLI. Pulling that down uninvited on somebody's
+ * laptop to save them a click is not a trade this script gets to make.
+ */
+async function ollama() {
+  const url = (process.env.CONTENTOS_OLLAMA_URL ?? "").trim().replace(/\/+$/, "") ||
+    "http://localhost:11434";
+  try {
+    const res = await fetch(`${url}/api/tags`, { signal: AbortSignal.timeout(1500) });
+    if (res.ok) {
+      const body = (await res.json()) as { models?: { name?: string }[] };
+      const names = (body.models ?? []).map((m) => m.name).filter(Boolean);
+      say(`
+Ollama is running at ${url}${names.length ? ` with ${names.length} model(s): ${names.slice(0, 4).join(", ")}` : " with no models pulled yet — try: ollama pull llama3.1"}.`);
+      return;
+    }
+  } catch {
+    // Not running, not installed, or a firewall. All three are the same
+    // sentence to whoever is reading, and none of them is an error here.
+  }
+  say(`
+Ollama is optional and is not answering at ${url}. It is only needed for the
+local-model brain; if you want it, it has its own installer at
+https://ollama.com/download and then: ollama pull llama3.1`);
+}
+
+/* ── Small things ───────────────────────────────────────────────────────── */
+
+async function readManifest(root: string): Promise {
+  try {
+    return JSON.parse(await fs.readFile(path.join(root, MANIFEST), "utf8")) as Manifest;
+  } catch {
+    return {};
+  }
+}
+
+/**
+ * Written after every tool rather than once at the end.
+ *
+ * A run that fails on the third download has still installed the first two,
+ * and a manifest that only exists on a clean finish would describe a folder
+ * that does not match it — which matters because `lib/server/tools.ts` reads
+ * this file to report the version of a whisper.cpp that has no --version flag.
+ */
+async function writeManifest(root: string, manifest: Manifest) {
+  manifest.updatedAt = new Date().toISOString();
+  await fs.mkdir(root, { recursive: true });
+  await fs.writeFile(path.join(root, MANIFEST), `${JSON.stringify(manifest, null, 2)}\n`, "utf8");
+}
+
+/**
+ * Ask, unless told not to.
+ *
+ * A prompt is only possible when somebody is there to answer it. With no
+ * terminal — a CI job, a Tauri sidecar, a scheduled task — the answer is not
+ * "assume yes": the whole rule is that nothing downloads without somebody
+ * having said so, and in that situation `--yes` is how they say it.
+ */
+async function confirm(yes: boolean) {
+  if (yes) return true;
+  if (!process.stdin.isTTY) {
+    say(
+      "There is no terminal here to ask, and nothing downloads unasked.\n" +
+        "Run it again with --yes if this is what you want:  npm run worker:setup -- --yes",
+    );
+    return false;
+  }
+  const rl = createInterface({ input: process.stdin, output: process.stdout });
+  try {
+    const answer = (await rl.question("Download these? [y/N] ")).trim().toLowerCase();
+    return answer === "y" || answer === "yes";
+  } finally {
+    rl.close();
+  }
+}
+
+function progress(label: string, got: number, total: number) {
+  if (!process.stdout.isTTY) return;
+  const mb = (n: number) => (n / 1_048_576).toFixed(1);
+  const pct = total ? ` ${Math.min(100, Math.round((got / total) * 100))}%` : "";
+  process.stdout.write(`\r  ${label}: ${mb(got)} / ${mb(total)} MB${pct}   `);
+}
+
+const firstLine = (text: string) => (text.split(/\r?\n/).find((l) => l.trim()) ?? "").trim();
+
+function say(line: string) {
+  console.log(line);
+}
+
+main().catch((e: unknown) => {
+  /*
+   * One sentence, no stack trace.
+   *
+   * Whoever is reading this is looking at a console window on their own
+   * laptop because somebody told them to run one command, and every failure
+   * this script can have — a wrong option, a checksum that did not match, no
+   * network — is already written as a sentence that says what to do next.
+   */
+  console.error(`\nSetup stopped: ${e instanceof Error ? e.message : String(e)}`);
+  process.exit(1);
+});
diff --git a/worker/temp.ts b/worker/temp.ts
new file mode 100644
index 0000000..6e97715
--- /dev/null
+++ b/worker/temp.ts
@@ -0,0 +1,50 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+/**
+ * Where a job's scratch files go — and, just as importantly, where they do not.
+ *
+ * NOT `cliHome()`. That directory is deliberately empty so that `claude`,
+ * `codex` and `gemini`, all of which read the project they are started in,
+ * have nothing to find; filling it with JPEGs would undo the reasoning in
+ * `docs/DECISIONS.md` one frame at a time and put a stray file in the model's
+ * context on the next run. The CLI still starts in `cliHome()`; the frames sit
+ * somewhere else entirely and are named to it by absolute path.
+ *
+ * Under one root rather than scattered through the temp directory, so a worker
+ * that was killed — a closed lid, a `taskkill`, a crash — leaves its litter
+ * somewhere the next start can find and clear in one line.
+ */
+export const TEMP_ROOT = path.join(os.tmpdir(), "contentos-worker");
+
+/** A directory of this job's own, named so a leftover says which job left it. */
+export async function jobDir(jobId: string, what: string) {
+  const dir = path.join(TEMP_ROOT, `${jobId}-${what}`);
+  await fs.mkdir(dir, { recursive: true });
+  return dir;
+}
+
+/**
+ * Remove a directory and never throw for it.
+ *
+ * Called from a `finally`, where an exception would replace the real failure
+ * with a housekeeping one — and on Windows the real failure is often exactly
+ * why the directory is still locked.
+ */
+export async function discard(dir: string) {
+  await fs.rm(dir, { recursive: true, force: true }).catch(() => {});
+}
+
+/**
+ * Everything left behind by an earlier run of this worker.
+ *
+ * Swept at start rather than only at shutdown, because the shutdown that
+ * matters is the one that did not happen: a killed process cleans up nothing,
+ * and a laptop that has been fetching videos for a month should not be storing
+ * a month of them in its temp directory.
+ */
+export async function sweepTemp() {
+  await fs.rm(TEMP_ROOT, { recursive: true, force: true }).catch(() => {});
+  await fs.mkdir(TEMP_ROOT, { recursive: true }).catch(() => {});
+}
diff --git a/worker/test-brain.ts b/worker/test-brain.ts
new file mode 100644
index 0000000..219033c
--- /dev/null
+++ b/worker/test-brain.ts
@@ -0,0 +1,51 @@
+import { callBrain } from "./brains";
+import type { Config } from "./config";
+import type { JobResult, TestPayload } from "./job-types";
+
+/**
+ * Can this machine write?
+ *
+ * The smallest job, and the one that makes the whole design legible: a person
+ * in Settings → Integrations picks a model and a machine and finds out whether
+ * that pair works. A model that writes on one teammate's laptop and not on
+ * another's is now a thing the console can express, and before the split it
+ * could not even ask the question.
+ *
+ * This never fails the job, and that is deliberate. The refusal IS the answer
+ * — "codex is not signed in on this machine" is what the button was pressed to
+ * discover — so it comes back as a result with `ok: false`, which is what the
+ * page polls for. Failing the job instead would put the sentence in `error`,
+ * spend an attempt, and leave the button spinning while the queue decided
+ * whether to try again.
+ */
+export async function testBrain(cfg: Config, payload: TestPayload): Promise {
+  const started = Date.now();
+  try {
+    const { text } = await callBrain(cfg, {
+      brain: payload.brain,
+      transport: payload.transport,
+      system: payload.system,
+      user: payload.user,
+      tier: payload.tier,
+      timeoutMs: payload.timeoutMs ?? 150_000,
+    });
+    const reply = text.trim();
+    return {
+      // Answering with nothing is not a pass. The button exists to prove the
+      // model writes, and an empty string proves the opposite.
+      ok: Boolean(reply),
+      ms: Date.now() - started,
+      reply: reply.slice(0, 2000),
+      error: reply ? "" : "The model answered with nothing.",
+      transport: payload.transport,
+    };
+  } catch (e) {
+    return {
+      ok: false,
+      ms: Date.now() - started,
+      reply: "",
+      error: (e instanceof Error ? e.message : String(e)).slice(0, 2000),
+      transport: payload.transport,
+    };
+  }
+}
diff --git a/worker/transcribe-audio.ts b/worker/transcribe-audio.ts
new file mode 100644
index 0000000..166dba7
--- /dev/null
+++ b/worker/transcribe-audio.ts
@@ -0,0 +1,258 @@
+import fs from "node:fs/promises";
+import os from "node:os";
+import path from "node:path";
+
+import {
+  run,
+  toolStatuses,
+  whisperModel,
+  whisperModelGap,
+  type ToolStatus,
+} from "../lib/server/tools";
+import type { Api } from "./api";
+import type { Config } from "./config";
+import { Refusal, type JobResult, type TranscribePayload } from "./job-types";
+import { discard, jobDir } from "./temp";
+
+/**
+ * The spoken audio, as text, on the machine that has a GPU.
+ *
+ * Its own job rather than the tail of an ingest, and the reason is a ratio.
+ * Audio is roughly a thirtieth of a video, so what crosses the network is a
+ * megabyte a minute rather than four hundred — which makes a round trip to a
+ * second desktop affordable, and transcription is exactly the step worth
+ * sending there: it is the one that wants a GPU and the one dependency the
+ * server should not carry. It also means an upload, whose bytes never left the
+ * server, gets transcribed by the same path as a link.
+ *
+ * The WAV and nothing else is downloaded. It is already 16 kHz mono because
+ * whoever produced it — this worker's own ingest, or the server's ffmpeg for
+ * an upload — made it that way on purpose: that is whisper.cpp's input format,
+ * so the conversion is a step that had to happen somewhere and it happened
+ * where the video already was.
+ *
+ * **Two programs answer to the name `whisper` and this runs whichever is
+ * here.** whisper.cpp is an executable plus a `.bin`; openai-whisper is a
+ * Python package with PyTorch behind it. They share no arguments and they do
+ * not write their output to the same place. The probe in `lib/server/tools.ts`
+ * says which one it found and prefers whisper.cpp, because that is the one
+ * `npm run worker:setup` can put on a machine belonging to somebody who does
+ * not have a Python.
+ */
+export async function transcribeAudio(
+  cfg: Config,
+  api: Api,
+  jobId: string,
+  payload: TranscribePayload,
+): Promise {
+  if (!payload.sourceId || !payload.audioUrl) {
+    throw new Refusal("The transcription job arrived without any audio to transcribe.");
+  }
+
+  if (cfg.toolsOff.has("whisper")) {
+    throw new Refusal("whisper is switched off on this machine (CONTENTOS_WORKER_TOOLS_OFF).");
+  }
+
+  /*
+   * However this machine can reach whisper.
+   *
+   * Two separate installs hide from a plain PATH lookup, and the probe knows
+   * about both: `pip install openai-whisper` often lands its script somewhere
+   * not on PATH, which is normal on Windows, so `python -m whisper` is tried
+   * too; and `npm run worker:setup` deliberately edits no PATH at all, so
+   * whisper.cpp is looked for in the app's own tools folder first. Running the
+   * invocation the probe recorded means an install it found is an install this
+   * can use, rather than telling somebody to install what they already have.
+   */
+  const tool = (await toolStatuses()).find((t) => t.id === "whisper");
+  if (!tool?.present) {
+    /*
+     * Three different absences, and only one of them is "install whisper".
+     *
+     * The probe already distinguishes them and puts the specific sentence in
+     * `error` with the specific remedy in `install` — most importantly the
+     * case where whisper.cpp IS here and its model file is not, which is a
+     * download rather than an installation and would otherwise send somebody
+     * to reinstall a program they can see in the folder.
+     */
+    throw new Refusal(
+      tool?.error
+        ? `${tool.error} Fix it with: ${tool.install}`
+        : `whisper is not installed on this machine. Install it with: ${tool?.install ?? "npm run worker:setup"}`,
+    );
+  }
+
+  const cpp = tool.flavor === "whisper.cpp";
+  const name = cpp ? "whisper.cpp" : "openai-whisper";
+
+  /*
+   * whisper.cpp is a program AND a file, and the file is the half that goes
+   * missing.
+   *
+   * Checked again here rather than trusted from the probe, because the probe
+   * only asked whether ANY model exists and this job asked for a particular
+   * one; and because a probe is up to a minute old, which is long enough for
+   * somebody to have moved the folder. `whisperModel` answers with the nearest
+   * size that is actually present, so a machine holding `base` still does the
+   * job when the server asked for `small` — at the accuracy it has, which is
+   * reported back rather than left for somebody to wonder about.
+   */
+  const model = cpp ? whisperModel(payload.model) : null;
+  if (cpp && !model) {
+    throw new Refusal(`${whisperModelGap()} Fix it with: npm run worker:setup -- --only whisper`);
+  }
+
+  const dir = await jobDir(jobId, "audio");
+  try {
+    const wav = path.join(dir, "audio.wav");
+    await api.download(payload.audioUrl, wav);
+
+    const timeout = payload.timeoutMs ?? 600_000;
+    const { code, err } = await run(tool.command, argsFor(tool, payload, wav, dir, model), {
+      env: tool.env,
+      timeout,
+    });
+
+    if (code !== 0) {
+      /*
+       * A timeout reads as its own thing, not as a crash.
+       *
+       * `run()` kills the child and returns -1 with "Timed out after ..." on
+       * stderr, and that is the likeliest real failure here: the base model on
+       * a CPU runs at roughly real time, so a long video runs out of clock
+       * rather than crashing. The remedy is a shorter clip or a faster
+       * machine, not a reinstall, so the sentence must not say what a crash
+       * would say — and it stays retryable, because the next machine to claim
+       * it may be the one with the GPU.
+       */
+      if (/Timed out after/.test(err)) {
+        throw new Error(
+          `${name} ran out of time after ${Math.round(timeout / 60_000)} minutes on this audio. A shorter clip will finish.`,
+        );
+      }
+      /*
+       * A crash, and the build is named in it.
+       *
+       * Two programs can fail here and their fixes have nothing in common —
+       * one wants a Visual C++ runtime or a different `.bin`, the other wants
+       * a working PyTorch — so a sentence that only said "whisper failed"
+       * would send somebody looking in the wrong half of their machine.
+       */
+      throw new Error(`${name} failed: ${lastUseful(err) || `exit ${code}`}`);
+    }
+
+    /*
+     * Whatever it wrote, wherever it decided to write it.
+     *
+     * openai-whisper names the file after the input inside `--output_dir`;
+     * whisper.cpp appends `.txt` to the input path. Both land in this job's own
+     * directory, which holds exactly one WAV and nothing else, so looking for
+     * the `.txt` is more robust than either program's naming rule and survives
+     * the next time one of them changes it.
+     */
+    const files = await fs.readdir(dir).catch(() => []);
+    const txt = files.find((f) => f.endsWith(".txt"));
+    if (!txt) {
+      // Exited clean and left nothing behind: a directory it could not write
+      // into, or a build that wrote some other format. Either way there is no
+      // transcript, and it does not get to look like the fast path.
+      throw new Error(`${name} finished but wrote no text file.`);
+    }
+
+    /*
+     * An empty transcript is a result, not a failure.
+     *
+     * A reel with music and no speech genuinely has no words in it, and the
+     * server says exactly that on the source. Failing the job instead would
+     * spend a second machine's GPU discovering the same silence. This is the
+     * one place where "wrote nothing" and "wrote an empty file" have to stay
+     * different sentences: the first is a broken install, the second is a
+     * silent video, and collapsing them cost a real afternoon.
+     */
+    const text = (await fs.readFile(path.join(dir, txt), "utf8").catch(() => "")).trim();
+    // `text` is what the server stores; the rest is the operational record on
+    // the job row — which machine's build did this, and at what accuracy.
+    return { text, using: name, model: model?.name ?? payload.model ?? "" };
+  } finally {
+    await discard(dir);
+  }
+}
+
+/**
+ * The same request, in whichever program's language.
+ *
+ * They agree on nothing. openai-whisper takes the audio as a bare argument, a
+ * model *name* it will download for itself, and an output directory.
+ * whisper.cpp takes `-f` for the audio and `-m` for a model *file* that has to
+ * already exist, and writes beside the input.
+ */
+function argsFor(
+  tool: ToolStatus,
+  payload: TranscribePayload,
+  wav: string,
+  dir: string,
+  model: { file: string; name: string } | null,
+) {
+  if (tool.flavor === "whisper.cpp" && model) {
+    return [
+      ...tool.lead,
+      "-m",
+      model.file,
+      "-f",
+      wav,
+      // Without this it writes no file at all — it prints to the console and
+      // exits 0, which arrives here as "finished but wrote no text file".
+      "-otxt",
+      /*
+       * Detect the language rather than assume one.
+       *
+       * whisper.cpp defaults to `-l en`, and openai-whisper defaults to
+       * detecting — so a Bangla source that transcribed correctly under the
+       * Python one would come back as English-shaped nonsense under this one,
+       * with nothing anywhere saying why. An `.en` model ignores this and says
+       * so on stderr, which is the honest outcome for a model that only knows
+       * one language.
+       */
+      "-l",
+      "auto",
+      "-t",
+      String(threads()),
+    ];
+  }
+
+  return [
+    ...tool.lead,
+    wav,
+    "--model",
+    payload.model || "base",
+    "--output_format",
+    "txt",
+    "--output_dir",
+    dir,
+    // CPU builds of PyTorch cannot do fp16 and warn about it on every run;
+    // saying so up front keeps the warning out of the stderr this reads for
+    // real failures.
+    "--fp16",
+    "False",
+  ];
+}
+
+/**
+ * How many cores to give it.
+ *
+ * whisper.cpp defaults to four, which is a decision made for laptops in 2022
+ * and leaves most of a modern desktop idle — and the failure this job actually
+ * has is running out of clock, not running out of cores. Capped at eight
+ * because the gain flattens there and the machine belongs to somebody who is
+ * probably also using it.
+ */
+const threads = () => Math.max(1, Math.min(8, os.cpus().length || 4));
+
+/** The last line of stderr that says something, not a progress bar. */
+function lastUseful(err: string) {
+  const lines = err
+    .split("\n")
+    .map((l) => l.trim())
+    .filter((l) => l && !/^\s*[|/\-\\]\s*$/.test(l));
+  return lines.slice(-3).join(" ").slice(0, 300);
+}
diff --git a/worker/write-section.ts b/worker/write-section.ts
new file mode 100644
index 0000000..dd6f051
--- /dev/null
+++ b/worker/write-section.ts
@@ -0,0 +1,114 @@
+import path from "node:path";
+
+import type { Api } from "./api";
+import { callBrain } from "./brains";
+import type { Config } from "./config";
+import { Refusal, type JobResult, type WritePayload } from "./job-types";
+import { discard, jobDir } from "./temp";
+
+/**
+ * Write one section, on this machine, exactly as the payload says.
+ *
+ * The server has already done all the thinking: resolved the pack, checked the
+ * dependencies, gathered the finished upstream sections, fetched the website
+ * or the source brief, and run `systemPrompt()` and `userPrompt()`. What
+ * arrives here is their output — text and a command — so there is nothing in
+ * this file that knows what a pack is, and that is the point: business logic
+ * shipped to fifteen desktops would need fifteen updates to change a prompt,
+ * and the prompt changes weekly.
+ *
+ * `ms` is measured here rather than on the server because the server's clock
+ * would include the queue wait: a section would report six minutes because a
+ * laptop was shut, and "how long does this model take" would stop being a
+ * question anybody could answer.
+ */
+export async function writeSection(
+  cfg: Config,
+  api: Api,
+  jobId: string,
+  payload: WritePayload,
+): Promise {
+  if (!payload.system?.trim() || !payload.user?.trim()) {
+    // A bad payload will be exactly as bad on the next machine, so it does not
+    // get a second attempt.
+    throw new Refusal("The job arrived with no prompt in it.");
+  }
+
+  const dir = payload.frames?.length ? await jobDir(jobId, "frames") : "";
+  try {
+    const imageFiles = dir ? await fetchFrames(api, dir, payload.frames ?? []) : [];
+
+    const started = Date.now();
+    const { text, using } = await callBrain(cfg, {
+      brain: payload.brain,
+      transport: payload.transport,
+      system: payload.system,
+      user: payload.user,
+      tier: payload.tier,
+      timeoutMs: payload.timeoutMs,
+      ...(imageFiles.length ? { imageFiles } : {}),
+      allowFrameRead: payload.allowFrameRead,
+    });
+    const ms = Date.now() - started;
+
+    /*
+     * Nothing back is a failure, not an empty section.
+     *
+     * The server draws the same conclusion in `sectionResult`, and it is worth
+     * drawing on both sides: a `write_section` result arrives whole or not at
+     * all, and posting an empty one as a success would put an approved-looking
+     * blank on the page for every downstream section to be written against.
+     */
+    if (!text.trim()) {
+      throw new Error("The model exited cleanly and wrote nothing.");
+    }
+
+    return { text, ms, using };
+  } finally {
+    /*
+     * The frames go, whether the call worked or not.
+     *
+     * In a `finally` because the interesting case is the failure: a CLI that
+     * timed out has still left eight JPEGs on somebody's desktop, and a worker
+     * that ran all week would leave a week of them.
+     */
+    if (dir) await discard(dir);
+  }
+}
+
+/**
+ * The stills, on this machine's disk, in the order the payload named them.
+ *
+ * None of the three CLIs takes bytes and each takes a file differently, so the
+ * frames have to become local absolute paths before a model can be asked about
+ * them — which is the whole reason the server can stay the only place they
+ * actually live. They are downloaded through the worker's own door
+ * (`/api/workers/sources/...`), because a bearer token is not a session and
+ * must not be able to walk the browser's routes.
+ *
+ * Named by position rather than by the name the server used. The order is the
+ * only thing the prompt depends on, and a filename taken off a URL and joined
+ * to a directory is a path traversal waiting for the day something upstream
+ * stops checking it. Positions cannot traverse anything.
+ */
+async function fetchFrames(
+  api: Api,
+  dir: string,
+  frames: { url: string; at: number }[],
+) {
+  const files: string[] = [];
+  for (const [i, frame] of frames.entries()) {
+    if (!frame?.url) continue;
+    const file = path.join(dir, `frame-${String(i + 1).padStart(2, "0")}.jpg`);
+    await api.download(frame.url, file);
+    files.push(file);
+  }
+  if (!files.length) {
+    // Frames were asked for and none arrived: the section is about what is on
+    // screen, and answering it from the words alone is the one outcome
+    // `docs/DECISIONS.md` calls worse than an error. Retryable, because a
+    // download that failed once may be a network that came back.
+    throw new Error("The frames this section is about could not be downloaded.");
+  }
+  return files;
+}