From 50ca528ff4bc4b3cd1d378527e2b28057c29b634 Mon Sep 17 00:00:00 2001 From: Ondrej Drapalik Date: Wed, 9 Sep 2026 22:36:39 +0200 Subject: [PATCH 1/5] feat: add plugin-owned harness sign-in --- src/auth-cli.js | 88 +++++++++++-- src/auth-menu.js | 8 +- src/connections.js | 59 ++++++++- src/fleet-auth.js | 6 + src/fleet-seed.js | 10 +- src/harness-auth.js | 4 +- src/harnesses.js | 25 +++- src/oauth.js | 270 +++++++++++++++++++++++++++++++++++++++ test/connections.test.js | 56 +++++++- test/fleet-auth.test.js | 8 ++ test/fleet-seed.test.js | 43 +++++++ test/harnesses.test.js | 15 ++- test/oauth.test.js | 139 ++++++++++++++++++++ 13 files changed, 706 insertions(+), 25 deletions(-) create mode 100644 src/oauth.js create mode 100644 test/oauth.test.js diff --git a/src/auth-cli.js b/src/auth-cli.js index b3c5ace..77025f5 100644 --- a/src/auth-cli.js +++ b/src/auth-cli.js @@ -10,9 +10,10 @@ import { CONFIG_PATH, AUTH_PATH, CONNECTIONS_DIR } from "./config-paths.js" import { runProbe } from "./harness-probe.js" import { HARNESSES, readHarnessFile } from "./harnesses.js" import { - classifySubscription, connectionId, connectionMaterial, localAccountLabel, readConnections, - removeConnection, saveConnection, selectConnection, suggestedConnectionId, validSetupToken, + OAUTH_HARNESSES, classifySubscription, connectionId, connectionMaterial, localAccountLabel, readConnections, readOauthSecret, + removeConnection, rewriteOauthSecret, saveConnection, selectConnection, suggestedConnectionId, validSetupToken, } from "./connections.js" +import { OAUTH_CONNECT, codexRefresh, validAmpAccessToken } from "./oauth.js" const HELP = `e2b-box auth — coding-agent connections @@ -23,6 +24,15 @@ const HELP = `e2b-box auth — coding-agent connections auth connect claude --token-stdin --yes accept an existing token over stdin auth connect codex [--user] borrow the current local Codex session + auth connect muse sign in with Meta's device flow; the Muse + API key it mints is stored privately + auth connect amp open ampcode.com/settings/security and paste + its access token; stored privately + auth connect codex --oauth sign in to ChatGPT for a Codex session of the + plugin's own (refreshed by reconnect, not boxes) + auth connect amp --token-stdin --yes + accept an access token from + ampcode.com/settings/security over stdin auth connect --name ID choose a connection name auth list [--json] list connections without reading tokens auth explain --template NAME [--connection ID] [--json] @@ -45,6 +55,9 @@ Claude names use the detected organization (e.g. claude-e2b), personal for Pro/M or local when unknown. --name overrides this; names never grant shared access. Check validates local material, not provider acceptance; it makes no model call. Codex borrowing excludes its real refresh token and lasts until bearer expiry. +An --oauth codex connection is a second ChatGPT login owned by this plugin: it never +touches ~/.codex, boxes still get the placeholder copy, and reconnect refreshes it +without a browser while its refresh token is good (ADR 0015). Reconnect/disconnect do not change or revoke credentials already inside a box. ` @@ -80,6 +93,20 @@ async function captureToken() { }) } +/** + * What a human sees during the plugin's own sign-in: the URL, the code to compare, + * and the browser opened when there is one. Codes are never secrets; tokens never + * come through here. + */ +async function showSignIn({ url, userCode, expiresIn }) { + console.log(`\n ${userCode || expiresIn ? "Sign in at " : "Open "} ${url}`) + if (userCode) console.log(` Code shown ${userCode} (confirm it matches in the browser)`) + if (userCode || url.includes("auth.")) console.log(` Waiting up to ${Math.round(expiresIn / 60)} min for the browser...`) + if (process.stdout.isTTY && process.platform === "darwin") { + try { spawn("open", [url], { stdio: "ignore", detached: true }).unref() } catch { /* the URL is printed */ } + } +} + async function detectClaude() { const probe = await runProbe("claude", ["auth", "status"]) try { return classifySubscription(JSON.parse(probe.stdout)) } catch { return classifySubscription() } @@ -89,7 +116,7 @@ function parse(argv) { const { values, positionals } = parseArgs({ args: argv, allowPositionals: true, options: { help: { type: "boolean", short: "h" }, yes: { type: "boolean", short: "y" }, user: { type: "boolean" }, org: { type: "string" }, name: { type: "string" }, - "token-stdin": { type: "boolean" }, json: { type: "boolean" }, + "token-stdin": { type: "boolean" }, oauth: { type: "boolean" }, json: { type: "boolean" }, template: { type: "string", short: "t" }, connection: { type: "string" }, } }) return { values, positionals } @@ -100,7 +127,7 @@ async function main() { if (options.help || positionals[0] === "help") { console.log(HELP); return } const [verb, target, ...rest] = positionals const allowed = { - connect: ["yes", "user", "org", "name", "token-stdin"], + connect: ["yes", "user", "org", "name", "token-stdin", "oauth"], reconnect: ["yes", "token-stdin"], disconnect: ["yes"], list: ["json"], check: ["json"], explain: ["json", "template", "connection"], preflight: ["template", "connection"], @@ -115,7 +142,7 @@ async function main() { const rows = records.map((c) => ({ id: c.id, agent: c.harness, owner: "Only you", method: c.method, detectedSubscription: c.detected?.subscription || "unknown", detectedOrganization: c.detected?.organization || null, detectionSource: c.detected?.source || null, localAccount: localAccountLabel(c.harness, c.detected), - status: c.method === "setup-token" && c.expiresAt && Date.parse(c.expiresAt) <= Date.now() ? "expired" : "configured", + status: c.method !== "borrowed-session" && c.expiresAt && Date.parse(c.expiresAt) <= Date.now() ? "expired" : "configured", expiresAt: c.expiresAt || null })) if (options.json) console.log(JSON.stringify(rows, null, 2)) else if (!rows.length) console.log("No connections. Run e2b-box auth connect claude or e2b-box auth discover.") @@ -166,20 +193,55 @@ async function main() { } const harness = verb === "reconnect" ? existing.harness : target - if (!["claude", "codex"].includes(harness)) throw new Error("Managed connections currently support claude and codex. Use auth discover for other agents.") - if (options["token-stdin"] && harness !== "claude") throw new Error("--token-stdin is only supported for Claude setup-token.") + // muse and amp have exactly one managed method, the plugin's own sign-in; codex + // has two and picks the borrowed local session unless --oauth says otherwise. + const oauth = verb === "reconnect" ? existing.method === "oauth" : options.oauth === true || ["muse", "amp"].includes(harness) + if (options.oauth && !OAUTH_HARNESSES.includes(harness)) throw new Error("--oauth is supported for muse, amp and codex.") + if (!(["claude", "codex"].includes(harness) || (oauth && OAUTH_HARNESSES.includes(harness)))) throw new Error("Managed connections support claude, codex, muse and amp. Use auth discover for other agents.") + if (options["token-stdin"] && !(harness === "claude" || (oauth && harness === "amp"))) throw new Error("--token-stdin is supported for Claude setup-token and amp access tokens.") + const method = oauth ? "oauth" : harness === "claude" ? "setup-token" : "borrowed-session" const detected = harness === "claude" ? await detectClaude() : undefined - const id = connectionId(verb === "reconnect" ? existing.id : options.name ?? suggestedConnectionId(harness, detected)) + const id = connectionId(verb === "reconnect" ? existing.id : options.name ?? (oauth ? `${harness}-personal` : suggestedConnectionId(harness, detected))) if (verb === "connect" && records.some((c) => c.id === id)) throw new Error(`Connection '${id}' already exists. Run e2b-box auth reconnect ${id}.`) - output({ Connection: id, ...(detected ? { "Local account": localAccountLabel(harness, detected) } : {}), - Access: "Only you", Method: harness === "claude" ? "Claude setup-token" : "Borrowed Codex session" }) + const methodLabel = { "setup-token": "Claude setup-token", "borrowed-session": "Borrowed Codex session", oauth: `${harness} sign-in owned by this plugin` }[method] + output({ Connection: id, ...(detected ? { "Local account": localAccountLabel(harness, detected) } : {}), Access: "Only you", Method: methodLabel }) if (detected) console.log(" Account check Local login detected; connection account unverified.") if (verb === "connect" && !options.name && detected) console.log(" Name source Local account suggestion; use --name to choose another.") - if (harness === "codex") console.log(" Refresh token Not copied; boxes use this session only until its bearer expires.") + if (method === "borrowed-session") console.log(" Refresh token Not copied; boxes use this session only until its bearer expires.") + if (method === "oauth" && harness === "codex") console.log(" Refresh token Kept by this plugin only; boxes receive a copy without it.") + if (method === "oauth" && harness === "amp") console.log(" Stored amp's long-lived access token, pasted from its Security page (amp reads it behind a recent-sign-in gate no CLI can pass, ADR 0015).") + if (method === "oauth" && harness === "muse") console.log(" Stored the Muse API key Meta mints for the signed-in account.") await confirm("Save this connection as the default when it is the only one for this agent?", options.yes) - let record = { id, harness, detected, method: harness === "claude" ? "setup-token" : "borrowed-session" } + let record = { id, harness, detected, method } let token - if (harness === "claude") { + if (method === "oauth") { + if (verb === "reconnect" && harness === "codex" && !options["token-stdin"]) { + // The refresh token is the plugin's alone, so reconnect tries the grant first + // and only opens a browser when the chain is dead. + try { + const next = await codexRefresh(readOauthSecret(existing), { fetch }) + rewriteOauthSecret(existing, next) + console.log(`\n✓ Refreshed ${existing.id} without a browser (bearer until ${connectionMaterial(existing).expiresAt}).`) + return + } catch (error) { + console.log(` Refresh ${error.message}. Signing in again.`) + } + } + if (harness === "amp" && options["token-stdin"]) { + const value = readFileSync(0, "utf8").trim() + if (!validAmpAccessToken(value)) throw new Error("Expected an amp access token (sgamp_…); nothing was saved.") + token = { kind: "api-key", AMP_API_KEY: value } + } else { + const prompt = async (question) => { + if (!process.stdin.isTTY) throw new Error("Pasting the amp access token needs a terminal. Use auth connect amp --token-stdin --yes.") + const rl = createInterface({ input: process.stdin, output: process.stdout }) + try { return await rl.question(` ${question}`) } finally { rl.close() } + } + const result = await OAUTH_CONNECT[harness]({ fetch, sleep: (ms) => new Promise((r) => setTimeout(r, ms)), onCode: showSignIn, prompt }) + token = result.secret + record.expiresAt = result.expiresAt + } + } else if (harness === "claude") { const startedAt = Date.now() token = options["token-stdin"] ? readFileSync(0, "utf8").trim() : await captureToken() if (!validSetupToken(token)) throw new Error("Expected a Claude setup-token value; nothing was saved.") diff --git a/src/auth-menu.js b/src/auth-menu.js index 28c113e..940bc6c 100644 --- a/src/auth-menu.js +++ b/src/auth-menu.js @@ -78,6 +78,10 @@ export function authMenuActions(agent, connections) { id: "reconnect", label: `Reconnect ${c.id}`, connection: c.id, })) if (["claude", "codex"].includes(agent)) actions.push({ id: "connect", label: agent === "claude" ? "Connect new claude account" : "Connect local codex session" }) + // The plugin's own sign-in (ADR 0015): the only managed method for muse and amp, + // the second one for codex. + if (agent === "codex") actions.push({ id: "connect-oauth", label: "Sign in to ChatGPT for a codex session of this plugin's own" }) + if (["muse", "amp"].includes(agent)) actions.push({ id: "connect-oauth", label: `Sign in to ${agent} and store its credential` }) actions.push({ id: "explain", label: "Inspect authentication" }, { id: "config", label: "Open config.toml (templates / API keys)" }, { id: "discovery", label: "Open auth.toml (generated sources)" }, @@ -135,8 +139,8 @@ export async function runAuthMenu({ initial, refresh, render, save }) { } else { let args if (action.id === "reconnect") args = ["reconnect", action.connection] - else if (action.id === "connect") { - args = ["connect", agent] + else if (action.id === "connect" || action.id === "connect-oauth") { + args = ["connect", agent, ...(action.id === "connect-oauth" && agent === "codex" ? ["--oauth"] : [])] if (state.cfg.connections.some((c) => c.harness === agent)) { const name = await question("\n New connection name (blank cancels): ") if (!name) continue diff --git a/src/connections.js b/src/connections.js index 33b9c80..9de21a1 100644 --- a/src/connections.js +++ b/src/connections.js @@ -41,12 +41,16 @@ export function suggestedConnectionId(harness, detected) { return detected?.classification === "personal" ? "claude-personal" : "claude-local" } +/** Harnesses `method: "oauth"` (the plugin's own sign-in, src/oauth.js, ADR 0015) is defined for. */ +export const OAUTH_HARNESSES = ["muse", "amp", "codex"] + function validate(record) { connectionId(record?.id) if (record.version !== 1 || record.owner !== "personal" || !/^[0-9a-f-]{36}$/.test(record.revision || "") || !((record.harness === "claude" && record.method === "setup-token") || - (record.harness === "codex" && record.method === "borrowed-session" && typeof record.path === "string"))) { + (record.harness === "codex" && record.method === "borrowed-session" && typeof record.path === "string") || + (record.method === "oauth" && OAUTH_HARNESSES.includes(record.harness)))) { throw new Error(`Invalid connection '${record.id}'.`) } return record @@ -95,6 +99,14 @@ export function saveConnection(record, token, { directory = CONNECTIONS_DIR, rep if (!validSetupToken(token)) throw new Error("Expected a Claude setup-token value; nothing was saved.") writeFileSync(secretPath(next, directory), token, { mode: 0o600, flag: "wx" }) } + // An OAuth connection's secret is the object src/oauth.js returned: a key for + // muse (minted by Meta) and amp (pasted from its Security page), codex's whole + // auth.json (real refresh token included - the plugin is its only custodian, + // boxes never see it; see connectionMaterial). + if (next.method === "oauth") { + if (!token || typeof token !== "object") throw new Error("Expected the sign-in's credential; nothing was saved.") + writeFileSync(secretPath(next, directory), `${JSON.stringify(token)}\n`, { mode: 0o600, flag: "wx" }) + } writeFileSync(temporary, `${JSON.stringify(next, null, 2)}\n`, { mode: 0o600, flag: "wx" }) renameSync(temporary, destination) committed = true @@ -139,7 +151,50 @@ export function selectConnection(cfg = {}, template) { return candidates[0] || null } +/** The stored secret of an OAuth connection, parsed. Throws the reconnect sentence when it is gone or unreadable. */ +export function readOauthSecret(record, directory = CONNECTIONS_DIR) { + try { + return JSON.parse(readFileSync(secretPath(record, directory), "utf8")) + } catch { + throw new Error(`Credential for '${record.id}' is unavailable. Run e2b-box auth reconnect ${record.id}.`) + } +} + +/** + * Rewrite an OAuth connection's secret in place - the refresh path. Same revision, + * atomic rename: a provisioner reading the old bytes sees either the old file or + * the new one, never a torn write. Only codex ever needs this (its bearer expires + * and its refresh token is single-use, so the plugin rotates it, nobody else). + */ +export function rewriteOauthSecret(record, secret, directory = CONNECTIONS_DIR) { + const dest = secretPath(record, directory) + const tmp = `${dest}.tmp` + writeFileSync(tmp, `${JSON.stringify(secret)}\n`, { mode: 0o600, flag: "wx" }) + try { + renameSync(tmp, dest) + } finally { + rmSync(tmp, { force: true }) + } +} + export function connectionMaterial(record, { directory = CONNECTIONS_DIR, readFile = readHarnessFile, now = Date.now() } = {}) { + if (record.method === "oauth") { + const secret = readOauthSecret(record, directory) + if (record.harness === "muse" || record.harness === "amp") { + const name = record.harness === "muse" ? "META_API_KEY" : "AMP_API_KEY" + if (typeof secret?.[name] !== "string" || !secret[name].trim()) throw new Error(`Credential for '${record.id}' is invalid. Reconnect it.`) + return { env: { [name]: secret[name] }, expiresAt: null } + } + // codex: the plugin holds the real refresh token; the box gets the placeholder + // copy (ADR 0010) and lives until the bearer expires. An expired bearer is a + // reconnect, not a silent refresh here: provisioning stays synchronous, and + // `auth check` / `auth reconnect` are where the plugin rotates the chain. + let session = null + try { session = HARNESSES.codex.sessionFile.read(JSON.stringify(secret)) } catch { /* not a session */ } + if (!session) throw new Error(`Credential for '${record.id}' is invalid. Reconnect it.`) + if (Date.parse(session.expires) <= now) throw new Error(`Connection '${record.id}' has expired. Run e2b-box auth reconnect ${record.id} (a browser is only needed if the refresh fails).`) + return { env: { CODEX_AUTH_JSON: session.value }, expiresAt: session.expires } + } if (record.method === "setup-token") { if (record.expiresAt && (!Number.isFinite(Date.parse(record.expiresAt)) || Date.parse(record.expiresAt) <= now)) { throw new Error(`Connection '${record.id}' has expired. Run e2b-box auth reconnect ${record.id}.`) @@ -162,6 +217,8 @@ export function connectionMaterial(record, { directory = CONNECTIONS_DIR, readFi export const CONFLICTING_AUTH = { claude: ["ANTHROPIC_API_KEY", "ANTHROPIC_AUTH_TOKEN", "CLAUDE_CODE_OAUTH_TOKEN", "CLAUDE_CODE_OAUTH_REFRESH_TOKEN", "ANTHROPIC_BASE_URL", "CLAUDE_CODE_USE_BEDROCK", "CLAUDE_CODE_USE_VERTEX", "CLAUDE_CODE_USE_FOUNDRY", "ANTHROPIC_PROFILE", "ANTHROPIC_FEDERATION_RULE_ID", "ANTHROPIC_ORGANIZATION_ID"], codex: ["OPENAI_API_KEY", "CODEX_API_KEY", "CODEX_AUTH_JSON", "OPENAI_BASE_URL"], + muse: ["META_API_KEY"], + amp: ["AMP_API_KEY"], } export function applyConnection(record, env, options) { diff --git a/src/fleet-auth.js b/src/fleet-auth.js index 158d0ac..5193cac 100644 --- a/src/fleet-auth.js +++ b/src/fleet-auth.js @@ -82,7 +82,13 @@ export function unauthenticatedMembers(members = [], cfg = {}, env = {}) { // Blank is missing. An empty credential is the failure that shows up furthest // from its cause — the agent boots, reads it and dies authenticating, which // reads as a broken key rather than an absent one. + // …and the row's own `boxVar`, which is not always among them: opencode has no + // host variable to forward (`hostVar: null`, its providers come from a file), so + // `credentialVars` is empty there and a member whose OPENCODE_AUTH_CONTENT was + // resolved from auth.json read as unauthenticated (observed live, fleet test, + // 2026-09-08: the warning named it, the box came up signed in). const boxVars = credentialVars(h).map((v) => v.boxVar) + if (h.boxVar) boxVars.push(h.boxVar) if (h.sessionFile) boxVars.push(h.sessionFile.boxVar) if (boxVars.some((v) => typeof resolved?.[v] === "string" && resolved[v].trim())) continue // The variable the remedy actually produces — the same one `e2b-box auth`'s own diff --git a/src/fleet-seed.js b/src/fleet-seed.js index 600a867..e57eca6 100644 --- a/src/fleet-seed.js +++ b/src/fleet-seed.js @@ -145,7 +145,15 @@ export const DEFAULT_SEEDS = { // is `canonicalize(findGitRootForPath(cwd) ?? cwd)` — realpath'd, walked up to // the nearest .git — so the seed computes exactly that for itself, in the box. // Merged, not written flat: the same file carries model settings and fallbacks. - droid: `node -e 'const fs=require("fs"),q=require("path"),d=process.env.HOME+"/.factory",f=d+"/settings.json",rp=function(x){try{return fs.realpathSync(x)}catch{return x}};let g=rp(process.cwd()),x=g;for(;;){if(fs.existsSync(q.join(x,".git"))){g=x;break}const n=q.dirname(x);if(n===x)break;x=n}let s={};try{s=JSON.parse(fs.readFileSync(f,"utf8"))}catch{}let t=s.trustedFolders;if(!t||typeof t!=="object"||Array.isArray(t))t={};if(t[g]){console.log("herdr-e2b: droid already trusts "+g+" - leaving it alone");process.exit(0)}t[g]={trustedAt:new Date().toISOString()};s.trustedFolders=t;fs.mkdirSync(d,{recursive:true});fs.writeFileSync(f,JSON.stringify(s))'`, + // + // The same write sets the session default `sessionDefaultSettings.autonomyLevel` + // to "high" when nothing set it: the interactive `droid` takes its Autonomy Level + // from settings.json, not from a flag (`--auto` is `droid exec`'s, docs.factory.ai + // /droid-cli/settings, and typed at the TUI it left "Auto (Off) · all actions + // require approval" — observed live, fleet test, 2026-09-08). Off, a member stops + // at its first edit, which is the droid half of what claude's + // `bypassPermissionsModeAccepted` and codex's `trust_level` are for. + droid: `node -e 'const fs=require("fs"),q=require("path"),d=process.env.HOME+"/.factory",f=d+"/settings.json",rp=function(x){try{return fs.realpathSync(x)}catch{return x}};let g=rp(process.cwd()),x=g;for(;;){if(fs.existsSync(q.join(x,".git"))){g=x;break}const n=q.dirname(x);if(n===x)break;x=n}let s={};try{s=JSON.parse(fs.readFileSync(f,"utf8"))}catch{}let t=s.trustedFolders;if(!t||typeof t!=="object"||Array.isArray(t))t={};let w=false;if(t[g])console.log("herdr-e2b: droid already trusts "+g+" - leaving it alone");else{t[g]={trustedAt:new Date().toISOString()};s.trustedFolders=t;w=true}let a=s.sessionDefaultSettings;if(!a||typeof a!=="object"||Array.isArray(a))a={};if(a.autonomyLevel===undefined){a.autonomyLevel="high";s.sessionDefaultSettings=a;w=true;console.log("herdr-e2b: droid autonomy set to high for new sessions")}if(w){fs.mkdirSync(d,{recursive:true});fs.writeFileSync(f,JSON.stringify(s))}'`, opencode: `node -e 'const fs=require("fs"),d=process.env.HOME+"/.config/opencode",f=d+"/opencode.json";let c={};try{c=JSON.parse(fs.readFileSync(f,"utf8"))}catch{}if(c.autoupdate===false){console.log("herdr-e2b: opencode autoupdate already off - leaving it alone");process.exit(0)}c.autoupdate=false;fs.mkdirSync(d,{recursive:true});fs.writeFileSync(f,JSON.stringify(c))'`, } diff --git a/src/harness-auth.js b/src/harness-auth.js index 36d699a..6239c40 100644 --- a/src/harness-auth.js +++ b/src/harness-auth.js @@ -405,9 +405,9 @@ export function buildSummary(rows, plan, cfg = {}) { connectionMaterial(selected, { directory: cfg.connectionsDir }) const account = selected.detected ? ` · ${localAccountLabel(selected.harness, selected.detected)}` : "" return { id: r.id, mark: "ok", managed: true, - auth: selected.method === "setup-token" ? "OAuth token" : "OAuth session", source: selected.id, + auth: selected.method === "setup-token" ? "OAuth token" : selected.method === "oauth" ? "Own sign-in" : "OAuth session", source: selected.id, details: [`${selected.id} · Only you`, ...(selected.detected ? [`Local account: ${localAccountLabel(selected.harness, selected.detected)}`] : []), - selected.method === "setup-token" ? "Setup token" : "Borrowed local session"], + selected.method === "setup-token" ? "Setup token" : selected.method === "oauth" ? "Signed in by this plugin (ADR 0015)" : "Borrowed local session"], method: `connection ${selected.id}${account} · Only you · ${selected.method} (locally ready)` } } } catch (error) { diff --git a/src/harnesses.js b/src/harnesses.js index 97827ea..bf2c45c 100644 --- a/src/harnesses.js +++ b/src/harnesses.js @@ -9,7 +9,7 @@ import path from "node:path" // agent, a harness is that same agent installed here, and this file is the only // place the two are mapped to each other. // -// Governed by docs/adr/0006: detection is a spawn-the-binary probe, reads are +// Governed by docs/adr/0009: detection is a spawn-the-binary probe, reads are // limited to environment variables and a harness's own DOCUMENTED config file, and // no Keychain item or OAuth token cache is ever opened. The per-harness facts below // were established against vendor source and live binaries — see @@ -31,6 +31,8 @@ import path from "node:path" * an accident that stops being true the first time a vendor moves one. */ const CSI = /\x1b\[[0-?]*[ -/]*[@-~]/g +/** Three base64url segments starting with `eyJ` (`{"` encoded): a JWT, so a session, not a key. */ +const isJwt = (v) => /^eyJ[A-Za-z0-9_-]+\.[A-Za-z0-9_-]+\.[A-Za-z0-9_-]*$/.test(String(v ?? "")) const stripAnsi = (s) => String(s || "").replace(CSI, "") /** @@ -447,15 +449,30 @@ export const HARNESSES = { // — and nothing at all when several are present, because picking one of two // servers' keys is the guess this table does not make. keyFile: { path: AMP_SECRETS, field: "apiKey@https://ampcode.com/" }, + // What `amp login` leaves under `apiKey@…` is NOT an access token: it is the + // browser sign-in's OAuth session, a JWT that amp itself refuses when handed back + // as AMP_API_KEY ("holds an OAuth session token, which is short-lived", amp + // 0.0.1788897629; the template's older amp says "Invalid or missing API key" for + // the same value — observed live, fleet test-amp, 2026-09-08). The bearer lives + // FIVE MINUTES and the refresh token rotates-invalidates (measured, ADR 0014), so + // neither 0010's placeholder trick nor 0011's verbatim rule applies: a JWT-shaped + // value reads as "signed in, nothing to borrow" and the remedy names the one + // thing that works, an access token the user mints. valueFile: { path: AMP_SECRETS, read: (text) => { const j = JSON.parse(text) - if (typeof j?.["apiKey@https://ampcode.com/"] === "string") return j["apiKey@https://ampcode.com/"] - const keys = Object.keys(j || {}).filter((k) => k.startsWith("apiKey@") && typeof j[k] === "string") - return keys.length === 1 ? j[keys[0]] : null + let v = null + if (typeof j?.["apiKey@https://ampcode.com/"] === "string") v = j["apiKey@https://ampcode.com/"] + else { + const keys = Object.keys(j || {}).filter((k) => k.startsWith("apiKey@") && typeof j[k] === "string") + v = keys.length === 1 ? j[keys[0]] : null + } + return v && !isJwt(v) ? v : null }, }, + advice: + "`amp login` leaves a short-lived OAuth session a box cannot use; create an access token at https://ampcode.com/settings/security#access-token and set AMP_API_KEY to it", parse: ({ stdout, stderr }) => { if (/^Signed in as /m.test(stripAnsi(stdout))) return { state: "no-key", source: "login" } if (/Invalid or missing API key/.test(stripAnsi(stderr))) return { state: "no-key", source: null } diff --git a/src/oauth.js b/src/oauth.js new file mode 100644 index 0000000..0c36443 --- /dev/null +++ b/src/oauth.js @@ -0,0 +1,270 @@ +// The plugin's OWN sign-in to a vendor, stored as a connection (ADR 0015). +// +// Every other credential this plugin handles is one it FOUND: a variable in the +// environment, a key in a harness's config file, a session a harness left on disk +// (ADR 0009, 0010, 0011). This module is the one place the plugin signs in itself, +// using the same public OAuth client each vendor's CLI uses, so what comes back is a +// login the laptop's own CLI never touches: it rotates nothing on the laptop and the +// laptop rotates nothing under it. The result is stored under `connections/` +// (ADR 0013) and delivered like any other connection. +// +// What each vendor yields, and why it is that and not a session: +// muse Meta's device flow, then Meta's own "mint a Muse API key" call. A key, +// no refresh, no expiry the plugin has to track. This is exactly what +// `muse login` does for itself. +// amp no OAuth at all: amp's long-lived access token (`sgamp_…`) can only be +// read behind the web app's recent-sign-in gate, so the plugin opens that +// page and stores what the human pastes. The reasons are measured, below. +// codex Codex CLI's PKCE flow against auth.openai.com, its own client id. A +// ten-day bearer plus a single-use refresh token, of which the plugin is the +// sole custodian: boxes get the placeholder rule's copy (ADR 0010), the +// plugin refreshes on `auth check` / `auth reconnect`, and never at +// box-create time, so provisioning stays synchronous and pure. +// +// The flows are written against injectable `fetch`, `open`, `now` and `sleep` so +// test/oauth.test.js runs every one of them against a scripted server with no +// network and no browser. Nothing here prints; the CLI decides what a human sees. +// +// Every request body below is data the vendor's own CLI sends. Client ids are +// PUBLIC clients (no secret exists), copied from the shipped CLIs — the same ids +// replicas-cli 0.2.634 uses for the same purpose. + +import { createHash, randomBytes } from "node:crypto" +import http from "node:http" + +/** `{ code_verifier, code_challenge }` for PKCE S256, base64url per RFC 7636. */ +export function pkcePair(bytes = randomBytes(32)) { + const b64url = (b) => b.toString("base64").replace(/\+/g, "-").replace(/\//g, "_").replace(/=+$/, "") + const verifier = b64url(bytes) + return { verifier, challenge: b64url(createHash("sha256").update(verifier).digest()) } +} + +/** `exp` of a JWT as ISO, or null when the token is not a decodable JWT. */ +export function jwtExpiryIso(token) { + try { + const seg = String(token).split(".")[1] + const pad = "=".repeat((4 - (seg.length % 4)) % 4) + const j = JSON.parse(Buffer.from(seg.replace(/-/g, "+").replace(/_/g, "/") + pad, "base64").toString("utf8")) + return Number.isFinite(j?.exp) ? new Date(j.exp * 1000).toISOString() : null + } catch { + return null + } +} + +const form = (o) => new URLSearchParams(o).toString() +const FORM = { "Content-Type": "application/x-www-form-urlencoded" } +const JSON_H = { "Content-Type": "application/json", Accept: "application/json" } + +async function readJson(res) { + const text = await res.text() + try { + return JSON.parse(text) + } catch { + return { _raw: text } + } +} + +/** + * RFC 8628 device flow, shared by muse (Meta) and amp (WorkOS): ask for a code, hand + * the human a URL, poll until the vendor answers with tokens. + * + * `onCode({ url, userCode, expiresIn })` is the CLI's hook to print and/or open the + * URL. `authorization_pending` and `slow_down` are the only errors that keep polling; + * anything else is the vendor saying no, and the message names it. + */ +export async function deviceFlow({ authorizeUrl, tokenUrl, clientId, extra = {}, onCode, fetch, sleep, now = Date.now }) { + const start = await fetch(authorizeUrl, { method: "POST", headers: FORM, body: form({ client_id: clientId, ...extra }) }) + const device = await readJson(start) + if (!start.ok || typeof device?.device_code !== "string" || typeof device?.user_code !== "string") { + throw new Error(`sign-in could not start (${start.status}${device?.error ? `: ${device.error}` : ""})`) + } + const url = typeof device.verification_uri_complete === "string" ? device.verification_uri_complete : device.verification_uri + const expiresIn = Number.isFinite(device.expires_in) ? device.expires_in : 600 + await onCode?.({ url, userCode: device.user_code, expiresIn }) + const deadline = now() + expiresIn * 1000 + let interval = Math.max(1, Number.isFinite(device.interval) ? device.interval : 5) * 1000 + while (now() < deadline) { + await sleep(interval) + const res = await fetch(tokenUrl, { + method: "POST", + headers: FORM, + body: form({ grant_type: "urn:ietf:params:oauth:grant-type:device_code", device_code: device.device_code, client_id: clientId }), + }) + const body = await readJson(res) + if (res.ok && typeof body?.access_token === "string") return body + if (body?.error === "slow_down") { + interval += 5000 + continue + } + if (body?.error === "authorization_pending") continue + throw new Error(body?.error === "access_denied" ? "sign-in was denied in the browser" : `sign-in failed: ${body?.error_description || body?.error || res.status}`) + } + throw new Error("sign-in expired before it was approved in the browser") +} + +// ─── muse ───────────────────────────────────────────────────────────────────── + +export const MUSE = { + clientId: "1031625952748946", + authorizeUrl: "https://auth.meta.com/oidc/device/authorization/", + tokenUrl: "https://auth.meta.com/oidc/device/token/", + keyUrl: "https://api.meta.ai/muse-code/key", +} + +/** + * Sign in with a Meta account, then have Meta mint a Muse API key for it. Returns + * `{ META_API_KEY }` — a key, which is what `[templates.muse.env]` and the muse + * harness row already know how to deliver. + */ +export async function museConnect({ fetch, sleep, onCode, now }) { + const tokens = await deviceFlow({ ...MUSE, extra: {}, onCode, fetch, sleep, now }) + const res = await fetch(MUSE.keyUrl, { + method: "POST", + headers: { ...JSON_H, Authorization: `Bearer ${tokens.access_token}`, "x-api-version": "1.0.0" }, + body: "{}", + }) + const body = await readJson(res) + if (!res.ok) throw new Error(`Muse did not issue an API key (${res.status}${body?.message ? `: ${body.message}` : ""})`) + if (typeof body?.api_key !== "string" || !body.api_key) { + throw new Error(`Muse did not issue an API key.${typeof body?.action_url === "string" ? ` Finish setup at ${body.action_url} and connect again.` : ""}`) + } + return { secret: { kind: "api-key", META_API_KEY: body.api_key }, expiresAt: null } +} + +// ─── amp ────────────────────────────────────────────────────────────────────── +// +// amp is the harness where the plugin's own sign-in buys nothing, and it is worth +// recording why so nobody re-derives it. `amp login` is a WorkOS device flow +// (`authapi.ampcode.com/user_management/authorize/device`, client +// `client_01JNSEYM3V0J5AXK4YXNXRXTGP`) and the plugin can run it; the result is the +// five-minute bearer plus rotating refresh token ADR 0014 measured. The credential a +// box needs is the long-lived ACCESS TOKEN (`sgamp_…`) the Security settings page +// shows, and that page reads it through a SvelteKit remote function +// (`POST /_app/remote/…/getAccessToken`) which (a) accepts only the browser's cookie +// session, never a bearer ("Authentication required"), (b) refuses cross-site +// callers, and (c) demands a RECENT sign-in ("REAUTH_REQUIRED") — all measured live, +// 2026-09-09. Nothing a CLI holds satisfies that, by design: it is amp's sudo gate. +// +// So `auth connect amp` opens that page and takes the token the human pastes. One +// paste, once; the token lives until they rotate it, and N boxes can hold it. + +export const AMP = { + settingsUrl: "https://ampcode.com/settings/security#access-token", +} + +/** `sgamp_…`: amp's long-lived access token, the only amp credential a connection stores. */ +export function validAmpAccessToken(value) { + return typeof value === "string" && /^sgamp_[A-Za-z0-9_-]{16,}$/.test(value) +} + +/** + * Open amp's Security page and take the pasted token. `prompt` is the CLI's + * readline (injected, so tests pass a value); `onCode` is the same hook the device + * flows use, here carrying the page URL and no code. + */ +export async function ampConnect({ onCode, prompt }) { + await onCode?.({ url: AMP.settingsUrl, userCode: null, expiresIn: 600 }) + const value = String((await prompt?.("Paste the access token from that page: ")) ?? "").trim() + if (!validAmpAccessToken(value)) throw new Error("Expected an amp access token (sgamp_…); nothing was saved.") + return { secret: { kind: "api-key", AMP_API_KEY: value }, expiresAt: null } +} + +// ─── codex ──────────────────────────────────────────────────────────────────── + +export const CODEX = { + clientId: "app_EMoamEEZ73f0CkXaXp7hrann", + authorizeUrl: "https://auth.openai.com/oauth/authorize", + tokenUrl: "https://auth.openai.com/oauth/token", + callbackPort: 1455, + callbackPath: "/auth/callback", + scope: "openid profile email offline_access", +} + +/** The browser URL for one PKCE attempt; `state` and `challenge` are the caller's. */ +export function codexAuthorizeUrl({ challenge, state, port = CODEX.callbackPort }) { + const q = new URLSearchParams({ + response_type: "code", + client_id: CODEX.clientId, + redirect_uri: `http://localhost:${port}${CODEX.callbackPath}`, + scope: CODEX.scope, + code_challenge: challenge, + code_challenge_method: "S256", + id_token_add_organizations: "true", + codex_cli_simplified_flow: "true", + state, + originator: "codex_cli_rs", + }) + return `${CODEX.authorizeUrl}?${q}` +} + +/** The `~/.codex/auth.json` shape codex reads, from a token response. */ +export function codexAuthJson(tokens, now = Date.now) { + return { + auth_mode: "chatgpt", + OPENAI_API_KEY: null, + tokens: { id_token: tokens.id_token, access_token: tokens.access_token, refresh_token: tokens.refresh_token, account_id: tokens.account_id }, + last_refresh: new Date(now()).toISOString(), + } +} + +/** + * Codex's refresh grant, the request codex-rs itself sends. Returns a NEW auth.json + * (new bearer, new single-use refresh token). Whoever calls this owns the chain: + * the old refresh token is dead afterwards, which is why only the plugin ever does. + */ +export async function codexRefresh(authJson, { fetch, now = Date.now }) { + const res = await fetch(CODEX.tokenUrl, { + method: "POST", + headers: JSON_H, + body: JSON.stringify({ client_id: CODEX.clientId, grant_type: "refresh_token", refresh_token: authJson?.tokens?.refresh_token, scope: "openid profile email" }), + }) + const body = await readJson(res) + if (!res.ok || typeof body?.access_token !== "string") throw new Error(`codex refresh failed (${res.status}${body?.error ? `: ${body.error}` : ""}) - sign in again with auth reconnect`) + return codexAuthJson({ ...authJson.tokens, ...body, account_id: authJson?.tokens?.account_id }, now) +} + +/** + * Codex CLI's own sign-in: PKCE, a localhost callback on codex's port, the token + * exchange. Resolves to `{ secret: , expiresAt }` where expiresAt + * is the bearer's. `listen` is injectable so the test can drive the callback + * without a socket; the default binds codex's port on 127.0.0.1. + */ +export async function codexConnect({ fetch, onCode, now = Date.now, listen = listenOnce, port = CODEX.callbackPort, pkce = pkcePair(), state = randomBytes(16).toString("hex") }) { + const url = codexAuthorizeUrl({ challenge: pkce.challenge, state, port }) + const callback = listen({ port, path: CODEX.callbackPath }) + await onCode?.({ url, userCode: null, expiresIn: 600 }) + const q = await callback + if (q.error) throw new Error(`sign-in failed: ${q.error}${q.error_description ? ` - ${q.error_description}` : ""}`) + if (q.state !== state) throw new Error("sign-in returned a different state than it was given - possible CSRF, nothing was saved") + if (!q.code) throw new Error("sign-in returned no authorization code") + const res = await fetch(CODEX.tokenUrl, { + method: "POST", + headers: FORM, + body: form({ grant_type: "authorization_code", code: q.code, redirect_uri: `http://localhost:${port}${CODEX.callbackPath}`, client_id: CODEX.clientId, code_verifier: pkce.verifier }), + }) + const tokens = await readJson(res) + if (!res.ok || typeof tokens?.access_token !== "string") throw new Error(`token exchange failed (${res.status}${tokens?.error ? `: ${tokens.error}` : ""})`) + return { secret: codexAuthJson(tokens, now), expiresAt: jwtExpiryIso(tokens.access_token) } +} + +/** One HTTP request on 127.0.0.1:, its query as an object, then the server closes. */ +export function listenOnce({ port, path: expected }) { + return new Promise((resolve, reject) => { + const server = http.createServer((req, res) => { + const u = new URL(req.url, `http://127.0.0.1:${port}`) + if (u.pathname !== expected) { + res.writeHead(404).end("Not Found") + return + } + res.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }) + res.end("herdr-e2b

Signed in. You can close this tab and return to the terminal.

") + server.close() + resolve(Object.fromEntries(u.searchParams)) + }) + server.on("error", (e) => reject(e.code === "EADDRINUSE" ? new Error(`port ${port} is in use - close any other codex sign-in and try again`) : e)) + server.listen(port, "127.0.0.1") + }) +} + +/** Which harnesses `auth connect --oauth` knows, and the connect function for each. */ +export const OAUTH_CONNECT = { muse: museConnect, amp: ampConnect, codex: codexConnect } diff --git a/test/connections.test.js b/test/connections.test.js index 661ae88..63e0407 100644 --- a/test/connections.test.js +++ b/test/connections.test.js @@ -5,7 +5,7 @@ import { tmpdir } from "node:os" import path from "node:path" import { spawnSync } from "node:child_process" import { fileURLToPath } from "node:url" -import { classifySubscription, connectionMaterial, readConnections, removeConnection, saveConnection, selectConnection, suggestedConnectionId } from "../src/connections.js" +import { classifySubscription, connectionMaterial, readConnections, readOauthSecret, removeConnection, rewriteOauthSecret, saveConnection, selectConnection, suggestedConnectionId } from "../src/connections.js" import { resolveEnv } from "../src/config.js" import { unauthenticatedMembers } from "../src/fleet-auth.js" import { seedCommand } from "../src/fleet-seed.js" @@ -346,3 +346,57 @@ sys.exit(os.waitstatus_to_exitcode(status))` assert.equal(record.detected.classification, "organization") assert.equal(connectionMaterial(record, { directory }).env.CLAUDE_CODE_OAUTH_TOKEN, TOKEN) }) + +// ── oauth connections: the plugin's own sign-in, stored locally (ADR 0015) ───── + +test("oauth connections: a key for muse/amp, a placeholdered auth.json for codex, all from one secret file", (t) => { + const directory = fixture(t) + const muse = saveConnection({ id: "muse-personal", harness: "muse", method: "oauth" }, { kind: "api-key", META_API_KEY: "muse-key" }, { directory }) + const amp = saveConnection({ id: "amp-personal", harness: "amp", method: "oauth" }, { kind: "api-key", AMP_API_KEY: "sgamp_user_abcdefghijklmnop" }, { directory }) + const exp = Math.floor(Date.now() / 1000) + 3600 + const bearer = `eyJhbGciOiJub25lIn0.${Buffer.from(JSON.stringify({ exp })).toString("base64url")}.sig` + const codex = saveConnection({ id: "codex-own", harness: "codex", method: "oauth" }, { auth_mode: "chatgpt", tokens: { id_token: "id", access_token: bearer, refresh_token: "real-refresh" } }, { directory }) + assert.deepEqual(readConnections(directory).map((c) => c.id), ["amp-personal", "codex-own", "muse-personal"]) + assert.deepEqual(connectionMaterial(muse, { directory }).env, { META_API_KEY: "muse-key" }) + assert.deepEqual(connectionMaterial(amp, { directory }).env, { AMP_API_KEY: "sgamp_user_abcdefghijklmnop" }) + const material = connectionMaterial(codex, { directory }) + const shipped = JSON.parse(material.env.CODEX_AUTH_JSON) + // The box never sees the real refresh token: the plugin is that chain's only custodian. + assert.equal(shipped.tokens.access_token, bearer) + assert.notEqual(shipped.tokens.refresh_token, "real-refresh") + assert.equal(material.expiresAt, new Date(exp * 1000).toISOString()) + // Secrets are private files; metadata never carries them. + for (const f of readdirSync(directory).filter((n) => n.endsWith(".secret"))) assert.equal(statSync(path.join(directory, f)).mode & 0o777, 0o600) + assert.doesNotMatch(readFileSync(path.join(directory, "codex-own.json"), "utf8"), /real-refresh|eyJ/) +}) + +test("oauth connections: an expired codex bearer asks for reconnect; a rewrite keeps the revision", (t) => { + const directory = fixture(t) + const old = `eyJhbGciOiJub25lIn0.${Buffer.from(JSON.stringify({ exp: 1 })).toString("base64url")}.sig` + const rec = saveConnection({ id: "codex-own", harness: "codex", method: "oauth" }, { auth_mode: "chatgpt", tokens: { access_token: old, refresh_token: "r1" } }, { directory }) + assert.throws(() => connectionMaterial(rec, { directory }), /has expired.*auth reconnect codex-own/) + const fresh = `eyJhbGciOiJub25lIn0.${Buffer.from(JSON.stringify({ exp: Math.floor(Date.now() / 1000) + 600 })).toString("base64url")}.sig` + rewriteOauthSecret(rec, { auth_mode: "chatgpt", tokens: { access_token: fresh, refresh_token: "r2" } }, directory) + assert.equal(readOauthSecret(rec, directory).tokens.refresh_token, "r2") + assert.ok(connectionMaterial(rec, { directory }).env.CODEX_AUTH_JSON) + assert.equal(readdirSync(directory).filter((n) => n.endsWith(".secret")).length, 1) +}) + +test("oauth connections: only muse, amp and codex; a bad or missing secret is a reconnect, never a fallback", (t) => { + const directory = fixture(t) + assert.throws(() => saveConnection({ id: "claude-x", harness: "claude", method: "oauth" }, { x: 1 }, { directory }), /Invalid connection/) + assert.throws(() => saveConnection({ id: "muse-x", harness: "muse", method: "oauth" }, "not-an-object", { directory }), /nothing was saved/) + const rec = saveConnection({ id: "muse-personal", harness: "muse", method: "oauth" }, { kind: "api-key", META_API_KEY: "k" }, { directory }) + rmSync(path.join(directory, readdirSync(directory).find((n) => n.endsWith(".secret")))) + assert.throws(() => connectionMaterial(rec, { directory }), /unavailable.*auth reconnect muse-personal/) + const amp = saveConnection({ id: "amp-personal", harness: "amp", method: "oauth" }, { kind: "api-key", AMP_API_KEY: "" }, { directory }) + assert.throws(() => connectionMaterial(amp, { directory }), /invalid/) +}) + +test("oauth connections: a selected muse or amp connection replaces the template's own credential", (t) => { + const directory = fixture(t) + const muse = saveConnection({ id: "muse-personal", harness: "muse", method: "oauth" }, { kind: "api-key", META_API_KEY: "from-connection" }, { directory }) + const cfg = { connections: [muse], connectionsDir: directory, envTemplates: { muse: { META_API_KEY: "hand-written" } } } + assert.equal(resolveEnv(cfg, "muse", {}).META_API_KEY, "from-connection") + assert.deepEqual(unauthenticatedMembers([{ template: "muse", label: "m" }], cfg, {}), []) +}) diff --git a/test/fleet-auth.test.js b/test/fleet-auth.test.js index 3d247b0..4e69ad3 100644 --- a/test/fleet-auth.test.js +++ b/test/fleet-auth.test.js @@ -37,6 +37,14 @@ test("a value discovered by `e2b-box auth` authenticates the member", () => { assert.deepEqual(unauthenticatedMembers([member("claude")], cfg, {}), []) }) +test("opencode's auth.json pointer authenticates the member (its boxVar has no host variable)", () => { + // opencode forwards a whole auth.json under OPENCODE_AUTH_CONTENT and has no + // `hostVar`, so the row's own `boxVar` must count: a member that came up signed + // in was being named as unauthenticated (fleet test, 2026-09-08). + const cfg = { envDiscovered: { opencode: { OPENCODE_AUTH_CONTENT: '{"openrouter":{"type":"api","key":"k"}}' } } } + assert.deepEqual(unauthenticatedMembers([{ template: "opencode", label: "m" }], cfg, {}), []) +}) + test("a forwarded NAME authenticates the member only when this shell holds it", () => { const cfg = { envForward: { claude: { ANTHROPIC_API_KEY: "ANTHROPIC_API_KEY" } } } assert.deepEqual(unauthenticatedMembers([member("claude")], cfg, { ANTHROPIC_API_KEY: "sk-ant-x" }), []) diff --git a/test/fleet-seed.test.js b/test/fleet-seed.test.js index 3a98160..13d1e3f 100644 --- a/test/fleet-seed.test.js +++ b/test/fleet-seed.test.js @@ -1,5 +1,9 @@ import test from "node:test" import assert from "node:assert/strict" +import { execFileSync } from "node:child_process" +import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs" +import os from "node:os" +import path from "node:path" import { DEFAULT_SEEDS, seedCommand } from "../src/fleet-seed.js" // --- which command a template is seeded with --------------------------------- @@ -57,3 +61,42 @@ test("a shipped command reads its credential from a variable, never a value", () assert.match(cmd, /leaving it alone/, `${template}: no no-clobber bail-out`) } }) + +// --- the droid seed, actually executed ----------------------------------------- +// Against a fake HOME and a fake project with a .git, the way provision.js runs it +// from the project dir in a box. Exercised rather than read because the trust key +// droid checks is computed in the box (realpath, nearest .git), and the autonomy +// default is what keeps an unattended member from stopping at its first edit. + +function runDroidSeed(home) { + const proj = path.join(home, "proj") + mkdirSync(path.join(proj, ".git"), { recursive: true }) + const out = execFileSync("bash", ["-c", DEFAULT_SEEDS.droid], { + cwd: proj, + env: { PATH: process.env.PATH, HOME: home }, + encoding: "utf8", + }) + return { out, settings: JSON.parse(readFileSync(path.join(home, ".factory/settings.json"), "utf8")) } +} + +test("droid: trusts the git root and defaults new sessions to high autonomy, once", () => { + const home = mkdtempSync(path.join(os.tmpdir(), "herdr-seed-")) + const { out, settings } = runDroidSeed(home) + assert.equal(Object.keys(settings.trustedFolders).length, 1) + assert.match(Object.keys(settings.trustedFolders)[0], /proj$/) + assert.equal(settings.sessionDefaultSettings.autonomyLevel, "high") + assert.match(out, /autonomy set to high/) + const again = runDroidSeed(home) + assert.match(again.out, /already trusts/) + assert.deepEqual(again.settings, settings) +}) + +test("droid: a chosen autonomy level and existing settings are left alone", () => { + const home = mkdtempSync(path.join(os.tmpdir(), "herdr-seed-")) + mkdirSync(path.join(home, ".factory")) + writeFileSync(path.join(home, ".factory/settings.json"), JSON.stringify({ model: "kimi-k3", sessionDefaultSettings: { autonomyLevel: "off" } })) + const { settings } = runDroidSeed(home) + assert.equal(settings.model, "kimi-k3") + assert.equal(settings.sessionDefaultSettings.autonomyLevel, "off") + assert.equal(Object.keys(settings.trustedFolders).length, 1) +}) diff --git a/test/harnesses.test.js b/test/harnesses.test.js index b1d180a..691c2ef 100644 --- a/test/harnesses.test.js +++ b/test/harnesses.test.js @@ -1,6 +1,6 @@ import test from "node:test" import assert from "node:assert/strict" -import { HARNESSES, interpretProbe } from "../src/harnesses.js" +import { HARNESSES, interpretProbe, remedyFor } from "../src/harnesses.js" // --- what the plugin can borrow ---------------------------------------------- // `state` answers one question only: can this plugin authenticate a box from what @@ -619,6 +619,19 @@ test("amp reads its key from the default server's entry", () => { assert.equal(f.read('{"apiKey@https://ampcode.com/":"amp-key"}'), "amp-key") }) +test("amp does not borrow the OAuth session `amp login` stores under the same key", () => { + // A JWT under `apiKey@…` is the browser sign-in's session, not an access token: amp + // itself rejects it as AMP_API_KEY ("holds an OAuth session token, which is + // short-lived"), so borrowing it boots a member that dies on its first request. + const jwt = "eyJhbGciOiJIUzI1NiJ9.eyJzdWIiOiJ1In0.c2lnbmF0dXJl" + const f = HARNESSES.amp.valueFile + assert.equal(f.read(JSON.stringify({ "apiKey@https://ampcode.com/": jwt })), null) + assert.equal(f.read(JSON.stringify({ "apiKey@https://amp.example/": jwt })), null) + // The remedy names the one credential that works, not `amp login` again. + assert.match(remedyFor("amp"), /ampcode\.com\/settings\/security/) + assert.match(remedyFor("amp"), /AMP_API_KEY/) +}) + test("amp falls back to a sole entry when the server is not the default one", () => { // amp's server is configurable, so the key name is not fixed. assert.equal(HARNESSES.amp.valueFile.read('{"apiKey@https://amp.internal/":"self-hosted"}'), "self-hosted") diff --git a/test/oauth.test.js b/test/oauth.test.js new file mode 100644 index 0000000..335e03e --- /dev/null +++ b/test/oauth.test.js @@ -0,0 +1,139 @@ +import test from "node:test" +import assert from "node:assert/strict" +import { createHash } from "node:crypto" +import { + AMP, CODEX, MUSE, ampConnect, codexAuthorizeUrl, codexConnect, codexRefresh, deviceFlow, + jwtExpiryIso, museConnect, pkcePair, validAmpAccessToken, +} from "../src/oauth.js" + +// Every flow runs against a scripted `fetch`: a list of handlers matched by URL, +// each returning { status, body }. Nothing here touches the network or a browser. +function scripted(routes) { + const calls = [] + const fetch = async (url, init = {}) => { + calls.push({ url, init }) + const route = routes.find((r) => (typeof r.url === "string" ? r.url === url : r.url.test(url))) + if (!route) throw new Error(`unexpected request to ${url}`) + const step = typeof route.reply === "function" ? route.reply(init, calls) : route.reply + return { ok: step.status < 400, status: step.status, text: async () => JSON.stringify(step.body) } + } + return { fetch, calls } +} +const sleep = async () => {} +const jwt = (exp) => `eyJhbGciOiJub25lIn0.${Buffer.from(JSON.stringify({ exp })).toString("base64url")}.sig` + +test("pkce pair: verifier is base64url of the bytes, challenge is its S256", () => { + const bytes = Buffer.alloc(32, 7) + const { verifier, challenge } = pkcePair(bytes) + assert.equal(verifier, bytes.toString("base64url")) + assert.equal(challenge, createHash("sha256").update(verifier).digest("base64url")) + assert.doesNotMatch(verifier + challenge, /[+/=]/) +}) + +test("jwtExpiryIso reads exp, and refuses anything that is not a JWT", () => { + assert.equal(jwtExpiryIso(jwt(1_800_000_000)), new Date(1_800_000_000 * 1000).toISOString()) + assert.equal(jwtExpiryIso("sgamp_notajwt"), null) + assert.equal(jwtExpiryIso(undefined), null) +}) + +test("device flow: hands the human the complete URL and code, polls through pending and slow_down, returns tokens", async () => { + let polls = 0 + const { fetch, calls } = scripted([ + { url: "https://x/auth", reply: { status: 200, body: { device_code: "dc", user_code: "AB-CD", verification_uri: "https://x/d", verification_uri_complete: "https://x/d?user_code=AB-CD", expires_in: 300, interval: 1 } } }, + { url: "https://x/token", reply: () => (++polls === 1 ? { status: 400, body: { error: "authorization_pending" } } : polls === 2 ? { status: 400, body: { error: "slow_down" } } : { status: 200, body: { access_token: "tok" } }) }, + ]) + const seen = [] + const tokens = await deviceFlow({ authorizeUrl: "https://x/auth", tokenUrl: "https://x/token", clientId: "cid", onCode: (c) => seen.push(c), fetch, sleep }) + assert.equal(tokens.access_token, "tok") + assert.deepEqual(seen, [{ url: "https://x/d?user_code=AB-CD", userCode: "AB-CD", expiresIn: 300 }]) + assert.equal(calls[0].init.body, "client_id=cid") + assert.match(calls[1].init.body, /grant_type=urn%3Aietf%3Aparams%3Aoauth%3Agrant-type%3Adevice_code&device_code=dc&client_id=cid/) + assert.equal(polls, 3) +}) + +test("device flow: a denial and an expiry are named, not retried forever", async () => { + const denied = scripted([ + { url: "https://x/auth", reply: { status: 200, body: { device_code: "dc", user_code: "A", verification_uri: "https://x/d", expires_in: 300, interval: 1 } } }, + { url: "https://x/token", reply: { status: 400, body: { error: "access_denied" } } }, + ]) + await assert.rejects(deviceFlow({ authorizeUrl: "https://x/auth", tokenUrl: "https://x/token", clientId: "c", fetch: denied.fetch, sleep }), /denied in the browser/) + let t = 0 + const expired = scripted([ + { url: "https://x/auth", reply: { status: 200, body: { device_code: "dc", user_code: "A", verification_uri: "https://x/d", expires_in: 10, interval: 5 } } }, + { url: "https://x/token", reply: { status: 400, body: { error: "authorization_pending" } } }, + ]) + await assert.rejects(deviceFlow({ authorizeUrl: "https://x/auth", tokenUrl: "https://x/token", clientId: "c", fetch: expired.fetch, sleep, now: () => (t += 6000) }), /expired before it was approved/) +}) + +test("muse: Meta device flow, then Meta mints the API key; only the key is kept", async () => { + const { fetch, calls } = scripted([ + { url: MUSE.authorizeUrl, reply: { status: 200, body: { device_code: "dc", user_code: "XCCF-DXML", verification_uri: "https://auth.meta.com/oauth/device", expires_in: 600, interval: 5 } } }, + { url: MUSE.tokenUrl, reply: { status: 200, body: { access_token: "meta-id-token" } } }, + { url: MUSE.keyUrl, reply: (init) => (init.headers.Authorization === "Bearer meta-id-token" && init.headers["x-api-version"] === "1.0.0" ? { status: 200, body: { api_key: "muse-key-1" } } : { status: 401, body: {} }) }, + ]) + const r = await museConnect({ fetch, sleep, onCode: () => {} }) + assert.deepEqual(r, { secret: { kind: "api-key", META_API_KEY: "muse-key-1" }, expiresAt: null }) + assert.equal(calls[0].init.body, `client_id=${MUSE.clientId}`) +}) + +test("muse: an account that has not finished setup is told where, and nothing is stored", async () => { + const { fetch } = scripted([ + { url: MUSE.authorizeUrl, reply: { status: 200, body: { device_code: "dc", user_code: "A", verification_uri: "https://m/d", expires_in: 600, interval: 5 } } }, + { url: MUSE.tokenUrl, reply: { status: 200, body: { access_token: "t" } } }, + { url: MUSE.keyUrl, reply: { status: 200, body: { action_url: "https://dev.meta.ai/setup" } } }, + ]) + await assert.rejects(museConnect({ fetch, sleep }), /Finish setup at https:\/\/dev\.meta\.ai\/setup/) +}) + +test("amp: opens the Security page and keeps exactly the pasted access token", async () => { + const seen = [] + const r = await ampConnect({ onCode: (c) => seen.push(c), prompt: async () => " sgamp_user_abcdefghijklmnop \n" }) + assert.deepEqual(seen, [{ url: AMP.settingsUrl, userCode: null, expiresIn: 600 }]) + assert.deepEqual(r, { secret: { kind: "api-key", AMP_API_KEY: "sgamp_user_abcdefghijklmnop" }, expiresAt: null }) + // A session JWT pasted by mistake is refused: it is the five-minute bearer of ADR 0014. + await assert.rejects(ampConnect({ prompt: async () => jwt(1_900_000_000) }), /Expected an amp access token/) + await assert.rejects(ampConnect({ prompt: async () => "" }), /nothing was saved/) + assert.equal(validAmpAccessToken("sgamp_user_0123456789abcdefXYZ"), true) + assert.equal(validAmpAccessToken(jwt(1)), false) +}) + +test("codex: authorize URL carries codex's own client id, PKCE and originator", () => { + const u = new URL(codexAuthorizeUrl({ challenge: "ch", state: "st" })) + assert.equal(u.origin + u.pathname, CODEX.authorizeUrl) + assert.equal(u.searchParams.get("client_id"), CODEX.clientId) + assert.equal(u.searchParams.get("redirect_uri"), "http://localhost:1455/auth/callback") + assert.equal(u.searchParams.get("code_challenge"), "ch") + assert.equal(u.searchParams.get("code_challenge_method"), "S256") + assert.equal(u.searchParams.get("originator"), "codex_cli_rs") + assert.equal(u.searchParams.get("scope"), CODEX.scope) +}) + +test("codex: PKCE callback, state check, exchange, auth.json shape with the bearer's expiry", async () => { + const exp = 1_900_000_000 + const { fetch, calls } = scripted([ + { url: CODEX.tokenUrl, reply: (init) => (/grant_type=authorization_code&code=the-code&redirect_uri=http%3A%2F%2Flocalhost%3A1455%2Fauth%2Fcallback&client_id=app_EMoamEEZ73f0CkXaXp7hrann&code_verifier=/.test(init.body) ? { status: 200, body: { id_token: "id", access_token: jwt(exp), refresh_token: "rt-1" } } : { status: 400, body: { error: "bad" } }) }, + ]) + const listen = () => Promise.resolve({ code: "the-code", state: "st" }) + const r = await codexConnect({ fetch, onCode: () => {}, listen, state: "st", now: () => 1_700_000_000_000 }) + assert.equal(r.secret.auth_mode, "chatgpt") + assert.equal(r.secret.tokens.refresh_token, "rt-1") + assert.equal(r.secret.last_refresh, new Date(1_700_000_000_000).toISOString()) + assert.equal(r.expiresAt, new Date(exp * 1000).toISOString()) + assert.equal(calls.length, 1) + await assert.rejects(codexConnect({ fetch, listen: () => Promise.resolve({ code: "c", state: "other" }), state: "st" }), /different state/) + await assert.rejects(codexConnect({ fetch, listen: () => Promise.resolve({ error: "access_denied", error_description: "no" }), state: "st" }), /access_denied - no/) +}) + +test("codex refresh: the request codex sends, and the rotated auth.json that comes back", async () => { + const { fetch, calls } = scripted([ + { url: CODEX.tokenUrl, reply: { status: 200, body: { id_token: "id2", access_token: jwt(2_000_000_000), refresh_token: "rt-2" } } }, + ]) + const before = { auth_mode: "chatgpt", tokens: { id_token: "id1", access_token: "old", refresh_token: "rt-1", account_id: "acct" } } + const after = await codexRefresh(before, { fetch, now: () => 1_700_000_000_000 }) + assert.deepEqual(JSON.parse(calls[0].init.body), { client_id: CODEX.clientId, grant_type: "refresh_token", refresh_token: "rt-1", scope: "openid profile email" }) + assert.equal(after.tokens.refresh_token, "rt-2") + assert.equal(after.tokens.account_id, "acct") + assert.equal(after.tokens.id_token, "id2") + const dead = scripted([{ url: CODEX.tokenUrl, reply: { status: 400, body: { error: "invalid_grant" } } }]) + await assert.rejects(codexRefresh(before, { fetch: dead.fetch }), /codex refresh failed \(400: invalid_grant\)/) +}) From 5947a508da1c76d41b663df12ec5c452fb33564c Mon Sep 17 00:00:00 2001 From: Ondrej Drapalik Date: Wed, 9 Sep 2026 22:36:39 +0200 Subject: [PATCH 2/5] feat: derive model pins from harness catalogs --- .agents/skills/release/SKILL.md | 17 + CONTRIBUTING.md | 11 + config/config.example.toml | 49 +- package.json | 4 +- scripts/harness-catalog.mjs | 267 +++++ src/catalog.js | 577 ++++++++++ src/config.js | 9 +- src/effort-cli.js | 23 + src/effort.js | 149 +++ src/fleet-name.js | 73 +- src/harness-catalog.generated.js | 1620 +++++++++++++++++++++++++++ src/model-pin.js | 30 +- src/provision.js | 34 +- test/catalog.test.js | 262 +++++ test/config.test.js | 4 +- test/effort.test.js | 106 ++ test/fixtures/catalog/amp.json | 28 + test/fixtures/catalog/claude.json | 23 + test/fixtures/catalog/codex.json | 18 + test/fixtures/catalog/droid.json | 28 + test/fixtures/catalog/grok.json | 28 + test/fixtures/catalog/muse.json | 23 + test/fixtures/catalog/opencode.json | 13 + test/fixtures/catalog/prime.json | 23 + test/fleet-name.test.js | 33 +- test/model-pin.test.js | 11 +- 26 files changed, 3419 insertions(+), 44 deletions(-) create mode 100644 scripts/harness-catalog.mjs create mode 100644 src/catalog.js create mode 100644 src/effort-cli.js create mode 100644 src/effort.js create mode 100644 src/harness-catalog.generated.js create mode 100644 test/catalog.test.js create mode 100644 test/effort.test.js create mode 100644 test/fixtures/catalog/amp.json create mode 100644 test/fixtures/catalog/claude.json create mode 100644 test/fixtures/catalog/codex.json create mode 100644 test/fixtures/catalog/droid.json create mode 100644 test/fixtures/catalog/grok.json create mode 100644 test/fixtures/catalog/muse.json create mode 100644 test/fixtures/catalog/opencode.json create mode 100644 test/fixtures/catalog/prime.json diff --git a/.agents/skills/release/SKILL.md b/.agents/skills/release/SKILL.md index 57b2e89..37619a2 100644 --- a/.agents/skills/release/SKILL.md +++ b/.agents/skills/release/SKILL.md @@ -159,6 +159,23 @@ node -e 'const f="package.json",fs=require("fs");const j=JSON.parse(fs.readFileS sed -i '' 's/^version = ".*"/version = "X.Y.Z"/' herdr-plugin.toml ``` +### Refresh the harness catalog + +```bash +npm run catalog # --check: boots one throwaway box per template, ~1 min, exit 2 on drift +npm run catalog -- --write # accept what the templates' CLIs now say; commit the result +``` + +The effort words and model ids each template accepts are read off the CLI IN the template +(`docs/adr/0016`), and templates get rebuilt with newer CLIs between releases. A release +should ship the catalog of the images it will boot, so run the check here; if it exits 2, +run `--write`, read the diff of `src/harness-catalog.generated.js` and the `` +block in `config/config.example.toml` (a word gained or lost, a model added or retired, +a version bump), and commit it with the version bump. Needs the E2B key and the agent +credentials your `config.toml` already holds; the output names variables, never values, +so it is safe to paste into the release notes. Exit 1 means a probe could not be trusted: +the committed row is kept and marked stale, so it is not a blocker, but say so. + ### Preflight, then tag ```bash diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 01eba50..cc62ab0 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -28,6 +28,17 @@ layers, the data flow, the state model, and the invariants worth checking). `auth.toml` reader and the loader test that proves a box boots from it, which writes a fixture under the OS temp dir and spawns one child `node` — never the developer's own config. +- `test/catalog.test.js`: the harness catalog (`interpretCatalog`) from + `test/fixtures/catalog/`, output captured off the CLI in each E2B template, and the + guard that `src/harness-catalog.generated.js` and the `` block in + `config/config.example.toml` are exactly what those fixtures render to. When it + fails after a parse-rule change, rerun + `node scripts/harness-catalog.mjs --from=fixture --write`; when the WORLD changed + (a template rebuilt with a newer CLI, a new model), `npm run catalog -- --write` + boots one throwaway sandbox per template, re-reads every row and refreshes all three + (see `docs/adr/0016`). `--check` (the default) exits 2 on drift and writes nothing; + a probe that fails keeps the committed row and marks it stale rather than blanking + it. Run it before a release; `.claude/skills/release` does. - `test/cli.test.sh` — `bash -n` / `node --check` lint across the scripts, plus offline `e2b-box` behavior (the `no sandbox tracked` messages and the non-interactive `pull` abort-without-clobber path). The template picker's auth diff --git a/config/config.example.toml b/config/config.example.toml index 1edad3f..cb16531 100644 --- a/config/config.example.toml +++ b/config/config.example.toml @@ -222,21 +222,34 @@ server_port = 3000 # on an image model with no tool use). Set here once instead of `-m` in every launch # command. # -# Both keys are optional, free strings, each harness's own vocabulary: -# -# template model reasoning -# claude a Claude model id or alias low | medium | high | xhigh -# codex an OpenAI model id minimal | low | medium | high | xhigh -# grok a grok model id (grok-build, grok-4.5) none … max -# droid a Factory model id (glm-5.2, kimi-k3) off | low | medium | high | max, per model -# prime "/", split at the prime's thinking levels (off … max) -# first slash (openai-codex/gpt-6-astra, -# prime-inference/z-ai/glm-5.2) -# opencode "/" as `opencode (not pinned: opencode does reasoning as -# models` prints it per-model variants) -# muse (not pinned: no verified settings key, none … ultra -# use --model in your agent command) -# amp (nothing: amp has no model setting, `--mode low|medium|high|ultra` is its knob) +# The pickers (`e2b-box open`'s template list, the fleet's roster table) show +# `model` and `reasoning` as cells that open on these pins and can be cycled per box +# or per member, from the catalog below; `e2b-box open --model gpt-5.5 --reasoning +# xhigh` and `e2b-fleet -t codex:gpt-5.5*3@ultra` say the same from a script. Those +# override the pin for THAT box only, and are checked against the catalog first. +# +# Both keys are optional, free strings, each harness's own vocabulary. How a model is +# SPELLED is the plugin's business: bare ids for claude, codex, grok and droid (as the +# CLI prints them); "/" for prime and opencode, split at the FIRST +# slash (openai-codex/gpt-6-astra, prime-inference/z-ai/glm-5.2); muse has no +# verified settings key for a model and amp no model setting at all, so for those +# two `model` is reported and not written. What each CLI ACCEPTS is read off the CLI +# in its template and lives in the block below: +# +# +# Generated by `npm run catalog -- --write` from the CLI shipped in each template. +# Do not edit this block: rerun the script. Per-model detail is in +# src/harness-catalog.generated.js. +# template cli model reasoning +# claude 2.1.265 14: claude-fable-5, claude-fable-5-1, … low | medium | high | xhigh | max (per model) +# codex 0.153.4 6: gpt-6-astra, gpt-5.6-sol, … low | medium | high | xhigh | max | ultra (per model) +# grok 1.0.5 2: grok-4.6, grok-4.5 low | medium | high | xhigh (per model) +# droid 0.213.0 50: auto, claude-fable-5.1, … none | off | minimal | low | medium | high | xhigh | max (per model) +# amp 0.0.1788926461-g32d216 42 listed, no pin route: alibaba/qwen3.8-max, … --mode low | medium | high | ultra (amp's knob is the mode) +# prime 0.7.0 104: prime-inference/anthropic/claude-fable-5, … off | minimal | low | medium | high | xhigh | max +# muse 0.2.1 (no pinnable model: use --model in your agent command) none | minimal | low | medium | high | xhigh | ultra +# opencode 1.18.18 any models.dev "/" (not pinned: opencode does reasoning as per-model variants in its UI) +# # # How it reaches the agent is the harness's own way (src/model-pin.js): claude and # opencode read it from the environment (ANTHROPIC_MODEL, CLAUDE_CODE_EFFORT_LEVEL, @@ -305,6 +318,12 @@ ignore = [".git", "node_modules", ".next", "dist", "build", ".turbo", ".cache", # already selected when the picker opens. # default_roster = ["claude", "codex", "grok"] +# Reusable counts, models and reasoning live in JSON files under presets/ beside +# this config. Select with p / Shift+P in the roster, or run headlessly: +# e2b-box fleet login-fix --preset compare --task "Fix the login loop" +# e2b-box fleet create --file fleet.json --dry-run +# See docs/fleet-presets.md for the JSON format; `fleet presets` lists names. + # Which command starts a member's agent once its box is ready, per template. # # The shipped defaults start each agent UNATTENDED — with that vendor's own diff --git a/package.json b/package.json index 0e3bee5..b49206e 100644 --- a/package.json +++ b/package.json @@ -26,7 +26,9 @@ "node": ">=22" }, "scripts": { - "test": "node --test test/*.test.js && bash test/cli.test.sh" + "test": "node --test test/*.test.js && bash test/cli.test.sh", + "catalog": "node scripts/harness-catalog.mjs", + "verify-keys": "node scripts/verify-keys.mjs" }, "dependencies": { "e2b": "^2.32.0", diff --git a/scripts/harness-catalog.mjs b/scripts/harness-catalog.mjs new file mode 100644 index 0000000..94708fd --- /dev/null +++ b/scripts/harness-catalog.mjs @@ -0,0 +1,267 @@ +// Which models and effort words does each shipped agent template ACTUALLY accept? +// +// node scripts/harness-catalog.mjs [template ...] [--check|--write] [--from=box|local|fixture] +// npm run catalog -- --write +// +// One throwaway sandbox per template (the default, `--from=box`), booted with the +// env a fleet member would get, running that harness's own catalog probes: its +// version, the rejection message that lists its effort words, the command or file +// that lists its models (src/catalog.js, CATALOG_PROBES). What it read is compared +// with src/harness-catalog.generated.js and, with `--write`, written there, into the +// table in config/config.example.toml, and into test/fixtures/catalog/ so `npm test` +// re-derives the same rows offline. +// +// Why the box and not this machine: the template ships an older CLI than a laptop +// in every case measured but one (docs/research/0003), and the CLI in the box is +// the one that will accept or reject a pin. `--from=local` asks the binaries on +// PATH instead: quick, and wrong for the box whenever the two differ, so rows it +// writes are marked `probedFrom: "local"`. `--from=fixture` re-reads the captured +// output and writes nothing new about the world; it is how a parse-rule change is +// replayed and what test/catalog.test.js does. +// +// Exit codes: 0 nothing moved · 2 drift found (a finding, not an error: the world +// changed and the committed rows have not) · 1 usage, or a probe that could not be +// trusted. THE TWO RULES: a credential is never printed and never on a command line +// (probes run unauthenticated or read a variable the box already holds; this output +// is safe to paste into a PR), and a failed probe never blanks a row (the committed +// one is kept and marked stale, since an unauthenticated run must not empty every scale). +import { execFile } from "node:child_process" +import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs" +import path from "node:path" +import { fileURLToPath } from "node:url" +import { promisify } from "node:util" + +import { + CATALOG_PROBES, + diffRow, + interpretCatalog, + mergeCatalog, + renderGeneratedModule, + renderTomlBlock, + spliceTomlBlock, +} from "../src/catalog.js" + +const ROOT = path.resolve(path.dirname(fileURLToPath(import.meta.url)), "..") +const GENERATED = path.join(ROOT, "src", "harness-catalog.generated.js") +const TOML = path.join(ROOT, "config", "config.example.toml") +const FIXTURES = path.join(ROOT, "test", "fixtures", "catalog") +const execFileP = promisify(execFile) + +const usage = (code, msg) => { + if (msg) console.error(`harness-catalog: ${msg}`) + console.error("usage: node scripts/harness-catalog.mjs [template ...] [--check|--write] [--from=box|local|fixture]") + process.exit(code) +} + +let mode = "check" +let from = "box" +const names = [] +for (const a of process.argv.slice(2)) { + if (a === "--check" || a === "--write") mode = a.slice(2) + else if (a.startsWith("--from=")) from = a.slice("--from=".length) + else if (a === "-h" || a === "--help") usage(0) + else if (a.startsWith("-")) usage(1, `unknown flag ${a}`) + else names.push(a) +} +if (!["box", "local", "fixture"].includes(from)) usage(1, `--from must be box, local or fixture, not ${from}`) +const templates = names.length ? names : Object.keys(CATALOG_PROBES) +for (const t of templates) if (!CATALOG_PROBES[t]) usage(1, `no catalog probes for '${t}' (known: ${Object.keys(CATALOG_PROBES).join(", ")})`) + +const committed = existsSync(GENERATED) ? (await import(GENERATED)).HARNESS_CATALOG : {} +const probedAt = new Date().toISOString() + +// --- transports --------------------------------------------------------------------- +// Each returns { stdout, stderr, status }; `status` is null when the command never ran +// or never came back. Parse rules read stdout+stderr and ignore status on purpose. + +const fetched = new Map() +async function get(url) { + if (!fetched.has(url)) { + fetched.set( + url, + fetch(url) + .then(async (res) => ({ stdout: await res.text(), stderr: "", status: res.status })) + .catch((e) => ({ stdout: "", stderr: String(e?.message || e), status: null })), + ) + } + return fetched.get(url) +} + +// `/bin/sh -c`, not the user's shell: an interactive zsh here shadows two of these +// binaries with functions that rewrite their arguments. +async function runLocal(cmd) { + try { + const { stdout, stderr } = await execFileP("/bin/sh", ["-c", cmd], { timeout: 90000, maxBuffer: 64 << 20 }) + return { stdout, stderr, status: 0 } + } catch (e) { + return { stdout: e?.stdout ?? "", stderr: e?.stderr ?? String(e?.message || e), status: typeof e?.code === "number" ? e.code : null } + } +} + +async function probeLocal(spec) { + const out = {} + for (const [name, p] of Object.entries(spec.probes)) out[name] = p.url ? await get(p.url) : await runLocal(p.cmd) + return out +} + +let box = null +async function boxDeps() { + if (box) return box + const [{ Sandbox }, { loadConfig, resolveEnv }, { seedCommand }] = await Promise.all([ + import("e2b"), + import("../src/config.js"), + import("../src/fleet-seed.js"), + ]) + const cfg = loadConfig() + if (!cfg.apiKey) usage(1, "no E2B API key (E2B_API_KEY, config.toml, or `e2b auth login`), needed for --from=box") + const conn = { apiKey: cfg.apiKey, ...(cfg.domain ? { domain: cfg.domain } : {}) } + box = { Sandbox, cfg, conn, resolveEnv, seedCommand } + return box +} + +async function runBox(sbx, cmd) { + // droid installs to ~/.local/bin, which a non-interactive shell does not have on + // PATH; harmless for the others. + const line = `export PATH="$HOME/.local/bin:$PATH"; ${cmd}` + try { + const r = await sbx.commands.run(line, { timeoutMs: 120000, cwd: "/home/user" }) + return { stdout: r.stdout ?? "", stderr: r.stderr ?? "", status: r.exitCode ?? 0 } + } catch (e) { + // A non-zero exit throws, with the output on the thrown object. Expected for + // every rejection oracle. + const r = e?.result + return { stdout: r?.stdout ?? "", stderr: r?.stderr ?? (r ? "" : String(e?.message || e)), status: typeof r?.exitCode === "number" ? r.exitCode : null } + } +} + +async function probeBox(template, spec) { + const { Sandbox, cfg, conn, resolveEnv, seedCommand } = await boxDeps() + const envs = resolveEnv(cfg, template) || {} + const sbx = await Sandbox.create(template, { ...conn, envs, timeoutMs: 600000 }) + try { + // Exactly as src/provision.js seeds a member: codex reads its credential from + // ~/.codex/auth.json, grok's cache only exists after an authenticated command. + const seed = seedCommand(template, cfg.fleetSeeds) + if (seed) { + await sbx.files.write("/home/user/.herdr-e2b-seed.sh", `${seed}\n`) + await sbx.commands.run('bash "$HOME/.herdr-e2b-seed.sh"', { cwd: "/home/user" }).catch(() => {}) + } + const out = {} + for (const [name, p] of Object.entries(spec.probes)) out[name] = p.url ? await get(p.url) : await runBox(sbx, p.cmd) + return out + } finally { + await sbx.kill().catch(() => {}) + } +} + +/** + * Which build of each template is live, so a row can say which image it was read + * from. The listing is this TEAM's templates: a name that resolves to the team's own + * image (`ondrejs-project/claude` behind the alias `claude`) gets its build; a public + * E2B template is not listed and gets none. Best-effort, never fatal. + */ +async function templateBuilds() { + if (from !== "box") return {} + const { cfg } = await boxDeps() + try { + const res = await fetch(`https://api.${cfg.domain || "e2b.app"}/templates`, { headers: { "X-API-KEY": cfg.apiKey } }) + if (!res.ok) return {} + const list = await res.json() + const out = {} + for (const t of templates) { + const hit = (Array.isArray(list) ? list : []).find( + (x) => x?.templateID === t || (x?.aliases || []).includes(t) || (x?.names || []).includes(t), + ) + if (hit) out[t] = { name: hit.names?.[0] ?? hit.aliases?.[0] ?? t, buildID: hit.buildID ?? null, updatedAt: hit.updatedAt ?? null } + } + return out + } catch { + return {} + } +} + +// --- fixtures ------------------------------------------------------------------------- + +const fixturePath = (id) => path.join(FIXTURES, `${id}.json`) +function readFixture(id) { + const f = fixturePath(id) + if (!existsSync(f)) return null + return JSON.parse(readFileSync(f, "utf8")) +} +function writeFixture(id, spec, results, version) { + const probes = {} + for (const [name, r] of Object.entries(results)) { + const trim = spec.probes[name]?.trim + let stdout = r.stdout + if (trim && r.stdout) { + try { + stdout = trim(r.stdout) + } catch {} + } + probes[name] = { stdout, stderr: r.stderr, status: r.status } + } + mkdirSync(FIXTURES, { recursive: true }) + writeFileSync(fixturePath(id), `${JSON.stringify({ harness: id, version, probedFrom: from, capturedAt: probedAt, probes }, null, 2)}\n`) +} + +// --- the run -------------------------------------------------------------------------- + +async function probe(id) { + const spec = CATALOG_PROBES[id] + try { + if (from === "fixture") { + const fx = readFixture(id) + if (!fx) return { id, results: null, reason: `no fixture at test/fixtures/catalog/${id}.json` } + // A replay says where the CAPTURE came from, not "fixture". + return { id, results: fx.probes, probedFrom: fx.probedFrom, probedAt: fx.capturedAt } + } + const results = from === "box" ? await probeBox(id, spec) : await probeLocal(spec) + return { id, results } + } catch (e) { + const body = `${e?.result?.stdout || e?.stdout || ""}${e?.result?.stderr || e?.stderr || ""}`.trim() + const tail = body ? body.split("\n").filter(Boolean).slice(-3).join(" / ") : String(e?.message || e) + return { id, results: null, reason: tail.slice(0, 240) } + } +} + +console.log(`reading ${templates.length} harness catalog(s) from ${from}${from === "box" ? ", one sandbox each, killed on the way out" : ""}\n`) +const [probed, builds] = await Promise.all([Promise.all(templates.map(probe)), templateBuilds()]) + +const fresh = {} +let failed = 0 +let drifted = 0 +const lines = [] +for (const { id, results, reason, probedFrom: rowFrom, probedAt: rowAt } of probed) { + const row = results ? interpretCatalog(id, results) : null + if (!row) { + failed++ + const why = reason || `probe output did not parse (${Object.keys(results || {}).join(", ")})` + fresh[id] = { row: null, reason: why } + const kept = committed[id] ? `; row kept from ${String(committed[id].probedAt || "").slice(0, 10) || "an earlier run"}, marked stale` : "; no committed row to keep" + lines.push(`✗ ${id.padEnd(9)} ${why}${kept}`) + continue + } + fresh[id] = { row, ...(rowFrom ? { probedFrom: rowFrom } : {}), ...(rowAt ? { probedAt: rowAt } : {}) } + const diff = diffRow(committed[id], row) + if (diff.length) drifted++ + const models = row.models ? `models ${row.models.length}` : "models -" + const efforts = row.modes ? `modes ${row.modes.length}` : row.efforts ? `efforts ${row.efforts.length}` : "efforts -" + const build = builds[id]?.buildID ? ` build ${String(builds[id].buildID).slice(0, 8)}` : "" + lines.push(`${diff.length ? "~" : "✓"} ${id.padEnd(9)} ${row.version.padEnd(14)} ${efforts.padEnd(10)} ${models.padEnd(11)}${build}${diff.length ? ` drift: ${diff.join("; ")}` : ""}`) + if (mode === "write" && from !== "fixture" && results) writeFixture(id, CATALOG_PROBES[id], results, row.version) +} +for (const l of lines) console.log(l) + +const merged = mergeCatalog(committed, fresh, { probedFrom: from, probedAt, templates: builds }) + +if (mode === "write") { + writeFileSync(GENERATED, renderGeneratedModule(merged, { probedAt })) + const toml = readFileSync(TOML, "utf8") + writeFileSync(TOML, spliceTomlBlock(toml, renderTomlBlock(merged))) + console.log(`\nwrote ${path.relative(ROOT, GENERATED)}, the block in ${path.relative(ROOT, TOML)}${from !== "fixture" ? `, ${path.relative(ROOT, FIXTURES)}/` : ""}`) + if (from === "local") console.log("note: rows read from this machine's binaries are marked probedFrom: \"local\"; the template's CLI is the one that counts; rerun with --from=box before shipping") +} else { + console.log(`\n${drifted ? `${drifted} row(s) drifted, rerun with --write to accept` : "no drift"}${failed ? `, ${failed} probe(s) failed` : ""}`) +} + +process.exit(failed ? 1 : drifted && mode === "check" ? 2 : 0) diff --git a/src/catalog.js b/src/catalog.js new file mode 100644 index 0000000..6220037 --- /dev/null +++ b/src/catalog.js @@ -0,0 +1,577 @@ +// What each harness's CLI offers: the model ids it knows and the words it takes for +// "how hard to think", read off the binary shipped in that harness's TEMPLATE. +// +// Two tables used to be typed by hand from a laptop (`EFFORT_SCALES` in +// src/effort.js, the template table in config/config.example.toml) and they had +// already drifted from each other and from the boxes by the day they were compared +// (docs/research/0003-model-and-effort-catalogs.md). The templates ship older CLIs +// than a developer's machine, in every case measured but one, so a laptop is the +// wrong oracle: `muse --help` on the box has no `max`, the box's `amp` has no +// per-model efforts key, and a row "verified against the CLI" was verified against +// the wrong CLI. What a box will accept is decided by the binary IN the box, so that +// is what gets asked. +// +// No registry answers this. models.dev knows every model id and, for a few +// providers, each model's effort values, but `ultra`, `dynamic`, `off` and `none` +// are words the coding CLIs invented and appear in no registry at all: registries +// are keyed by inference provider, this table by coding-agent CLI. So model ids +// come from the CLI where the CLI enumerates them (codex, grok, droid, amp, prime) +// and from models.dev where it does not (claude); efforts always come from the CLI. +// Five of eight print their own vocabulary when fed a sentinel value, which is the +// ADR 0009 spawn-the-binary shape applied to a different question. +// +// Same shape as src/harnesses.js and for the same reason: a shipped data table beside +// a PURE function. `interpretCatalog` never spawns and never fetches, it reads probe +// RESULTS, so every parse rule is testable from captured output on a machine with +// none of these installed. The spawning lives in scripts/harness-catalog.mjs, which +// boots the template, runs each probe, and writes what it read into +// src/harness-catalog.generated.js (committed, never hand-edited) and the table in +// config/config.example.toml. Two rules that script keeps and this module makes +// possible: a failed probe never blanks a row (`mergeCatalog` keeps the committed one +// and marks it stale), and a credential is never on a command line (every probe here +// runs unauthenticated, or reads a variable the box already has). + +/** + * The words in the order every scale is written: nothing, then more. Every harness + * spells its own subset, and two print theirs descending, so a generated row would + * otherwise flip order with the vendor's whim. Unknown words keep their own order + * after these. + */ +export const EFFORT_ORDER = ["none", "dynamic", "off", "minimal", "low", "medium", "high", "xhigh", "max", "ultra"] + +const text = (p) => `${p?.stdout ?? ""}\n${p?.stderr ?? ""}` +const uniq = (xs) => [...new Set(xs)] +const json = (p) => { + try { + return JSON.parse(String(p?.stdout ?? "")) + } catch { + return null + } +} + +/** An effort word as a CLI prints one: lowercase letters, digits, dashes. */ +const isWord = (s) => /^[a-z][a-z0-9-]*$/.test(s) + +/** `values` sorted by EFFORT_ORDER, de-duplicated; `null` for nothing usable. */ +export function orderEfforts(values) { + const words = uniq((values || []).map((v) => String(v ?? "").trim()).filter(isWord)) + if (!words.length) return null + const rank = (w) => (EFFORT_ORDER.includes(w) ? EFFORT_ORDER.indexOf(w) : EFFORT_ORDER.length + words.indexOf(w)) + return words.sort((a, b) => rank(a) - rank(b)) +} + +/** + * The comma- or pipe-separated list that follows `re`'s first capture in `s`, + * as ordered effort words. This is how a rejection message is read: "Valid values: + * low, medium, high." → ["low", "medium", "high"]. + */ +export function enumAfter(s, re) { + const m = String(s ?? "").match(re) + if (!m?.[1]) return null + return orderEfforts(m[1].split(/[,|]/).map((w) => w.trim().replace(/^'|'$/g, "").replace(/^and\s+/, ""))) +} + +/** The union of every model's efforts, ordered; `null` when no model names any. */ +export function unionEfforts(models) { + return orderEfforts((models || []).flatMap((m) => m.efforts || [])) +} + +/** Does at least one model take a different set than the union? (Then a flat row lies.) */ +export function perModel(row) { + const all = row?.efforts || [] + return !!row?.models?.some((m) => m.efforts && (m.efforts.length !== all.length || m.efforts.some((e) => !all.includes(e)))) +} + +/** models.dev's `api.json`: one provider's models as catalog rows, with each model's effort values when it has any. */ +function modelsDev(p, provider) { + const j = json(p) + const models = j?.[provider]?.models + if (!models || typeof models !== "object") return null + const rows = Object.values(models) + .filter((m) => m?.id) + .map((m) => { + const eff = (m.reasoning_options || []).find((o) => o?.type === "effort")?.values + const ordered = orderEfforts(eff) + return { id: String(m.id), ...(ordered ? { efforts: ordered } : {}) } + }) + .sort((a, b) => a.id.localeCompare(b.id)) + return rows.length ? rows : null +} + +/** The first `x.y.z[-tag]` in a `--version` answer, or null. */ +export function versionIn(s) { + return String(s ?? "").match(/\d+\.\d+\.\d+(?:-[A-Za-z0-9.]+)?/)?.[0] ?? null +} + +// The droid handshake. Factory's own envelope, not plain JSON-RPC and not ACP: every +// field below is required (`type`, the literal `factoryApiVersion`, a STRING id, +// `machineId`) and the two `--*-format` flags must both be given. It scaffolds +// ~/.factory and opens a session, so it runs against a throwaway HOME. Verified to +// return the full model list with no credential at all. +const DROID_HELLO = JSON.stringify({ + jsonrpc: "2.0", + factoryApiVersion: "1.0.0", + type: "request", + id: "1", + method: "droid.initialize_session", + params: { cwd: "/tmp", machineId: "herdr-e2b-catalog" }, +}) +const THROWAWAY_HOME = 'H="$(mktemp -d)"; export HOME="$H";' + +/** + * Per harness: the probes to run (in this order: grok's cache is written by the + * probe before it) and the pure rule that reads their results. + * + * probes. { cmd } a shell line run in the box's login shell, or + * { url } an unauthenticated GET; `trim` reduces a large answer + * to the fields `parse` reads before it is saved as a fixture. + * needs which field a usable row must carry: "efforts", "modes" or null. + * modelsFrom where the model column comes from, for the docs; null = not + * enumerable (muse has no pinnable model; opencode's universe is + * all of models.dev and its own `models` lists only CONFIGURED + * providers, 7 lines on a clean HOME against 439 with keys, so a + * box probe would delete 432 valid ids). + * parse(results) → { efforts, models, modes? } or null when the row cannot be + * trusted. `efforts` is the flat set the CLI accepts (what a pin may + * say); each model may carry its own `efforts` and `default`. + * + * Every rule reads stdout AND stderr and ignores the exit code: the rejection + * oracles exit non-zero by design, and droid's `Invalid model` exits 0. + */ +export const CATALOG_PROBES = { + claude: { + needs: "efforts", + modelsFrom: "models.dev (anthropic)", + probes: { + version: { cmd: "claude --version" }, + // An empty prompt: the CLI validates --effort, warns with the valid set, and + // then dies for want of input: exit 1, no session, no tokens. + efforts: { cmd: `claude -p --effort __probe__ ""` }, + models: { + url: "https://models.dev/api.json", + trim: (s) => JSON.stringify({ anthropic: JSON.parse(s).anthropic }), + }, + }, + parse: ({ efforts, models }) => { + const e = enumAfter(text(efforts), /Valid values:\s*([^.\n]+)/) + return e ? { efforts: e, models: modelsDev(models, "anthropic") } : null + }, + }, + codex: { + needs: "efforts", + modelsFrom: "codex debug models --bundled", + probes: { + version: { cmd: "codex --version" }, + // `--bundled` is the catalog compiled into this binary, offline. Without it + // codex silently falls back to the same thing when unauthenticated, so asking + // for it outright is the honest version of the same answer. + catalog: { + cmd: "codex debug models --bundled", + trim: (s) => + JSON.stringify({ + models: (JSON.parse(s).models || []).map((m) => ({ + slug: m.slug, + visibility: m.visibility, + default_reasoning_level: m.default_reasoning_level, + supported_reasoning_levels: (m.supported_reasoning_levels || []).map((l) => ({ effort: l.effort })), + })), + }), + }, + }, + parse: ({ catalog }) => { + const j = json(catalog) + if (!Array.isArray(j?.models)) return null + // `visibility: "list"` is what codex's own picker shows; hidden slugs are + // retired or internal. + const models = j.models + .filter((m) => m?.slug && m.visibility === "list") + .map((m) => { + const eff = orderEfforts((m.supported_reasoning_levels || []).map((l) => l?.effort)) + return { id: String(m.slug), ...(eff ? { efforts: eff } : {}), ...(m.default_reasoning_level ? { default: m.default_reasoning_level } : {}) } + }) + const efforts = unionEfforts(models) + return efforts ? { efforts, models } : null + }, + }, + grok: { + needs: "efforts", + modelsFrom: "~/.grok/models_cache.json after `grok models`", + probes: { + version: { cmd: "grok --version" }, + // Needs a credential: the catalog is team-scoped and fetched on the first + // authenticated command, which writes the cache the next probe reads. The box + // has XAI_API_KEY in its environment; nothing here names its value. + models: { cmd: "grok models" }, + cache: { cmd: 'cat "$HOME/.grok/models_cache.json"' }, + // A non-empty prompt on purpose: the empty-prompt guard runs BEFORE effort + // validation, so `-p ""` would pass every value. + efforts: { cmd: "grok --reasoning-effort __probe__ -p x" }, + }, + parse: ({ models, cache, efforts }) => { + const j = json(cache) + if (j?.models && typeof j.models === "object") { + const rows = Object.entries(j.models) + .filter(([, m]) => !m?.info?.hidden) + .map(([id, m]) => { + const list = m?.info?.reasoning_efforts || [] + const eff = orderEfforts(list.map((e) => e?.value)) + const def = list.find((e) => e?.default)?.value + return { id, ...(eff ? { efforts: eff } : {}), ...(def ? { default: def } : {}) } + }) + const all = unionEfforts(rows) + if (all) return { efforts: all, models: rows } + } + // No cache (unauthenticated, or an older grok): the flat set from the + // rejection and whatever ids `grok models` listed. + const e = enumAfter(text(efforts), /use one of:\s*([^\n;]+)/) + if (!e) return null + const ids = uniq([...text(models).matchAll(/^\s+[*-]\s+(\S+)/gm)].map((m) => m[1])) + return { efforts: e, models: ids.length ? ids.map((id) => ({ id })) : null } + }, + }, + droid: { + needs: "efforts", + modelsFrom: "droid.initialize_session (stream-jsonrpc)", + probes: { + version: { cmd: "droid --version" }, + catalog: { + cmd: `${THROWAWAY_HOME} printf '%s\\n' '${DROID_HELLO}' | droid exec --input-format stream-jsonrpc --output-format stream-jsonrpc`, + // Only the response that carries the models, and only the fields read. + trim: (s) => { + for (const line of String(s).split("\n")) { + try { + const o = JSON.parse(line) + if (o?.type === "response" && Array.isArray(o.result?.availableModels)) { + return JSON.stringify({ + type: "response", + result: { + availableModels: o.result.availableModels.map((m) => ({ + id: m.id, + supportedReasoningEfforts: m.supportedReasoningEfforts, + defaultReasoningEffort: m.defaultReasoningEffort, + deprecated: m.deprecated, + })), + }, + }) + } + } catch {} + } + return s + }, + }, + efforts: { cmd: `${THROWAWAY_HOME} droid exec --reasoning-effort __bogus__ x` }, + models: { cmd: `${THROWAWAY_HOME} droid exec --model __bogus__ x` }, + }, + parse: ({ catalog, efforts, models }) => { + let rows = null + for (const line of text(catalog).split("\n")) { + let o + try { + o = JSON.parse(line) + } catch { + continue + } + if (o?.type === "response" && Array.isArray(o.result?.availableModels)) { + rows = o.result.availableModels + .filter((m) => m?.id) + .map((m) => { + const eff = orderEfforts(m.supportedReasoningEfforts) + return { + id: String(m.id), + ...(eff ? { efforts: eff } : {}), + ...(m.defaultReasoningEffort ? { default: m.defaultReasoningEffort } : {}), + ...(m.deprecated ? { deprecated: true } : {}), + } + }) + break + } + } + // The live union, not the syntactic enum: droid accepts nine words at the + // flag and then silently coerces one no model supports (`dynamic`, today) to + // the model's default, exit 0. A word no model takes is not a choice. + const live = unionEfforts(rows) + if (live) return { efforts: live, models: rows } + const e = enumAfter(text(efforts), /Allowed values:\s*([^\n]+)/) + if (!e) return null + const ids = text(models).match(/Available built-in models:\s*\n\s*([^\n]+)/)?.[1] + return { efforts: e, models: ids ? ids.split(",").map((s) => ({ id: s.trim() })).filter((m) => m.id) : null } + }, + }, + amp: { + needs: "modes", + modelsFrom: "amp plugins show-agent-options --json", + probes: { + version: { cmd: "amp --version" }, + options: { + cmd: "amp plugins show-agent-options --json", + trim: (s) => + JSON.stringify({ + models: (JSON.parse(s).models || []).map((m) => ({ + id: m.id, + capabilities: m.capabilities?.efforts ? { efforts: m.capabilities.efforts } : {}, + })), + }), + }, + help: { cmd: "amp --help" }, + // amp's own --reasoning-effort vocabulary, from its rejection. Recorded, and + // kept APART from the modes: `ultra` is a mode only, `xhigh`/`max` efforts + // only, and `low|medium|high` mean different things in each. Never unioned. + efforts: { cmd: "amp config model-providers check-access --provider-model a/b --reasoning-effort bogus-zzz" }, + }, + parse: ({ options, help, efforts }) => { + // "Set the agent mode (low, medium, high, ultra, or a plugin mode by key or + // label, case-insensitive)": the bare words up to the first phrase. + const inParens = text(help).match(/--mode [\s\S]*?\(([^)]*)\)/)?.[1] + const modes = [] + for (const w of (inParens || "").split(",").map((s) => s.trim())) { + if (!/^[a-z]+$/.test(w)) break + modes.push(w) + } + if (!modes.length) return null + const j = json(options) + const models = Array.isArray(j?.models) + ? j.models + .filter((m) => m?.id) + .map((m) => { + const eff = orderEfforts(m.capabilities?.efforts) + return { id: String(m.id), ...(eff ? { efforts: eff } : {}) } + }) + : null + return { modes, efforts: enumAfter(text(efforts), /Allowed choices are\s*([^.\n]+)/), models } + }, + }, + prime: { + needs: "efforts", + modelsFrom: "prime-agent model list", + probes: { + version: { cmd: "prime-agent --version" }, + help: { cmd: "prime-agent --help" }, + models: { cmd: "prime-agent model list" }, + }, + parse: ({ help, models }) => { + const e = enumAfter(text(help), /--thinking \s+Set reasoning:\s*([^\n]+)/) + if (!e) return null + // A table: `provider model context max-out thinking images`. The id is + // spelled the way the pin is: `/`, split at the first slash. + const rows = [] + for (const line of text(models).split("\n")) { + const m = line.match(/^([a-z0-9][a-z0-9._-]*)\s+(\S+)\s+\S+\s+\S+\s+(yes|no)\b/i) + if (!m || m[1] === "provider") continue + rows.push({ id: `${m[1]}/${m[2]}`, ...(m[3].toLowerCase() === "yes" ? { reasoning: true } : {}) }) + } + return { efforts: e, models: rows.length ? rows : null } + }, + }, + muse: { + needs: "efforts", + modelsFrom: null, + probes: { + version: { cmd: "muse --version" }, + help: { cmd: "muse exec --help" }, + // clap validates the value before anything else happens: "invalid value + // '__bogus__' … expected none|minimal|…". No config, no network. + oracle: { cmd: "muse --reasoning-effort __bogus__" }, + }, + parse: ({ help, oracle }) => { + const e = enumAfter(text(oracle), /expected\s+([a-z|]+)/) || enumAfter(text(help), /reasoning effort:\s*([a-z|]+)/i) + return e ? { efforts: e, models: null } : null + }, + }, + opencode: { + needs: null, + modelsFrom: null, + probes: { version: { cmd: "opencode --version" } }, + // Reasoning is a per-model VARIANT picked in its UI, not a setting the plugin + // writes (src/effort.js gives it the generic scale); its model universe is all + // of models.dev. Only the version is worth reading. + parse: () => ({ efforts: null, models: null }), + }, +} + +/** + * Read one harness's probe results into a catalog row, or null when they cannot be + * trusted (no version, or the field the row needs is missing). Pure: takes results, + * never a spawner. A null here is what makes `mergeCatalog` keep the committed row. + * + * @param {string} id key into CATALOG_PROBES + * @param {object} results { : { stdout, stderr, status } } + * @returns {{version:string, efforts:string[]|null, models:object[]|null, modelsFrom:string|null, modes?:string[]}|null} + */ +export function interpretCatalog(id, results = {}) { + const spec = CATALOG_PROBES[id] + if (!spec) return null + const version = versionIn(text(results.version)) + if (!version) return null + let parsed + try { + parsed = spec.parse(results) + } catch { + parsed = null + } + if (!parsed) return null + const row = { + version, + efforts: parsed.efforts ?? null, + ...(parsed.modes ? { modes: parsed.modes } : {}), + models: parsed.models ?? null, + modelsFrom: parsed.models ? spec.modelsFrom : null, + } + if (spec.needs && !row[spec.needs]?.length) return null + return row +} + +/** A row without its provenance, for comparing what the CLI said across runs. */ +export function stripMeta(row) { + if (!row) return null + const { probedFrom, probedAt, template, stale, ...rest } = row + return rest +} + +/** + * The next committed catalog: every harness the probes cover, taken from `fresh` + * where a row was read and kept from `committed` where it was not. + * + * `fresh[id]` is `{ row, reason, probedFrom?, probedAt? }`: `row` null means the + * probe failed and `reason` says how; the two provenance fields, when given, say + * where THAT row came from (a fixture replay carries the box's, not "fixture"). A + * failed probe never blanks a row: an unauthenticated run of the script would + * otherwise empty every scale and turn every user's pin invalid at once. The kept + * row is marked `stale: { since, reason }` so the report and the docs can say so. + * + * @param {object} committed the current HARNESS_CATALOG + * @param {object} fresh { : { row, reason, probedFrom, probedAt } } + * @param {object} meta { probedFrom, probedAt, templates: { : { name, buildID, updatedAt } } } + */ +export function mergeCatalog(committed = {}, fresh = {}, meta = {}) { + const out = {} + const now = meta.probedAt ?? new Date().toISOString() + for (const id of Object.keys(CATALOG_PROBES)) { + const got = fresh[id] + const template = meta.templates?.[id] ?? committed[id]?.template ?? null + if (got?.row) { + out[id] = { ...got.row, probedFrom: got.probedFrom ?? meta.probedFrom ?? "box", probedAt: got.probedAt ?? now, template } + } else if (committed[id]) { + const { stale, ...kept } = committed[id] + out[id] = got ? { ...kept, template, stale: { since: now, reason: got.reason || "probe failed" } } : committed[id] + } + } + return out +} + +/** What changed between two rows of the same harness, as short phrases; [] when nothing did. */ +export function diffRow(before, after) { + const a = stripMeta(before) + const b = stripMeta(after) + if (!a) return b ? ["new row"] : [] + if (!b) return ["row gone"] + const out = [] + if (a.version !== b.version) out.push(`version ${a.version} → ${b.version}`) + const list = (name, x, y) => { + const xs = x || [] + const ys = y || [] + const plus = ys.filter((v) => !xs.includes(v)) + const minus = xs.filter((v) => !ys.includes(v)) + if (plus.length || minus.length) out.push(`${name} ${[...plus.map((v) => `+${v}`), ...minus.map((v) => `-${v}`)].join(" ")}`) + else if (xs.join(",") !== ys.join(",")) out.push(`${name} reordered`) + } + list("efforts", a.efforts, b.efforts) + if (a.modes || b.modes) list("modes", a.modes, b.modes) + const ids = (ms) => (ms || []).map((m) => m.id) + const am = ids(a.models) + const bm = ids(b.models) + const plus = bm.filter((v) => !am.includes(v)) + const minus = am.filter((v) => !bm.includes(v)) + if (plus.length || minus.length) { + const show = (xs, sign) => xs.slice(0, 4).map((v) => `${sign}${v}`).join(" ") + (xs.length > 4 ? ` ${sign}${xs.length - 4} more` : "") + out.push(`models ${[show(plus, "+"), show(minus, "-")].filter(Boolean).join(" ")}`) + } else { + const changed = (b.models || []).filter((m) => { + const o = (a.models || []).find((x) => x.id === m.id) + return o && ((o.efforts || []).join(",") !== (m.efforts || []).join(",") || o.default !== m.default) + }) + if (changed.length) out.push(`per-model efforts changed: ${changed.slice(0, 4).map((m) => m.id).join(", ")}${changed.length > 4 ? ` +${changed.length - 4} more` : ""}`) + } + return out +} + +// --- what gets written ------------------------------------------------------------- + +/** Source of the committed module. JSON with 2-space indent, one export, a header that says who writes it. */ +export function renderGeneratedModule(catalog, { probedAt = new Date().toISOString() } = {}) { + const head = [ + "// GENERATED by scripts/harness-catalog.mjs. Do not edit, rerun:", + "// npm run catalog -- --write (boots each template, the authoritative read)", + "// npm run catalog -- --from=fixture --write (re-derives from test/fixtures/catalog)", + "//", + "// What each harness's CLI accepts, read off the binary in its E2B template: the", + "// effort words a pin may say (`efforts`; amp's `modes`), the model ids it lists", + "// (`models`, each with its own efforts and default where the CLI says so), the", + "// version that answered, and the template build it was read from. src/effort.js", + "// derives EFFORT_SCALES from this; config/config.example.toml's table is rendered", + "// from it. Parse rules and the reasoning behind every field: src/catalog.js.", + `// Last run: ${probedAt}`, + "", + ] + return `${head.join("\n")}export const HARNESS_CATALOG = ${JSON.stringify(catalog, null, 2)}\n` +} + +export const TOML_BEGIN = "# " +export const TOML_END = "# " + +/** The first ids of `xs` that fit in `width` characters, at least one: "a, b, …". */ +const examples = (xs, width = 36) => { + const out = [] + for (const x of xs) { + if (out.length && `${[...out, x].join(", ")}, …`.length > width) break + out.push(x) + } + return out.length < xs.length ? `${out.join(", ")}, …` : out.join(", ") +} + +/** + * The comment table in config/config.example.toml, between the two sentinels: one + * line per harness with the CLI version, the model column (count and first ids, or + * why there is none) and the effort scale. Rendered as text and spliced in as + * text: the file is nearly all comments, and re-stringifying TOML would drop them. + */ +export function renderTomlBlock(catalog) { + const rows = Object.keys(CATALOG_PROBES) + .filter((id) => catalog[id]) + .map((id) => { + const r = catalog[id] + const ids = (r.models || []).map((m) => m.id) + let models + if (id === "opencode") models = 'any models.dev "/"' + else if (id === "muse") models = "(no pinnable model: use --model in your agent command)" + else if (id === "amp") models = ids.length ? `${ids.length} listed, no pin route: ${examples(ids, 24)}` : "(no pin route)" + else models = ids.length ? `${ids.length}: ${examples(ids)}` : "(not enumerated)" + let reasoning + if (r.modes) reasoning = `--mode ${r.modes.join(" | ")} (amp's knob is the mode)` + else if (r.efforts) reasoning = `${r.efforts.join(" | ")}${perModel(r) ? " (per model)" : ""}` + else reasoning = "(not pinned: opencode does reasoning as per-model variants in its UI)" + const cli = `${r.version}${r.stale ? " (stale)" : ""}` + return [id, cli, models, reasoning] + }) + const w = [0, 1, 2].map((i) => Math.max(...rows.map((r) => r[i].length), ["template", "cli", "model"][i].length)) + const line = (cols) => `# ${cols[0].padEnd(w[0])} ${cols[1].padEnd(w[1])} ${cols[2].padEnd(w[2])} ${cols[3]}`.trimEnd() + const stale = Object.entries(catalog).filter(([, r]) => r.stale) + return [ + TOML_BEGIN, + "# Generated by `npm run catalog -- --write` from the CLI shipped in each template.", + "# Do not edit this block: rerun the script. Per-model detail is in", + "# src/harness-catalog.generated.js.", + line(["template", "cli", "model", "reasoning"]), + ...rows.map(line), + ...stale.map(([id, r]) => `# (${id}: kept from an earlier run, probe failed ${r.stale.since.slice(0, 10)}: ${r.stale.reason})`), + TOML_END, + ].join("\n") +} + +/** `text` with whatever sits between the sentinels replaced by `block`. Throws when the sentinels are missing. */ +export function spliceTomlBlock(text, block) { + const s = String(text) + const i = s.indexOf(TOML_BEGIN) + const j = s.indexOf(TOML_END) + if (i < 0 || j < i) throw new Error(`config.example.toml: sentinels ${TOML_BEGIN} … ${TOML_END} not found`) + return s.slice(0, i) + block + s.slice(j + TOML_END.length) +} diff --git a/src/config.js b/src/config.js index fca890f..0e1dc76 100644 --- a/src/config.js +++ b/src/config.js @@ -7,6 +7,7 @@ import { HARNESSES, readHarnessFile } from "./harnesses.js" import { modelEnv, normalizePin } from "./model-pin.js" import { AUTH_PATH, CONFIG_PATH, CONNECTIONS_DIR } from "./config-paths.js" import { applyConnection, readConnections, selectConnection } from "./connections.js" +import { harnessHint } from "./effort.js" export { AUTH_PATH, CONFIG_PATH } from "./config-paths.js" // Where the `e2b` CLI keeps its login (@e2b/cli USER_CONFIG_PATH — hardcoded @@ -79,7 +80,11 @@ const DEFAULTS = { // network and writes outside the workspace for no added safety. codex: "codex --dangerously-bypass-approvals-and-sandbox", grok: "grok --always-approve", - amp: "amp --dangerously-allow-all", + // amp has no reasoning knob; `--mode low|medium|high|ultra` is the closest + // thing, so an effort picked for an amp member (HERDR_E2B_REASONING, set by the + // pin or the picker's cell) becomes its mode. The `${…:+…}` expands in the BOX's + // shell: no effort, no flag, amp's own default mode. + amp: 'amp --dangerously-allow-all ${HERDR_E2B_REASONING:+--mode "$HERDR_E2B_REASONING"}', // Ends in `--prompt` ON PURPOSE: the fleet appends its task as one positional // argument, and opencode's default-command positional is a PROJECT DIRECTORY // (`opencode [project]`), so `opencode --auto "fix the bug"` means "cd into @@ -447,7 +452,7 @@ export function resolveEnv(cfg, template, env = {}, now = Date.now(), readFile = // harness reads plus the two shared ones the file-route pin command reads // (src/model-pin.js). Below the user's own `env` table, so a hand-written // ANTHROPIC_MODEL still wins over the pin. - ...modelEnv(template, cfg?.templateModels?.[template]), + ...modelEnv(template, cfg?.templateModels?.[template], harnessHint(template, cfg) ?? template), // The discovered rung has two halves, and forwarding wins between them: a name // resolves from THIS run's environment, while a stored value is a copy taken // whenever `e2b-box auth` last ran. When both name the same box variable the diff --git a/src/effort-cli.js b/src/effort-cli.js new file mode 100644 index 0000000..9734ac1 --- /dev/null +++ b/src/effort-cli.js @@ -0,0 +1,23 @@ +// `node src/effort-cli.js --fleet |