From 762020bfafde4303f4cebb15e219fd7f6d9ac275 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 19 Oct 2026 02:15:48 +0000 Subject: [PATCH 1/9] feat(cli): add recoverable local Prometheus teams --- SECURITY.md | 12 + docs/TESTING.md | 12 +- docs/harness-modes.md | 3 +- .../2026-10-18-prometheus-local-design.md | 366 ++++++ docs/prometheus-local.md | 146 +++ .../plans/2026-10-18-prometheus-local-v1.md | 509 ++++++++ packages/cli/README.md | 11 + packages/cli/src/cli.test.ts | 82 +- packages/cli/src/cli.ts | 108 +- packages/cli/src/guest.test.ts | 9 +- packages/cli/src/guest.ts | 14 +- packages/cli/src/process-identity.test.ts | 30 +- packages/cli/src/process-identity.ts | 23 + packages/cli/src/prometheus-protocol.test.ts | 605 +++++++++ packages/cli/src/prometheus-protocol.ts | 728 +++++++++++ packages/cli/src/prometheus-runtime.test.ts | 258 ++++ packages/cli/src/prometheus-runtime.ts | 419 +++++++ packages/cli/src/prometheus-state.test.ts | 653 ++++++++++ packages/cli/src/prometheus-state.ts | 1093 +++++++++++++++++ packages/cli/src/prometheus.test.ts | 412 +++++++ packages/cli/src/prometheus.ts | 609 +++++++++ packages/cli/src/storage.test.ts | 24 +- packages/cli/src/storage.ts | 61 +- scripts/export-manifest.json | 13 +- scripts/package-smoke.mjs | 20 +- 25 files changed, 6186 insertions(+), 34 deletions(-) create mode 100644 docs/plans/2026-10-18-prometheus-local-design.md create mode 100644 docs/prometheus-local.md create mode 100644 docs/superpowers/plans/2026-10-18-prometheus-local-v1.md create mode 100644 packages/cli/src/prometheus-protocol.test.ts create mode 100644 packages/cli/src/prometheus-protocol.ts create mode 100644 packages/cli/src/prometheus-runtime.test.ts create mode 100644 packages/cli/src/prometheus-runtime.ts create mode 100644 packages/cli/src/prometheus-state.test.ts create mode 100644 packages/cli/src/prometheus-state.ts create mode 100644 packages/cli/src/prometheus.test.ts create mode 100644 packages/cli/src/prometheus.ts diff --git a/SECURITY.md b/SECURITY.md index 08316ba..6e52f02 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -24,5 +24,17 @@ seat resumes that session with the Room's lines in its prompt and the session's own permissions. Join only Rooms you trust, or join with `--no-wake`; `sharednet serve --stop` stops all waking on the machine. +Prometheus local teams run three persistent Codex sessions in Docker against a +shared workspace. Their Room and Goal content is untrusted input with the same +authority implications. Prometheus metadata is stored in owner-only, +no-follow, atomically replaced files under the SharedNet state directory; it +must never contain API keys, member or invite tokens, Codex tokens, +`auth.json`, or secret-bearing environments and arguments. SharedNet +credentials stay in the normal config directory, while Codex authentication is +only mounted or forwarded to the role containers. Local state protection does +not sandbox the agents from the shared workspace: use only trusted Rooms and +give the container runtime no host permissions you would not give those +agents. + Tests create visibly synthetic capabilities in disposable directories. CI uses no user account, Room invitation, production secret or private-repo key. diff --git a/docs/TESTING.md b/docs/TESTING.md index f9b6a9b..05df066 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -5,7 +5,7 @@ | Export audit | `pnpm check:export` | Explicit file allowlist, private-path denylist, CLI ESM import boundaries and symlink refusal | | Types | `pnpm typecheck` | Compiled CLI source typechecks | | Tests | `pnpm test` | All client unit tests, executable Skill read examples, export checker negative cases, real-process wait test | -| Package | `pnpm test:package` | Real tarball installs offline without install scripts; exports and executable work without TS stripping; its declarations type-check in a consumer without `@types/node`; artifact SHA-256 recorded | +| Package | `pnpm test:package` | Real tarball installs offline without install scripts; exports and executable work without TS stripping; its declarations type-check in a consumer without `@types/node`; local Prometheus no-team status needs no Docker, network, or credentials; artifact SHA-256 recorded | | Secrets | `gitleaks git . --redact --no-banner` | Complete committed history, default detectors plus SharedNet capability detector | CI runs on Ubuntu with Node 22.18.0, 24.x and 26.x. The local development @@ -52,6 +52,16 @@ service's pull requests: Claude Code seats have not run live. +`prometheus-protocol.test.ts`, `prometheus-state.test.ts`, +`prometheus-runtime.test.ts`, and `prometheus.test.ts` cover policy parsing, +compiled evidence and terminal receipts, owner-only durable state, stable +three-role runtime reuse, listener status, lifecycle checkpoints, and the +completion barrier. Docker, SharedNet, and Codex are fakes in those unit tests. +The installed-package smoke runs `prometheus status` with isolated config and +state directories and no team; it proves that this local-only path needs no +credentials, Docker, network, TypeScript stripping, or install script. It does +not prove a live three-agent run. + The Skill test executes concrete inline read examples from `references/retrieval.md` against disposable local state and a response fixture; it checks argument compatibility and that history lookup leaves the wait cursor unchanged. It diff --git a/docs/harness-modes.md b/docs/harness-modes.md index feff532..c53ee16 100644 --- a/docs/harness-modes.md +++ b/docs/harness-modes.md @@ -58,7 +58,8 @@ implementation cannot silently redefine `@1`. ## Serve-backed runtime modes available in this build -The runtime family describes the serve-backed runner developed in PR #48. Its +The runtime family reserves names for the serve-backed runner first developed +in PR #48. Its three agents each own a production `sharednet serve` listener and persistent Codex thread; the runner provisions the Room and containers, observes the run, checks the result, and exports evidence. diff --git a/docs/plans/2026-10-18-prometheus-local-design.md b/docs/plans/2026-10-18-prometheus-local-design.md new file mode 100644 index 0000000..1501797 --- /dev/null +++ b/docs/plans/2026-10-18-prometheus-local-design.md @@ -0,0 +1,366 @@ +# Prometheus local v1 design + +**Status:** Approved for implementation + +**Date:** 2026-10-18 + +## Purpose + +Prometheus local v1 adds a top-level `sharednet prometheus` workflow for one +long-lived local Codex team. It gives one workspace three stable roles, reuses +their Codex homes, threads, and production `sharednet serve` listeners across +runs, and gives every run a fresh compiled Goal Room. + +The command is an opinionated local product path. It is separate from Goal +mode's research harnesses. It does not change any frozen `research/*@1` mode or +claim either reserved `runtime/*@1` mode. + +## Command surface + +Local v1 exposes four actions: + +```sh +sharednet prometheus run GOAL.md \ + --model \ + --check '' \ + --timeout 2h + +sharednet prometheus status +sharednet prometheus resume +sharednet prometheus cancel +``` + +The ordinary global `--session` and `--json` options continue to work. `run`, +`status`, `resume`, and `cancel` accept `--workspace`; it defaults to the +current directory. The workspace's real path identifies its one local team. + +The first `run` creates the team. `--model` applies to all three roles and is +recorded with the team. Later runs must name the same model; changing the model +or rebuilding the team is outside v1. `--check` and `--timeout` are required on +every new run. The timeout is a hard bound between one minute and 24 hours and +is stored as an absolute deadline, so `resume` cannot reset it. + +`run` refuses when that workspace already has an active run and points to +`status`, `resume`, and `cancel`. `resume` takes over an active run whose prior +monitor is gone. It does not create a Room, join another seat, make a new Codex +thread, repost the goal, or extend the deadline. `cancel` is idempotent and may +be issued from a second terminal while a monitor is running. + +There is no local-v1 command to add a fourth role, replace a role, discover +other teams, or operate a team on another machine. + +## Workspace contract + +`PROMETHEUS.md` is required at the workspace root before any Docker or network +work. It is UTF-8, no larger than 64 KiB, and must contain these level-two +headings exactly once, in this order, with non-empty content: + +```markdown +## Motto + +## Rules + +## Preferences + +## Handoff + +## Completion +``` + +Other prose and headings are allowed. A heading inside a fenced code block is +not a section. The parser rejects missing, duplicated, reordered, or +case-changed required headings. A run records the SHA-256 of the exact file +bytes and gives the complete current file to every role. A later run may use a +changed `PROMETHEUS.md`; the persistent threads are told that the new hash and +content supersede the earlier run's policy. + +`GOAL.md` remains the run-specific objective. It must not address `@all`, a +role name, or a remembered member id. The Goal line is deliberately +unaddressed. The runner's later kickoff is the only initial wake. + +## Fixed team + +Every workspace has exactly these roles: + +| Role | Stable seat name | Responsibility | +| --- | --- | --- | +| Coordinator | `coordinator` | Decompose the goal, issue typed work, route exact handoffs, resolve the final work, and synthesize the result. | +| Builder | `builder` | Accept the typed work, change the shared workspace, upload the final `submission` artifact, and report its result reference and digest. | +| Reviewer | `reviewer` | Independently download and inspect the exact submitted artifact, upload review evidence, and issue `verification.pass` for the current result. | + +All three roles use Codex in v1. One model is shared by the team. The number of +roles, role names, drivers, and topology are constants rather than flags. + +Each role owns a stable Docker container and persistent Docker volume. Its +volume holds its Codex home, thread, SharedNet seat files, checkpoints, and +listener logs. The host's existing Codex login is mounted read-only; it is not +copied into Prometheus state. Each role has its own SharedNet storage root, so +its seat credentials and cursor cannot be selected by another role. + +The initial bootstrap turn creates one Codex thread per role and returns +`READY`; it does no Goal work. One production `sharednet serve` process then +runs for that role without a `--rooms` filter. The listener persists after a +run and discovers the role's seat in each later Room. If a container or +listener stopped, the next action may restart it around the same persistent +home and thread. It must never silently replace a missing thread with a new +one. + +## One fresh compiled Room per run + +A new run proceeds in this order: + +1. Validate all arguments, `PROMETHEUS.md`, `GOAL.md`, owner credentials, + Docker, the saved team, and the absence of an active run. +2. Atomically reserve the active-run slot in local state. +3. Ensure the three persistent containers, threads, and listeners are healthy. +4. Create a new Room with `type: "compiled"` and a Goal whose only server end + condition is the stored hard timeout. +5. Join the three existing threads as fresh `coordinator`, `builder`, and + `reviewer` seats. Bind their existing threads and let their existing + listeners discover the new seats. +6. Wait until all three exact Room seats report `listening` under + `wake-plugins-v1`. +7. Post one runner message addressed only to the coordinator's exact member id. + +The kickoff contains the run id, Goal sequence, exact member-id roster, +`PROMETHEUS.md` content and hash, the typed-work recipe, the terminal-receipt +format, the semantic-check description, and the absolute deadline. No kickoff +uses `@all` or an addressable name. Every inter-role request, answer, correction, +and handoff uses one exact `@` supplied by that run's roster. + +The coordinator creates a run-scoped compiled work item addressed to the +builder. It requires a deliverable named `submission` and a `verified_by` +requirement naming the reviewer member id. The builder accepts it and publishes +`work.result` with the uploaded artifact. The reviewer downloads the artifact, +checks the returned SHA-256, uploads a review report, and publishes +`verification.pass` with the current `result_ref` and report artifact id. The +coordinator then publishes `work.resolve`. + +Compiled work cannot replace a result in place. If review or the semantic check +finds a defect, the coordinator creates a successor work item using +`supersedes`; the old item remains terminal evidence. A candidate is the unique +unsuperseded head of this chain. + +## Terminal receipts + +Chat remains nonbinding, including the word `DONE`. A role declares that it has +finished with a single-line structured message: + +```text +[sharednet-prometheus:v1] {"type":"terminal.receipt","run_id":"","role":"builder","revision_sha256":"<64 lowercase hex>"} +``` + +The three allowed role values are `coordinator`, `builder`, and `reviewer`. +The runner accepts a receipt only when: + +- the Room message sender is that run's saved member id for the claimed role; +- the JSON has exactly the v1 fields and values shown above; +- `run_id` is the active run; +- `revision_sha256` is the runner's current candidate revision. + +A malformed line, a forged role field, an old run id, a stale revision, free +text such as `DONE`, or a model's ordinary final response has no completion +effect. After the coordinator resolves the final head, it directly asks the +builder and reviewer for terminal receipts, then posts its own. A change to +compiled state creates a different candidate revision and makes all older +receipts stale. + +## Candidate revision + +The client keeps its protocol parsing independent of the hosted service source. +It validates the public compiled-state wire object returned by +`GET /rooms//state` and fails closed on an unknown or malformed shape. + +For the unique head work item, the client finds the canonical `work.result` +event whose event hash equals `projection.work[work_id].result`. It requires an +artifact id for every declared deliverable and a matching +`refs.artifact_receipts[artifact_id]` object with that id and a 64-character +SHA-256. It also finds, for every `verify` requirement on the head, an accepted +`verification.pass` event that: + +- was authored by the saved reviewer member id; +- names that work item and requirement; +- names the head's exact current `result_ref`; +- carries a review-evidence artifact receipt with an exact SHA-256; and +- corresponds to a requirement whose projected status is `satisfied`. + +The compiled protocol's `result_ref` is the canonical result-event hash. That +event contains the artifact ids and the service-attested artifact receipts, so +the review is transitively tied to the exact delivered hashes rather than a +filename or a mutable workspace path. + +The runner computes `revision_sha256` as SHA-256 over canonical JSON containing: + +- compiled state digest; +- head work id and `result_ref`; +- sorted deliverable name, artifact id, and artifact SHA-256 triples; and +- sorted verification event hash, requirement id, evidence artifact id, and + evidence SHA-256 tuples. + +All non-head work must be terminal. The head must be `RESOLVED`, have at least +one deliverable, and have every requirement satisfied. `OPEN`, `COMMITTED`, or +`REVIEW` work anywhere in the run keeps the barrier closed. A rejected, +cancelled, declined, or expired head is not success. + +## Completion barrier + +Prometheus never passes `said: DONE` to Goal mode and never scans raw chat for +completion. It closes a run successfully only after one candidate satisfies +all of these conditions in order: + +1. Compiled state has one resolved head, no nonterminal work, complete artifact + receipts, and reviewer-authored verification bound to the exact current + result. +2. Coordinator, builder, and reviewer each have an authenticated terminal + receipt for the computed candidate revision. +3. Each active-room seat is `listening`; none is `starting`, `working`, + `retrying`, or `stopped` in `sharednet serve --status --json`. +4. `sharednet handed --json` reports no unsettled handoff for any of the three + active-room seats. +5. The user-supplied semantic check has passed for this revision. +6. A post-check read returns the same compiled digest and candidate revision, + the same three receipts, and the same idle/no-handoff state. + +The semantic check runs only after conditions 1 through 4 hold. It runs in the +fresh checker container inherited from PR 48, with the workspace mounted at +`/workspace` and these non-secret environment variables: + +```text +SHAREDNET_PROMETHEUS_RUN_ID +SHAREDNET_PROMETHEUS_WORK_ID +SHAREDNET_PROMETHEUS_RESULT_REF +SHAREDNET_PROMETHEUS_REVISION_SHA256 +``` + +The run record stores its start and finish times, exit code, bounded output, +output SHA-256, and candidate revision. A nonzero check blocks that exact +revision and produces one bounded, exact-`@coordinator` feedback message. It is +not retried until compiled work produces a different revision. A passing check +is discarded if the post-check read sees a different revision or any new +working/unsettled turn. + +This barrier proves a coherent terminal snapshot. It does not claim that a +model understood every Room message or that the semantic check is sufficient +for every project. + +## Local state and concurrency + +The workspace key is the first 24 hex characters of SHA-256 over the +workspace's real absolute path. Non-secret Prometheus metadata lives at: + +```text +$XDG_STATE_HOME/sharednet/prometheus/local-v1// + team.json + active.json + lock.json + runs//run.json + runs//events.ndjson + runs//checks.ndjson +``` + +On systems without `XDG_STATE_HOME`, the existing SharedNet default +`~/.local/state/sharednet` applies. Directories are mode `0700`, files are mode +`0600`, writes are atomic, and reads reject symlinks, non-owner files, and +group/other permissions. The short-held state lock records a random lock id, +PID, and OS process start identity; PID reuse cannot authorize stale-lock +recovery. + +`team.json` stores schema version, workspace path/key, fixed model, stable +container and volume names, three thread ids, policy hash last seen, and +timestamps. `run.json` stores the run configuration, Room id, role member ids, +absolute deadline, monitor lease, outcome, candidate, and check summaries. +`active.json` contains only the active run id and monitor identity. None of +these files stores a SharedNet API key, instance token, member token, invite, +Codex token, or copied `auth.json`. + +Existing credential owners remain responsible for credentials: the host CLI +reads its ordinary owner-only SharedNet credential/session files, the invite is +passed only to each `join` process, each role's ordinary seat credential stays +inside that role's persistent home, and Codex reads the host login through a +read-only mount. Logs redact known capability formats and never serialize +process environments or Docker arguments containing a secret. Agent-generated +Room or tool content is still sensitive and the entire state root should be +treated as private. + +Only state mutation holds the short lock. A monitor owns a renewable lease in +`active.json`, so `status` can read and `cancel` can set `cancel_requested` +without waiting for a long-running process. A second `run` always loses the +atomic active-slot race. `resume` may replace a monitor lease only after the +recorded process identity is no longer alive. + +## Outcomes and recovery + +A run has one of these persisted states: + +| State | Meaning | +| --- | --- | +| `starting` | The active slot is reserved and the fresh Room/team seats are being prepared. | +| `running` | The kickoff was posted and the completion monitor is active or resumable. | +| `succeeded` | The full completion barrier passed and the Room was closed with the candidate revision. | +| `timed_out` | The original absolute deadline arrived before success. | +| `cancelled` | An explicit `prometheus cancel` closed the Room. | +| `failed` | Startup, protocol, checker, listener, or Room state made safe continuation impossible. | + +`succeeded`, `timed_out`, `cancelled`, and `failed` are terminal. The terminal +record includes `finished_at` and a bounded reason. Clearing `active.json` is +the last state transition; a crash before that is repaired idempotently by +`status` or the next command. + +SIGINT, SIGTERM, loss of the terminal, or a monitor crash does not cancel a +run, remove the team containers, close the Room, or clear the active slot. The +monitor lease becomes stale and `resume` continues from `run.json`, the Room, +compiled state, listener status, and handed records. Startup is step-recorded, +so resume can distinguish a Room that was never created, a Room created before +seats joined, and a kickoff already committed; it never repeats a committed +external write with a new idempotency key. + +Timeout and cancel attempt to close the Room, write the terminal record, and +leave the persistent team healthy for its next fresh Room. An unexpected Room +closure is `failed` unless its recorded close trigger is the stored timeout or +cancel request. Semantic-check failure alone is not `failed`; it blocks the +candidate and lets the team create a successor until timeout or cancellation. + +## PR 48 and existing modes + +PR 48 (`feat/serve-goal-demo`) is rebased onto the current client before the +Prometheus work. Its reviewed real-listener ordering, exact-member kickoff, +Codex bootstrap, checker container, auth boundary, and fake-Docker tests become +the starting point. The implementation then extracts reusable runtime helpers +instead of copying the 519-line runner. + +`goal serve-demo` keeps its one-run experimental behavior and cleanup contract. +Prometheus uses stable role containers, all-room listeners, durable state, and +resume semantics through the extracted layer. Refactoring must preserve the PR +48 tests before adding Prometheus behavior. + +The following compatibility constraints are absolute: + +- The six `research/{baseline,mentions,tasks,both,routing-only,receipt-only}@1` + specs, profiles, hashes, and behavior do not change. +- `runtime/serve-first@1` and `runtime/serve-all@1` remain reserved and + unavailable through `goal run --harness-mode`. Prometheus does not execute or + report either mode. +- Immutable registry provenance, including the reviewed PR 48 dependency head, + is not rewritten to describe Prometheus. +- The mutable catalog text may say that PR 48 code is present while the two + runtime modes remain unwired; it must not present them as runnable. +- Existing `goal run`, `goal serve-demo`, `serve`, compiled Room commands, and + credential paths retain their public behavior. + +## Deferred work + +Local v1 deliberately defers: + +- a Prometheus Cloud control plane or hosted durable runner; +- global team discovery, team listing, naming, sharing, or marketplace flows; +- cross-machine role placement, synchronization, or failover; +- variable team size, custom roles, non-Codex drivers, or per-role models; +- simultaneous runs for one team or scheduling/queuing multiple goals; +- automatic team reset, model migration, container-volume garbage collection, + or import/export of a team; +- making either reserved `runtime/*@1` harness mode available; and +- new server routes or changes to the compiled Room protocol. + +Those features require their own versioned design. Local v1 proves the fixed +three-role lifecycle and completion barrier using existing public service +contracts. diff --git a/docs/prometheus-local.md b/docs/prometheus-local.md new file mode 100644 index 0000000..ff859c3 --- /dev/null +++ b/docs/prometheus-local.md @@ -0,0 +1,146 @@ +# Local Prometheus teams + +Prometheus local v1 runs one persistent three-agent Codex team for one local +workspace. The fixed roles are `coordinator`, `builder`, and `reviewer`. Their +containers, home volumes, and Codex thread ids are reused between runs, while +every run gets a fresh compiled SharedNet Room and fresh Room member ids. + +This is a local CLI runtime. It is not the hosted Prometheus control plane, an +elastic agent pool, or either reserved `runtime/*@1` research harness. + +## Prerequisites + +- Node.js 22.18 or newer and this build of the `sharednet` CLI; +- Docker, with permission to create containers and named volumes; +- a SharedNet login for creating the Room and Goal; +- a working Codex login or `OPENAI_API_KEY` for the three role containers; and +- a real workspace directory containing `PROMETHEUS.md` and the Goal file. + +The configured SharedNet base URL must be reachable from both the host CLI and +the containers. Local v1 does not rewrite or bridge a loopback-only +`localhost` server address into Docker. + +`PROMETHEUS.md` is capped at 64 KiB and must contain these non-empty level-two +sections exactly once, in this order and case: + +```markdown +## Motto +Build the smallest verified result. + +## Rules +Use exact member-id handoffs and keep compiled work current. + +## Preferences +Prefer tests and artifacts over chat assurances. + +## Handoff +Name the requested work, evidence, and recipient. + +## Completion +Require the reviewed artifact, terminal receipts, and semantic check. +``` + +The Goal file must not address `@all`, a fixed role name, or a remembered +member id. Prometheus sends one exact kickoff to the new Room's coordinator. + +## Preview limitations + +Local v1 records every startup checkpoint and replays unfinished external +writes with the run's original idempotency keys. It also verifies the exact +saved image, user, labels, mounts, container names, volumes, and thread markers +before reuse. A verified container left before bootstrap completion is resumed +from bootstrap; an identity mismatch terminates the run instead of adopting an +untrusted container. + +The remaining preview boundaries are operational: + +- This remains a foreground controller. A crash needs an explicit `resume`; + it does not launch a host service automatically. +- A transient Docker, listener, authentication, or SharedNet transport error + leaves the checkpoint active for repair and `resume`. Stable external-write + keys prevent a repaired resume from creating another Room, Goal, role seat, + or kickoff. +- Partially created named volumes are retained. Verified containers and saved + thread markers are reconciled, but unused volumes may require later cleanup. +- `status` always returns durable state. For an active fully seeded run it also + attempts live container, Room, listener, handoff, and barrier inspection; a + failed preflight is reported as live-unavailable rather than hiding the + durable record. + +These limits are why local v1 is a preview rather than a hosted background Goal +controller. + +## Run and operate a team + +From any directory, identify the workspace explicitly: + +```bash +sharednet prometheus run GOAL.md \ + --workspace /absolute/path/to/workspace \ + --model gpt-6.1-sol \ + --check 'python3 check.py' \ + --timeout 2h + +sharednet prometheus status --workspace /absolute/path/to/workspace --json +sharednet prometheus resume --workspace /absolute/path/to/workspace +sharednet prometheus cancel --workspace /absolute/path/to/workspace +``` + +`run` remains in the foreground while it monitors the run. Only one run may be +active for a workspace. A later run must use the saved team's model; local v1 +does not replace a persistent team with a different model. + +With no saved team, `status` returns +`{"workspace":"","team":null,"run":null}` without credentials, +Docker, or network access. With an active run it preserves that durable answer +even when live inspection cannot start. `cancel` records a durable request; a +live monitor observes it, while `cancel` itself reclaims a dead monitor and +closes the Room. `resume` is safe at every recorded startup checkpoint and +retains the original absolute deadline. + +Terminal outcomes are `succeeded`, `timed_out`, `cancelled`, and `failed`. +Timeout and cancellation do not delete the stable team containers or volumes. + +## Evidence and completion + +The coordinator assigns typed compiled work to the builder. The builder +publishes a result artifact. The reviewer must independently download and +inspect that exact artifact, publish separate review evidence, and issue a +`verification.pass` bound to the current result reference. A valid candidate +has no open work or unmet verification requirement. + +Plain chat remains nonbinding. In particular, `DONE` has no effect. Completion +also requires one revision-bound terminal receipt from every role, all three +listeners idle and healthy, no unsettled handoff, a passing semantic `--check`, +and an unchanged second read of the compiled candidate after the check. Any +new compiled revision makes older receipts stale. + +## Local state and credentials + +Metadata is keyed by the workspace's real absolute path and stored under: + +```text +$XDG_STATE_HOME/sharednet/prometheus/local-v1// +``` + +When `XDG_STATE_HOME` is unset, the platform's normal SharedNet state directory +is used. Directories and JSON files are owner-only, no-follow, and atomically +replaced. State records the fixed runtime identity, run checkpoints, Room and +member ids, evidence summaries, check output, and terminal reason. It rejects +API keys, member tokens, invite tokens, Codex tokens, copied `auth.json` +contents or paths, and secret-bearing environment or argument values. + +SharedNet credentials remain in the normal owner-only SharedNet config +directory. Codex authentication is mounted or forwarded only while preparing +the containers; it is not copied into Prometheus metadata. The agents share +the workspace and run with broad permissions inside their containers, so use a +workspace and Rooms whose content you trust. + +## Deliberate local-v1 limits + +Local v1 has exactly three local Codex roles and one active run per workspace. +It does not provide Cloud scheduling, cross-machine discovery, dynamic team +size, model migration, unattended host-process restart, or deletion of +persistent team homes. The official or hidden evaluator +must remain outside the runtime; use `--check` only for an agent-visible, +predeclared semantic verifier. diff --git a/docs/superpowers/plans/2026-10-18-prometheus-local-v1.md b/docs/superpowers/plans/2026-10-18-prometheus-local-v1.md new file mode 100644 index 0000000..1e00f62 --- /dev/null +++ b/docs/superpowers/plans/2026-10-18-prometheus-local-v1.md @@ -0,0 +1,509 @@ +# Prometheus Local v1 Implementation Plan + +> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking. + +**Goal:** Add `sharednet prometheus` as a resumable local workflow for exactly three persistent Codex roles, with a fresh compiled Goal Room and a fail-closed completion barrier for every run. + +**Architecture:** Rebase PR 48 first and extract its tested Docker, Codex bootstrap, real-listener, and checker mechanics into a reusable runtime. A protocol module validates `PROMETHEUS.md`, compiled state, artifact receipts, reviewer verification, and terminal receipts; a separate owner-only XDG store holds non-secret team/run metadata and monitor leases. The command layer creates or resumes one active run and closes it only after the same candidate revision passes typed-state, terminal-receipt, listener-idle, handed-empty, and semantic-check gates. + +**Tech Stack:** TypeScript 7, Node.js 22.18+ built-ins, Docker CLI, Codex CLI, existing SharedNet HTTP/compiled Room/serve interfaces, Vitest 4, pnpm 11. + +**Spec:** `docs/plans/2026-10-18-prometheus-local-design.md` + +## Global Constraints + +- Expose exactly `sharednet prometheus run|status|resume|cancel`; local v1 has exactly `coordinator`, `builder`, and `reviewer`, all Codex, all on one saved model. +- Require a workspace-root `PROMETHEUS.md` with non-empty `## Motto`, `## Rules`, `## Preferences`, `## Handoff`, and `## Completion` sections before Docker or network work. +- Reuse persistent role homes, Codex threads, and real `sharednet serve` listeners across runs; create one fresh compiled Goal Room per run. +- Allow one active run per real workspace path. Persist its original absolute deadline and make interruption resumable rather than destructive. +- Address only exact run member ids. Do not use `@all`, role-name routing, broadcast kickoff, runner-owned `wait`, or runner-owned `ack` for role delivery. +- Never treat raw chat, a model final response, or `DONE` as completion. Accept only authenticated v1 terminal receipts matching the current candidate revision. +- Success requires a resolved compiled head, no nonterminal typed work, service-attested artifact SHA-256 receipts, reviewer-authored `verification.pass` for the exact current `result_ref`, three terminal receipts, no working seat, no unsettled handed wake, and a semantic check followed by an unchanged re-read. +- Store Prometheus metadata beneath `$XDG_STATE_HOME/sharednet/prometheus/local-v1`; use owner-only, no-follow, atomic files and never persist API keys, instance/member tokens, invites, Codex tokens, copied `auth.json`, or secret-bearing environments/arguments. +- Preserve the six frozen `research/*@1` profiles and hashes exactly. Keep `runtime/serve-first@1` and `runtime/serve-all@1` unavailable and do not report Prometheus as either mode. +- Keep the published CLI at zero runtime dependencies. Add every intentional file to `scripts/export-manifest.json`; do not import hosted service or protocol source. +- Keep PR 48's `goal serve-demo` behavior and tests while extracting shared implementation. Do not merge, publish, or change repository visibility as part of this plan. +- Deferred: Cloud, global team discovery, cross-machine operation/failover, variable team size, custom roles/drivers, concurrent runs, reset/migration, and runtime-mode activation. + +## Review Focus + +- Missing, duplicated, reordered, fenced, oversized, or case-changed `PROMETHEUS.md` sections must fail before Docker/network calls; Task 2 covers parsing and Task 5 proves CLI ordering. +- Concurrent `run` calls, PID reuse, a live monitor, and a dead monitor must never create two active Rooms or let `resume` steal a live lease; Task 3 pins the lock/lease races. +- Malformed compiled projections, missing artifact receipts, stale `result_ref`, forged reviewer actors, and ambiguous successor heads must keep the barrier closed; Task 2 covers each wire shape. +- Raw `DONE`, stale/forged terminal receipts, a revision change during a passing check, and a second check on a known-failing revision must never close the Room; Task 5 exercises all four cases. +- A `working`, `retrying`, or stopped seat, an unsettled handed wake, an interrupted monitor, and a previously posted kickoff must remain safe across `resume`; Tasks 4 and 5 test those lifecycle edges. + +--- + +### Task 1: Rebase PR 48 and preserve the frozen registry + +**Files:** +- Rebase/retain: `packages/cli/src/goal-serve-demo.ts` +- Rebase/retain: `packages/cli/src/goal-serve-demo.test.ts` +- Rebase/retain: `packages/cli/src/cli.ts` +- Rebase/retain: `packages/cli/src/cli.test.ts` +- Rebase/retain: `packages/cli/src/goal-run.ts` +- Rebase/retain: `packages/cli/src/goal-run.test.ts` +- Rebase/retain: `packages/cli/src/goal.ts` +- Rebase/retain: `packages/cli/src/goal.test.ts` +- Rebase/retain: `docs/goal-mode.md` +- Rebase/retain: `docs/superpowers/plans/2026-10-14-serve-goal-demo.md` +- Modify: `scripts/export-manifest.json` +- Verify unchanged: `packages/cli/src/harness-modes.ts` +- Verify unchanged: `packages/cli/src/harness-modes.test.ts` +- Verify unchanged: `docs/harness-modes.md` + +**Interfaces:** +- Consumes: PR 48 commits `2d4cc36`, `e77597d`, `10aecb6`, and `8154322`; current `feat/prometheus-local-v1` based on `1f0ef66`. +- Produces: rebased `parseServeDemoAgents(raw)`, `assertServeDemoGoalUnaddressed(content)`, `serveKickoff(mode, seats, goalSequence)`, `serveSessionPrompt(seat, seats, until, kickoff)`, and `runServeGoalDemo(client, memberToken, apiKey, input, dependencies)` with their existing behavior. +- Preserves: all values returned by `harnessModeSpecHash`, and `resolveGoalRunHarnessMode("runtime/serve-first@1")` / `serve-all` continuing to throw `harness_mode_unavailable` before side effects. + +- [ ] **Step 1: Record the pre-rebase registry contract** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/harness-modes.test.ts packages/cli/src/cli.test.ts +``` + +Expected: PASS on the six research profiles, their exact hashes, and both unavailable runtime reservations. + +- [ ] **Step 2: Rebase the existing PR 48 worktree onto this feature branch** + +Run: + +```bash +git -C /mnt/data0/xisen/workspace/repos/sharednet-client.worktrees/serve-goal-demo rebase feat/prometheus-local-v1 +``` + +Resolve conflicts additively: retain the current harness-mode imports, validation, provenance, episode fields, read-only Codex-auth mount/copy fix, and export entries; then retain PR 48's `goal serve-demo` command and files. Do not change registry semantics or hashes. + +- [ ] **Step 3: Fast-forward the Prometheus branch to the rebased PR head** + +Run in this worktree: + +```bash +git merge --ff-only feat/serve-goal-demo +``` + +Expected: the four PR 48 commits sit after the Prometheus design commit; no merge commit. + +- [ ] **Step 4: Run the rebased focused suite** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/goal-serve-demo.test.ts packages/cli/src/goal-run.test.ts packages/cli/src/goal.test.ts packages/cli/src/harness-modes.test.ts packages/cli/src/cli.test.ts +``` + +Expected: PASS, including listener-before-goal ordering, exact kickoff ids, cleanup, immutable research hashes, and early runtime-mode refusal. + +- [ ] **Step 5: Check the rebase diff** + +Run: + +```bash +git diff --check 1f0ef66..HEAD +git diff 1f0ef66..HEAD -- packages/cli/src/harness-modes.ts packages/cli/src/harness-modes.test.ts +``` + +Expected: no whitespace errors; registry changes are conflict resolution only and preserve all frozen values. + +### Task 2: Parse policy, compiled evidence, and terminal receipts + +**Files:** +- Create: `packages/cli/src/prometheus-protocol.ts` +- Create: `packages/cli/src/prometheus-protocol.test.ts` +- Modify: `packages/cli/src/compiled.ts` + +**Interfaces:** +- Consumes: `CompiledState` from `compiled.ts` and Room messages shaped as `{ sequence, content, sender_instance_id?, sender?: { member_id? } }`. +- Produces: `PROMETHEUS_ROLES`, `type PrometheusRole`, `parsePrometheusPolicy(raw: string): PrometheusPolicy`, `assertPrometheusGoalUnaddressed(raw: string): void`, `parsePrometheusCandidate(state: CompiledState, reviewerMemberId: string): CandidateResult`, `parseTerminalReceipts(messages: readonly PrometheusMessage[], roster: PrometheusRoster, runId: string, revisionSha256: string): TerminalReceiptResult`. +- `PrometheusPolicy` is `{ raw, sha256, sections: Record<"Motto" | "Rules" | "Preferences" | "Handoff" | "Completion", string> }`. +- A successful `CandidateResult` is `{ ok: true, candidate: { stateDigest, workId, resultRef, artifacts, verifications, revisionSha256 } }`; a refusal is `{ ok: false, reasons: string[] }` and never throws for merely incomplete work. + +- [ ] **Step 1: Write failing policy and goal tests** + +Add tests named: + +```ts +it("parses the five ordered nonempty policy sections outside fences and hashes exact bytes", () => { /* assert exact sections and SHA-256 */ }); +it.each(["missing", "duplicate", "reordered", "wrong-case", "empty", "oversized"])("rejects %s PROMETHEUS.md before side effects", ...); +it("rejects goals that address all, a fixed role name, or a remembered member id", () => { /* assert invalid_arguments */ }); +``` + +- [ ] **Step 2: Run the policy tests to verify they fail** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/prometheus-protocol.test.ts -t 'policy|goals' +``` + +Expected: FAIL because `prometheus-protocol.ts` does not exist. + +- [ ] **Step 3: Implement the policy and unaddressed-goal parsers** + +Use a fence-aware line scan with a 64 KiB byte limit and exact case-sensitive level-two headings. Hash the untouched UTF-8 bytes. Reject `@all`, `@coordinator`, `@builder`, `@reviewer`, and `@i_…`/`@mem_…` in the goal. + +- [ ] **Step 4: Write failing candidate tests with public wire fixtures** + +Build minimal `CompiledState` fixtures and assert: + +```ts +it("derives one revision from a resolved successor head, exact result artifact receipts, and reviewer passes", ...); +it.each(["nonterminal-work", "ambiguous-head", "rejected-head", "missing-result-event", "missing-artifact-receipt", "bad-artifact-sha256", "stale-result-ref", "wrong-reviewer", "missing-review-evidence", "unsatisfied-requirement"])("keeps the candidate closed for %s", ...); +it("sorts work artifacts and verification evidence before hashing the revision", ...); +``` + +The passing fixture must include `work.result.refs.artifacts`, matching `refs.artifact_receipts`, a reviewer `verification.pass` whose `refs.result_ref` equals the projected result, and an evidence receipt. + +- [ ] **Step 5: Run the candidate tests to verify they fail** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/prometheus-protocol.test.ts -t 'candidate|revision' +``` + +Expected: FAIL because the candidate parser is absent. + +- [ ] **Step 6: Implement strict standalone wire parsing and canonical revision hashing** + +Do not import from the service/protocol repository. Validate objects, arrays, ids, event hashes, and lowercase 64-hex artifact SHA-256 values before use. Select the unique unsuperseded head; require every other item terminal and the head `RESOLVED`; bind every verify requirement to the saved reviewer and exact current result. + +- [ ] **Step 7: Write and pass terminal-receipt tests** + +Test one valid receipt from each saved sender and rejection of raw `DONE`, ordinary final text, malformed/extra JSON fields, a forged role, stale run id, stale revision, duplicate older receipts, and a receipt from an unknown member. The newest valid receipt per role wins. + +Run: + +```bash +pnpm exec vitest run packages/cli/src/prometheus-protocol.test.ts +``` + +Expected: PASS. + +- [ ] **Step 8: Commit the protocol boundary** + +```bash +git add packages/cli/src/prometheus-protocol.ts packages/cli/src/prometheus-protocol.test.ts packages/cli/src/compiled.ts +git commit -m "feat(cli): validate Prometheus completion evidence" +``` + +### Task 3: Add owner-only team and run state + +**Files:** +- Create: `packages/cli/src/prometheus-state.ts` +- Create: `packages/cli/src/prometheus-state.test.ts` +- Modify: `packages/cli/src/storage.ts` +- Modify: `packages/cli/src/storage.test.ts` +- Modify: `packages/cli/src/process-identity.ts` +- Test: `packages/cli/src/process-identity.test.ts` + +**Interfaces:** +- Consumes: `getStoragePaths(env).stateDir`, `ProcessProbe`, real workspace path, and `PrometheusRole`. +- Produces: `prometheusPaths(env, workspace): Promise`, `readTeam(paths): Promise`, `writeTeam(paths, state): Promise`, `readRun(paths, runId): Promise`, `writeRun(paths, state): Promise`, `reserveActiveRun(paths, state, probe): Promise`, `claimMonitor(paths, runId, probe): Promise`, `requestCancel(paths, runId): Promise`, `finishRun(paths, finalState, lease): Promise`. +- `PrometheusTeamState` schema v1 fixes workspace key/path, model, three container/volume/thread records, last policy hash, and timestamps. +- `PrometheusRunState` schema v1 holds configuration, startup checkpoint, Room/roster, deadline, monitor, outcome, candidate/check summaries, and never a credential. +- `readOwnerOnlyFile`, `writeOwnerOnlyJson`, and `ensureOwnerOnlyDirectory` become exported storage primitives with their existing no-follow/ownership semantics. + +- [ ] **Step 1: Write failing path, mode, symlink, and redaction tests** + +Assert the state root is `/prometheus/local-v1/`, directories are `0700`, files are `0600`, atomic replacement works, symlink/group-readable targets fail, malformed schema fails, and serialized team/run fixtures contain none of `snk_`, `sni_`, `rit_`, `member_token`, `api_key`, `invite`, `OPENAI_API_KEY`, or `auth.json` content. + +- [ ] **Step 2: Run state tests to verify they fail** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/prometheus-state.test.ts packages/cli/src/storage.test.ts +``` + +Expected: FAIL on missing state APIs. + +- [ ] **Step 3: Export the existing secure-file primitives and implement schemas** + +Keep credential callers unchanged. Validate every enum, id, absolute path, timestamp, role key, and terminal transition on read; use temporary file, `fsync`, chmod, atomic rename, and parent-directory `fsync` on write. + +- [ ] **Step 4: Write failing one-active-run and monitor-lease races** + +Use concurrent promises plus a fake `ProcessProbe` to assert: + +```ts +it("lets exactly one concurrent caller reserve the active slot", ...); +it("refuses a second run while an active or resumable run exists", ...); +it("does not let resume replace a live monitor or a reused pid", ...); +it("lets resume replace a dead exact process identity without changing the deadline", ...); +it("lets cancel set cancel_requested while the monitor lease is live", ...); +it("clears active only after the matching lease writes a terminal run", ...); +``` + +- [ ] **Step 5: Implement the short lock and renewable monitor lease** + +The lock record is `{ schema_version: 1, lock_id, pid, start_identity, created_at }`. Recover it only when the same process identity is provably dead; never treat a live/reused PID as the owner. Hold the lock only around local state reads/writes. + +- [ ] **Step 6: Run focused state tests** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/prometheus-state.test.ts packages/cli/src/storage.test.ts packages/cli/src/process-identity.test.ts +``` + +Expected: PASS. + +- [ ] **Step 7: Commit secure state** + +```bash +git add packages/cli/src/prometheus-state.ts packages/cli/src/prometheus-state.test.ts packages/cli/src/storage.ts packages/cli/src/storage.test.ts packages/cli/src/process-identity.ts packages/cli/src/process-identity.test.ts +git commit -m "feat(cli): persist local Prometheus state" +``` + +### Task 4: Refactor PR 48 into a persistent three-role runtime + +**Files:** +- Create: `packages/cli/src/prometheus-runtime.ts` +- Create: `packages/cli/src/prometheus-runtime.test.ts` +- Modify: `packages/cli/src/goal-serve-demo.ts` +- Modify: `packages/cli/src/goal-serve-demo.test.ts` +- Modify: `packages/cli/src/goal-run.ts` + +**Interfaces:** +- Consumes: `DockerRunner`, `RunPlan`/image/auth preflight pieces from `goal-run.ts`, `PrometheusTeamState`, `PrometheusRunState`, and PR 48's tested container mechanics. +- Produces: `ensurePrometheusTeam(input: EnsureTeamInput, deps: PrometheusRuntimeDependencies): Promise`, `attachPrometheusRoom(team, input: AttachRoomInput, deps): Promise`, `inspectPrometheusRuntime(team, roomId, deps): Promise`, `runPrometheusCheck(team, check, env, deps): Promise`. +- `PrometheusRoster` is an exact record keyed by the three roles with `{ memberId, threadId, containerName }`. +- `PrometheusRuntimeStatus` reports each active seat's serve state and room-scoped `unanswered` handed entries; no caller parses Docker stdout itself. + +- [ ] **Step 1: Write failing stable-team and second-run tests** + +Starting from PR 48's fake Docker, assert the first call creates exactly three stable containers/volumes, bootstraps exactly three threads, starts one unfiltered `sharednet serve` per role, and stores no secret in Docker args or Prometheus state. A second Room call must reuse the same containers, volumes, thread ids, and listener processes while joining three fresh member ids; it must not issue another `codex exec` bootstrap or another listener start. + +- [ ] **Step 2: Run the runtime tests to verify they fail** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/prometheus-runtime.test.ts +``` + +Expected: FAIL because the persistent runtime does not exist. + +- [ ] **Step 3: Extract PR 48's reusable mechanics** + +Move shared Docker exec, wrapper, host-forwarder, Codex-config, join/wake, listener-health, and checker code behind `prometheus-runtime.ts`. Leave thin adapters in `goal-serve-demo.ts` so its per-run names, `first|all` kickoff, copy-out, signal cleanup, and removal behavior stay unchanged. + +- [ ] **Step 4: Implement stable roles and listener reuse** + +Use fixed role constants and workspace-keyed container/volume names. Mount the host Codex login read-only and keep credential bytes out of XDG state. Run each role under the host numeric uid/gid. Start `sharednet serve` with no `--rooms` filter and require the exact new seat to reach `listening` before returning. Refuse a missing/mismatched saved thread rather than bootstrapping over it. + +- [ ] **Step 5: Add runtime-status and checker tests** + +Assert `inspectPrometheusRuntime` distinguishes `listening`, `working`, `retrying`, and `stopped`, filters handed records by active Room/seat, and returns malformed status as unhealthy. Assert the checker gets only the four documented non-secret environment variables and bounded output. + +- [ ] **Step 6: Run PR 48 and runtime regression suites** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/prometheus-runtime.test.ts packages/cli/src/goal-serve-demo.test.ts packages/cli/src/goal-run.test.ts +``` + +Expected: PASS; PR 48 still removes only containers it created, while Prometheus leaves stable team containers and volumes running. + +- [ ] **Step 7: Commit the runtime refactor** + +```bash +git add packages/cli/src/prometheus-runtime.ts packages/cli/src/prometheus-runtime.test.ts packages/cli/src/goal-serve-demo.ts packages/cli/src/goal-serve-demo.test.ts packages/cli/src/goal-run.ts +git commit -m "refactor(cli): reuse persistent Codex team runtime" +``` + +### Task 5: Add the CLI, completion barrier, and recovery lifecycle + +**Files:** +- Create: `packages/cli/src/prometheus.ts` +- Create: `packages/cli/src/prometheus.test.ts` +- Modify: `packages/cli/src/cli.ts` +- Modify: `packages/cli/src/cli.test.ts` +- Modify: `packages/cli/src/prometheus-protocol.ts` +- Modify: `packages/cli/src/prometheus-protocol.test.ts` +- Modify: `packages/cli/src/prometheus-state.ts` +- Modify: `packages/cli/src/prometheus-state.test.ts` + +**Interfaces:** +- Consumes: protocol, state, and runtime interfaces from Tasks 2-4; `ApiClient`, `startGoal`, `readGoalFile`, `parseDuration`, `resolveApiKey`, and the selected owner session. +- Produces: `prometheusCommand(action: string | undefined, args: string[], context: PrometheusCommandContext): Promise`, `startPrometheusRun(input, deps): Promise`, `evaluatePrometheusBarrier(input): PrometheusBarrierResult`, `monitorPrometheusRun(run, deps): Promise`, `resumePrometheusRun(workspace, deps)`, `cancelPrometheusRun(workspace, deps)`, and `prometheusStatus(workspace, deps)`. +- CLI syntax: `prometheus run GOAL.md --model MODEL --check COMMAND --timeout DURATION [--workspace DIR]`; `status`, `resume`, and `cancel` accept only `--workspace`. +- `PrometheusBarrierResult` is `{ ready: false, reasons, revisionSha256? }` or `{ ready: true, candidate, receipts }`. Terminal run outcomes are `succeeded | timed_out | cancelled | failed`. + +- [ ] **Step 1: Write failing early-validation and fresh-run tests** + +For missing/duplicate options, bad duration, missing or malformed policy, +addressed goal, missing workspace/login/Docker, saved-model mismatch, and an +active run, assert nonzero exit, no HTTP requests, and no Docker calls. Add one +successful test that asserts this exact order: reserve active state; ensure +team; create `{ type: "compiled" }` Room; start a Goal with only the stored +`after` bound; mint invite; attach the three existing roles; observe all exact +seats `listening`; post one kickoff containing only +`@`. Assert no runner `wait` or `ack`. + +- [ ] **Step 2: Run the CLI/start tests to verify they fail** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/cli.test.ts packages/cli/src/prometheus.test.ts -t 'prometheus|fresh compiled run' +``` + +Expected: FAIL with `unknown_command` or missing Prometheus functions. + +- [ ] **Step 3: Implement top-level parsing and checkpointed startup** + +Route `resource === "prometheus"` without changing global `--session` or +credential rules. Validate all local inputs before side effects. Persist +`reserved`, `room_created`, `goal_started`, `roles_joined`, +`listeners_ready`, and `kickoff_posted` with returned ids/sequences and stable +idempotency keys. Per-run prompts include exact roster, policy/goal hashes, +compiled successor recipe, receipt JSON, check description, and deadline. + +- [ ] **Step 4: Write the failing completion truth table** + +Start with a valid candidate and remove one condition at a time: resolved head, +terminal prior work, artifact receipt/hash, reviewer pass/evidence, each role's +receipt, listening state, and empty handed state. Assert each returns +`ready: false` with a stable reason. Assert raw `DONE`, an ordinary final +response, and stale/forged receipts never affect readiness. + +- [ ] **Step 5: Write failing semantic-check and post-check race tests** + +Add tests named: + +```ts +it("runs the check only after typed, receipt, and idle gates are ready", ...); +it("posts one bounded exact-coordinator failure and never reruns the same failed revision", ...); +it("discards a passing check when result, artifact, digest, receipt, working, or handed state changes", ...); +it("closes success only after an identical post-check candidate and records the exact revision", ...); +``` + +- [ ] **Step 6: Implement the barrier and revision-keyed check cache** + +Combine already parsed evidence in the pure evaluator. The monitor records a +check before feedback or close. A nonzero result blocks that revision and posts +one bounded exact-`@coordinator` message. A zero result triggers a complete +state/messages/status/handed re-read; only an identical snapshot closes with +`trigger: "prometheus_success"` and the revision in bounded detail. + +- [ ] **Step 7: Write failing resume and outcome tests** + +For each startup checkpoint, assert `resume` performs only missing steps, +reuses idempotency keys, never creates a second Room/thread/seat or reposts a +committed kickoff, and retains the deadline. Assert a live monitor cannot be +stolen; dead exact identity can be. Assert timeout, concurrent/idempotent +cancel, unexpected Room close, malformed protocol, checker launch failure, and +irreparable team map respectively to the specified outcome; a failed semantic +check remains `running`; every terminal transition preserves the team and +clears `active.json` last. + +- [ ] **Step 8: Implement status, resume, cancel, signals, and outcomes** + +`status` reports non-secret team/health/run/deadline/checkpoint/barrier/monitor +data. `cancel` sets `cancel_requested` under the short lock; the monitor or a +cancel-owned recovery pass closes once. `resume` reconciles stored checkpoints +and remote state before entering the same monitor. SIGINT/SIGTERM leave the run +active and resumable; no outcome removes role containers or volumes. + +- [ ] **Step 9: Run and commit the complete command lifecycle** + +Run: + +```bash +pnpm exec vitest run packages/cli/src/prometheus-protocol.test.ts packages/cli/src/prometheus-state.test.ts packages/cli/src/prometheus-runtime.test.ts packages/cli/src/prometheus.test.ts packages/cli/src/cli.test.ts +``` + +Expected: PASS. + +```bash +git add packages/cli/src/prometheus.ts packages/cli/src/prometheus.test.ts packages/cli/src/cli.ts packages/cli/src/cli.test.ts packages/cli/src/prometheus-protocol.ts packages/cli/src/prometheus-protocol.test.ts packages/cli/src/prometheus-state.ts packages/cli/src/prometheus-state.test.ts +git commit -m "feat(cli): run persistent local Prometheus teams" +``` + +### Task 6: Document, export, package, and verify local v1 + +**Files:** +- Create: `docs/prometheus-local.md` +- Modify: `packages/cli/README.md` +- Modify: `docs/TESTING.md` +- Modify: `docs/harness-modes.md` +- Modify: `SECURITY.md` +- Modify: `scripts/export-manifest.json` +- Test: `scripts/check-export.test.mjs` +- Test: `scripts/package-smoke.mjs` + +**Interfaces:** +- Consumes: final CLI syntax, state/outcome schemas, fixed roles, and barrier behavior from Tasks 2-5. +- Produces: an operator guide; package/export allowlisting for every new source, test, and document; explicit security and research-mode boundaries. +- Preserves: zero runtime dependencies, existing package exports, six frozen research hashes, and both runtime reservations remaining unavailable. + +- [ ] **Step 1: Write user, testing, and security documentation** + +Document prerequisites, the five policy headings, first/subsequent runs, +status/resume/cancel, role behavior, compiled artifacts/review, terminal +receipts, why `DONE` is inert, state paths/modes, credential locations, +outcomes, and deferred features. Update the runtime catalog wording from +"unmerged PR 48" to "reserved and not wired" without changing registry specs, +hashes, or availability. + +- [ ] **Step 2: Add every intentional file to the export manifest** + +Add the three `prometheus-*.ts` source/test pairs, the design, this plan, the +operator guide, and every PR 48 file introduced by the rebase. Do not edit +imported source hashes for modified existing files. + +- [ ] **Step 3: Extend package smoke with a no-team status check** + +From a clean offline tarball install with isolated `HOME`, `XDG_CONFIG_HOME`, +and `XDG_STATE_HOME`, assert `sharednet prometheus status --json` returns the +documented no-team result without TypeScript stripping, install scripts, +Docker, network, or credentials. Assert the tarball contains the new runtime +modules/declarations and no credential/state fixture. + +- [ ] **Step 4: Run the complete verification set** + +Run: + +```bash +pnpm check:export +pnpm typecheck +pnpm test +pnpm test:package +gitleaks git . --redact --no-banner +``` + +Expected: every command exits 0; package smoke records an artifact SHA-256; +gitleaks reports no leak. + +- [ ] **Step 5: Inspect frozen compatibility and the final diff** + +Run: + +```bash +git diff origin/codex/bootstrap-base...HEAD -- packages/cli/src/harness-modes.ts docs/harness-modes.md +git diff --check origin/codex/bootstrap-base...HEAD +git status --short +``` + +Expected: immutable registry values/hashes are unchanged; catalog edits only +describe current availability; no whitespace errors; only intentional files. + +- [ ] **Step 6: Request review, fix findings, and commit the boundary** + +Use `superpowers:requesting-code-review` with the design, plan, and full diff. +Resolve every correctness/security finding and rerun the affected focused test +plus Step 4 whenever source changes. + +```bash +git add docs/prometheus-local.md packages/cli/README.md docs/TESTING.md docs/harness-modes.md SECURITY.md scripts/export-manifest.json scripts/check-export.test.mjs scripts/package-smoke.mjs +git commit -m "docs(cli): document local Prometheus teams" +``` + +Leave the branch ready for a pull request; do not merge or publish. diff --git a/packages/cli/README.md b/packages/cli/README.md index 562d446..b4a2d0c 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -99,6 +99,17 @@ npx sharednet goal run TASK.md --agent codex --agent claude-code --until 'check: `goal run` also starts the Agents, each in its own Docker container; the containers share the workspace. The guide: https://github.com/systemind-team/sharednet-client/blob/HEAD/docs/goal-mode.md +The local Prometheus preview runs a persistent three-Codex team: +`sharednet prometheus run GOAL.md +--workspace --model --check --timeout ` reuses +the same coordinator, builder, and reviewer threads across fresh compiled +Rooms. It accepts completion only after typed work, exact-artifact review, +revision-bound receipts, an idle team, and the semantic check agree; a chat +message saying `DONE` is inert. See the +[local Prometheus guide](https://github.com/systemind-team/sharednet-client/blob/HEAD/docs/prometheus-local.md) +for `PROMETHEUS.md`, status/resume/cancel, state, security, and its crash, +localhost, cleanup, and durable-status limitations. + Needs Node 22.18 or newer and has no runtime dependencies. The npm package ships compiled JavaScript. Credentials live in `~/.config/sharednet` (owner-only) and per-project state in `./.sharednet/`, which ignores itself in diff --git a/packages/cli/src/cli.test.ts b/packages/cli/src/cli.test.ts index 06cfb30..dda90b5 100644 --- a/packages/cli/src/cli.test.ts +++ b/packages/cli/src/cli.test.ts @@ -1,12 +1,15 @@ // @vitest-environment node -import { mkdtemp, readFile, writeFile } from "node:fs/promises"; +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, readFile, writeFile } from "node:fs/promises"; import { hostname, tmpdir } from "node:os"; import { join } from "node:path"; import { afterEach, describe, expect, it, vi } from "vitest"; import { runCli } from "./cli.ts"; +import { prometheusPaths, reserveActiveRun, writeTeam, type PrometheusRunState, type PrometheusTeamState } from "./prometheus-state.ts"; +import type { ProcessProbe } from "./process-identity.ts"; const cleanup: string[] = []; @@ -731,3 +734,80 @@ it("goal start selects a compiled Room before posting the goal", async () => { expect(sentBody(result.requests[0]!)).toMatchObject({ name: file, type: "compiled" }); expect(sentBody(result.requests[1]!)).toEqual({ content: "Deliver a checked patch.", until: ["after 1h"] }); }); + +it("routes top-level prometheus status without credentials, Docker, or HTTP", async () => { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-status-")); + cleanup.push(root); + const stdout: string[] = []; + const requests: string[] = []; + const exitCode = await runCli(["prometheus", "status", "--workspace", root, "--json"], { + cwd: root, + env: { HOME: root, XDG_STATE_HOME: join(root, "state"), XDG_CONFIG_HOME: join(root, "config") }, + fetch: async (input) => { requests.push(String(input)); throw new Error("unexpected HTTP"); }, + stdout: (value) => stdout.push(value), + stderr: () => undefined, + }); + expect(exitCode).toBe(0); + expect(JSON.parse(stdout.join(""))).toEqual({ workspace: root, team: null, run: null }); + expect(requests).toEqual([]); +}); + +it("returns durable Prometheus status when live runtime preflight fails", async () => { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-status-active-")); + cleanup.push(root); + const workspace = join(root, "workspace"); + await mkdir(workspace); + const env = { HOME: root, XDG_STATE_HOME: join(root, "state"), XDG_CONFIG_HOME: join(root, "config") }; + const paths = await prometheusPaths(env, workspace); + const roles = Object.fromEntries((["coordinator", "builder", "reviewer"] as const).map((role) => [role, { + container_name: `sharednet-prometheus-${paths.workspaceKey}-${role}`, + volume_name: `sharednet-prometheus-${paths.workspaceKey}-${role}-home`, + thread_id: `thr-${role}`, + }])) as PrometheusTeamState["roles"]; + const team: PrometheusTeamState = { + schema_version: 1, workspace_key: paths.workspaceKey, workspace_path: paths.workspacePath, model: "gpt-6-sol", roles, + last_policy_sha256: "a".repeat(64), created_at: "2026-10-18T00:00:00.000Z", updated_at: "2026-10-18T00:00:00.000Z", + }; + const run: PrometheusRunState = { + schema_version: 1, run_id: "run_status", workspace_key: paths.workspaceKey, workspace_path: paths.workspacePath, + configuration: { model: "gpt-6-sol", goal_path: join(workspace, "GOAL.md"), goal_sha256: createHash("sha256").update("goal").digest("hex"), policy_sha256: team.last_policy_sha256, check_command: "true" }, + startup_checkpoint: "reserved", room: null, roster: null, + idempotency_keys: { room: "status-room", goal: "status-goal", roles: { coordinator: "status-coordinator", builder: "status-builder", reviewer: "status-reviewer" }, kickoff: "status-kickoff" }, + deadline_at: "2026-10-18T04:00:00.000Z", monitor: null, cancel_requested: false, outcome: "starting", candidate: null, checks: [], + created_at: "2026-10-18T00:00:00.000Z", updated_at: "2026-10-18T00:00:00.000Z", finished_at: null, reason: null, + }; + const probe: ProcessProbe = { supported: true, alive: () => true, identity: () => "test", argv: () => null }; + await writeTeam(paths, team); + await reserveActiveRun(paths, run, probe); + const stdout: string[] = []; + const requests: string[] = []; + const exitCode = await runCli(["prometheus", "status", "--workspace", workspace, "--json"], { + cwd: workspace, env, + docker: async () => { throw new Error("Docker unavailable"); }, + fetch: async (input) => { requests.push(String(input)); throw new Error("unexpected HTTP"); }, + stdout: (value) => stdout.push(value), stderr: () => undefined, + }); + + expect(exitCode).toBe(0); + expect(JSON.parse(stdout.join(""))).toMatchObject({ + workspace, + run: { run_id: "run_status", startup_checkpoint: "reserved" }, + live: { available: false, error: { code: "service_unavailable", message: "SharedNet could not complete the request." } }, + }); + expect(requests).toEqual([]); +}); + +it("routes top-level prometheus resume and reports an absent run before external work", async () => { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-resume-")); + cleanup.push(root); + const stderr: string[] = []; + const exitCode = await runCli(["prometheus", "resume", "--workspace", root, "--json"], { + cwd: root, + env: { HOME: root, XDG_STATE_HOME: join(root, "state"), XDG_CONFIG_HOME: join(root, "config") }, + fetch: async () => { throw new Error("unexpected HTTP"); }, + stdout: () => undefined, + stderr: (value) => stderr.push(value), + }); + expect(exitCode).toBe(2); + expect(JSON.parse(stderr.join("")).error.code).toBe("prometheus_run_not_active"); +}); diff --git a/packages/cli/src/cli.ts b/packages/cli/src/cli.ts index 593d3b2..761cefe 100644 --- a/packages/cli/src/cli.ts +++ b/packages/cli/src/cli.ts @@ -20,6 +20,7 @@ import type { ProcessProbe } from "./process-identity.ts"; import type { WakeServiceControl } from "./serve-service.ts"; import type { CheckOptions, CheckResult, SeatWake, TurnRunner } from "./wake-driver.ts"; import { validTurnLimit } from "./wake-driver.ts"; +import { cancelPrometheusRun, prometheusStatus, resumePrometheusRun, startPrometheusRun } from "./prometheus.ts"; type Environment = Record; @@ -135,6 +136,9 @@ const optionValueNames = new Set([ "harness-mode", "kickoff", "type", + "model", + "check", + "timeout", ]); const booleanOptionNames = new Set(["new", "private", "quiet-checks", "mention-gate", "task-gate"]); @@ -881,9 +885,111 @@ async function execute( if (resource === "goal") { return goalCommand(action, commandArgs, globals, dependencies); } + if (resource === "prometheus") { + const parsed = parseArguments(commandArgs); + const workspace = resolvePathFrom(dependencies.cwd, option(parsed, "workspace") ?? "."); + if (action === "status") { + assertPositionals(parsed, 0); + assertOnlyOptions(parsed, ["workspace"]); + const local = await prometheusStatus(workspace, dependencies.env); + if (!local.run || !local.team) return local; + try { + const docker = dependencies.docker ?? dockerRunner(dependencies.env); + const cliEntry = dependencies.cliEntry ?? realpathSync(process.argv[1]!); + const plan = await prepareRun({ agents: (["coordinator", "builder", "reviewer"] as const).map((name) => ({ name, driver: "codex" as const, model: local.run!.configuration.model })), image: null, workspace, env: dependencies.env, home: dependencies.env.HOME ?? homedir(), entry: cliEntry }, docker, dependencies.stderr); + let connection: Awaited[1]["connect"]>>> | undefined; + return prometheusStatus(workspace, dependencies.env, { + docker, plan, + ...(dependencies.processProbe ? { processProbe: dependencies.processProbe } : {}), + connect: async () => { + if (connection) return connection; + const baseUrl = resolveBaseUrl(dependencies.env.SHAREDNET_BASE_URL); + const apiKey = await resolveApiKey(dependencies.env, getStoragePaths(dependencies.env), baseUrl); + connection = await withSelectedSession(globals, dependencies, async (client, session) => ({ client, memberToken: session.instance_token, apiKey, baseUrl })); + return connection; + }, + }); + } catch (error) { + const failure = asCliError(error); + return { ...local, live: { available: false, error: { code: failure.code, message: failure.message } } }; + } + } + if (action === "cancel") { + assertPositionals(parsed, 0); + assertOnlyOptions(parsed, ["workspace"]); + let connection: Awaited[1]["connect"]>>> | undefined; + return cancelPrometheusRun(workspace, { + env: dependencies.env, cwd: dependencies.cwd, now: dependencies.now, + ...(dependencies.processProbe ? { processProbe: dependencies.processProbe } : {}), + connect: async () => { + if (connection) return connection; + const baseUrl = resolveBaseUrl(dependencies.env.SHAREDNET_BASE_URL); + const apiKey = await resolveApiKey(dependencies.env, getStoragePaths(dependencies.env), baseUrl); + connection = await withSelectedSession(globals, dependencies, async (client, session) => ({ client, memberToken: session.instance_token, apiKey, baseUrl })); + return connection; + }, + }); + } + if (action === "resume") { + assertPositionals(parsed, 0); + assertOnlyOptions(parsed, ["workspace"]); + const status = await prometheusStatus(workspace, dependencies.env); + const model = status.run?.configuration.model ?? "gpt-6-sol"; + const docker = dependencies.docker ?? dockerRunner(dependencies.env); + const cliEntry = dependencies.cliEntry ?? realpathSync(process.argv[1]!); + let connection: Awaited[1]["connect"]>>> | undefined; + return resumePrometheusRun(workspace, { + env: dependencies.env, cwd: dependencies.cwd, now: dependencies.now, docker, + ...(dependencies.sleep ? { sleep: dependencies.sleep } : {}), + ...(dependencies.processProbe ? { processProbe: dependencies.processProbe } : {}), + preparePlan: () => prepareRun({ agents: (["coordinator", "builder", "reviewer"] as const).map((name) => ({ name, driver: "codex" as const, model })), image: null, workspace, env: dependencies.env, home: dependencies.env.HOME ?? homedir(), entry: cliEntry }, docker, dependencies.stderr), + connect: async () => { + if (connection) return connection; + const baseUrl = resolveBaseUrl(dependencies.env.SHAREDNET_BASE_URL); + const apiKey = await resolveApiKey(dependencies.env, getStoragePaths(dependencies.env), baseUrl); + connection = await withSelectedSession(globals, dependencies, async (client, session) => ({ client, memberToken: session.instance_token, apiKey, baseUrl })); + return connection; + }, + }); + } + if (action === "run") { + assertPositionals(parsed, 1); + assertOnlyOptions(parsed, ["workspace", "model", "check", "timeout"]); + const model = requiredOption(parsed, "model"); + const check = requiredOption(parsed, "check"); + const timeoutMs = parseDuration(requiredOption(parsed, "timeout"), "--timeout"); + const docker = dependencies.docker ?? dockerRunner(dependencies.env); + const cliEntry = dependencies.cliEntry ?? realpathSync(process.argv[1]!); + let connection: Awaited[1]["connect"]>>> | undefined; + return startPrometheusRun({ workspace, goalPath: parsed.positionals[0]!, model, check, timeoutMs }, { + env: dependencies.env, + cwd: dependencies.cwd, + now: dependencies.now, + docker, + ...(dependencies.sleep ? { sleep: dependencies.sleep } : {}), + ...(dependencies.processProbe ? { processProbe: dependencies.processProbe } : {}), + preparePlan: () => prepareRun({ + agents: (["coordinator", "builder", "reviewer"] as const).map((name) => ({ name, driver: "codex" as const, model })), + image: null, + workspace, + env: dependencies.env, + home: dependencies.env.HOME ?? homedir(), + entry: cliEntry, + }, docker, dependencies.stderr), + connect: async () => { + if (connection) return connection; + const baseUrl = resolveBaseUrl(dependencies.env.SHAREDNET_BASE_URL); + const apiKey = await resolveApiKey(dependencies.env, getStoragePaths(dependencies.env), baseUrl); + connection = await withSelectedSession(globals, dependencies, async (client, session) => ({ client, memberToken: session.instance_token, apiKey, baseUrl })); + return connection; + }, + }); + } + throw localError("unknown_command", "Use sharednet prometheus run|status|resume|cancel."); + } throw localError( "unknown_command", - "Use login, whoami, join/say/read/wait/ack/watch/handed/add/rooms/requests/accept/deny/reach/timer, task claim/done/list, wake/wake set/off/check, balance/redeem/pay/ledger, upload/download/files, act/open/deliver, or session start/status, room create/list/invite/add/join/post/messages, goal start/run/serve-demo/watch/export, and decision list/approve/deny.", + "Use login, whoami, join/say/read/wait/ack/watch/handed/add/rooms/requests/accept/deny/reach/timer, task claim/done/list, wake/wake set/off/check, balance/redeem/pay/ledger, upload/download/files, act/open/deliver, or session start/status, room create/list/invite/add/join/post/messages, goal start/run/serve-demo/watch/export, prometheus run/status/resume/cancel, and decision list/approve/deny.", ); } diff --git a/packages/cli/src/guest.test.ts b/packages/cli/src/guest.test.ts index e3e17bf..519008c 100644 --- a/packages/cli/src/guest.test.ts +++ b/packages/cli/src/guest.test.ts @@ -620,7 +620,11 @@ describe("sharednet join", () => { }, { status: 200, body: { items: [message(1, "Welcome")], next_cursor: "1", has_more: false } }, ], - { SHAREDNET_API_KEY: `snk_${"K".repeat(43)}` }, + { + SHAREDNET_API_KEY: `snk_${"K".repeat(43)}`, + SHAREDNET_REUSE_RUNTIME_SESSION: "1", + SHAREDNET_JOIN_IDEMPOTENCY_KEY: "run-1-coordinator", + }, ); // The only thing on stderr is the seat this session got, never a token. @@ -639,7 +643,8 @@ describe("sharednet join", () => { expect(JSON.parse(String(result.requests[1]!.init.body))).toEqual({ invite: INVITE_TOKEN }); expect(result.stderr).toContain("No --name was given, so the Room shows this seat by its id, i_AccountSeat1."); expect(JSON.parse(String(result.requests[0]!.init.body)).runtime_kind).toBe("claude-code"); - expect(header(result.requests[1]!, "idempotency-key")).toMatch(/^[0-9a-f-]{36}$/); + expect(JSON.parse(String(result.requests[0]!.init.body)).local_instance_key).toMatch(/^[A-Za-z0-9_-]{20,}$/); + expect(header(result.requests[1]!, "idempotency-key")).toBe("run-1-coordinator"); const output = JSON.parse(result.stdout); expect(output).toMatchObject({ as: "account", member_id: "i_AccountSeat1", principal_id: "p_AcCoUnT0001", last_sequence: 1 }); diff --git a/packages/cli/src/guest.ts b/packages/cli/src/guest.ts index 01f4a5e..360f266 100644 --- a/packages/cli/src/guest.ts +++ b/packages/cli/src/guest.ts @@ -45,6 +45,15 @@ import { type Environment = Record; +function joinIdempotencyKey(env: Environment): string { + const configured = env.SHAREDNET_JOIN_IDEMPOTENCY_KEY?.trim(); + if (configured === undefined || configured.length === 0) return randomUUID(); + if (!/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(configured)) { + throw localError("invalid_arguments", "SHAREDNET_JOIN_IDEMPOTENCY_KEY is not a safe idempotency key."); + } + return configured; +} + /** A path the human typed, resolved against the directory the CLI was run in. */ function resolvePath(cwd: string, path: string): string { return isAbsolute(path) ? path : resolvePathFrom(cwd, path); @@ -505,6 +514,7 @@ async function join(args: string[], dependencies: GuestDependencies): Promise () => { throw Object.assign(new Error(code), { code }); }; describe("is the process that wrote the lock still there", () => { + it("distinguishes the exact live process from a dead, reused, or unverifiable pid", () => { + const statuses = new Map([ + [10, { alive: true, identity: "same-start" }], + [11, { alive: false, identity: null }], + [12, { alive: true, identity: "new-start" }], + [13, { alive: true, identity: null }], + ]); + const probe: ProcessProbe = { + supported: true, + alive: (pid) => statuses.get(pid)?.alive ?? false, + identity: (pid) => statuses.get(pid)?.identity ?? null, + argv: () => null, + }; + + expect(inspectProcessIdentity(probe, { pid: 10, start_identity: "same-start" })).toBe("alive"); + expect(inspectProcessIdentity(probe, { pid: 11, start_identity: "old-start" })).toBe("dead"); + expect(inspectProcessIdentity(probe, { pid: 12, start_identity: "old-start" })).toBe("reused"); + expect(inspectProcessIdentity(probe, { pid: 13, start_identity: "old-start" })).toBe("unknown"); + }); + it("counts EPERM from signal 0 as alive (someone else's process), and only ESRCH or a bad pid as gone", () => { expect(processAlive(4242, () => true)).toBe(true); expect(processAlive(4242, failing("EPERM"))).toBe(true); diff --git a/packages/cli/src/process-identity.ts b/packages/cli/src/process-identity.ts index a92cccf..efe7869 100644 --- a/packages/cli/src/process-identity.ts +++ b/packages/cli/src/process-identity.ts @@ -33,6 +33,29 @@ export interface ProcessProbe { argv(pid: number): string | null; } +export interface RecordedProcessIdentity { + pid: number; + start_identity: string; +} + +export type ProcessIdentityStatus = "alive" | "dead" | "reused" | "unknown"; + +/** + * Classify the exact process recorded in a lock. A live PID with another + * start identity is a reused PID: the recorded process is gone, and the new + * process must never inherit its authority. + */ +export function inspectProcessIdentity( + probe: ProcessProbe, + recorded: RecordedProcessIdentity, +): ProcessIdentityStatus { + if (!probe.supported) return "unknown"; + if (!probe.alive(recorded.pid)) return "dead"; + const current = probe.identity(recorded.pid); + if (current === null) return "unknown"; + return current === recorded.start_identity ? "alive" : "reused"; +} + /** Signal 0 probes without signalling; EPERM means the process exists but belongs to someone else. */ export function processAlive(pid: number, kill: (pid: number, signal: 0) => unknown = (target, signal) => process.kill(target, signal)): boolean { if (!Number.isSafeInteger(pid) || pid <= 0) return false; diff --git a/packages/cli/src/prometheus-protocol.test.ts b/packages/cli/src/prometheus-protocol.test.ts new file mode 100644 index 0000000..8cc4fcb --- /dev/null +++ b/packages/cli/src/prometheus-protocol.test.ts @@ -0,0 +1,605 @@ +// @vitest-environment node + +import { describe, expect, it } from "vitest"; + +import { + assertPrometheusGoalUnaddressed, + parsePrometheusCandidate, + parsePrometheusPolicy, + parseTerminalReceipts, +} from "./prometheus-protocol.ts"; +import type { CompiledState } from "./compiled.ts"; + +const VALID_POLICY = [ + "# Prometheus", + "", + "```markdown", + "## Rules", + "not real", + "```", + "", + "## Motto", + "Ship evidence.", + "", + "## Rules", + "- Keep receipts strict.", + "", + "## Preferences", + "Prefer small patches.", + "", + "## Handoff", + "Name the exact member.", + "", + "## Completion", + "All checks pass.", + "", +].join("\n"); + +function expectInvalidArguments(action: () => unknown): void { + try { + action(); + throw new Error("expected invalid_arguments"); + } catch (error) { + expect(error).toMatchObject({ code: "invalid_arguments" }); + } +} + +describe("Prometheus policy", () => { + it("parses the five ordered nonempty policy sections outside fences and hashes exact bytes", () => { + const policy = parsePrometheusPolicy(VALID_POLICY); + + expect(policy).toEqual({ + raw: VALID_POLICY, + sha256: "e3a40d32e60d9d280f4664b44b620034925fbed4fa8cb33d18c956b940080c75", + sections: { + Motto: "Ship evidence.", + Rules: "- Keep receipts strict.", + Preferences: "Prefer small patches.", + Handoff: "Name the exact member.", + Completion: "All checks pass.", + }, + }); + }); + + it("accepts bare CR line endings while hashing the untouched bytes", () => { + const raw = VALID_POLICY.replaceAll("\n", "\r"); + + expect(parsePrometheusPolicy(raw)).toEqual({ + raw, + sha256: "1e5756195ac80f246df30080412a4fc3806c7a24621a99a7186c89110f1d648c", + sections: { + Motto: "Ship evidence.", + Rules: "- Keep receipts strict.", + Preferences: "Prefer small patches.", + Handoff: "Name the exact member.", + Completion: "All checks pass.", + }, + }); + }); + + it.each([ + ["missing", VALID_POLICY.replace(/\n## Completion\nAll checks pass\.\n$/, "\n")], + ["duplicate", `${VALID_POLICY}\n## Rules\nAgain.\n`], + [ + "reordered", + VALID_POLICY.replace( + "## Motto\nShip evidence.\n\n## Rules\n- Keep receipts strict.", + "## Rules\n- Keep receipts strict.\n\n## Motto\nShip evidence.", + ), + ], + ["wrong-case", VALID_POLICY.replace("## Rules\n- Keep receipts strict.", "## rules\n- Keep receipts strict.")], + ["empty", VALID_POLICY.replace("## Preferences\nPrefer small patches.", "## Preferences\n ")], + ["oversized", `${VALID_POLICY}${"x".repeat(65_536)}`], + ])("rejects %s PROMETHEUS.md before side effects", (_case, raw) => { + expectInvalidArguments(() => parsePrometheusPolicy(raw)); + }); +}); + +describe("Prometheus goals", () => { + it("rejects goals that address all, a fixed role name, or a remembered member id", () => { + for (const goal of [ + "@all finish it", + "Ask @coordinator to plan", + "@builder implement it", + "Review this, @reviewer", + "Continue with @i_AbCdEfGhIj", + "Continue with @mem_ZyXwVuTsRq", + ]) { + expectInvalidArguments(() => assertPrometheusGoalUnaddressed(goal)); + } + + expect(() => assertPrometheusGoalUnaddressed("Build the feature described here.")).not.toThrow(); + }); +}); + +const COORDINATOR = "i_Coord00001"; +const BUILDER = "i_Build00001"; +const REVIEWER = "i_Review0001"; +const SUBMISSION = "art_Submit0001"; +const REVIEW_REPORT = "art_Review0001"; +const RESULT_HASH = `sha256:${"5".repeat(24)}`; +const VERIFICATION_HASH = `sha256:${"6".repeat(24)}`; +const STATE_DIGEST = `sha256:${"d".repeat(24)}`; +const SUBMISSION_SHA256 = "a".repeat(64); +const REVIEW_SHA256 = "b".repeat(64); +const REQUIREMENT = `verify:${REVIEWER}`; +const ROSTER = { + coordinator: { memberId: COORDINATOR }, + builder: { memberId: BUILDER }, + reviewer: { memberId: REVIEWER }, +} as const; + +interface TestRequirement { + kind: string; + eligible: string[]; + q: number; + predicate: null; + pos: string[]; + neg: string[]; + status: string; +} + +interface TestWork { + state: string; + requester: string; + assignee_addressed: string[]; + responsible: string | null; + accepted_by: string | null; + result: string | null; + artifacts: Record; + deliverables: string[]; + requirements: Record; + veto: string[]; + guards: unknown[]; + supersedes: string | null; + resolver: string | null; +} + +interface TestEvent { + type: string; + actor: string; + work_id: string | null; + to: string[]; + payload: Record; + refs: Record; + clauses: Array>; + parents: string[]; + idempotency_key: string | null; + version: string; + seq: number; + hash: string; +} + +interface TestCompiledState extends CompiledState { + projection: { work: Record; clock: number }; + events: TestEvent[]; +} + +function testEvent( + seq: number, + type: string, + actor: string, + workId: string, + refs: Record = {}, + clauses: Array> = [], +): TestEvent { + return { + type, + actor, + work_id: workId, + to: type === "work.request" ? [BUILDER] : [], + payload: type === "work.request" ? { task: "Produce the submission" } : {}, + refs, + clauses, + parents: [], + idempotency_key: null, + version: "sha256:protocol-v1-d680a891c4c8063b", + seq, + hash: `sha256:${String(seq).repeat(24)}`, + }; +} + +function candidateFixture(): TestCompiledState { + const oldWork: TestWork = { + state: "CANCELLED", + requester: COORDINATOR, + assignee_addressed: [BUILDER], + responsible: null, + accepted_by: null, + result: null, + artifacts: {}, + deliverables: [], + requirements: {}, + veto: [], + guards: [], + supersedes: null, + resolver: null, + }; + const head: TestWork = { + state: "RESOLVED", + requester: COORDINATOR, + assignee_addressed: [BUILDER], + responsible: null, + accepted_by: BUILDER, + result: RESULT_HASH, + artifacts: { submission: SUBMISSION }, + deliverables: ["submission"], + requirements: { + [REQUIREMENT]: { + kind: "verify", + eligible: [REVIEWER], + q: 1, + predicate: null, + pos: [REVIEWER], + neg: [], + status: "satisfied", + }, + }, + veto: [], + guards: [], + supersedes: "W0", + resolver: null, + }; + const events = [ + testEvent(1, "work.request", COORDINATOR, "W0"), + testEvent(2, "work.cancel", COORDINATOR, "W0"), + testEvent(3, "work.request", COORDINATOR, "W1", { supersedes: "W0" }, [ + { kind: "deliverables", names: ["submission"] }, + { kind: "verify", id: REQUIREMENT, verifier: REVIEWER }, + ]), + testEvent(4, "work.accept", BUILDER, "W1"), + testEvent(5, "work.result", BUILDER, "W1", { + artifacts: { submission: SUBMISSION }, + artifact_receipts: { [SUBMISSION]: { id: SUBMISSION, sha256: SUBMISSION_SHA256 } }, + }), + testEvent(6, "verification.pass", REVIEWER, "W1", { + result_ref: RESULT_HASH, + requirement: REQUIREMENT, + evidence: REVIEW_REPORT, + artifact_receipts: { [REVIEW_REPORT]: { id: REVIEW_REPORT, sha256: REVIEW_SHA256 } }, + }), + testEvent(7, "work.resolve", COORDINATOR, "W1"), + ]; + return { + protocol_version: "sha256:protocol-v1-d680a891c4c8063b", + sequence: events.length, + digest: STATE_DIGEST, + projection: { work: { W0: oldWork, W1: head }, clock: 0 }, + obligations: "No outstanding obligations.", + events, + }; +} + +describe("Prometheus candidate revisions", () => { + it("derives one revision from a resolved successor head, exact result artifact receipts, and reviewer passes", () => { + expect(parsePrometheusCandidate(candidateFixture(), ROSTER)).toEqual({ + ok: true, + candidate: { + stateDigest: STATE_DIGEST, + workId: "W1", + resultRef: RESULT_HASH, + artifacts: [{ name: "submission", artifactId: SUBMISSION, sha256: SUBMISSION_SHA256 }], + verifications: [{ + eventHash: VERIFICATION_HASH, + requirementId: REQUIREMENT, + evidenceArtifactId: REVIEW_REPORT, + evidenceSha256: REVIEW_SHA256, + }], + revisionSha256: "bcff959b214dbe461e44be212f7e2df82c5f84cd76f38cbc07291563977afa2c", + }, + }); + }); + + it.each([ + ["nonterminal-work", (state: TestCompiledState) => { state.projection.work.W0!.state = "OPEN"; }], + ["ambiguous-head", (state: TestCompiledState) => { + state.projection.work.W2 = { ...structuredClone(state.projection.work.W0!), supersedes: null }; + }], + ["rejected-head", (state: TestCompiledState) => { state.projection.work.W1!.state = "REJECTED"; }], + ["missing-result-event", (state: TestCompiledState) => { state.events[4]!.hash = `sha256:${"9".repeat(24)}`; }], + ["missing-artifact-receipt", (state: TestCompiledState) => { + state.events[4]!.refs.artifact_receipts = {}; + }], + ["bad-artifact-sha256", (state: TestCompiledState) => { + state.events[4]!.refs.artifact_receipts = { [SUBMISSION]: { id: SUBMISSION, sha256: "A".repeat(64) } }; + }], + ["stale-result-ref", (state: TestCompiledState) => { + state.events[5]!.refs.result_ref = `sha256:${"0".repeat(24)}`; + }], + ["wrong-reviewer", (state: TestCompiledState) => { state.events[5]!.actor = BUILDER; }], + ["missing-review-evidence", (state: TestCompiledState) => { + delete state.events[5]!.refs.evidence; + delete state.events[5]!.refs.artifact_receipts; + }], + ["unsatisfied-requirement", (state: TestCompiledState) => { + state.projection.work.W1!.requirements[REQUIREMENT]!.status = "open"; + }], + ])("keeps the candidate closed for %s", (reason, mutate) => { + const state = candidateFixture(); + mutate(state); + + const parsed = parsePrometheusCandidate(state, ROSTER); + + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.reasons).toContain(reason); + }); + + it("sorts work artifacts and verification evidence before hashing the revision", () => { + const first = candidateFixture(); + const head = first.projection.work.W1!; + const result = first.events[4]!; + const report = "art_Report0001"; + const reportSha256 = "c".repeat(64); + const secondRequirement = "verify:secondary"; + const secondEvidence = "art_Proof00001"; + const secondEvidenceSha256 = "e".repeat(64); + head.deliverables = ["report", "submission"]; + head.artifacts = { report, submission: SUBMISSION }; + result.refs.artifacts = { report, submission: SUBMISSION }; + result.refs.artifact_receipts = { + [report]: { id: report, sha256: reportSha256 }, + [SUBMISSION]: { id: SUBMISSION, sha256: SUBMISSION_SHA256 }, + }; + head.requirements = { + [secondRequirement]: { + kind: "verify", eligible: [REVIEWER], q: 1, predicate: null, + pos: [REVIEWER], neg: [], status: "satisfied", + }, + [REQUIREMENT]: head.requirements[REQUIREMENT]!, + }; + const resolve = first.events.pop()!; + first.events.push( + testEvent(7, "verification.pass", REVIEWER, "W1", { + result_ref: RESULT_HASH, + requirement: secondRequirement, + evidence: secondEvidence, + artifact_receipts: { [secondEvidence]: { id: secondEvidence, sha256: secondEvidenceSha256 } }, + }), + { ...resolve, seq: 8, hash: `sha256:${"8".repeat(24)}` }, + ); + first.sequence = first.events.length; + + const second = structuredClone(first); + const secondHead = second.projection.work.W1!; + secondHead.artifacts = { submission: SUBMISSION, report }; + secondHead.requirements = { + [REQUIREMENT]: secondHead.requirements[REQUIREMENT]!, + [secondRequirement]: secondHead.requirements[secondRequirement]!, + }; + second.events[4]!.refs.artifacts = { submission: SUBMISSION, report }; + second.events[4]!.refs.artifact_receipts = { + [SUBMISSION]: { id: SUBMISSION, sha256: SUBMISSION_SHA256 }, + [report]: { id: report, sha256: reportSha256 }, + }; + + const one = parsePrometheusCandidate(first, ROSTER); + const two = parsePrometheusCandidate(second, ROSTER); + + expect(one.ok).toBe(true); + expect(two.ok).toBe(true); + if (one.ok && two.ok) { + expect(one.candidate.artifacts.map((artifact) => artifact.name)).toEqual(["report", "submission"]); + expect(one.candidate.verifications.map((verification) => verification.eventHash)).toEqual([ + VERIFICATION_HASH, + `sha256:${"7".repeat(24)}`, + ]); + expect(two.candidate.revisionSha256).toBe(one.candidate.revisionSha256); + } + }); + + it("fails closed on malformed compiled objects, arrays, ids, hashes, and receipts", () => { + const badProjection = { ...candidateFixture(), projection: [] } as unknown as CompiledState; + const badEvents = { ...candidateFixture(), events: {} } as unknown as CompiledState; + const badVersion = candidateFixture(); + badVersion.protocol_version = "sha256:protocol-v2-0000000000000000"; + const badMember = candidateFixture(); + badMember.projection.work.W1!.requester = "coordinator"; + const badHash = candidateFixture(); + badHash.events[0]!.hash = "sha256:not-a-hash"; + const badArtifact = candidateFixture(); + badArtifact.projection.work.W1!.artifacts.submission = "submission.zip"; + const duplicateVerification = candidateFixture(); + const resolve = duplicateVerification.events.pop()!; + duplicateVerification.events.push( + testEvent(7, "verification.pass", REVIEWER, "W1", structuredClone(duplicateVerification.events[5]!.refs)), + { ...resolve, seq: 8, hash: `sha256:${"8".repeat(24)}` }, + ); + duplicateVerification.sequence = duplicateVerification.events.length; + + for (const state of [badProjection, badEvents, badVersion, badMember, badHash, badArtifact, duplicateVerification]) { + const result = parsePrometheusCandidate(state, ROSTER); + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reasons).toContain("invalid-compiled-state"); + } + }); + + it.each([ + ["requester", (state: TestCompiledState) => { state.projection.work.W1!.requester = REVIEWER; }], + ["assignee", (state: TestCompiledState) => { state.projection.work.W1!.assignee_addressed = [REVIEWER]; }], + ["acceptor", (state: TestCompiledState) => { state.projection.work.W1!.accepted_by = REVIEWER; }], + ["result actor", (state: TestCompiledState) => { state.events[4]!.actor = COORDINATOR; }], + ])("binds the head's %s to the exact coordinator or builder roster member", (_field, mutate) => { + const state = candidateFixture(); + mutate(state); + + const result = parsePrometheusCandidate(state, ROSTER); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reasons).toContain("invalid-compiled-state"); + }); + + it("requires distinct coordinator, builder, and reviewer members", () => { + const state = candidateFixture(); + state.projection.work.W1!.requirements[REQUIREMENT]!.eligible = [BUILDER]; + state.projection.work.W1!.requirements[REQUIREMENT]!.pos = [BUILDER]; + state.events[5]!.actor = BUILDER; + const duplicateRoster = { + ...ROSTER, + reviewer: { memberId: BUILDER }, + } as const; + + const result = parsePrometheusCandidate(state, duplicateRoster); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reasons).toContain("invalid-compiled-state"); + }); + + it("rejects review evidence that reuses any result artifact", () => { + const state = candidateFixture(); + state.events[5]!.refs.evidence = SUBMISSION; + state.events[5]!.refs.artifact_receipts = { + [SUBMISSION]: { id: SUBMISSION, sha256: SUBMISSION_SHA256 }, + }; + + const result = parsePrometheusCandidate(state, ROSTER); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reasons).toContain("missing-review-evidence"); + }); + + it("rejects a requirement when the same member appears in both positive and negative evidence", () => { + const state = candidateFixture(); + state.projection.work.W1!.requirements[REQUIREMENT]!.neg = [REVIEWER]; + + const result = parsePrometheusCandidate(state, ROSTER); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reasons).toContain("invalid-compiled-state"); + }); + + it.each([ + ["extra positive", [REVIEWER, BUILDER], []], + ["ineligible negative", [REVIEWER], [BUILDER]], + ])("rejects %s votes on a reviewer-only verify requirement", (_case, pos, neg) => { + const state = candidateFixture(); + state.projection.work.W1!.requirements[REQUIREMENT]!.pos = pos; + state.projection.work.W1!.requirements[REQUIREMENT]!.neg = neg; + + const result = parsePrometheusCandidate(state, ROSTER); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reasons).toContain("wrong-reviewer"); + }); + + it("rejects result artifacts that were not declared as deliverables", () => { + const state = candidateFixture(); + const report = "art_Report0001"; + const reportSha256 = "c".repeat(64); + state.projection.work.W1!.artifacts.report = report; + state.events[4]!.refs.artifacts = { submission: SUBMISSION, report }; + state.events[4]!.refs.artifact_receipts = { + [SUBMISSION]: { id: SUBMISSION, sha256: SUBMISSION_SHA256 }, + [report]: { id: report, sha256: reportSha256 }, + }; + + const result = parsePrometheusCandidate(state, ROSTER); + + expect(result.ok).toBe(false); + if (!result.ok) expect(result.reasons).toContain("invalid-compiled-state"); + }); +}); + +const RUN_ID = "run_20261018_abcdef"; +const REVISION_SHA256 = "f".repeat(64); + +function terminalReceipt( + role: "coordinator" | "builder" | "reviewer", + overrides: Partial<{ type: string; run_id: string; role: string; revision_sha256: string; extra: string }> = {}, +): string { + return `[sharednet-prometheus:v1] ${JSON.stringify({ + type: "terminal.receipt", + run_id: RUN_ID, + role, + revision_sha256: REVISION_SHA256, + ...overrides, + })}`; +} + +describe("Prometheus terminal receipts", () => { + it("accepts one current receipt from each saved sender", () => { + const parsed = parseTerminalReceipts([ + { sequence: 2, content: terminalReceipt("coordinator"), sender_instance_id: COORDINATOR }, + { sequence: 3, content: terminalReceipt("builder"), sender: { member_id: BUILDER } }, + { sequence: 4, content: terminalReceipt("reviewer"), sender_instance_id: REVIEWER, sender: { member_id: REVIEWER } }, + ], ROSTER, RUN_ID, REVISION_SHA256); + + expect(parsed).toEqual({ + ok: true, + receipts: { + coordinator: { sequence: 2, role: "coordinator", memberId: COORDINATOR, runId: RUN_ID, revisionSha256: REVISION_SHA256 }, + builder: { sequence: 3, role: "builder", memberId: BUILDER, runId: RUN_ID, revisionSha256: REVISION_SHA256 }, + reviewer: { sequence: 4, role: "reviewer", memberId: REVIEWER, runId: RUN_ID, revisionSha256: REVISION_SHA256 }, + }, + missingRoles: [], + }); + }); + + it("rejects raw DONE, ordinary final text, malformed or extra JSON, forged roles, stale values, and unknown members", () => { + const invalid = [ + { sequence: 1, content: "DONE", sender: { member_id: COORDINATOR } }, + { sequence: 2, content: "The work is complete and all checks pass.", sender: { member_id: COORDINATOR } }, + { sequence: 3, content: "[sharednet-prometheus:v1] {", sender: { member_id: COORDINATOR } }, + { sequence: 4, content: terminalReceipt("coordinator", { extra: "not allowed" }), sender: { member_id: COORDINATOR } }, + { + sequence: 5, + content: `[sharednet-prometheus:v1] {"type":"terminal.receipt","run_id":"${RUN_ID}","role":"coordinator","revision_sha256":"${REVISION_SHA256}","__proto__":"not allowed"}`, + sender: { member_id: COORDINATOR }, + }, + { + sequence: 6, + content: `[sharednet-prometheus:v1] {"type":"terminal.receipt","run_id":"${RUN_ID}","role":"coordinator","role":"coordinator","revision_sha256":"${REVISION_SHA256}"}`, + sender: { member_id: COORDINATOR }, + }, + { sequence: 7, content: terminalReceipt("reviewer"), sender: { member_id: BUILDER } }, + { sequence: 8, content: terminalReceipt("coordinator", { run_id: "run_stale" }), sender: { member_id: COORDINATOR } }, + { sequence: 9, content: terminalReceipt("coordinator", { revision_sha256: "0".repeat(64) }), sender: { member_id: COORDINATOR } }, + { sequence: 10, content: terminalReceipt("coordinator"), sender: { member_id: "i_Unknown001" } }, + { + sequence: 11, + content: terminalReceipt("coordinator"), + sender_instance_id: COORDINATOR, + sender: { member_id: BUILDER }, + }, + ]; + + expect(parseTerminalReceipts(invalid, ROSTER, RUN_ID, REVISION_SHA256)).toEqual({ + ok: false, + receipts: {}, + missingRoles: ["coordinator", "builder", "reviewer"], + }); + }); + + it("keeps the newest valid receipt per role even with duplicate older and newer invalid messages", () => { + const reorderedCoordinator = `[sharednet-prometheus:v1] {"role":"coordinator","revision_sha256":"${REVISION_SHA256}","type":"terminal.receipt","run_id":"${RUN_ID}"}`; + const parsed = parseTerminalReceipts([ + { sequence: 12, content: terminalReceipt("builder"), sender: { member_id: BUILDER } }, + { sequence: 4, content: terminalReceipt("coordinator"), sender: { member_id: COORDINATOR } }, + { sequence: 15, content: terminalReceipt("reviewer"), sender: { member_id: REVIEWER } }, + { sequence: 11, content: reorderedCoordinator, sender: { member_id: COORDINATOR } }, + { sequence: 20, content: terminalReceipt("coordinator", { revision_sha256: "0".repeat(64) }), sender: { member_id: COORDINATOR } }, + ], ROSTER, RUN_ID, REVISION_SHA256); + + expect(parsed.ok).toBe(true); + expect(parsed.receipts.coordinator?.sequence).toBe(11); + }); + + it.each([ + ["extra", { ...ROSTER, observer: { memberId: "i_Observe001" } }], + ["inherited", Object.assign(Object.create({ coordinator: ROSTER.coordinator }), { + builder: ROSTER.builder, + reviewer: ROSTER.reviewer, + })], + ])("rejects a roster with %s role keys instead of exact own role keys", (_case, roster) => { + const parsed = parseTerminalReceipts([ + { sequence: 2, content: terminalReceipt("coordinator"), sender_instance_id: COORDINATOR }, + { sequence: 3, content: terminalReceipt("builder"), sender_instance_id: BUILDER }, + { sequence: 4, content: terminalReceipt("reviewer"), sender_instance_id: REVIEWER }, + ], roster as typeof ROSTER, RUN_ID, REVISION_SHA256); + + expect(parsed).toEqual({ + ok: false, + receipts: {}, + missingRoles: ["coordinator", "builder", "reviewer"], + }); + }); +}); diff --git a/packages/cli/src/prometheus-protocol.ts b/packages/cli/src/prometheus-protocol.ts new file mode 100644 index 0000000..d08ad38 --- /dev/null +++ b/packages/cli/src/prometheus-protocol.ts @@ -0,0 +1,728 @@ +import { createHash } from "node:crypto"; + +import type { CompiledState } from "./compiled.ts"; +import { localError } from "./errors.ts"; + +export const PROMETHEUS_ROLES = ["coordinator", "builder", "reviewer"] as const; +export type PrometheusRole = (typeof PROMETHEUS_ROLES)[number]; + +const POLICY_SECTIONS = ["Motto", "Rules", "Preferences", "Handoff", "Completion"] as const; +type PrometheusPolicySection = (typeof POLICY_SECTIONS)[number]; + +export interface PrometheusPolicy { + raw: string; + sha256: string; + sections: Record; +} + +const MAX_POLICY_BYTES = 64 * 1024; + +interface MarkdownHeading { + line: number; + title: string; +} + +function markdownLevelTwoHeadings(lines: readonly string[]): MarkdownHeading[] { + const headings: MarkdownHeading[] = []; + let fence: { marker: "`" | "~"; length: number } | null = null; + + for (const [line, value] of lines.entries()) { + if (fence) { + const close = new RegExp(`^ {0,3}\\${fence.marker}{${fence.length},}[ \\t]*$`); + if (close.test(value)) fence = null; + continue; + } + + const opened = /^ {0,3}(`{3,}|~{3,})(.*)$/.exec(value); + if (opened) { + const run = opened[1]!; + fence = { marker: run[0] as "`" | "~", length: run.length }; + continue; + } + + const heading = /^ {0,3}##(?!#)[ \t]+(.+?)[ \t]*$/.exec(value); + if (heading) headings.push({ line, title: heading[1]! }); + } + + return headings; +} + +export function parsePrometheusPolicy(raw: string): PrometheusPolicy { + if (Buffer.byteLength(raw, "utf8") > MAX_POLICY_BYTES) { + throw localError("invalid_arguments", "PROMETHEUS.md must be no larger than 64 KiB."); + } + + const lines = raw.split(/\r\n|\n|\r/); + const headings = markdownLevelTwoHeadings(lines); + const required = headings.filter((heading) => + POLICY_SECTIONS.some((name) => name.toLowerCase() === heading.title.toLowerCase()), + ); + + if ( + required.length !== POLICY_SECTIONS.length || + required.some((heading, index) => heading.title !== POLICY_SECTIONS[index]) + ) { + throw localError( + "invalid_arguments", + "PROMETHEUS.md needs exactly one ## Motto, ## Rules, ## Preferences, ## Handoff, and ## Completion section, in that order and case.", + ); + } + + const sections = Object.create(null) as Record; + for (const heading of required) { + const next = headings.find((candidate) => candidate.line > heading.line)?.line ?? lines.length; + const content = lines.slice(heading.line + 1, next).join("\n").trim(); + if (!content) { + throw localError("invalid_arguments", `PROMETHEUS.md section ## ${heading.title} must not be empty.`); + } + sections[heading.title as PrometheusPolicySection] = content; + } + + return { + raw, + sha256: createHash("sha256").update(raw, "utf8").digest("hex"), + sections, + }; +} + +export function assertPrometheusGoalUnaddressed(raw: string): void { + const fixedRole = /@(all|coordinator|builder|reviewer)(?![\p{L}\p{N}_-])/iu; + const rememberedMember = /@(?:i|mem)_[0-9A-Za-z]{10}(?![\p{L}\p{N}_-])/u; + if (fixedRole.test(raw) || rememberedMember.test(raw)) { + throw localError( + "invalid_arguments", + "GOAL.md must be unaddressed; Prometheus sends one kickoff to the run's exact coordinator member id.", + ); + } +} + +export interface CandidateArtifact { + name: string; + artifactId: string; + sha256: string; +} + +export interface CandidateVerification { + eventHash: string; + requirementId: string; + evidenceArtifactId: string; + evidenceSha256: string; +} + +export interface PrometheusCandidate { + stateDigest: string; + workId: string; + resultRef: string; + artifacts: CandidateArtifact[]; + verifications: CandidateVerification[]; + revisionSha256: string; +} + +export type CandidateResult = + | { ok: true; candidate: PrometheusCandidate } + | { ok: false; reasons: string[] }; + +const MEMBER_ID = /^(?:i|mem)_[0-9A-Za-z]{10}$/; +const ARTIFACT_ID = /^art_[0-9A-Za-z]{10}$/; +const WORK_ID = /^[A-Za-z0-9_.:-]{1,64}$/; +const REQUIREMENT_ID = /^[A-Za-z0-9_.:#-]{1,256}$/; +const EVENT_HASH = /^sha256:[0-9a-f]{24}$/; +const ARTIFACT_SHA256 = /^[0-9a-f]{64}$/; +const COMPILED_PROTOCOL_VERSION = "sha256:protocol-v1-d680a891c4c8063b"; +const WORK_STATES = new Set(["OPEN", "COMMITTED", "REVIEW", "DECLINED", "REJECTED", "CANCELLED", "EXPIRED", "RESOLVED"]); +const TERMINAL_WORK_STATES = new Set(["DECLINED", "REJECTED", "CANCELLED", "EXPIRED", "RESOLVED"]); +const EVENT_TYPES = new Set([ + "message", + "work.request", + "work.accept", + "work.decline", + "work.result", + "work.cancel", + "work.resolve", + "verification.pass", + "verification.fail", + "human.approve", + "human.reject", + "clock.tick", +]); + +interface WireReceipt { + id: string; + sha256: string; +} + +interface WireRequirement { + kind: "verify" | "human_review" | "quorum"; + eligible: string[]; + q: number; + predicate: unknown; + pos: string[]; + neg: string[]; + status: "open" | "satisfied" | "failed"; +} + +interface WireWork { + state: string; + requester: string; + assignee_addressed: string[]; + responsible: string | null; + accepted_by: string | null; + result: string | null; + artifacts: Record; + deliverables: string[]; + requirements: Record; + veto: string[]; + guards: unknown[]; + supersedes: string | null; + resolver: string | null; +} + +interface WireEvent { + type: string; + actor: string; + work_id: string | null; + refs: Record; + seq: number; + hash: string; +} + +interface WireState { + digest: string; + work: Record; + events: WireEvent[]; +} + +function isRecord(value: unknown): value is Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) return false; + const prototype = Object.getPrototypeOf(value); + return prototype === Object.prototype || prototype === null; +} + +function hasExactKeys(value: Record, required: readonly string[], optional: readonly string[] = []): boolean { + const allowed = new Set([...required, ...optional]); + return required.every((key) => Object.hasOwn(value, key)) && Object.keys(value).every((key) => allowed.has(key)); +} + +function isStringArray(value: unknown, pattern?: RegExp): value is string[] { + return Array.isArray(value) && value.every((item) => typeof item === "string" && (pattern?.test(item) ?? true)); +} + +function isUnique(values: readonly string[]): boolean { + return new Set(values).size === values.length; +} + +function areDisjoint(left: readonly string[], right: readonly string[]): boolean { + const rightValues = new Set(right); + return left.every((value) => !rightValues.has(value)); +} + +function isJsonValue(value: unknown, depth = 0): boolean { + if (depth > 20) return false; + if (value === null || typeof value === "string" || typeof value === "boolean") return true; + if (typeof value === "number") return Number.isFinite(value); + if (Array.isArray(value)) return value.every((item) => isJsonValue(item, depth + 1)); + return isRecord(value) && Object.values(value).every((item) => isJsonValue(item, depth + 1)); +} + +function validNamedArtifacts(value: unknown): value is Record { + return isRecord(value) && Object.entries(value).every(([name, id]) => + name.length > 0 && name.length <= 256 && typeof id === "string" && ARTIFACT_ID.test(id)); +} + +function validReceipts(value: unknown): value is Record { + if (!isRecord(value)) return false; + return Object.entries(value).every(([key, receipt]) => + ARTIFACT_ID.test(key) && isRecord(receipt) && hasExactKeys(receipt, ["id", "sha256"]) && + receipt.id === key && typeof receipt.sha256 === "string"); +} + +function validRequirement(value: unknown): value is WireRequirement { + if (!isRecord(value) || !hasExactKeys(value, ["kind", "eligible", "q", "predicate", "pos", "neg", "status"])) return false; + return (value.kind === "verify" || value.kind === "human_review" || value.kind === "quorum") && + isStringArray(value.eligible, MEMBER_ID) && isUnique(value.eligible) && + Number.isSafeInteger(value.q) && Number(value.q) >= 1 && + (value.predicate === null || isJsonValue(value.predicate)) && + isStringArray(value.pos, MEMBER_ID) && isUnique(value.pos) && + isStringArray(value.neg, MEMBER_ID) && isUnique(value.neg) && + areDisjoint(value.pos, value.neg) && + (value.status === "open" || value.status === "satisfied" || value.status === "failed"); +} + +function validWork(value: unknown): value is WireWork { + if (!isRecord(value) || !hasExactKeys(value, [ + "state", "requester", "assignee_addressed", "responsible", "accepted_by", "result", + "artifacts", "deliverables", "requirements", "veto", "guards", "supersedes", "resolver", + ])) return false; + if (typeof value.state !== "string" || !WORK_STATES.has(value.state) || + typeof value.requester !== "string" || !MEMBER_ID.test(value.requester) || + !isStringArray(value.assignee_addressed, MEMBER_ID) || !isUnique(value.assignee_addressed) || + !(value.responsible === null || (typeof value.responsible === "string" && MEMBER_ID.test(value.responsible))) || + !(value.accepted_by === null || (typeof value.accepted_by === "string" && MEMBER_ID.test(value.accepted_by))) || + !(value.result === null || (typeof value.result === "string" && EVENT_HASH.test(value.result))) || + !validNamedArtifacts(value.artifacts) || + !isStringArray(value.deliverables) || !isUnique(value.deliverables) || + value.deliverables.some((name) => name.length === 0 || name.length > 256) || + !isRecord(value.requirements) || !Object.entries(value.requirements).every(([id, requirement]) => + REQUIREMENT_ID.test(id) && validRequirement(requirement)) || + !isStringArray(value.veto, MEMBER_ID) || !isUnique(value.veto) || + !Array.isArray(value.guards) || !value.guards.every((guard) => isRecord(guard) && isJsonValue(guard)) || + !(value.supersedes === null || (typeof value.supersedes === "string" && WORK_ID.test(value.supersedes))) || + !(value.resolver === null || (typeof value.resolver === "string" && MEMBER_ID.test(value.resolver)))) return false; + return true; +} + +function validEventRefs(value: unknown): value is Record { + if (!isRecord(value) || !hasExactKeys(value, [], [ + "supersedes", "result_ref", "requirement", "evidence", "artifacts", "artifact_receipts", + ])) return false; + if (value.supersedes !== undefined && value.supersedes !== null && + (typeof value.supersedes !== "string" || !WORK_ID.test(value.supersedes))) return false; + if (value.result_ref !== undefined && value.result_ref !== null && + (typeof value.result_ref !== "string" || !EVENT_HASH.test(value.result_ref))) return false; + if (value.requirement !== undefined && + (typeof value.requirement !== "string" || !REQUIREMENT_ID.test(value.requirement))) return false; + if (value.evidence !== undefined && + (typeof value.evidence !== "string" || !ARTIFACT_ID.test(value.evidence))) return false; + if (value.artifacts !== undefined && !validNamedArtifacts(value.artifacts)) return false; + if (value.artifact_receipts !== undefined && !validReceipts(value.artifact_receipts)) return false; + return true; +} + +function validClause(value: unknown): boolean { + if (!isRecord(value) || typeof value.kind !== "string" || !value.kind || !isJsonValue(value)) return false; + if (value.verifier !== undefined && (typeof value.verifier !== "string" || !MEMBER_ID.test(value.verifier))) return false; + if (value.names !== undefined && (!isStringArray(value.names) || !isUnique(value.names) || + value.names.some((name) => name.length === 0 || name.length > 256))) return false; + return true; +} + +function parseWireEvent(value: unknown, index: number): WireEvent | null { + if (!isRecord(value) || !hasExactKeys(value, [ + "type", "actor", "work_id", "to", "payload", "refs", "clauses", "parents", + "idempotency_key", "seq", "hash", + ], ["version"])) return null; + if (typeof value.type !== "string" || !EVENT_TYPES.has(value.type) || + typeof value.actor !== "string" || !(MEMBER_ID.test(value.actor) || (value.type === "clock.tick" && value.actor === "room_service")) || + !(value.work_id === null || (typeof value.work_id === "string" && WORK_ID.test(value.work_id))) || + !isStringArray(value.to, MEMBER_ID) || !isUnique(value.to) || + !isRecord(value.payload) || !isJsonValue(value.payload) || !validEventRefs(value.refs) || + !Array.isArray(value.clauses) || !value.clauses.every(validClause) || + !isStringArray(value.parents, EVENT_HASH) || !isUnique(value.parents) || + !(value.idempotency_key === null || (typeof value.idempotency_key === "string" && value.idempotency_key.length > 0)) || + !(value.version === undefined || value.version === COMPILED_PROTOCOL_VERSION) || + value.seq !== index + 1 || typeof value.hash !== "string" || !EVENT_HASH.test(value.hash)) return null; + return value as unknown as WireEvent; +} + +function parseWireState(state: CompiledState): WireState | null { + const value: unknown = state; + if (!isRecord(value) || !hasExactKeys(value, [ + "protocol_version", "sequence", "digest", "projection", "obligations", "events", + ]) || value.protocol_version !== COMPILED_PROTOCOL_VERSION || + !Number.isSafeInteger(value.sequence) || Number(value.sequence) < 0 || + typeof value.digest !== "string" || !EVENT_HASH.test(value.digest) || + typeof value.obligations !== "string" || !Array.isArray(value.events) || + value.events.length !== value.sequence || !isRecord(value.projection) || + !hasExactKeys(value.projection, ["work", "clock"]) || !isRecord(value.projection.work) || + !Number.isFinite(value.projection.clock)) return null; + + const work: Record = Object.create(null); + for (const [id, item] of Object.entries(value.projection.work)) { + if (!WORK_ID.test(id) || !validWork(item)) return null; + work[id] = item; + } + for (const [id, item] of Object.entries(work)) { + if (item.supersedes !== null && (!Object.hasOwn(work, item.supersedes) || item.supersedes === id)) { + return null; + } + } + + const events: WireEvent[] = []; + const hashes = new Set(); + for (const [index, item] of value.events.entries()) { + const event = parseWireEvent(item, index); + if (!event || hashes.has(event.hash)) return null; + hashes.add(event.hash); + events.push(event); + } + return { digest: value.digest, work, events }; +} + +const CANDIDATE_REASON_ORDER = [ + "invalid-compiled-state", + "nonterminal-work", + "ambiguous-head", + "rejected-head", + "missing-result-event", + "missing-artifact-receipt", + "bad-artifact-sha256", + "stale-result-ref", + "wrong-reviewer", + "missing-review-evidence", + "unsatisfied-requirement", +] as const; + +function refused(reasons: ReadonlySet): CandidateResult { + return { ok: false, reasons: CANDIDATE_REASON_ORDER.filter((reason) => reasons.has(reason)) }; +} + +function compareTuple(left: readonly string[], right: readonly string[]): number { + for (let index = 0; index < Math.min(left.length, right.length); index += 1) { + if (left[index] === right[index]) continue; + return left[index]! < right[index]! ? -1 : 1; + } + return left.length - right.length; +} + +function canonicalJson(value: unknown): string { + if (value === null || typeof value !== "object") return JSON.stringify(value); + if (Array.isArray(value)) return `[${value.map(canonicalJson).join(",")}]`; + const object = value as Record; + return `{${Object.keys(object).sort().map((key) => `${JSON.stringify(key)}:${canonicalJson(object[key])}`).join(",")}}`; +} + +function sameRecord(left: Record, right: Record): boolean { + const leftKeys = Object.keys(left).sort(); + const rightKeys = Object.keys(right).sort(); + return leftKeys.length === rightKeys.length && leftKeys.every((key, index) => + key === rightKeys[index] && left[key] === right[key]); +} + +function wouldCycle(work: Record): boolean { + for (const id of Object.keys(work)) { + const seen = new Set([id]); + let next = work[id]!.supersedes; + while (next !== null) { + if (seen.has(next)) return true; + seen.add(next); + next = work[next]!.supersedes; + } + } + return false; +} + +function parseRosterMembers(roster: PrometheusRoster): Record | null { + const value: unknown = roster; + if (!isRecord(value) || !hasExactKeys(value, PROMETHEUS_ROLES)) return null; + + const members = Object.create(null) as Record; + const seen = new Set(); + for (const role of PROMETHEUS_ROLES) { + const seat = value[role]; + if (!isRecord(seat) || !Object.hasOwn(seat, "memberId") || + typeof seat.memberId !== "string" || !MEMBER_ID.test(seat.memberId) || seen.has(seat.memberId)) return null; + members[role] = seat.memberId; + seen.add(seat.memberId); + } + return members; +} + +export function parsePrometheusCandidate(state: CompiledState, roster: PrometheusRoster): CandidateResult { + const wire = parseWireState(state); + const members = parseRosterMembers(roster); + if (!wire || !members || wouldCycle(wire.work)) { + return { ok: false, reasons: ["invalid-compiled-state"] }; + } + const reviewerMemberId = members.reviewer; + + const reasons = new Set(); + const entries = Object.entries(wire.work); + if (entries.some(([, work]) => !TERMINAL_WORK_STATES.has(work.state))) reasons.add("nonterminal-work"); + + const superseded = new Set(entries.flatMap(([, work]) => work.supersedes === null ? [] : [work.supersedes])); + const heads = entries.filter(([id]) => !superseded.has(id)); + if (heads.length !== 1) { + reasons.add("ambiguous-head"); + return refused(reasons); + } + const [workId, head] = heads[0]!; + if (head.state !== "RESOLVED") { + if (TERMINAL_WORK_STATES.has(head.state)) reasons.add("rejected-head"); + return refused(reasons); + } + if (head.requester !== members.coordinator || head.assignee_addressed.length !== 1 || + head.assignee_addressed[0] !== members.builder || head.accepted_by !== members.builder) { + reasons.add("invalid-compiled-state"); + } + + const resultRef = head.result; + if (resultRef === null) { + reasons.add("missing-result-event"); + return refused(reasons); + } + const resultEvents = wire.events.filter((event) => event.hash === resultRef); + const resultEvent = resultEvents.length === 1 && resultEvents[0]!.type === "work.result" && resultEvents[0]!.work_id === workId + ? resultEvents[0]! + : null; + if (!resultEvent) { + reasons.add("missing-result-event"); + return refused(reasons); + } + if (resultEvent.actor !== members.builder) reasons.add("invalid-compiled-state"); + + const resultArtifacts = resultEvent.refs.artifacts; + const resultReceipts = resultEvent.refs.artifact_receipts; + const artifacts: CandidateArtifact[] = []; + const resultArtifactIds = new Set(); + if (!validNamedArtifacts(resultArtifacts) || !validReceipts(resultReceipts) || + head.deliverables.length === 0 || !head.deliverables.every((name) => Object.hasOwn(resultArtifacts, name)) || + !sameRecord(head.artifacts, resultArtifacts)) { + reasons.add("missing-artifact-receipt"); + } else { + for (const artifactId of Object.values(resultArtifacts)) resultArtifactIds.add(artifactId); + if (Object.keys(resultArtifacts).length !== head.deliverables.length || + Object.keys(resultReceipts).some((id) => !resultArtifactIds.has(id))) reasons.add("invalid-compiled-state"); + for (const [name, artifactId] of Object.entries(resultArtifacts)) { + const receipt = resultReceipts[artifactId]; + if (!receipt || receipt.id !== artifactId) { + reasons.add("missing-artifact-receipt"); + continue; + } + if (!ARTIFACT_SHA256.test(receipt.sha256)) { + reasons.add("bad-artifact-sha256"); + continue; + } + artifacts.push({ name, artifactId, sha256: receipt.sha256 }); + } + } + + const requirements = Object.entries(head.requirements); + if (requirements.length === 0 || requirements.some(([, requirement]) => requirement.status !== "satisfied")) { + reasons.add("unsatisfied-requirement"); + } + const verifyRequirements = requirements.filter(([, requirement]) => requirement.kind === "verify"); + if (verifyRequirements.length === 0) reasons.add("unsatisfied-requirement"); + + const verifications: CandidateVerification[] = []; + for (const [requirementId, requirement] of verifyRequirements) { + if (requirement.q !== 1 || requirement.eligible.length !== 1 || requirement.eligible[0] !== reviewerMemberId || + requirement.pos.length !== 1 || requirement.pos[0] !== reviewerMemberId || requirement.neg.length !== 0) { + reasons.add("wrong-reviewer"); + } + + const forRequirement = wire.events.filter((event) => + event.type === "verification.pass" && event.work_id === workId && event.refs.requirement === requirementId); + const forResult = forRequirement.filter((event) => event.refs.result_ref === resultRef); + if (forRequirement.length > 0 && forResult.length === 0) reasons.add("stale-result-ref"); + const byReviewer = forResult.filter((event) => event.actor === reviewerMemberId); + if (forResult.length > 0 && byReviewer.length === 0) reasons.add("wrong-reviewer"); + if (byReviewer.length > 1) reasons.add("invalid-compiled-state"); + if (byReviewer.length !== 1) { + if (forRequirement.length === 0) reasons.add("missing-review-evidence"); + continue; + } + + const verification = byReviewer[0]!; + const evidence = verification.refs.evidence; + const evidenceReceipts = verification.refs.artifact_receipts; + if (typeof evidence !== "string" || !ARTIFACT_ID.test(evidence) || !validReceipts(evidenceReceipts)) { + reasons.add("missing-review-evidence"); + continue; + } + if (resultArtifactIds.has(evidence)) { + reasons.add("missing-review-evidence"); + continue; + } + const receipt = evidenceReceipts[evidence]; + if (!receipt || receipt.id !== evidence || Object.keys(evidenceReceipts).length !== 1) { + reasons.add("missing-review-evidence"); + continue; + } + if (!ARTIFACT_SHA256.test(receipt.sha256)) { + reasons.add("bad-artifact-sha256"); + continue; + } + verifications.push({ + eventHash: verification.hash, + requirementId, + evidenceArtifactId: evidence, + evidenceSha256: receipt.sha256, + }); + } + + if (reasons.size > 0) return refused(reasons); + artifacts.sort((left, right) => compareTuple( + [left.name, left.artifactId, left.sha256], + [right.name, right.artifactId, right.sha256], + )); + verifications.sort((left, right) => compareTuple( + [left.eventHash, left.requirementId, left.evidenceArtifactId, left.evidenceSha256], + [right.eventHash, right.requirementId, right.evidenceArtifactId, right.evidenceSha256], + )); + + const revisionInput = { + state_digest: wire.digest, + work_id: workId, + result_ref: resultRef, + artifacts: artifacts.map((artifact) => ({ + name: artifact.name, + artifact_id: artifact.artifactId, + sha256: artifact.sha256, + })), + verifications: verifications.map((verification) => ({ + event_hash: verification.eventHash, + requirement_id: verification.requirementId, + evidence_artifact_id: verification.evidenceArtifactId, + evidence_sha256: verification.evidenceSha256, + })), + }; + const revisionSha256 = createHash("sha256").update(canonicalJson(revisionInput), "utf8").digest("hex"); + return { + ok: true, + candidate: { + stateDigest: wire.digest, + workId, + resultRef, + artifacts, + verifications, + revisionSha256, + }, + }; +} + +export interface PrometheusMessage { + sequence: number; + content: string; + sender_instance_id?: string; + sender?: { member_id?: string }; +} + +export type PrometheusRoster = { + readonly [Role in PrometheusRole]: { readonly memberId: string }; +}; + +export interface TerminalReceipt { + sequence: number; + role: PrometheusRole; + memberId: string; + runId: string; + revisionSha256: string; +} + +export interface TerminalReceiptResult { + ok: boolean; + receipts: Partial>; + missingRoles: PrometheusRole[]; +} + +const TERMINAL_RECEIPT_PREFIX = "[sharednet-prometheus:v1] "; + +function skipJsonWhitespace(raw: string, start: number): number { + let index = start; + while (index < raw.length && /[ \t\r\n]/.test(raw[index]!)) index += 1; + return index; +} + +function readJsonString(raw: string, start: number): { value: string; next: number } | null { + if (raw[start] !== "\"") return null; + let index = start + 1; + while (index < raw.length) { + if (raw[index] === "\\") { + index += 2; + continue; + } + if (raw[index] === "\"") { + const encoded = raw.slice(start, index + 1); + try { + const value: unknown = JSON.parse(encoded); + return typeof value === "string" ? { value, next: index + 1 } : null; + } catch { + return null; + } + } + index += 1; + } + return null; +} + +/** The receipt object is deliberately string-only, which lets duplicate keys be rejected before JSON can erase them. */ +function parseReceiptObject(raw: string): Record | null { + let index = skipJsonWhitespace(raw, 0); + if (raw[index] !== "{") return null; + index = skipJsonWhitespace(raw, index + 1); + const result = Object.create(null) as Record; + if (raw[index] === "}") return skipJsonWhitespace(raw, index + 1) === raw.length ? result : null; + + while (index < raw.length) { + const key = readJsonString(raw, index); + if (!key || Object.hasOwn(result, key.value)) return null; + index = skipJsonWhitespace(raw, key.next); + if (raw[index] !== ":") return null; + index = skipJsonWhitespace(raw, index + 1); + const value = readJsonString(raw, index); + if (!value) return null; + result[key.value] = value.value; + index = skipJsonWhitespace(raw, value.next); + if (raw[index] === "}") { + index = skipJsonWhitespace(raw, index + 1); + return index === raw.length ? result : null; + } + if (raw[index] !== ",") return null; + index = skipJsonWhitespace(raw, index + 1); + } + return null; +} + +function parsedTerminalReceipt(content: string): { + type: "terminal.receipt"; + run_id: string; + role: PrometheusRole; + revision_sha256: string; +} | null { + if (content.includes("\n") || content.includes("\r") || !content.startsWith(TERMINAL_RECEIPT_PREFIX)) return null; + const value = parseReceiptObject(content.slice(TERMINAL_RECEIPT_PREFIX.length)); + if (!value || !hasExactKeys(value, ["type", "run_id", "role", "revision_sha256"]) || + value.type !== "terminal.receipt" || !PROMETHEUS_ROLES.includes(value.role as PrometheusRole) || + !ARTIFACT_SHA256.test(value.revision_sha256!)) return null; + return value as { + type: "terminal.receipt"; + run_id: string; + role: PrometheusRole; + revision_sha256: string; + }; +} + +export function parseTerminalReceipts( + messages: readonly PrometheusMessage[], + roster: PrometheusRoster, + runId: string, + revisionSha256: string, +): TerminalReceiptResult { + const receipts: Partial> = {}; + const members = parseRosterMembers(roster); + if (!members) return { ok: false, receipts, missingRoles: [...PROMETHEUS_ROLES] }; + const memberRoles = new Map(); + for (const role of PROMETHEUS_ROLES) { + const memberId = members[role]; + memberRoles.set(memberId, role); + } + if (!ARTIFACT_SHA256.test(revisionSha256)) { + return { ok: false, receipts, missingRoles: [...PROMETHEUS_ROLES] }; + } + + for (const message of messages) { + if (!Number.isSafeInteger(message.sequence) || message.sequence < 1 || typeof message.content !== "string") continue; + const direct = message.sender_instance_id; + const nested = message.sender?.member_id; + if (direct !== undefined && nested !== undefined && direct !== nested) continue; + const memberId = nested ?? direct; + if (typeof memberId !== "string") continue; + const savedRole = memberRoles.get(memberId); + if (!savedRole) continue; + + const receipt = parsedTerminalReceipt(message.content); + if (!receipt || receipt.role !== savedRole || receipt.run_id !== runId || receipt.revision_sha256 !== revisionSha256) continue; + const previous = receipts[savedRole]; + if (previous && previous.sequence >= message.sequence) continue; + receipts[savedRole] = { + sequence: message.sequence, + role: savedRole, + memberId, + runId, + revisionSha256, + }; + } + + const missingRoles = PROMETHEUS_ROLES.filter((role) => receipts[role] === undefined); + return { ok: missingRoles.length === 0, receipts, missingRoles }; +} diff --git a/packages/cli/src/prometheus-runtime.test.ts b/packages/cli/src/prometheus-runtime.test.ts new file mode 100644 index 0000000..370de2d --- /dev/null +++ b/packages/cli/src/prometheus-runtime.test.ts @@ -0,0 +1,258 @@ +// @vitest-environment node + +import { describe, expect, it } from "vitest"; + +import { DEFAULT_IMAGE, type DockerRunner, type RunPlan } from "./goal-run.ts"; +import { ensurePrometheusListeners, ensurePrometheusTeam, inspectPrometheusRuntime, runPrometheusCheck } from "./prometheus-runtime.ts"; +import type { PrometheusTeamState } from "./prometheus-state.ts"; + +const plan: RunPlan = { + image: DEFAULT_IMAGE, + agents: [ + { name: "coordinator", driver: "codex", model: "gpt-6-sol" }, + { name: "builder", driver: "codex", model: "gpt-6-sol" }, + { name: "reviewer", driver: "codex", model: "gpt-6-sol" }, + ], + workspace: "/work/project", + cli: { root: "/opt/client", entry: "src/main.ts" }, + codexAuth: { kind: "login", file: "/owner/.codex/auth.json" }, + claudeAuth: null, +}; + +const WORKSPACE_KEY = "0".repeat(24); + +function inspectPayload(role: "coordinator" | "builder" | "reviewer", overrides: Record = {}) { + const container = `sharednet-prometheus-${WORKSPACE_KEY}-${role}`; + return JSON.stringify([{ + Config: { + Image: DEFAULT_IMAGE, + User: "1000:1000", + Cmd: ["sleep", "infinity"], + Labels: { + "io.sharednet.prometheus": "local-v1", + "io.sharednet.prometheus.workspace": WORKSPACE_KEY, + "io.sharednet.prometheus.role": role, + }, + }, + Mounts: [ + { Type: "volume", Name: `${container}-home`, Destination: "/home/prometheus", RW: true }, + { Type: "bind", Source: plan.workspace, Destination: "/workspace", RW: true }, + { Type: "bind", Source: plan.cli.root, Destination: "/opt/sharednet", RW: false }, + { Type: "bind", Source: plan.codexAuth!.file, Destination: "/run/codex/auth.json", RW: false }, + ], + ...overrides, + }]); +} + +describe("persistent Prometheus runtime", () => { + it("creates three stable role homes and reuses their saved threads", async () => { + const calls: string[][] = []; + const created = new Set(); + const docker: DockerRunner = async (args) => { + calls.push([...args]); + const role = (["coordinator", "builder", "reviewer"] as const).find((candidate) => args.join(" ").includes(candidate)); + if (args[0] === "container" && args[1] === "inspect") { + return role && created.has(role) + ? { code: 0, stdout: inspectPayload(role), stderr: "" } + : { code: 1, stdout: "", stderr: "missing" }; + } + if (args[0] === "create" && role) created.add(role); + if (args[0] === "exec" && args.includes("cat") && role) { + return { code: 0, stdout: `thr-${role}\n`, stderr: "" }; + } + if (args[0] === "exec" && args.includes("codex")) { + const container = args.find((value) => value.startsWith("sharednet-prometheus-"))!; + return { code: 0, stdout: `${JSON.stringify({ type: "thread.started", thread_id: `thr-${container.split("-").at(-1)}` })}\n`, stderr: "" }; + } + return { code: 0, stdout: "", stderr: "" }; + }; + const input = { + workspaceKey: WORKSPACE_KEY, + workspacePath: plan.workspace, + model: "gpt-6-sol", + policySha256: "a".repeat(64), + policy: "## Motto\nShip it\n", + plan, + }; + + const first = await ensurePrometheusTeam(input, { docker, now: () => new Date("2026-10-18T00:00:00.000Z"), uid: 1000, gid: 1000 }); + const beforeReuse = calls.length; + const second = await ensurePrometheusTeam({ ...input, existing: first }, { docker, now: () => new Date("2026-10-18T00:01:00.000Z"), uid: 1000, gid: 1000 }); + + expect(first.roles.coordinator).toMatchObject({ container_name: `sharednet-prometheus-${WORKSPACE_KEY}-coordinator`, volume_name: `sharednet-prometheus-${WORKSPACE_KEY}-coordinator-home` }); + expect(Object.values(first.roles).map((role) => role.thread_id)).toEqual(["thr-coordinator", "thr-builder", "thr-reviewer"]); + expect(second.roles).toEqual(first.roles); + const creates = calls.filter((args) => args[0] === "create"); + expect(creates).toHaveLength(3); + expect(creates.every((args) => args.includes("--user") && args.includes("1000:1000"))).toBe(true); + expect(calls.slice(beforeReuse).some((args) => args.includes("codex"))).toBe(false); + expect(calls.flat().some((value) => value.includes("snk_") || value.includes("rit_"))).toBe(false); + }); + + it("rejects a saved container whose image, user, labels, mounts, auth mount, or thread marker changed", async () => { + const existing: PrometheusTeamState = { + schema_version: 1, + workspace_key: WORKSPACE_KEY, + workspace_path: plan.workspace, + model: "gpt-6-sol", + roles: { + coordinator: { container_name: `sharednet-prometheus-${WORKSPACE_KEY}-coordinator`, volume_name: `sharednet-prometheus-${WORKSPACE_KEY}-coordinator-home`, thread_id: "thr-coordinator" }, + builder: { container_name: `sharednet-prometheus-${WORKSPACE_KEY}-builder`, volume_name: `sharednet-prometheus-${WORKSPACE_KEY}-builder-home`, thread_id: "thr-builder" }, + reviewer: { container_name: `sharednet-prometheus-${WORKSPACE_KEY}-reviewer`, volume_name: `sharednet-prometheus-${WORKSPACE_KEY}-reviewer-home`, thread_id: "thr-reviewer" }, + }, + last_policy_sha256: "a".repeat(64), + created_at: "2026-10-18T00:00:00.000Z", + updated_at: "2026-10-18T00:00:00.000Z", + }; + const docker: DockerRunner = async (args) => { + const role = (["coordinator", "builder", "reviewer"] as const).find((candidate) => args.join(" ").includes(candidate))!; + if (args[0] === "container" && args[1] === "inspect") { + return { code: 0, stdout: inspectPayload(role, { Config: { Image: "attacker/image", User: "0:0", Cmd: ["sleep", "infinity"], Labels: {} } }), stderr: "" }; + } + if (args[0] === "exec" && args.includes("cat")) return { code: 0, stdout: `thr-${role}\n`, stderr: "" }; + return { code: 0, stdout: "", stderr: "" }; + }; + + await expect(ensurePrometheusTeam({ + workspaceKey: WORKSPACE_KEY, workspacePath: plan.workspace, model: "gpt-6-sol", + policySha256: "a".repeat(64), policy: "## Motto\nShip\n", plan, existing, + }, { docker, uid: 1000, gid: 1000 })).rejects.toMatchObject({ code: "prometheus_team_invalid" }); + }); + + it("reconciles a partially bootstrapped team from verified containers and saved thread markers", async () => { + const calls: string[][] = []; + const started = new Set(); + const docker: DockerRunner = async (args) => { + calls.push([...args]); + const role = (["coordinator", "builder", "reviewer"] as const).find((candidate) => args.join(" ").includes(candidate))!; + if (args[0] === "container" && args[1] === "inspect") return { code: 0, stdout: inspectPayload(role), stderr: "" }; + if (args[0] === "start") started.add(role); + if (args[0] === "exec" && args.includes("cat")) return started.has(role) + ? { code: 0, stdout: `thr-${role}\n`, stderr: "" } + : { code: 1, stdout: "", stderr: "container is stopped" }; + return { code: 0, stdout: "", stderr: "" }; + }; + + const recovered = await ensurePrometheusTeam({ + workspaceKey: WORKSPACE_KEY, workspacePath: plan.workspace, model: "gpt-6-sol", + policySha256: "a".repeat(64), policy: "## Motto\nShip\n", plan, + }, { docker, now: () => new Date("2026-10-18T00:00:00.000Z"), uid: 1000, gid: 1000 }); + + expect(Object.values(recovered.roles).map((role) => role.thread_id)).toEqual(["thr-coordinator", "thr-builder", "thr-reviewer"]); + expect(calls.some((args) => args[0] === "create")).toBe(false); + expect(calls.some((args) => args.includes("codex"))).toBe(false); + }); + + it("starts a verified stopped pre-bootstrap container before checking its marker, then completes bootstrap", async () => { + const calls: string[][] = []; + const started = new Set(); + const marked = new Set(); + const docker: DockerRunner = async (args) => { + calls.push([...args]); + const role = (["coordinator", "builder", "reviewer"] as const).find((candidate) => args.join(" ").includes(candidate))!; + if (args[0] === "container" && args[1] === "inspect") return { code: 0, stdout: inspectPayload(role), stderr: "" }; + if (args[0] === "start") { started.add(role); return { code: 0, stdout: "", stderr: "" }; } + if (args[0] === "exec" && args.includes("cat")) { + if (!started.has(role)) return { code: 1, stdout: "", stderr: "container is stopped" }; + return marked.has(role) ? { code: 0, stdout: `thr-${role}\n`, stderr: "" } : { code: 1, stdout: "", stderr: "missing" }; + } + if (args[0] === "exec" && args.includes("codex")) { + return { code: 0, stdout: `${JSON.stringify({ type: "thread.started", thread_id: `thr-${role}` })}\n`, stderr: "" }; + } + if (args.join(" ").includes(".prometheus-thread-id")) marked.add(role); + return { code: 0, stdout: "", stderr: "" }; + }; + + const recovered = await ensurePrometheusTeam({ + workspaceKey: WORKSPACE_KEY, workspacePath: plan.workspace, model: "gpt-6-sol", + policySha256: "a".repeat(64), policy: "## Motto\nShip\n", plan, + }, { docker, now: () => new Date("2026-10-18T00:00:00.000Z"), uid: 1000, gid: 1000 }); + + expect(Object.values(recovered.roles).map((role) => role.thread_id)).toEqual(["thr-coordinator", "thr-builder", "thr-reviewer"]); + expect(calls.filter((args) => args[0] === "create")).toHaveLength(0); + expect(calls.filter((args) => args.includes("codex"))).toHaveLength(3); + for (const role of ["coordinator", "builder", "reviewer"] as const) { + const start = calls.findIndex((args) => args[0] === "start" && args.join(" ").includes(role)); + const markerRead = calls.findIndex((args) => args.includes("cat") && args.join(" ").includes(role)); + expect(start).toBeGreaterThanOrEqual(0); + expect(markerRead).toBeGreaterThan(start); + } + }); + + it("passes check metadata through the environment and bounds output", async () => { + const seen: Array<{ args: readonly string[]; env?: Record }> = []; + const docker: DockerRunner = async (args, options) => { + seen.push({ args, env: options?.env }); + return { code: 7, stdout: "x".repeat(70_000), stderr: "tail" }; + }; + const result = await runPrometheusCheck( + { image: DEFAULT_IMAGE, workspace: plan.workspace }, + "pnpm test", + { + SHAREDNET_PROMETHEUS_RUN_ID: "run-1", + SHAREDNET_PROMETHEUS_WORK_ID: "work-1", + SHAREDNET_PROMETHEUS_RESULT_REF: "evt-1", + SHAREDNET_PROMETHEUS_REVISION_SHA256: "b".repeat(64), + }, + { docker, now: () => new Date("2026-10-18T00:00:00.000Z") }, + ); + expect(seen[0]!.env).toEqual({ + SHAREDNET_PROMETHEUS_RUN_ID: "run-1", + SHAREDNET_PROMETHEUS_WORK_ID: "work-1", + SHAREDNET_PROMETHEUS_RESULT_REF: "evt-1", + SHAREDNET_PROMETHEUS_REVISION_SHA256: "b".repeat(64), + }); + expect(seen[0]!.args.join(" ")).not.toContain("run-1"); + expect(result.exitCode).toBe(7); + expect(Buffer.byteLength(result.output)).toBeLessThanOrEqual(64_000); + expect(result.outputSha256).toMatch(/^[0-9a-f]{64}$/); + }); + + it("restarts unfiltered listeners for saved seats without joining again", async () => { + const calls: string[][] = []; + const team: PrometheusTeamState = { + schema_version: 1, workspace_key: "0".repeat(24), workspace_path: "/work/project", model: "gpt-6-sol", + roles: { + coordinator: { container_name: "c-coordinator", volume_name: "v1", thread_id: "t1" }, + builder: { container_name: "c-builder", volume_name: "v2", thread_id: "t2" }, + reviewer: { container_name: "c-reviewer", volume_name: "v3", thread_id: "t3" }, + }, + last_policy_sha256: "a".repeat(64), created_at: "2026-10-18T00:00:00.000Z", updated_at: "2026-10-18T00:00:00.000Z", + }; + const roster = { + coordinator: { member_id: "i_coord", thread_id: "t1", container_name: "c-coordinator" }, + builder: { member_id: "i_build", thread_id: "t2", container_name: "c-builder" }, + reviewer: { member_id: "i_review", thread_id: "t3", container_name: "c-reviewer" }, + }; + const docker: DockerRunner = async (args) => { + calls.push([...args]); + if (args.includes("--status")) { + const role = String(args.find((arg) => arg.startsWith("c-"))).slice(2); + const member = role === "coordinator" ? "i_coord" : role === "builder" ? "i_build" : "i_review"; + const checks = calls.filter((call) => call.includes("--status") && call.includes(`c-${role}`)).length; + return { code: 0, stdout: JSON.stringify({ running: checks > 1, wake_protocol: "wake-plugins-v1", seats: checks > 1 ? [{ room_id: "rom_AbCdEfGhIj", member_id: member, state: "listening" }] : [] }), stderr: "" }; + } + return { code: 0, stdout: "", stderr: "" }; + }; + await ensurePrometheusListeners(team, "rom_AbCdEfGhIj", roster, "https://www.sharednet.ai", { docker, sleep: async () => undefined }); + expect(calls.some((call) => call.includes("join"))).toBe(false); + expect(calls.filter((call) => call.includes("sh") && call.join(" ").includes("sharednet serve"))).toHaveLength(3); + expect(calls.filter((call) => call.includes("--rooms"))).toHaveLength(0); + }); + + it("keeps only the active Room's unsettled handed records", async () => { + const team = { + schema_version: 1 as const, workspace_key: "0".repeat(24), workspace_path: "/work/project", model: "gpt-6-sol", + roles: { coordinator: { container_name: "c1", volume_name: "v1", thread_id: "t1" }, builder: { container_name: "c2", volume_name: "v2", thread_id: "t2" }, reviewer: { container_name: "c3", volume_name: "v3", thread_id: "t3" } }, + last_policy_sha256: "a".repeat(64), created_at: "2026-10-18T00:00:00.000Z", updated_at: "2026-10-18T00:00:00.000Z", + }; + const roster = { coordinator: { member_id: "i_coord", thread_id: "t1", container_name: "c1" }, builder: { member_id: "i_build", thread_id: "t2", container_name: "c2" }, reviewer: { member_id: "i_review", thread_id: "t3", container_name: "c3" } }; + const members: Record = { c1: "i_coord", c2: "i_build", c3: "i_review" }; + const docker: DockerRunner = async (args) => args.includes("--status") + ? { code: 0, stdout: JSON.stringify({ running: true, wake_protocol: "wake-plugins-v1", seats: [{ room_id: "rom_AbCdEfGhIj", member_id: members[String(args.find((arg) => /^c\d$/.test(arg)))], state: "listening" }] }), stderr: "" } + : { code: 0, stdout: JSON.stringify({ unanswered: [{ room_id: "rom_AbCdEfGhIj", through: 2 }, { room_id: "rom_OtherRoom1", through: 3 }] }), stderr: "" }; + const status = await inspectPrometheusRuntime(team, "rom_AbCdEfGhIj", roster, { docker }); + expect(status.healthy).toBe(true); + expect(status.roles.coordinator.unanswered).toEqual([{ room_id: "rom_AbCdEfGhIj", through: 2 }]); + }); +}); diff --git a/packages/cli/src/prometheus-runtime.ts b/packages/cli/src/prometheus-runtime.ts new file mode 100644 index 0000000..fa16ecf --- /dev/null +++ b/packages/cli/src/prometheus-runtime.ts @@ -0,0 +1,419 @@ +import { createHash } from "node:crypto"; + +import { localError } from "./errors.ts"; +import { type DockerRunner, type RunPlan } from "./goal-run.ts"; +import { PROMETHEUS_ROLES, type PrometheusRole } from "./prometheus-protocol.ts"; +import type { PrometheusRoster, PrometheusTeamState } from "./prometheus-state.ts"; + +const CONTAINER_WORKSPACE = "/workspace"; +const CONTAINER_HOME = "/home/prometheus"; +const CONTAINER_CLI = "/opt/sharednet"; +const CONTAINER_AUTH = "/run/codex/auth.json"; +const THREAD_MARKER = `${CONTAINER_HOME}/.prometheus-thread-id`; +const MAX_OUTPUT_BYTES = 64_000; +const PROMETHEUS_LABEL = "io.sharednet.prometheus"; + +export interface PrometheusRuntimeDependencies { + docker: DockerRunner; + now?: () => Date; + sleep?: (milliseconds: number) => Promise; + uid?: number | null; + gid?: number | null; +} + +export interface EnsureTeamInput { + workspaceKey: string; + workspacePath: string; + model: string; + policySha256: string; + policy: string; + plan: RunPlan; + existing?: PrometheusTeamState | null; +} + +export interface AttachRoomInput { + roomId: string; + inviteToken: string; + baseUrl: string; + turnLimitMs?: number; + idempotencyKeys?: Partial>; +} + +export type PrometheusServeState = "listening" | "working" | "retrying" | "stopped" | "unhealthy"; + +export interface PrometheusRuntimeStatus { + healthy: boolean; + roles: Record; +} + +export interface SemanticCheckResult { + startedAt: string; + finishedAt: string; + exitCode: number; + output: string; + outputSha256: string; +} + +function stableName(workspaceKey: string, role: PrometheusRole): string { + return `sharednet-prometheus-${workspaceKey}-${role}`; +} + +interface DockerInspect { + Config?: { Image?: unknown; User?: unknown; Cmd?: unknown; Labels?: Record | null }; + Mounts?: Array<{ Type?: unknown; Name?: unknown; Source?: unknown; Destination?: unknown; RW?: unknown }>; +} + +function expectedUser(dependencies: PrometheusRuntimeDependencies): string { + const uid = dependencies.uid ?? (typeof process.getuid === "function" ? process.getuid() : null); + const gid = dependencies.gid ?? (typeof process.getgid === "function" ? process.getgid() : null); + return Number.isInteger(uid) && Number.isInteger(gid) ? `${uid}:${gid}` : "node"; +} + +function hasMount(inspected: DockerInspect, expected: { type: "bind" | "volume"; source: string; destination: string; rw: boolean }): boolean { + return (inspected.Mounts ?? []).some((mount) => + mount.Type === expected.type && + (expected.type === "volume" ? mount.Name === expected.source : mount.Source === expected.source) && + mount.Destination === expected.destination && + mount.RW === expected.rw); +} + +function parseInspect(raw: string): DockerInspect | null { + try { + const parsed = JSON.parse(raw) as unknown; + return Array.isArray(parsed) && parsed.length === 1 && typeof parsed[0] === "object" && parsed[0] !== null + ? parsed[0] as DockerInspect + : null; + } catch { + return null; + } +} + +async function verifiedContainer( + input: EnsureTeamInput, + role: PrometheusRole, + dependencies: PrometheusRuntimeDependencies, +): Promise<{ exists: false } | { exists: true }> { + const container = stableName(input.workspaceKey, role); + const volume = `${container}-home`; + const result = await dependencies.docker(["container", "inspect", container]); + if (result.code !== 0) return { exists: false }; + const inspected = parseInspect(result.stdout); + const labels = inspected?.Config?.Labels ?? {}; + const authFile = input.plan.codexAuth?.kind === "login" ? input.plan.codexAuth.file : null; + const identityMatches = inspected !== null && + inspected.Config?.Image === input.plan.image && + inspected.Config?.User === expectedUser(dependencies) && + JSON.stringify(inspected.Config?.Cmd) === JSON.stringify(["sleep", "infinity"]) && + labels[PROMETHEUS_LABEL] === "local-v1" && + labels[`${PROMETHEUS_LABEL}.workspace`] === input.workspaceKey && + labels[`${PROMETHEUS_LABEL}.role`] === role && + hasMount(inspected, { type: "volume", source: volume, destination: CONTAINER_HOME, rw: true }) && + hasMount(inspected, { type: "bind", source: input.workspacePath, destination: CONTAINER_WORKSPACE, rw: true }) && + hasMount(inspected, { type: "bind", source: input.plan.cli.root, destination: CONTAINER_CLI, rw: false }) && + (authFile !== null + ? hasMount(inspected, { type: "bind", source: authFile, destination: CONTAINER_AUTH, rw: false }) + : !(inspected.Mounts ?? []).some((mount) => mount.Destination === CONTAINER_AUTH)); + if (!identityMatches) { + throw localError("prometheus_team_invalid", `The saved ${role} container does not match the Prometheus runtime identity.`); + } + return { exists: true }; +} + +/** Verify saved container identity without starting it or reading mutable runtime state. */ +export async function verifyPrometheusTeamIdentity( + input: EnsureTeamInput & { existing: PrometheusTeamState }, + dependencies: PrometheusRuntimeDependencies, +): Promise { + if (input.existing.model !== input.model || input.existing.workspace_key !== input.workspaceKey || input.existing.workspace_path !== input.workspacePath) { + throw localError("prometheus_team_mismatch", "The saved Prometheus team does not match this workspace and model."); + } + for (const role of PROMETHEUS_ROLES) { + const expectedContainer = stableName(input.workspaceKey, role); + const saved = input.existing.roles[role]; + if (saved.container_name !== expectedContainer || saved.volume_name !== `${expectedContainer}-home` || !saved.thread_id) { + throw localError("prometheus_team_invalid", `The saved ${role} runtime cannot be reused safely.`); + } + const inspected = await verifiedContainer(input, role, dependencies); + if (!inspected.exists) throw localError("prometheus_team_missing", `The saved ${role} container is missing; Prometheus will not replace its thread.`); + } +} + +function temporaryTeam(input: EnsureTeamInput): PrometheusTeamState { + return { + schema_version: 1 as const, + workspace_key: input.workspaceKey, + workspace_path: input.workspacePath, + model: input.model, + roles: Object.fromEntries(PROMETHEUS_ROLES.map((candidate) => [candidate, { + container_name: stableName(input.workspaceKey, candidate), + volume_name: `${stableName(input.workspaceKey, candidate)}-home`, + thread_id: "pending", + }])) as PrometheusTeamState["roles"], + last_policy_sha256: input.policySha256, + created_at: new Date(0).toISOString(), + updated_at: new Date(0).toISOString(), + }; +} + +async function savedThreadMarker( + input: EnsureTeamInput, + role: PrometheusRole, + dependencies: PrometheusRuntimeDependencies, +): Promise { + const marker = await roleExec(dependencies.docker, temporaryTeam(input), role, ["cat", THREAD_MARKER]); + const threadId = marker.stdout.trim(); + return marker.code === 0 && /^[A-Za-z0-9][A-Za-z0-9._-]{0,255}$/.test(threadId) ? threadId : null; +} + +function roleExec( + docker: DockerRunner, + team: PrometheusTeamState, + role: PrometheusRole, + command: readonly string[], + options: { env?: Record; detach?: boolean; input?: string } = {}, +) { + const environment = options.env ?? {}; + return docker([ + "exec", + ...(options.detach ? ["--detach"] : []), + "--workdir", CONTAINER_WORKSPACE, + "-e", `HOME=${CONTAINER_HOME}`, + "-e", `CODEX_HOME=${CONTAINER_HOME}/.codex`, + ...Object.keys(environment).flatMap((name) => ["-e", name]), + team.roles[role].container_name, + ...command, + ], { env: environment, ...(options.input === undefined ? {} : { input: options.input }) }); +} + +function threadId(stdout: string): string | null { + for (const line of stdout.split("\n")) { + try { + const item = JSON.parse(line) as { type?: unknown; thread_id?: unknown }; + if (item.type === "thread.started" && typeof item.thread_id === "string" && item.thread_id.length > 0) return item.thread_id; + } catch {} + } + return null; +} + +function bootstrapPrompt(role: PrometheusRole, policy: string): string { + const duty = role === "coordinator" + ? "decompose typed work, route exact member-id handoffs, and resolve the verified result" + : role === "builder" + ? "accept typed work, modify the shared workspace, and publish the submission artifact" + : "independently inspect the exact submission artifact and publish verification evidence"; + return [ + `You are the persistent Prometheus ${role}. Your duty is to ${duty}.`, + "The current PROMETHEUS.md policy follows. A later run may supersede it.", + policy, + "This bootstrap turn creates your persistent thread only. Do no Goal work and reply exactly READY.", + ].join("\n\n"); +} + +/** Create the fixed local team once; subsequent calls verify and reuse its exact containers and threads. */ +export async function ensurePrometheusTeam( + input: EnsureTeamInput, + dependencies: PrometheusRuntimeDependencies, +): Promise { + const now = (dependencies.now ?? (() => new Date()))().toISOString(); + const user = expectedUser(dependencies); + if (input.existing) { + await verifyPrometheusTeamIdentity({ ...input, existing: input.existing }, dependencies); + for (const role of PROMETHEUS_ROLES) { + const saved = input.existing.roles[role]; + const started = await dependencies.docker(["start", saved.container_name]); + if (started.code !== 0) throw localError("container_failed", `The saved ${role} container could not be started.`); + const marker = await savedThreadMarker(input, role, dependencies); + if (marker !== saved.thread_id) throw localError("prometheus_team_invalid", `The saved ${role} thread marker does not match team.json.`); + } + return { ...input.existing, last_policy_sha256: input.policySha256, updated_at: now }; + } + + const roles = {} as PrometheusTeamState["roles"]; + for (const role of PROMETHEUS_ROLES) { + const container = stableName(input.workspaceKey, role); + const volume = `${container}-home`; + const recovered = await verifiedContainer(input, role, dependencies); + if (recovered.exists) { + const started = await dependencies.docker(["start", container]); + if (started.code !== 0) throw localError("container_failed", `The saved ${role} container could not be started.`); + const marker = await savedThreadMarker(input, role, dependencies); + if (marker !== null) { + roles[role] = { container_name: container, volume_name: volume, thread_id: marker }; + continue; + } + } else { + const volumeResult = await dependencies.docker(["volume", "create", volume]); + if (volumeResult.code !== 0) throw localError("container_failed", `The ${role} home volume could not be created.`); + const created = await dependencies.docker([ + "create", "--name", container, + "--user", user, + "--label", `${PROMETHEUS_LABEL}=local-v1`, + "--label", `${PROMETHEUS_LABEL}.workspace=${input.workspaceKey}`, + "--label", `${PROMETHEUS_LABEL}.role=${role}`, + "--add-host", "host.docker.internal:host-gateway", + "--volume", `${volume}:${CONTAINER_HOME}`, + "--volume", `${input.workspacePath}:${CONTAINER_WORKSPACE}`, + "--volume", `${input.plan.cli.root}:${CONTAINER_CLI}:ro`, + ...(input.plan.codexAuth?.kind === "login" ? ["--volume", `${input.plan.codexAuth.file}:${CONTAINER_AUTH}:ro`] : []), + input.plan.image, "sleep", "infinity", + ]); + if (created.code !== 0) throw localError("container_failed", `The ${role} container could not be created: ${created.stderr.trim().slice(-400)}`); + const started = await dependencies.docker(["start", container]); + if (started.code !== 0) throw localError("container_failed", `The ${role} container could not be started.`); + } + const entry = `${CONTAINER_CLI}/${input.plan.cli.entry}`; + const wrapper = `#!/bin/sh\nexec node ${entry.endsWith(".ts") ? "--experimental-strip-types --no-warnings " : ""}${entry} "$@"\n`; + const installed = await dependencies.docker(["exec", "--user", "root", "--interactive", container, "sh", "-c", "cat > /usr/local/bin/sharednet && chmod 755 /usr/local/bin/sharednet"], { input: wrapper }); + if (installed.code !== 0) throw localError("container_failed", `The sharednet command could not be installed for ${role}.`); + const prepared = await dependencies.docker(["exec", "--user", "root", container, "sh", "-c", `mkdir -p '${CONTAINER_HOME}/.codex' && chown -R ${user} '${CONTAINER_HOME}'${input.plan.codexAuth?.kind === "login" ? ` && ln -sf '${CONTAINER_AUTH}' '${CONTAINER_HOME}/.codex/auth.json'` : ""}`]); + if (prepared.code !== 0) throw localError("container_failed", `The ${role} Codex home could not be prepared.`); + const configured = await dependencies.docker(["exec", "--user", user, "--interactive", container, "sh", "-c", `cat > '${CONTAINER_HOME}/.codex/config.toml'`], { input: `model = "${input.model}"\napproval_policy = "never"\nsandbox_mode = "danger-full-access"\n` }); + if (configured.code !== 0) throw localError("container_failed", `The ${role} Codex configuration could not be written.`); + const bootstrapped = await dependencies.docker([ + "exec", "--user", user, "--workdir", CONTAINER_WORKSPACE, + "-e", `HOME=${CONTAINER_HOME}`, "-e", `CODEX_HOME=${CONTAINER_HOME}/.codex`, + ...(input.plan.codexAuth?.kind === "api-key" ? ["-e", "OPENAI_API_KEY"] : []), + container, "timeout", "120", "codex", "exec", "--json", "--skip-git-repo-check", "--dangerously-bypass-approvals-and-sandbox", "-m", input.model, bootstrapPrompt(role, input.policy), + ]); + const thread = threadId(bootstrapped.stdout); + if (bootstrapped.code !== 0 || thread === null) throw localError("agent_bootstrap_failed", `The ${role} persistent Codex thread could not be created.`); + const marked = await dependencies.docker(["exec", "--user", user, "--interactive", container, "sh", "-c", `cat > '${THREAD_MARKER}'`], { input: `${thread}\n` }); + if (marked.code !== 0) throw localError("agent_bootstrap_failed", `The ${role} persistent thread marker could not be saved.`); + roles[role] = { container_name: container, volume_name: volume, thread_id: thread }; + } + return { + schema_version: 1, + workspace_key: input.workspaceKey, + workspace_path: input.workspacePath, + model: input.model, + roles, + last_policy_sha256: input.policySha256, + created_at: now, + updated_at: now, + }; +} + +/** Join the saved threads to one fresh Room and wait for each exact seat to be listening. */ +export async function attachPrometheusRoom( + team: PrometheusTeamState, + input: AttachRoomInput, + dependencies: PrometheusRuntimeDependencies, +): Promise { + const roster = {} as PrometheusRoster; + for (const role of PROMETHEUS_ROLES) { + const saved = team.roles[role]; + const joined = await roleExec(dependencies.docker, team, role, ["sharednet", "join", input.roomId, "--name", role, "--no-wake", "--json"], { + env: { + SHAREDNET_INVITE_TOKEN: input.inviteToken, + CODEX_SESSION_ID: saved.thread_id, + SHAREDNET_BASE_URL: input.baseUrl, + SHAREDNET_REUSE_RUNTIME_SESSION: "1", + ...(input.idempotencyKeys?.[role] ? { SHAREDNET_JOIN_IDEMPOTENCY_KEY: input.idempotencyKeys[role]! } : {}), + }, + }); + let memberId: string | null = null; + try { memberId = (JSON.parse(joined.stdout) as { member_id?: string }).member_id ?? null; } catch {} + if (joined.code !== 0 || memberId === null) throw localError("join_failed", `The ${role} thread could not join the Prometheus Room.`); + roster[role] = { member_id: memberId, thread_id: saved.thread_id, container_name: saved.container_name }; + const wake = await roleExec(dependencies.docker, team, role, ["sharednet", "wake", "set", "codex", "--resume", saved.thread_id, "--command", "codex", "--as", memberId, "--json"], { env: { SHAREDNET_BASE_URL: input.baseUrl } }); + if (wake.code !== 0) throw localError("wake_setup_failed", `The ${role} wake binding could not be set.`); + await roleExec(dependencies.docker, team, role, ["sharednet", "wake", "limit", `${(input.turnLimitMs ?? 1_200_000) / 1000}s`, "--as", memberId, "--json"], { env: { SHAREDNET_BASE_URL: input.baseUrl } }); + } + await ensurePrometheusListeners(team, input.roomId, roster, input.baseUrl, dependencies); + return roster; +} + +/** Restart the saved all-Room serve processes without creating seats or Codex threads. */ +export async function ensurePrometheusListeners( + team: PrometheusTeamState, + roomId: string, + roster: PrometheusRoster, + baseUrl: string, + dependencies: PrometheusRuntimeDependencies, +): Promise { + const sleep = dependencies.sleep ?? ((milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds))); + for (const role of PROMETHEUS_ROLES) { + let started = false; + const initial = await roleExec(dependencies.docker, team, role, ["sharednet", "serve", "--status", "--json"], { env: { SHAREDNET_BASE_URL: baseUrl } }); + try { started = initial.code === 0 && (JSON.parse(initial.stdout) as { running?: unknown }).running === true; } catch {} + if (!started) { + const served = await roleExec(dependencies.docker, team, role, ["sh", "-c", "exec sharednet serve >>\"$HOME/serve.log\" 2>&1"], { env: { SHAREDNET_BASE_URL: baseUrl }, detach: true }); + if (served.code !== 0) throw localError("serve_failed", `The ${role} listener could not be started.`); + } + let listening = false; + for (let attempt = 0; attempt < 20 && !listening; attempt += 1) { + const status = await roleExec(dependencies.docker, team, role, ["sharednet", "serve", "--status", "--json"], { env: { SHAREDNET_BASE_URL: baseUrl } }); + try { + const parsed = JSON.parse(status.stdout) as { running?: unknown; wake_protocol?: unknown; seats?: Array<{ room_id?: unknown; member_id?: unknown; state?: unknown }> }; + listening = status.code === 0 && parsed.running === true && parsed.wake_protocol === "wake-plugins-v1" && (parsed.seats ?? []).some((seat) => + seat.room_id === roomId && seat.member_id === roster[role].member_id && seat.state === "listening"); + } catch {} + if (!listening) await sleep(250); + } + if (!listening) throw localError("serve_not_listening", `The ${role} listener did not reach listening.`); + } +} + +export async function inspectPrometheusRuntime( + team: PrometheusTeamState, + roomId: string, + roster: PrometheusRoster, + dependencies: Pick & { baseUrl?: string }, +): Promise { + const roles = {} as PrometheusRuntimeStatus["roles"]; + let healthy = true; + for (const role of PROMETHEUS_ROLES) { + const env = dependencies.baseUrl ? { SHAREDNET_BASE_URL: dependencies.baseUrl } : undefined; + const result = await roleExec(dependencies.docker, team, role, ["sharednet", "serve", "--status", "--json"], { env }); + let state: PrometheusServeState = "unhealthy"; + try { + const parsed = JSON.parse(result.stdout) as { running?: unknown; wake_protocol?: unknown; seats?: Array<{ room_id?: unknown; member_id?: unknown; state?: unknown }> }; + const seat = (parsed.seats ?? []).find((candidate) => candidate.room_id === roomId && candidate.member_id === roster[role].member_id); + const candidate = seat?.state; + state = result.code === 0 && parsed.running === true && parsed.wake_protocol === "wake-plugins-v1" && ["listening", "working", "retrying", "stopped"].includes(String(candidate)) ? candidate as PrometheusServeState : "unhealthy"; + } catch {} + const handed = await roleExec(dependencies.docker, team, role, ["sharednet", "handed", "--as", roster[role].member_id, "--json"], { env }); + let unanswered: unknown[] = []; + try { + const parsed = JSON.parse(handed.stdout) as unknown; + const list = Array.isArray(parsed) + ? parsed + : typeof parsed === "object" && parsed !== null && Array.isArray((parsed as { unanswered?: unknown }).unanswered) + ? (parsed as { unanswered: unknown[] }).unanswered + : []; + unanswered = list.filter((item) => typeof item !== "object" || item === null || (item as { room_id?: unknown }).room_id === undefined || (item as { room_id?: unknown }).room_id === roomId); + } catch { state = "unhealthy"; } + if (state !== "listening" || handed.code !== 0) healthy = false; + roles[role] = { state, unanswered }; + } + return { healthy, roles }; +} + +function bounded(raw: string): string { + const bytes = Buffer.from(raw); + return bytes.byteLength <= MAX_OUTPUT_BYTES ? raw : bytes.subarray(bytes.byteLength - MAX_OUTPUT_BYTES).toString("utf8"); +} + +export async function runPrometheusCheck( + runtime: { image: string; workspace: string }, + check: string, + environment: Record<"SHAREDNET_PROMETHEUS_RUN_ID" | "SHAREDNET_PROMETHEUS_WORK_ID" | "SHAREDNET_PROMETHEUS_RESULT_REF" | "SHAREDNET_PROMETHEUS_REVISION_SHA256", string>, + dependencies: Pick, +): Promise { + const now = dependencies.now ?? (() => new Date()); + const startedAt = now().toISOString(); + const result = await dependencies.docker([ + "run", "--rm", "--volume", `${runtime.workspace}:${CONTAINER_WORKSPACE}`, + "--workdir", CONTAINER_WORKSPACE, + ...Object.keys(environment).flatMap((name) => ["-e", name]), + runtime.image, "sh", "-c", check, + ], { env: environment }); + const output = bounded(`${result.stdout}${result.stderr}`); + return { + startedAt, + finishedAt: now().toISOString(), + exitCode: result.code, + output, + outputSha256: createHash("sha256").update(output).digest("hex"), + }; +} diff --git a/packages/cli/src/prometheus-state.test.ts b/packages/cli/src/prometheus-state.test.ts new file mode 100644 index 0000000..9ad2da0 --- /dev/null +++ b/packages/cli/src/prometheus-state.test.ts @@ -0,0 +1,653 @@ +// @vitest-environment node + +import { createHash } from "node:crypto"; +import { + chmod, + mkdir, + mkdtemp, + readFile, + readdir, + realpath, + rm, + stat, + symlink, + unlink, + writeFile, +} from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { + claimMonitor, + finishRun, + prometheusPaths, + readActiveRun, + readRun, + readTeam, + requestCancel, + reserveActiveRun, + writeRun, + writeTeam, + type MonitorLease, + type PrometheusRunState, + type PrometheusTeamState, +} from "./prometheus-state.ts"; +import type { ProcessProbe } from "./process-identity.ts"; + +const cleanup: string[] = []; +const CREATED_AT = "2026-10-18T00:00:00.000Z"; +const UPDATED_AT = "2026-10-18T00:01:00.000Z"; +const DEADLINE_AT = "2026-10-18T02:00:00.000Z"; +const SHA_A = "a".repeat(64); +const SHA_B = "b".repeat(64); + +afterEach(async () => { + await Promise.all(cleanup.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-state-")); + cleanup.push(root); + const workspace = join(root, "workspace"); + await import("node:fs/promises").then(({ mkdir }) => mkdir(workspace)); + const env = { + HOME: root, + XDG_CONFIG_HOME: join(root, "config"), + XDG_STATE_HOME: join(root, "state"), + }; + const paths = await prometheusPaths(env, workspace); + return { root, workspace, env, paths }; +} + +function team(paths: Awaited>): PrometheusTeamState { + return { + schema_version: 1, + workspace_key: paths.workspaceKey, + workspace_path: paths.workspacePath, + model: "gpt-6-sol", + roles: { + coordinator: { + container_name: "sharednet-prometheus-coordinator", + volume_name: "sharednet-prometheus-coordinator-home", + thread_id: "019a-coordinator-thread", + }, + builder: { + container_name: "sharednet-prometheus-builder", + volume_name: "sharednet-prometheus-builder-home", + thread_id: "019a-builder-thread", + }, + reviewer: { + container_name: "sharednet-prometheus-reviewer", + volume_name: "sharednet-prometheus-reviewer-home", + thread_id: "019a-reviewer-thread", + }, + }, + last_policy_sha256: SHA_A, + created_at: CREATED_AT, + updated_at: UPDATED_AT, + }; +} + +function run(paths: Awaited>): PrometheusRunState { + return { + schema_version: 1, + run_id: "run_20261018_demo", + workspace_key: paths.workspaceKey, + workspace_path: paths.workspacePath, + configuration: { + model: "gpt-6-sol", + goal_path: join(paths.workspacePath, "GOAL.md"), + goal_sha256: SHA_B, + policy_sha256: SHA_A, + check_command: "pnpm test", + }, + startup_checkpoint: "reserved", + room: null, + roster: null, + idempotency_keys: { + room: "run_20261018_demo-room", + goal: "run_20261018_demo-goal", + roles: { + coordinator: "run_20261018_demo-coordinator", + builder: "run_20261018_demo-builder", + reviewer: "run_20261018_demo-reviewer", + }, + kickoff: "run_20261018_demo-kickoff", + }, + deadline_at: DEADLINE_AT, + monitor: null, + cancel_requested: false, + outcome: "starting", + candidate: null, + checks: [], + created_at: CREATED_AT, + updated_at: UPDATED_AT, + finished_at: null, + reason: null, + }; +} + +function anotherRun( + paths: Awaited>, + runId: string, +): PrometheusRunState { + const state = run(paths); + return { + ...state, + run_id: runId, + idempotency_keys: { + room: `${runId}-room`, + goal: `${runId}-goal`, + roles: { + coordinator: `${runId}-coordinator`, + builder: `${runId}-builder`, + reviewer: `${runId}-reviewer`, + }, + kickoff: `${runId}-kickoff`, + }, + }; +} + +function rosteredRun( + paths: Awaited>, +): PrometheusRunState { + return { + ...run(paths), + startup_checkpoint: "roles_joined", + room: { room_id: "rom_OneRoom", goal_sequence: 1, kickoff_sequence: null }, + roster: { + coordinator: { + member_id: "i_Coordinator", + thread_id: "coordinator-thread", + container_name: "prometheus-coordinator", + }, + builder: { + member_id: "i_Builder", + thread_id: "builder-thread", + container_name: "prometheus-builder", + }, + reviewer: { + member_id: "i_Reviewer", + thread_id: "reviewer-thread", + container_name: "prometheus-reviewer", + }, + }, + }; +} + +function fakeProbe(identity = "test-process-start") { + const alive = new Map([[process.pid, true]]); + const identities = new Map([[process.pid, identity]]); + const probe: ProcessProbe = { + supported: true, + alive: (pid) => alive.get(pid) ?? false, + identity: (pid) => identities.get(pid) ?? null, + argv: () => null, + }; + return { probe, alive, identities }; +} + +function terminalRun( + state: PrometheusRunState, + lease: MonitorLease, +): PrometheusRunState { + const finishedAt = new Date(Date.parse(state.updated_at) + 1_000).toISOString(); + return { + ...state, + monitor: lease, + cancel_requested: true, + outcome: "cancelled", + updated_at: finishedAt, + finished_at: finishedAt, + reason: "Cancellation requested by the owner.", + }; +} + +describe("Prometheus owner-only state", () => { + it("keys the local-v1 state root by the real workspace path", async () => { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-path-")); + cleanup.push(root); + const workspace = join(root, "workspace"); + const alias = join(root, "workspace-link"); + await import("node:fs/promises").then(({ mkdir }) => mkdir(workspace)); + await symlink(workspace, alias); + const env = { HOME: root, XDG_STATE_HOME: join(root, "state") }; + + const paths = await prometheusPaths(env, alias); + const resolved = await realpath(workspace); + const key = createHash("sha256").update(resolved).digest("hex").slice(0, 24); + + expect(paths.workspacePath).toBe(resolved); + expect(paths.workspaceKey).toBe(key); + expect(paths.rootDir).toBe(join(env.XDG_STATE_HOME, "sharednet", "prometheus", "local-v1", key)); + expect((await stat(paths.rootDir)).mode & 0o777).toBe(0o700); + expect((await stat(paths.runsDir)).mode & 0o777).toBe(0o700); + }); + + it("rejects a symlink anywhere in the managed Prometheus directory hierarchy", async () => { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-ancestor-")); + cleanup.push(root); + const workspace = join(root, "workspace"); + const stateDir = join(root, "state", "sharednet"); + const outside = join(root, "outside-prometheus"); + await mkdir(workspace); + await mkdir(stateDir, { recursive: true, mode: 0o700 }); + await mkdir(outside, { mode: 0o700 }); + await symlink(outside, join(stateDir, "prometheus")); + + await expect( + prometheusPaths({ HOME: root, XDG_STATE_HOME: join(root, "state") }, workspace), + ).rejects.toMatchObject({ code: "unsafe_credential_storage" }); + }); + + it("writes team and run files owner-only and atomically replaces a team", async () => { + const { paths } = await fixture(); + const first = team(paths); + const replacement = { ...first, last_policy_sha256: SHA_B, updated_at: "2026-10-18T00:02:00.000Z" }; + const state = run(paths); + + await writeTeam(paths, first); + await writeTeam(paths, replacement); + await reserveActiveRun(paths, state, fakeProbe().probe); + + expect(await readTeam(paths)).toEqual(replacement); + expect(await readRun(paths, state.run_id)).toEqual(state); + expect((await stat(paths.teamFile)).mode & 0o777).toBe(0o600); + expect((await stat(paths.runFile(state.run_id))).mode & 0o777).toBe(0o600); + expect((await stat(paths.runDir(state.run_id))).mode & 0o777).toBe(0o700); + expect((await readdir(paths.rootDir)).filter((name) => name.includes(".tmp"))).toEqual([]); + }); + + it("rejects symlink and group-readable state targets", async () => { + const { root, paths } = await fixture(); + const state = run(paths); + await writeTeam(paths, team(paths)); + await reserveActiveRun(paths, state, fakeProbe().probe); + + const target = join(root, "team-target.json"); + await writeFile(target, "{}\n", { mode: 0o600 }); + await unlink(paths.teamFile); + await symlink(target, paths.teamFile); + await expect(readTeam(paths)).rejects.toMatchObject({ code: "unsafe_credential_storage" }); + await expect(writeTeam(paths, team(paths))).rejects.toMatchObject({ code: "unsafe_credential_storage" }); + + await chmod(paths.runFile(state.run_id), 0o640); + await expect(readRun(paths, state.run_id)).rejects.toMatchObject({ + code: "unsafe_credential_storage", + }); + }); + + it("rejects a symlinked run directory before reading its owner-only file", async () => { + const { root, paths } = await fixture(); + const state = run(paths); + await reserveActiveRun(paths, state, fakeProbe().probe); + const serialized = await readFile(paths.runFile(state.run_id), "utf8"); + const outside = join(root, "outside-run"); + await mkdir(outside, { mode: 0o700 }); + await writeFile(join(outside, "run.json"), serialized, { mode: 0o600 }); + await rm(paths.runDir(state.run_id), { recursive: true }); + await symlink(outside, paths.runDir(state.run_id)); + + await expect(readRun(paths, state.run_id)).rejects.toMatchObject({ + code: "unsafe_credential_storage", + }); + }); + + it("fails closed on malformed schemas, role maps, ids, and terminal state", async () => { + const { paths } = await fixture(); + await writeFile( + paths.teamFile, + `${JSON.stringify({ ...team(paths), roles: { coordinator: team(paths).roles.coordinator } })}\n`, + { mode: 0o600 }, + ); + await expect(readTeam(paths)).rejects.toMatchObject({ code: "invalid_local_state" }); + + const state = run(paths); + await import("node:fs/promises").then(({ mkdir }) => + mkdir(paths.runDir(state.run_id), { recursive: true, mode: 0o700 }), + ); + await writeFile( + paths.runFile(state.run_id), + `${JSON.stringify({ ...state, outcome: "succeeded", finished_at: null, reason: null })}\n`, + { mode: 0o600 }, + ); + await expect(readRun(paths, state.run_id)).rejects.toMatchObject({ code: "invalid_local_state" }); + await expect(readRun(paths, "../escape")).rejects.toMatchObject({ code: "invalid_local_state" }); + }); + + it("never serializes credentials or caller-supplied secret fields", async () => { + const { paths } = await fixture(); + const state = run(paths); + const syntheticCapability = ["sn", "k_visibly-synthetic-test-only"].join(""); + const secretNames = [ + ["sn", "k_"].join(""), + ["sn", "i_"].join(""), + ["ri", "t_"].join(""), + "member_token", + "api_key", + "invite", + "OPENAI_API_KEY", + "auth.json", + ]; + + await writeTeam(paths, team(paths)); + await reserveActiveRun(paths, state, fakeProbe().probe); + const serialized = `${await readFile(paths.teamFile, "utf8")}\n${await readFile(paths.runFile(state.run_id), "utf8")}`; + for (const forbidden of secretNames) expect(serialized).not.toContain(forbidden); + + await expect( + writeTeam(paths, { ...team(paths), api_key: syntheticCapability } as PrometheusTeamState), + ).rejects.toMatchObject({ code: "invalid_local_state" }); + await expect( + writeRun(paths, { + ...state, + environment: { OPENAI_API_KEY: syntheticCapability }, + } as PrometheusRunState), + ).rejects.toMatchObject({ code: "invalid_local_state" }); + }); + + it("rejects synthetic OpenAI and Codex credentials embedded in free text", async () => { + const openAiKey = ["s", "k-proj-visibly-synthetic-test-credential"].join(""); + const codexAuth = JSON.stringify({ + access_token: "visibly-synthetic-access-token", + refresh_token: "visibly-synthetic-refresh-token", + id_token: "visibly-synthetic-id-token", + }); + const syntheticJwt = [ + "eyJhbGciOiJIUzI1NiJ9", + "eyJzdWIiOiJ2aXNpYmx5LXN5bnRoZXRpYyJ9", + "visibly-synthetic-signature", + ].join("."); + const cases: Array<(state: PrometheusRunState) => PrometheusRunState> = [ + (state) => ({ + ...state, + configuration: { ...state.configuration, check_command: `test ${openAiKey}` }, + }), + (state) => ({ + ...state, + checks: [ + { + revision_sha256: SHA_A, + started_at: CREATED_AT, + finished_at: UPDATED_AT, + exit_code: 1, + output: codexAuth, + output_sha256: SHA_B, + }, + ], + }), + (state) => ({ + ...state, + outcome: "failed", + finished_at: UPDATED_AT, + reason: `checker exposed ${codexAuth}`, + }), + (state) => ({ + ...state, + configuration: { ...state.configuration, check_command: `inspect ${syntheticJwt}` }, + }), + (state) => ({ + ...state, + checks: [ + { + revision_sha256: SHA_A, + started_at: CREATED_AT, + finished_at: UPDATED_AT, + exit_code: 1, + output: `${["Bear", "er"].join("")} ${syntheticJwt}`, + output_sha256: SHA_B, + }, + ], + }), + ]; + + for (const makeState of cases) { + const { paths } = await fixture(); + await expect(writeRun(paths, makeState(run(paths)))).rejects.toMatchObject({ + code: "invalid_local_state", + }); + } + }); + + it("rejects cross-role identity collisions in teams and run rosters", async () => { + const { paths } = await fixture(); + for (const field of ["container_name", "volume_name", "thread_id"] as const) { + const state = team(paths); + state.roles.builder = { + ...state.roles.builder, + [field]: state.roles.coordinator[field], + }; + await expect(writeTeam(paths, state)).rejects.toMatchObject({ code: "invalid_local_state" }); + } + for (const field of ["member_id", "thread_id", "container_name"] as const) { + const state = rosteredRun(paths); + state.roster!.builder = { + ...state.roster!.builder, + [field]: state.roster!.coordinator[field], + }; + await expect(writeRun(paths, state)).rejects.toMatchObject({ code: "invalid_local_state" }); + } + }); + + it("rejects a terminal timestamp before the last persisted update", async () => { + const { paths } = await fixture(); + await expect( + writeRun(paths, { + ...run(paths), + outcome: "failed", + finished_at: CREATED_AT, + reason: "Visibly synthetic failure.", + }), + ).rejects.toMatchObject({ code: "invalid_local_state" }); + }); +}); + +describe("Prometheus active run and monitor lease", () => { + it("requires a matching active record for every public nonterminal update", async () => { + const { paths } = await fixture(); + const state = run(paths); + + await expect(writeRun(paths, state)).rejects.toMatchObject({ + code: "prometheus_run_not_active", + }); + expect(await readRun(paths, state.run_id)).toBeNull(); + }); + + it("lets exactly one concurrent caller reserve the active slot", async () => { + const { paths } = await fixture(); + const { probe } = fakeProbe(); + const first = run(paths); + const second = anotherRun(paths, "run_20261018_second"); + + const attempts = await Promise.allSettled([ + reserveActiveRun(paths, first, probe), + reserveActiveRun(paths, second, probe), + ]); + + expect(attempts.filter((result) => result.status === "fulfilled")).toHaveLength(1); + const rejected = attempts.find((result) => result.status === "rejected"); + expect(rejected).toMatchObject({ reason: { code: "prometheus_run_active" } }); + const active = JSON.parse(await readFile(paths.activeFile, "utf8")) as { run_id: string }; + expect([first.run_id, second.run_id]).toContain(active.run_id); + expect(await readRun(paths, active.run_id)).not.toBeNull(); + }); + + it("refuses a second run while an active or resumable run exists", async () => { + const { paths } = await fixture(); + const holder = fakeProbe("holder-start"); + const first = run(paths); + await reserveActiveRun(paths, first, holder.probe); + await claimMonitor(paths, first.run_id, holder.probe); + + holder.alive.set(process.pid, false); + holder.identities.set(process.pid, "replacement-start"); + await expect( + reserveActiveRun(paths, anotherRun(paths, "run_20261018_second"), holder.probe), + ).rejects.toMatchObject({ code: "prometheus_run_active" }); + expect((await readRun(paths, first.run_id))?.deadline_at).toBe(DEADLINE_AT); + }); + + it("does not let resume replace a live monitor or a reused pid", async () => { + const { paths } = await fixture(); + const holder = fakeProbe("first-process-start"); + const state = run(paths); + await reserveActiveRun(paths, state, holder.probe); + const first = await claimMonitor(paths, state.run_id, holder.probe); + + const renewed = await claimMonitor(paths, state.run_id, holder.probe); + expect(renewed.lease_id).toBe(first.lease_id); + + holder.identities.set(process.pid, "reused-pid-start"); + const replacement = await claimMonitor(paths, state.run_id, holder.probe); + expect(replacement.lease_id).not.toBe(first.lease_id); + expect(replacement.start_identity).toBe("reused-pid-start"); + expect((await readRun(paths, state.run_id))?.monitor).toEqual(replacement); + }); + + it("lets resume replace a dead exact process identity without changing the deadline", async () => { + const { paths } = await fixture(); + const holder = fakeProbe("first-process-start"); + const state = run(paths); + await reserveActiveRun(paths, state, holder.probe); + const first = await claimMonitor(paths, state.run_id, holder.probe); + + holder.alive.set(first.pid, false); + holder.identities.set(process.pid, "replacement-process-start"); + const replacement = await claimMonitor(paths, state.run_id, holder.probe); + + expect(replacement.lease_id).not.toBe(first.lease_id); + expect((await readRun(paths, state.run_id))?.deadline_at).toBe(DEADLINE_AT); + }); + + it("lets cancel set cancel_requested while the monitor lease is live", async () => { + const { paths } = await fixture(); + const { probe } = fakeProbe(); + const state = run(paths); + await reserveActiveRun(paths, state, probe); + const lease = await claimMonitor(paths, state.run_id, probe); + + await Promise.all([requestCancel(paths, state.run_id), requestCancel(paths, state.run_id)]); + + expect(await readRun(paths, state.run_id)).toMatchObject({ + cancel_requested: true, + monitor: lease, + }); + expect(JSON.parse(await readFile(paths.activeFile, "utf8"))).toMatchObject({ + run_id: state.run_id, + cancel_requested: true, + monitor: lease, + }); + }); + + it("serializes public updates and preserves a concurrent cancellation and monitor lease", async () => { + const { paths } = await fixture(); + const { probe } = fakeProbe(); + const initial = run(paths); + await reserveActiveRun(paths, initial, probe); + const lease = await claimMonitor(paths, initial.run_id, probe); + const stale = (await readRun(paths, initial.run_id))!; + await requestCancel(paths, initial.run_id); + const current = (await readRun(paths, initial.run_id))!; + + await writeRun(paths, { + ...stale, + startup_checkpoint: "room_created", + room: { room_id: "rom_OneRoom", goal_sequence: null, kickoff_sequence: null }, + updated_at: new Date(Date.parse(current.updated_at) + 1_000).toISOString(), + }); + + expect(await readRun(paths, initial.run_id)).toMatchObject({ + startup_checkpoint: "room_created", + cancel_requested: true, + monitor: lease, + }); + }); + + it("requires finishRun for terminal transitions and never moves a checkpoint backwards", async () => { + const { paths } = await fixture(); + const { probe } = fakeProbe(); + const initial = run(paths); + await reserveActiveRun(paths, initial, probe); + const lease = await claimMonitor(paths, initial.run_id, probe); + const current = (await readRun(paths, initial.run_id))!; + const advanced: PrometheusRunState = { + ...current, + startup_checkpoint: "room_created", + room: { room_id: "rom_OneRoom", goal_sequence: null, kickoff_sequence: null }, + updated_at: new Date(Date.parse(current.updated_at) + 1_000).toISOString(), + }; + await writeRun(paths, advanced); + + await expect(writeRun(paths, terminalRun(advanced, lease))).rejects.toMatchObject({ + code: "prometheus_terminal_requires_lease", + }); + await expect( + writeRun(paths, { + ...advanced, + startup_checkpoint: "reserved", + room: null, + updated_at: new Date(Date.parse(advanced.updated_at) + 2_000).toISOString(), + }), + ).rejects.toMatchObject({ code: "invalid_local_state" }); + expect((await readRun(paths, initial.run_id))?.startup_checkpoint).toBe("room_created"); + }); + + it("preserves a cancellation that wins the race with a terminal write", async () => { + const { paths } = await fixture(); + const { probe } = fakeProbe(); + const initial = run(paths); + await reserveActiveRun(paths, initial, probe); + const lease = await claimMonitor(paths, initial.run_id, probe); + const stale = (await readRun(paths, initial.run_id))!; + await requestCancel(paths, initial.run_id); + const current = (await readRun(paths, initial.run_id))!; + const finishedAt = new Date(Date.parse(current.updated_at) + 1_000).toISOString(); + const finalState: PrometheusRunState = { + ...stale, + outcome: "failed", + updated_at: finishedAt, + finished_at: finishedAt, + reason: "The monitor stopped after cancellation was requested.", + }; + + await finishRun(paths, finalState, lease); + expect(await readRun(paths, initial.run_id)).toMatchObject({ + outcome: "failed", + cancel_requested: true, + monitor: lease, + }); + await finishRun(paths, finalState, lease); + }); + + it("clears active only after the matching lease writes a terminal run", async () => { + const { paths } = await fixture(); + const { probe } = fakeProbe(); + const state = run(paths); + await reserveActiveRun(paths, state, probe); + const lease = await claimMonitor(paths, state.run_id, probe); + await requestCancel(paths, state.run_id); + const current = (await readRun(paths, state.run_id))!; + const finalState = terminalRun(current, lease); + + await expect( + finishRun(paths, finalState, { ...lease, lease_id: "wrong-lease" }), + ).rejects.toMatchObject({ code: "prometheus_monitor_lease_lost" }); + expect(await readFile(paths.activeFile, "utf8")).toContain(state.run_id); + expect((await readRun(paths, state.run_id))?.outcome).toBe("starting"); + + await finishRun(paths, finalState, lease); + expect(await readRun(paths, state.run_id)).toEqual(finalState); + await expect(readFile(paths.activeFile, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + }); + + it("returns the active run without exposing the private active record", async () => { + const { paths } = await fixture(); + const { probe } = fakeProbe(); + const state = run(paths); + expect(await readActiveRun(paths)).toBeNull(); + await reserveActiveRun(paths, state, probe); + expect(await readActiveRun(paths)).toEqual(state); + }); +}); diff --git a/packages/cli/src/prometheus-state.ts b/packages/cli/src/prometheus-state.ts new file mode 100644 index 0000000..bc8adee --- /dev/null +++ b/packages/cli/src/prometheus-state.ts @@ -0,0 +1,1093 @@ +import { createHash, randomUUID } from "node:crypto"; +import { open, realpath, unlink } from "node:fs/promises"; +import { dirname, isAbsolute, join } from "node:path"; + +import { localError } from "./errors.ts"; +import { + assertOwnerOnlyDirectory, + createOwnerOnlyJsonIfAbsent, + ensureOwnerOnlyDirectory, + getStoragePaths, + readOwnerOnlyFile, + writeOwnerOnlyJson, +} from "./storage.ts"; +import { + inspectProcessIdentity, + systemProcessProbe, + type ProcessProbe, + type RecordedProcessIdentity, +} from "./process-identity.ts"; +import { PROMETHEUS_ROLES, type PrometheusRole } from "./prometheus-protocol.ts"; + +type Environment = Record; + +const STARTUP_CHECKPOINTS = [ + "reserved", + "room_created", + "goal_started", + "roles_joined", + "listeners_ready", + "kickoff_posted", +] as const; +const RUN_OUTCOMES = ["starting", "running", "succeeded", "timed_out", "cancelled", "failed"] as const; +const TERMINAL_OUTCOMES = new Set([ + "succeeded", + "timed_out", + "cancelled", + "failed", +]); +const HEX_SHA256 = /^[0-9a-f]{64}$/; +const WORKSPACE_KEY = /^[0-9a-f]{24}$/; +const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; +const ROOM_ID = /^rom_[A-Za-z0-9]+$/; +const MEMBER_ID = /^(?:i|mem)_[A-Za-z0-9]+$/; +const DOCKER_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; +const MODEL_NAME = /^[A-Za-z0-9._/-]{1,100}$/; +const FORBIDDEN_STATE_TEXT = /(?:snk_|sni_|rit_|member_token|api_key|invite|OPENAI_API_KEY|auth\.json|access_token|refresh_token|id_token|client_secret|\bsk-(?:proj-|svcacct-)?[A-Za-z0-9_-]{16,})/i; +const FORBIDDEN_JWT = /(?:\bBearer\s+[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}|\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})/i; + +export type PrometheusStartupCheckpoint = (typeof STARTUP_CHECKPOINTS)[number]; +export type PrometheusRunOutcome = (typeof RUN_OUTCOMES)[number]; +export type PrometheusTerminalOutcome = Exclude; + +export interface PrometheusPaths { + workspaceKey: string; + workspacePath: string; + rootDir: string; + teamFile: string; + activeFile: string; + lockFile: string; + runsDir: string; + runDir(runId: string): string; + runFile(runId: string): string; + eventsFile(runId: string): string; + checksFile(runId: string): string; +} + +export interface PrometheusTeamRoleState { + container_name: string; + volume_name: string; + thread_id: string; +} + +export interface PrometheusTeamState { + schema_version: 1; + workspace_key: string; + workspace_path: string; + model: string; + roles: Record; + last_policy_sha256: string; + created_at: string; + updated_at: string; +} + +export interface PrometheusRunConfiguration { + model: string; + goal_path: string; + goal_sha256: string; + policy_sha256: string; + check_command: string; +} + +export interface PrometheusRoomState { + room_id: string; + goal_sequence: number | null; + kickoff_sequence: number | null; +} + +export interface PrometheusRosterEntry { + member_id: string; + thread_id: string; + container_name: string; +} + +export type PrometheusRoster = Record; + +export interface PrometheusIdempotencyKeys { + room: string; + goal: string; + roles: Record; + kickoff: string; +} + +export interface MonitorLease { + lease_id: string; + pid: number; + start_identity: string; + claimed_at: string; + renewed_at: string; +} + +export interface PrometheusCandidateSummary { + compiled_digest: string; + work_id: string; + result_ref: string; + revision_sha256: string; + observed_at: string; +} + +export interface PrometheusCheckSummary { + revision_sha256: string; + started_at: string; + finished_at: string; + exit_code: number; + output: string; + output_sha256: string; +} + +export interface PrometheusRunState { + schema_version: 1; + run_id: string; + workspace_key: string; + workspace_path: string; + configuration: PrometheusRunConfiguration; + startup_checkpoint: PrometheusStartupCheckpoint; + room: PrometheusRoomState | null; + roster: PrometheusRoster | null; + idempotency_keys: PrometheusIdempotencyKeys; + deadline_at: string; + monitor: MonitorLease | null; + cancel_requested: boolean; + outcome: PrometheusRunOutcome; + candidate: PrometheusCandidateSummary | null; + checks: PrometheusCheckSummary[]; + created_at: string; + updated_at: string; + finished_at: string | null; + reason: string | null; +} + +interface PrometheusActiveRun { + schema_version: 1; + run_id: string; + monitor: MonitorLease | null; + cancel_requested: boolean; +} + +interface PrometheusStateLock extends RecordedProcessIdentity { + schema_version: 1; + lock_id: string; + created_at: string; +} + +function invalidState(message = "Prometheus local state is invalid."): never { + throw localError("invalid_local_state", message); +} + +function record(value: unknown, keys: readonly string[], source: string): Record { + if (typeof value !== "object" || value === null || Array.isArray(value)) invalidState(source); + const object = value as Record; + const actual = Object.keys(object).sort(); + const expected = [...keys].sort(); + if (actual.length !== expected.length || actual.some((key, index) => key !== expected[index])) { + invalidState(source); + } + return object; +} + +function string(value: unknown, source: string, max = 16_384): string { + if (typeof value !== "string" || value.length === 0 || value.length > max || value.includes("\0")) { + invalidState(source); + } + return value; +} + +function nullableString(value: unknown, source: string, max = 16_384): string | null { + return value === null ? null : string(value, source, max); +} + +function pattern(value: unknown, expected: RegExp, source: string): string { + const parsed = string(value, source); + if (!expected.test(parsed)) invalidState(source); + return parsed; +} + +function timestamp(value: unknown, source: string): string { + const parsed = string(value, source, 64); + const time = Date.parse(parsed); + if (!Number.isFinite(time) || new Date(time).toISOString() !== parsed) invalidState(source); + return parsed; +} + +function nullableTimestamp(value: unknown, source: string): string | null { + return value === null ? null : timestamp(value, source); +} + +function safeInteger(value: unknown, source: string, minimum = 0): number { + if (!Number.isSafeInteger(value) || (value as number) < minimum) invalidState(source); + return value as number; +} + +function exactRoleRecord( + value: unknown, + source: string, + parse: (value: unknown, role: PrometheusRole) => T, +): Record { + const object = record(value, PROMETHEUS_ROLES, source); + return { + coordinator: parse(object.coordinator, "coordinator"), + builder: parse(object.builder, "builder"), + reviewer: parse(object.reviewer, "reviewer"), + }; +} + +function noCredentials(value: unknown): void { + const serialized = JSON.stringify(value); + if (FORBIDDEN_STATE_TEXT.test(serialized) || FORBIDDEN_JWT.test(serialized)) { + invalidState("Prometheus state must not contain credentials or credential locations."); + } +} + +function parseJson(raw: string, source: string): unknown { + try { + return JSON.parse(raw); + } catch { + invalidState(`${source} is not valid JSON.`); + } +} + +function parseTeamRole(value: unknown, source: string): PrometheusTeamRoleState { + const object = record(value, ["container_name", "volume_name", "thread_id"], source); + return { + container_name: pattern(object.container_name, DOCKER_NAME, source), + volume_name: pattern(object.volume_name, DOCKER_NAME, source), + thread_id: pattern(object.thread_id, SAFE_ID, source), + }; +} + +function parseTeam(value: unknown, paths: PrometheusPaths): PrometheusTeamState { + noCredentials(value); + const source = "The Prometheus team file"; + const object = record( + value, + [ + "schema_version", + "workspace_key", + "workspace_path", + "model", + "roles", + "last_policy_sha256", + "created_at", + "updated_at", + ], + `${source} is not valid Prometheus state.`, + ); + if (object.schema_version !== 1) invalidState(`${source} has an unsupported version.`); + const workspaceKey = pattern(object.workspace_key, WORKSPACE_KEY, source); + const workspacePath = string(object.workspace_path, source); + if (!isAbsolute(workspacePath) || workspaceKey !== paths.workspaceKey || workspacePath !== paths.workspacePath) { + invalidState(`${source} does not belong to this workspace.`); + } + const createdAt = timestamp(object.created_at, source); + const updatedAt = timestamp(object.updated_at, source); + if (Date.parse(updatedAt) < Date.parse(createdAt)) invalidState(source); + const roles = exactRoleRecord(object.roles, source, (entry, role) => + parseTeamRole(entry, `${source} role ${role}`), + ); + for (const field of ["container_name", "volume_name", "thread_id"] as const) { + const values = PROMETHEUS_ROLES.map((role) => roles[role][field]); + if (new Set(values).size !== PROMETHEUS_ROLES.length) invalidState(source); + } + return { + schema_version: 1, + workspace_key: workspaceKey, + workspace_path: workspacePath, + model: pattern(object.model, MODEL_NAME, source), + roles, + last_policy_sha256: pattern(object.last_policy_sha256, HEX_SHA256, source), + created_at: createdAt, + updated_at: updatedAt, + }; +} + +function parseConfiguration(value: unknown, source: string): PrometheusRunConfiguration { + const object = record( + value, + ["model", "goal_path", "goal_sha256", "policy_sha256", "check_command"], + source, + ); + const goalPath = string(object.goal_path, source); + if (!isAbsolute(goalPath)) invalidState(source); + return { + model: pattern(object.model, MODEL_NAME, source), + goal_path: goalPath, + goal_sha256: pattern(object.goal_sha256, HEX_SHA256, source), + policy_sha256: pattern(object.policy_sha256, HEX_SHA256, source), + check_command: string(object.check_command, source), + }; +} + +function parseRoom(value: unknown, source: string): PrometheusRoomState | null { + if (value === null) return null; + const object = record(value, ["room_id", "goal_sequence", "kickoff_sequence"], source); + return { + room_id: pattern(object.room_id, ROOM_ID, source), + goal_sequence: + object.goal_sequence === null ? null : safeInteger(object.goal_sequence, source, 1), + kickoff_sequence: + object.kickoff_sequence === null ? null : safeInteger(object.kickoff_sequence, source, 1), + }; +} + +function parseRosterEntry(value: unknown, source: string): PrometheusRosterEntry { + const object = record(value, ["member_id", "thread_id", "container_name"], source); + return { + member_id: pattern(object.member_id, MEMBER_ID, source), + thread_id: pattern(object.thread_id, SAFE_ID, source), + container_name: pattern(object.container_name, DOCKER_NAME, source), + }; +} + +function parseRoster(value: unknown, source: string): PrometheusRoster | null { + if (value === null) return null; + const roster = exactRoleRecord(value, source, (entry, role) => + parseRosterEntry(entry, `${source} role ${role}`), + ); + for (const field of ["member_id", "thread_id", "container_name"] as const) { + const values = PROMETHEUS_ROLES.map((role) => roster[role][field]); + if (new Set(values).size !== PROMETHEUS_ROLES.length) invalidState(source); + } + return roster; +} + +function parseIdempotencyKeys(value: unknown, source: string): PrometheusIdempotencyKeys { + const object = record(value, ["room", "goal", "roles", "kickoff"], source); + return { + room: pattern(object.room, SAFE_ID, source), + goal: pattern(object.goal, SAFE_ID, source), + roles: exactRoleRecord(object.roles, source, (entry) => pattern(entry, SAFE_ID, source)), + kickoff: pattern(object.kickoff, SAFE_ID, source), + }; +} + +function parseMonitor(value: unknown, source: string): MonitorLease | null { + if (value === null) return null; + const object = record( + value, + ["lease_id", "pid", "start_identity", "claimed_at", "renewed_at"], + source, + ); + const claimedAt = timestamp(object.claimed_at, source); + const renewedAt = timestamp(object.renewed_at, source); + if (Date.parse(renewedAt) < Date.parse(claimedAt)) invalidState(source); + return { + lease_id: pattern(object.lease_id, SAFE_ID, source), + pid: safeInteger(object.pid, source, 1), + start_identity: string(object.start_identity, source, 512), + claimed_at: claimedAt, + renewed_at: renewedAt, + }; +} + +function parseCandidate(value: unknown, source: string): PrometheusCandidateSummary | null { + if (value === null) return null; + const object = record( + value, + ["compiled_digest", "work_id", "result_ref", "revision_sha256", "observed_at"], + source, + ); + return { + compiled_digest: pattern(object.compiled_digest, HEX_SHA256, source), + work_id: pattern(object.work_id, SAFE_ID, source), + result_ref: pattern(object.result_ref, SAFE_ID, source), + revision_sha256: pattern(object.revision_sha256, HEX_SHA256, source), + observed_at: timestamp(object.observed_at, source), + }; +} + +function parseCheck(value: unknown, source: string): PrometheusCheckSummary { + const object = record( + value, + ["revision_sha256", "started_at", "finished_at", "exit_code", "output", "output_sha256"], + source, + ); + const startedAt = timestamp(object.started_at, source); + const finishedAt = timestamp(object.finished_at, source); + if (Date.parse(finishedAt) < Date.parse(startedAt)) invalidState(source); + return { + revision_sha256: pattern(object.revision_sha256, HEX_SHA256, source), + started_at: startedAt, + finished_at: finishedAt, + exit_code: safeInteger(object.exit_code, source), + output: typeof object.output === "string" && object.output.length <= 64_000 ? object.output : invalidState(source), + output_sha256: pattern(object.output_sha256, HEX_SHA256, source), + }; +} + +function parseRun(value: unknown, paths: PrometheusPaths, expectedRunId: string): PrometheusRunState { + noCredentials(value); + const source = "The Prometheus run file"; + const object = record( + value, + [ + "schema_version", + "run_id", + "workspace_key", + "workspace_path", + "configuration", + "startup_checkpoint", + "room", + "roster", + "idempotency_keys", + "deadline_at", + "monitor", + "cancel_requested", + "outcome", + "candidate", + "checks", + "created_at", + "updated_at", + "finished_at", + "reason", + ], + `${source} is not valid Prometheus state.`, + ); + if (object.schema_version !== 1) invalidState(`${source} has an unsupported version.`); + const runId = pattern(object.run_id, SAFE_ID, source); + const workspaceKey = pattern(object.workspace_key, WORKSPACE_KEY, source); + const workspacePath = string(object.workspace_path, source); + if ( + runId !== expectedRunId || + !isAbsolute(workspacePath) || + workspaceKey !== paths.workspaceKey || + workspacePath !== paths.workspacePath + ) { + invalidState(`${source} does not belong to this run and workspace.`); + } + const checkpoint = string(object.startup_checkpoint, source) as PrometheusStartupCheckpoint; + if (!(STARTUP_CHECKPOINTS as readonly string[]).includes(checkpoint)) invalidState(source); + const outcome = string(object.outcome, source) as PrometheusRunOutcome; + if (!(RUN_OUTCOMES as readonly string[]).includes(outcome)) invalidState(source); + const room = parseRoom(object.room, source); + const roster = parseRoster(object.roster, source); + const checkpointIndex = STARTUP_CHECKPOINTS.indexOf(checkpoint); + if ((checkpointIndex === 0) !== (room === null)) invalidState(source); + if (checkpointIndex >= 2 && room?.goal_sequence === null) invalidState(source); + if (checkpointIndex < 2 && room !== null && room.goal_sequence !== null) invalidState(source); + if ((checkpointIndex >= 3) !== (roster !== null)) invalidState(source); + if (checkpointIndex === 5 && room?.kickoff_sequence === null) invalidState(source); + if (checkpointIndex < 5 && room !== null && room.kickoff_sequence !== null) invalidState(source); + if (outcome === "running" && checkpoint !== "kickoff_posted") invalidState(source); + + const createdAt = timestamp(object.created_at, source); + const updatedAt = timestamp(object.updated_at, source); + const deadlineAt = timestamp(object.deadline_at, source); + const finishedAt = nullableTimestamp(object.finished_at, source); + const reason = nullableString(object.reason, source, 4_000); + if (Date.parse(updatedAt) < Date.parse(createdAt) || Date.parse(deadlineAt) <= Date.parse(createdAt)) { + invalidState(source); + } + const terminal = TERMINAL_OUTCOMES.has(outcome as PrometheusTerminalOutcome); + if (terminal !== (finishedAt !== null && reason !== null)) invalidState(source); + if (!terminal && (finishedAt !== null || reason !== null)) invalidState(source); + if (finishedAt !== null && Date.parse(finishedAt) < Date.parse(updatedAt)) invalidState(source); + + const candidate = parseCandidate(object.candidate, source); + if (outcome === "succeeded" && candidate === null) invalidState(source); + if (!Array.isArray(object.checks)) invalidState(source); + if (typeof object.cancel_requested !== "boolean") invalidState(source); + + return { + schema_version: 1, + run_id: runId, + workspace_key: workspaceKey, + workspace_path: workspacePath, + configuration: parseConfiguration(object.configuration, source), + startup_checkpoint: checkpoint, + room, + roster, + idempotency_keys: parseIdempotencyKeys(object.idempotency_keys, source), + deadline_at: deadlineAt, + monitor: parseMonitor(object.monitor, source), + cancel_requested: object.cancel_requested, + outcome, + candidate, + checks: object.checks.map((entry, index) => parseCheck(entry, `${source} check ${index}`)), + created_at: createdAt, + updated_at: updatedAt, + finished_at: finishedAt, + reason, + }; +} + +function assertSafeRunId(runId: string): void { + if (!SAFE_ID.test(runId)) invalidState("The Prometheus run ID is invalid."); +} + +export async function prometheusPaths( + env: Environment = process.env, + workspace: string, +): Promise { + const workspacePath = await realpath(workspace); + if (!isAbsolute(workspacePath)) invalidState("The Prometheus workspace path is invalid."); + const workspaceKey = createHash("sha256").update(workspacePath).digest("hex").slice(0, 24); + const stateDir = getStoragePaths(env).stateDir; + const prometheusDir = join(stateDir, "prometheus"); + const versionDir = join(prometheusDir, "local-v1"); + const rootDir = join(versionDir, workspaceKey); + const runsDir = join(rootDir, "runs"); + for (const directory of [stateDir, prometheusDir, versionDir, rootDir]) { + await ensureOwnerOnlyDirectory(directory); + } + await ensureOwnerOnlyDirectory(runsDir); + const checkedRunDir = (runId: string) => { + assertSafeRunId(runId); + return join(runsDir, runId); + }; + return { + workspaceKey, + workspacePath, + rootDir, + teamFile: join(rootDir, "team.json"), + activeFile: join(rootDir, "active.json"), + lockFile: join(rootDir, "lock.json"), + runsDir, + runDir: checkedRunDir, + runFile: (runId) => join(checkedRunDir(runId), "run.json"), + eventsFile: (runId) => join(checkedRunDir(runId), "events.ndjson"), + checksFile: (runId) => join(checkedRunDir(runId), "checks.ndjson"), + }; +} + +function prometheusRootDirectories(paths: PrometheusPaths): string[] { + const versionDir = dirname(paths.rootDir); + const prometheusDir = dirname(versionDir); + const stateDir = dirname(prometheusDir); + return [stateDir, prometheusDir, versionDir, paths.rootDir]; +} + +async function ownerOnlyDirectoriesExist(directories: readonly string[]): Promise { + for (const directory of directories) { + if (!(await assertOwnerOnlyDirectory(directory))) return false; + } + return true; +} + +async function ensurePrometheusDirectories(paths: PrometheusPaths): Promise { + for (const directory of [...prometheusRootDirectories(paths), paths.runsDir]) { + await ensureOwnerOnlyDirectory(directory); + } +} + +export async function readTeam(paths: PrometheusPaths): Promise { + if (!(await ownerOnlyDirectoriesExist(prometheusRootDirectories(paths)))) return null; + const raw = await readOwnerOnlyFile(paths.teamFile); + return raw === null ? null : parseTeam(parseJson(raw, "The Prometheus team file"), paths); +} + +async function writeTeamUnlocked( + paths: PrometheusPaths, + state: PrometheusTeamState, +): Promise { + const parsed = parseTeam(state, paths); + const existing = await readTeam(paths); + if (existing !== null) { + const immutable = (value: PrometheusTeamState) => ({ + workspace_key: value.workspace_key, + workspace_path: value.workspace_path, + model: value.model, + roles: value.roles, + created_at: value.created_at, + }); + if (JSON.stringify(immutable(existing)) !== JSON.stringify(immutable(parsed))) { + invalidState("A Prometheus team's fixed identity cannot be changed."); + } + if (Date.parse(parsed.updated_at) < Date.parse(existing.updated_at)) { + invalidState("A Prometheus team update cannot move backwards in time."); + } + } + await writeOwnerOnlyJson(paths.teamFile, parsed); +} + +export async function writeTeam(paths: PrometheusPaths, state: PrometheusTeamState): Promise { + const parsed = parseTeam(state, paths); + await withStateLock(paths, systemProcessProbe, () => writeTeamUnlocked(paths, parsed)); +} + +export async function readRun( + paths: PrometheusPaths, + runId: string, +): Promise { + assertSafeRunId(runId); + if ( + !(await ownerOnlyDirectoriesExist([ + ...prometheusRootDirectories(paths), + paths.runsDir, + paths.runDir(runId), + ])) + ) { + return null; + } + const raw = await readOwnerOnlyFile(paths.runFile(runId)); + return raw === null ? null : parseRun(parseJson(raw, "The Prometheus run file"), paths, runId); +} + +interface WriteRunOptions { + allowTerminal?: boolean; +} + +async function writeRunUnlocked( + paths: PrometheusPaths, + state: PrometheusRunState, + options: WriteRunOptions = {}, +): Promise { + const parsed = parseRun(state, paths, state.run_id); + const existing = await readRun(paths, parsed.run_id); + if (existing !== null) { + if (isTerminal(existing.outcome)) { + if (JSON.stringify(existing) === JSON.stringify(parsed)) return; + invalidState("A terminal Prometheus run cannot be changed."); + } + if ( + STARTUP_CHECKPOINTS.indexOf(parsed.startup_checkpoint) < + STARTUP_CHECKPOINTS.indexOf(existing.startup_checkpoint) + ) { + invalidState("A Prometheus run cannot move backwards through startup."); + } + if (existing.outcome === "running" && parsed.outcome === "starting") { + invalidState("A Prometheus run cannot return to startup."); + } + const immutable = (value: PrometheusRunState) => ({ + run_id: value.run_id, + workspace_key: value.workspace_key, + workspace_path: value.workspace_path, + configuration: value.configuration, + idempotency_keys: value.idempotency_keys, + deadline_at: value.deadline_at, + created_at: value.created_at, + }); + if (JSON.stringify(immutable(existing)) !== JSON.stringify(immutable(parsed))) { + invalidState("A Prometheus run's configuration and deadline cannot be changed."); + } + if (Date.parse(parsed.updated_at) < Date.parse(existing.updated_at)) { + invalidState("A Prometheus run update cannot move backwards in time."); + } + } + if (isTerminal(parsed.outcome) && !options.allowTerminal) { + throw localError( + "prometheus_terminal_requires_lease", + `Prometheus run ${parsed.run_id} must be finished through its monitor lease.`, + ); + } + await ensurePrometheusDirectories(paths); + await ensureOwnerOnlyDirectory(paths.runDir(parsed.run_id)); + await writeOwnerOnlyJson(paths.runFile(parsed.run_id), parsed); +} + +export async function writeRun(paths: PrometheusPaths, state: PrometheusRunState): Promise { + const parsed = parseRun(state, paths, state.run_id); + await withStateLock(paths, systemProcessProbe, async () => { + const existing = await readRun(paths, parsed.run_id); + if (isTerminal(parsed.outcome)) { + if ( + existing !== null && + isTerminal(existing.outcome) && + JSON.stringify(existing) === JSON.stringify(parsed) + ) { + return; + } + throw localError( + "prometheus_terminal_requires_lease", + `Prometheus run ${parsed.run_id} must be finished through its monitor lease.`, + ); + } + + const active = await readActive(paths); + if (active === null || active.run_id !== parsed.run_id) { + throw localError( + "prometheus_run_not_active", + `Prometheus run ${parsed.run_id} is not active.`, + ); + } + if ( + (active.monitor === null && parsed.monitor !== null) || + (active.monitor !== null && !sameLease(parsed.monitor, active.monitor)) + ) { + throw localError( + "prometheus_monitor_lease_lost", + `Prometheus run ${parsed.run_id} no longer has this monitor lease.`, + ); + } + const reconciled = { + ...parsed, + monitor: active.monitor, + cancel_requested: active.cancel_requested, + }; + await writeRunUnlocked(paths, reconciled); + }); +} + +function parseActive(value: unknown): PrometheusActiveRun { + noCredentials(value); + const source = "The Prometheus active-run file"; + const object = record( + value, + ["schema_version", "run_id", "monitor", "cancel_requested"], + source, + ); + if (object.schema_version !== 1 || typeof object.cancel_requested !== "boolean") { + invalidState(source); + } + return { + schema_version: 1, + run_id: pattern(object.run_id, SAFE_ID, source), + monitor: parseMonitor(object.monitor, source), + cancel_requested: object.cancel_requested, + }; +} + +function parseStateLock(value: unknown): PrometheusStateLock { + const source = "The Prometheus state lock"; + const object = record( + value, + ["schema_version", "lock_id", "pid", "start_identity", "created_at"], + source, + ); + if (object.schema_version !== 1) invalidState(source); + return { + schema_version: 1, + lock_id: pattern(object.lock_id, SAFE_ID, source), + pid: safeInteger(object.pid, source, 1), + start_identity: string(object.start_identity, source, 512), + created_at: timestamp(object.created_at, source), + }; +} + +async function readActive(paths: PrometheusPaths): Promise { + const raw = await readOwnerOnlyFile(paths.activeFile); + return raw === null ? null : parseActive(parseJson(raw, "The Prometheus active-run file")); +} + +/** Read the durable active run through the validated public run schema. */ +export async function readActiveRun(paths: PrometheusPaths): Promise { + const active = await readActive(paths); + if (active === null) return null; + const state = await readRun(paths, active.run_id); + if (state === null) invalidState("The Prometheus active-run file names a missing run."); + return state; +} + +async function removeOwnerOnlyFile(path: string): Promise { + if ((await readOwnerOnlyFile(path)) === null) return; + await unlink(path); + const directory = await open(dirname(path), "r"); + try { + await directory.sync(); + } finally { + await directory.close(); + } +} + +function currentProcessIdentity(probe: ProcessProbe): RecordedProcessIdentity { + if (!probe.supported) { + throw localError( + "prometheus_process_identity_unavailable", + "Prometheus state mutation needs an OS process start identity on this platform.", + ); + } + const startIdentity = probe.identity(process.pid); + if (startIdentity === null || startIdentity.length === 0) { + throw localError( + "prometheus_process_identity_unavailable", + "Prometheus could not read this process's start identity.", + ); + } + return { pid: process.pid, start_identity: startIdentity }; +} + +function pause(milliseconds: number): Promise { + return new Promise((resolve) => setTimeout(resolve, milliseconds)); +} + +type ReclaimResult = "won" | "busy" | "retry"; + +async function reclaimStateLock( + paths: PrometheusPaths, + observedRaw: string, + observed: PrometheusStateLock, + replacement: PrometheusStateLock, + probe: ProcessProbe, +): Promise { + let token: string | null = null; + for (let index = 0; index < 16 && token === null; index += 1) { + const candidate = `${paths.lockFile}.reclaim.${observed.lock_id}.${index}`; + if (await createOwnerOnlyJsonIfAbsent(candidate, replacement)) { + token = candidate; + break; + } + const tokenRaw = await readOwnerOnlyFile(candidate); + if (tokenRaw === null) continue; + const creator = parseStateLock(parseJson(tokenRaw, "A Prometheus lock-reclaim token")); + const status = inspectProcessIdentity(probe, creator); + if (status === "alive" || status === "unknown") return "busy"; + } + if (token === null) return "busy"; + try { + const currentRaw = await readOwnerOnlyFile(paths.lockFile); + if (currentRaw !== observedRaw) return currentRaw === null ? "retry" : "busy"; + await writeOwnerOnlyJson(paths.lockFile, replacement); + return "won"; + } finally { + await removeOwnerOnlyFile(token).catch(() => undefined); + } +} + +async function acquireStateLock( + paths: PrometheusPaths, + probe: ProcessProbe, +): Promise { + await ensurePrometheusDirectories(paths); + const identity = currentProcessIdentity(probe); + const lock: PrometheusStateLock = { + schema_version: 1, + lock_id: randomUUID(), + ...identity, + created_at: new Date().toISOString(), + }; + for (let attempt = 0; attempt < 1_000; attempt += 1) { + if (await createOwnerOnlyJsonIfAbsent(paths.lockFile, lock)) return lock; + const raw = await readOwnerOnlyFile(paths.lockFile); + if (raw === null) continue; + const current = parseStateLock(parseJson(raw, "The Prometheus state lock")); + const status = inspectProcessIdentity(probe, current); + if (status === "dead" || status === "reused") { + const reclaimed = await reclaimStateLock(paths, raw, current, lock, probe); + if (reclaimed === "won") return lock; + if (reclaimed === "retry") continue; + } + await pause(5); + } + throw localError( + "prometheus_state_busy", + "Another local Prometheus command is changing this workspace's state.", + ); +} + +async function releaseStateLock(paths: PrometheusPaths, lockId: string): Promise { + const raw = await readOwnerOnlyFile(paths.lockFile); + if (raw === null) return; + const current = parseStateLock(parseJson(raw, "The Prometheus state lock")); + if (current.lock_id === lockId) await removeOwnerOnlyFile(paths.lockFile); +} + +async function withStateLock( + paths: PrometheusPaths, + probe: ProcessProbe, + operation: () => Promise, +): Promise { + const lock = await acquireStateLock(paths, probe); + try { + return await operation(); + } finally { + await releaseStateLock(paths, lock.lock_id); + } +} + +function isTerminal(outcome: PrometheusRunOutcome): outcome is PrometheusTerminalOutcome { + return TERMINAL_OUTCOMES.has(outcome as PrometheusTerminalOutcome); +} + +function stateTimestamp(previous: string): string { + return new Date(Math.max(Date.now(), Date.parse(previous))).toISOString(); +} + +function sameLease(left: MonitorLease | null, right: MonitorLease): boolean { + return left !== null && + left.lease_id === right.lease_id && + left.pid === right.pid && + left.start_identity === right.start_identity && + left.claimed_at === right.claimed_at; +} + +/** Atomically reserve this workspace's single active-run slot. */ +export async function reserveActiveRun( + paths: PrometheusPaths, + state: PrometheusRunState, + probe: ProcessProbe, +): Promise { + const parsed = parseRun(state, paths, state.run_id); + if ( + parsed.outcome !== "starting" || + parsed.startup_checkpoint !== "reserved" || + parsed.monitor !== null || + parsed.cancel_requested + ) { + invalidState("A new Prometheus active run must be in its reserved startup state."); + } + await withStateLock(paths, probe, async () => { + const active = await readActive(paths); + if (active !== null) { + const existing = await readRun(paths, active.run_id); + if (existing === null) { + invalidState("The Prometheus active-run file names a missing run."); + } + if (isTerminal(existing.outcome)) { + await removeOwnerOnlyFile(paths.activeFile); + } else { + throw localError( + "prometheus_run_active", + `Prometheus run ${active.run_id} is already active or resumable for this workspace.`, + ); + } + } + await writeRunUnlocked(paths, parsed); + await writeOwnerOnlyJson(paths.activeFile, { + schema_version: 1, + run_id: parsed.run_id, + monitor: null, + cancel_requested: false, + } satisfies PrometheusActiveRun); + }); +} + +/** Claim or renew the active run's monitor lease using an exact OS process identity. */ +export async function claimMonitor( + paths: PrometheusPaths, + runId: string, + probe: ProcessProbe, +): Promise { + assertSafeRunId(runId); + return withStateLock(paths, probe, async () => { + const active = await readActive(paths); + if (active === null || active.run_id !== runId) { + throw localError("prometheus_run_not_active", `Prometheus run ${runId} is not active.`); + } + const state = await readRun(paths, runId); + if (state === null || isTerminal(state.outcome)) { + invalidState("The Prometheus active run is missing or terminal."); + } + const me = currentProcessIdentity(probe); + const now = new Date().toISOString(); + let lease: MonitorLease; + if (active.monitor === null) { + lease = { + lease_id: randomUUID(), + ...me, + claimed_at: now, + renewed_at: now, + }; + } else { + const status = inspectProcessIdentity(probe, active.monitor); + if ( + status === "alive" && + active.monitor.pid === me.pid && + active.monitor.start_identity === me.start_identity + ) { + lease = { ...active.monitor, renewed_at: now }; + } else if (status === "dead" || status === "reused") { + lease = { + lease_id: randomUUID(), + ...me, + claimed_at: now, + renewed_at: now, + }; + } else { + throw localError( + "prometheus_monitor_active", + `Prometheus run ${runId} already has a live or unverifiable monitor.`, + ); + } + } + await writeOwnerOnlyJson(paths.activeFile, { ...active, monitor: lease }); + await writeRunUnlocked(paths, { + ...state, + monitor: lease, + cancel_requested: active.cancel_requested, + updated_at: stateTimestamp(state.updated_at), + }); + return lease; + }); +} + +/** Set the active run's durable cancellation bit without waiting on its monitor. */ +export async function requestCancel(paths: PrometheusPaths, runId: string): Promise { + assertSafeRunId(runId); + await withStateLock(paths, systemProcessProbe, async () => { + const active = await readActive(paths); + const state = await readRun(paths, runId); + if (active === null) { + if (state !== null && isTerminal(state.outcome)) return; + throw localError("prometheus_run_not_active", `Prometheus run ${runId} is not active.`); + } + if (active.run_id !== runId || state === null) { + throw localError("prometheus_run_not_active", `Prometheus run ${runId} is not active.`); + } + if (isTerminal(state.outcome)) { + await removeOwnerOnlyFile(paths.activeFile); + return; + } + const next = { + ...state, + monitor: active.monitor, + cancel_requested: true, + updated_at: stateTimestamp(state.updated_at), + }; + await writeOwnerOnlyJson(paths.activeFile, { ...active, cancel_requested: true }); + await writeRunUnlocked(paths, next); + }); +} + +/** Write a terminal run under its matching lease, then clear the active slot last. */ +export async function finishRun( + paths: PrometheusPaths, + finalState: PrometheusRunState, + lease: MonitorLease, +): Promise { + const parsed = parseRun(finalState, paths, finalState.run_id); + if (!isTerminal(parsed.outcome)) { + invalidState("A finished Prometheus run needs a terminal outcome."); + } + if (!sameLease(parsed.monitor, lease)) { + throw localError( + "prometheus_monitor_lease_lost", + `Prometheus run ${parsed.run_id} no longer has this monitor lease.`, + ); + } + await withStateLock(paths, systemProcessProbe, async () => { + const active = await readActive(paths); + const current = await readRun(paths, parsed.run_id); + if (active === null) { + if ( + current !== null && + isTerminal(current.outcome) && + sameLease(current.monitor, lease) && + JSON.stringify(current) === + JSON.stringify({ + ...parsed, + monitor: current.monitor, + cancel_requested: current.cancel_requested, + }) + ) { + return; + } + throw localError( + "prometheus_monitor_lease_lost", + `Prometheus run ${parsed.run_id} no longer has this monitor lease.`, + ); + } + if ( + active.run_id !== parsed.run_id || + !sameLease(active.monitor, lease) || + current === null || + (parsed.outcome === "cancelled" && !active.cancel_requested) + ) { + throw localError( + "prometheus_monitor_lease_lost", + `Prometheus run ${parsed.run_id} no longer has this monitor lease.`, + ); + } + const canonical = { + ...parsed, + monitor: active.monitor, + cancel_requested: active.cancel_requested, + }; + if (isTerminal(current.outcome)) { + if (JSON.stringify(current) !== JSON.stringify(canonical)) { + invalidState("A terminal Prometheus run cannot be replaced by another outcome."); + } + } else { + await writeRunUnlocked(paths, canonical, { allowTerminal: true }); + } + await removeOwnerOnlyFile(paths.activeFile); + }); +} diff --git a/packages/cli/src/prometheus.test.ts b/packages/cli/src/prometheus.test.ts new file mode 100644 index 0000000..1f8bef3 --- /dev/null +++ b/packages/cli/src/prometheus.test.ts @@ -0,0 +1,412 @@ +// @vitest-environment node + +import { createHash } from "node:crypto"; +import { mkdir, mkdtemp, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it } from "vitest"; + +import { DEFAULT_IMAGE, type DockerRunner, type RunPlan } from "./goal-run.ts"; +import { cancelPrometheusRun, evaluatePrometheusBarrier, prometheusKickoff, prometheusStatus, readPrometheusMessages, resumePrometheusRun } from "./prometheus.ts"; +import { claimMonitor, prometheusPaths, readActiveRun, reserveActiveRun, writeRun, writeTeam, type PrometheusRunState, type PrometheusTeamState } from "./prometheus-state.ts"; +import type { ProcessProbe } from "./process-identity.ts"; + +const candidate = { + stateDigest: "a".repeat(64), + workId: "work-1", + resultRef: "evt-1", + artifacts: [], + verifications: [], + revisionSha256: "b".repeat(64), +}; +const receipts = { + ok: true as const, + receipts: { + coordinator: { sequence: 10, role: "coordinator" as const, memberId: "i_coord", runId: "run-1", revisionSha256: candidate.revisionSha256 }, + builder: { sequence: 11, role: "builder" as const, memberId: "i_build", runId: "run-1", revisionSha256: candidate.revisionSha256 }, + reviewer: { sequence: 12, role: "reviewer" as const, memberId: "i_review", runId: "run-1", revisionSha256: candidate.revisionSha256 }, + }, + missingRoles: [], +}; + +const cleanup: string[] = []; +afterEach(async () => Promise.all(cleanup.splice(0).map((path) => rm(path, { recursive: true, force: true })))); + +const plan: RunPlan = { + image: DEFAULT_IMAGE, + agents: (["coordinator", "builder", "reviewer"] as const).map((name) => ({ name, driver: "codex" as const, model: "gpt-6-sol" })), + workspace: "/unused", + cli: { root: "/opt/client", entry: "src/main.ts" }, + codexAuth: { kind: "login", file: "/owner/.codex/auth.json" }, + claudeAuth: null, +}; + +async function lifecycleFixture(checkpoint: PrometheusRunState["startup_checkpoint"] = "reserved", deadlineAt = "2026-10-18T02:00:00.000Z") { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-lifecycle-")); + cleanup.push(root); + const workspace = join(root, "workspace"); + await mkdir(workspace); + const policy = "## Motto\nShip\n\n## Rules\nBe exact\n\n## Preferences\nUse evidence\n\n## Handoff\nName the recipient\n\n## Completion\nRequire review\n"; + const goal = "Build the requested artifact.\n"; + await writeFile(join(workspace, "PROMETHEUS.md"), policy); + await writeFile(join(workspace, "GOAL.md"), goal); + const env = { HOME: root, XDG_CONFIG_HOME: join(root, "config"), XDG_STATE_HOME: join(root, "state") }; + const paths = await prometheusPaths(env, workspace); + const container = (role: string) => `sharednet-prometheus-${paths.workspaceKey}-${role}`; + const roles = Object.fromEntries((["coordinator", "builder", "reviewer"] as const).map((role) => [role, { + container_name: container(role), volume_name: `${container(role)}-home`, thread_id: `thr-${role}`, + }])) as PrometheusTeamState["roles"]; + const team: PrometheusTeamState = { + schema_version: 1, workspace_key: paths.workspaceKey, workspace_path: paths.workspacePath, + model: "gpt-6-sol", roles, last_policy_sha256: createHash("sha256").update(policy).digest("hex"), + created_at: "2026-10-18T00:00:00.000Z", updated_at: "2026-10-18T00:00:01.000Z", + }; + const checkpointIndex = ["reserved", "room_created", "goal_started", "roles_joined", "listeners_ready", "kickoff_posted"].indexOf(checkpoint); + const roster = Object.fromEntries((["coordinator", "builder", "reviewer"] as const).map((role) => [role, { + member_id: `i_${role}`, thread_id: `thr-${role}`, container_name: container(role), + }])) as NonNullable; + const run: PrometheusRunState = { + schema_version: 1, run_id: "run_lifecycle", workspace_key: paths.workspaceKey, workspace_path: paths.workspacePath, + configuration: { + model: "gpt-6-sol", goal_path: join(workspace, "GOAL.md"), + goal_sha256: createHash("sha256").update(goal).digest("hex"), policy_sha256: team.last_policy_sha256, + check_command: "pnpm test", + }, + startup_checkpoint: checkpoint, + room: checkpointIndex >= 1 ? { room_id: "rom_Lifecycle", goal_sequence: checkpointIndex >= 2 ? 1 : null, kickoff_sequence: checkpointIndex >= 5 ? 2 : null } : null, + roster: checkpointIndex >= 3 ? roster : null, + idempotency_keys: { room: "run-room", goal: "run-goal", roles: { coordinator: "run-coordinator", builder: "run-builder", reviewer: "run-reviewer" }, kickoff: "run-kickoff" }, + deadline_at: deadlineAt, monitor: null, cancel_requested: false, + outcome: checkpoint === "kickoff_posted" ? "running" : "starting", candidate: null, checks: [], + created_at: "2026-10-18T00:00:00.000Z", updated_at: "2026-10-18T00:00:01.000Z", finished_at: null, reason: null, + }; + const processState = { alive: true, identity: "test-process-start" }; + const probe: ProcessProbe = { supported: true, alive: () => processState.alive, identity: () => processState.identity, argv: () => null }; + await writeTeam(paths, team); + const reservedRun: PrometheusRunState = { + ...run, + startup_checkpoint: "reserved", + room: null, + roster: null, + outcome: "starting", + }; + await reserveActiveRun(paths, reservedRun, probe); + if (checkpoint !== "reserved") await writeRun(paths, run); + return { root, workspace, env, paths, team, run, probe, processState }; +} + +function inspectJson(team: PrometheusTeamState, role: "coordinator" | "builder" | "reviewer") { + const saved = team.roles[role]; + return JSON.stringify([{ + Config: { Image: DEFAULT_IMAGE, User: `${process.getuid?.() ?? "node"}:${process.getgid?.() ?? "node"}`, Cmd: ["sleep", "infinity"], Labels: { "io.sharednet.prometheus": "local-v1", "io.sharednet.prometheus.workspace": team.workspace_key, "io.sharednet.prometheus.role": role } }, + Mounts: [ + { Type: "volume", Name: saved.volume_name, Destination: "/home/prometheus", RW: true }, + { Type: "bind", Source: team.workspace_path, Destination: "/workspace", RW: true }, + { Type: "bind", Source: plan.cli.root, Destination: "/opt/sharednet", RW: false }, + { Type: "bind", Source: plan.codexAuth!.file, Destination: "/run/codex/auth.json", RW: false }, + ], + }]); +} + +function lifecycleDocker(team: PrometheusTeamState, calls: Array<{ args: readonly string[]; env?: Record }>, liveTruth = false): DockerRunner { + return async (args, options) => { + calls.push({ args: [...args], env: options?.env }); + const role = (["coordinator", "builder", "reviewer"] as const).find((candidate) => args.join(" ").includes(candidate)) ?? "coordinator"; + if (args[0] === "container" && args[1] === "inspect") return { code: 0, stdout: inspectJson(team, role), stderr: "" }; + if (args[0] === "exec" && args.includes("cat")) return { code: 0, stdout: `${team.roles[role].thread_id}\n`, stderr: "" }; + if (args.includes("join")) return { code: 0, stdout: JSON.stringify({ member_id: `i_${role}` }), stderr: "" }; + if (args.includes("--status")) return { code: 0, stdout: JSON.stringify({ running: true, wake_protocol: "wake-plugins-v1", seats: [{ room_id: "rom_Lifecycle", member_id: `i_${role}`, state: liveTruth && role === "builder" ? "working" : "listening" }] }), stderr: "" }; + if (args.includes("handed")) return { code: 0, stdout: JSON.stringify({ unanswered: liveTruth && role === "reviewer" ? [{ room_id: "rom_Lifecycle", through: 7 }] : [] }), stderr: "" }; + return { code: 0, stdout: "", stderr: "" }; + }; +} + +describe("Prometheus completion barrier", () => { + it("reads every message page so late terminal receipts remain visible", async () => { + const paths: string[] = []; + const client = { + request: async (_method: string, path: string) => { + paths.push(path); + if (path.includes("after=0")) { + return { + items: Array.from({ length: 100 }, (_, index) => ({ sequence: index + 1, content: `message ${index + 1}` })), + has_more: true, + }; + } + return { + items: [{ sequence: 101, content: "terminal receipt" }], + has_more: false, + }; + }, + }; + + const messages = await readPrometheusMessages(client as never, "rom_123", "token"); + + expect(messages).toHaveLength(101); + expect(messages.at(-1)?.sequence).toBe(101); + expect(paths).toEqual([ + "/rooms/rom_123/messages?after=0&order=asc&limit=100", + "/rooms/rom_123/messages?after=100&order=asc&limit=100", + ]); + }); + + it("requires candidate, all authenticated receipts, listening seats, and no handed work", () => { + const ready = evaluatePrometheusBarrier({ + candidate: { ok: true, candidate }, + receipts, + runtime: { + healthy: true, + roles: { + coordinator: { state: "listening", unanswered: [] }, + builder: { state: "listening", unanswered: [] }, + reviewer: { state: "listening", unanswered: [] }, + }, + }, + }); + expect(ready).toMatchObject({ ready: true, candidate }); + + expect(evaluatePrometheusBarrier({ + candidate: { ok: true, candidate }, receipts, + runtime: { healthy: true, roles: { coordinator: { state: "working", unanswered: [] }, builder: { state: "listening", unanswered: [] }, reviewer: { state: "listening", unanswered: [] } } }, + })).toEqual({ ready: false, reasons: ["coordinator is working"], revisionSha256: candidate.revisionSha256 }); + expect(evaluatePrometheusBarrier({ + candidate: { ok: true, candidate }, receipts, + runtime: { healthy: true, roles: { coordinator: { state: "listening", unanswered: [{}] }, builder: { state: "listening", unanswered: [] }, reviewer: { state: "listening", unanswered: [] } } }, + })).toEqual({ ready: false, reasons: ["coordinator has unsettled handoff work"], revisionSha256: candidate.revisionSha256 }); + expect(evaluatePrometheusBarrier({ + candidate: { ok: true, candidate }, receipts, + runtime: { healthy: false, roles: { coordinator: { state: "listening", unanswered: [] }, builder: { state: "listening", unanswered: [] }, reviewer: { state: "listening", unanswered: [] } } }, + })).toEqual({ ready: false, reasons: ["runtime status is unhealthy"], revisionSha256: candidate.revisionSha256 }); + }); + + it("builds one exact-member coordinator kickoff with the policy and typed-work recipe", () => { + const kickoff = prometheusKickoff({ + runId: "run-1", goalSequence: 3, deadlineAt: "2026-10-18T02:00:00.000Z", check: "pnpm test", + goalSha256: "c".repeat(64), policy: { raw: "## Motto\nShip\n", sha256: "d".repeat(64), sections: { Motto: "Ship", Rules: "R", Preferences: "P", Handoff: "H", Completion: "C" } }, + roster: { coordinator: { member_id: "i_coord", thread_id: "t1", container_name: "c1" }, builder: { member_id: "i_build", thread_id: "t2", container_name: "c2" }, reviewer: { member_id: "i_review", thread_id: "t3", container_name: "c3" } }, + }); + expect(kickoff.startsWith("@i_coord ")).toBe(true); + expect(kickoff).toContain("i_build"); + expect(kickoff).toContain("i_review"); + expect(kickoff).toContain("verification.pass"); + expect(kickoff).not.toContain("@all"); + expect(kickoff).not.toContain("@builder"); + }); +}); + +describe("Prometheus lifecycle recovery", () => { + it("resumes a reserved run through every missing startup checkpoint with the original idempotency keys", async () => { + const fixture = await lifecycleFixture("reserved"); + const dockerCalls: Array<{ args: readonly string[]; env?: Record }> = []; + const requests: Array<{ method: string; path: string; headers?: Record }> = []; + let startupFinished = false; + const client = { request: async (method: string, path: string, _credential: string, _body?: unknown, headers?: Record) => { + requests.push({ method, path, headers }); + if (path === "/rooms") return { room: { id: "rom_Lifecycle" } }; + if (path.endsWith("/goal")) return { message: { sequence: 1 } }; + if (path.endsWith("/invites")) return { token: `rit_${"t".repeat(43)}` }; + if (path.endsWith("/owner-messages")) { startupFinished = true; return { message: { sequence: 2 } }; } + return { room: { state: "closed" } }; + } }; + + const result = await resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date(startupFinished ? "2026-10-18T03:00:00.000Z" : "2026-10-18T01:00:00.000Z"), + docker: lifecycleDocker(fixture.team, dockerCalls), plan: { ...plan, workspace: fixture.workspace }, + connect: async () => ({ client: client as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), + processProbe: fixture.probe, sleep: async () => undefined, + }); + + expect(result).toMatchObject({ startup_checkpoint: "kickoff_posted", outcome: "timed_out", room: { room_id: "rom_Lifecycle", goal_sequence: 1, kickoff_sequence: 2 } }); + expect(await readActiveRun(fixture.paths)).toBeNull(); + expect(requests.filter((request) => request.path === "/rooms")).toHaveLength(1); + expect(requests.find((request) => request.path === "/rooms")?.headers?.["idempotency-key"]).toBe("run-room"); + expect(requests.find((request) => request.path.endsWith("/goal"))?.headers?.["idempotency-key"]).toBe("run-goal"); + expect(requests.find((request) => request.path.endsWith("/owner-messages"))?.headers?.["idempotency-key"]).toBe("run-kickoff"); + for (const role of ["coordinator", "builder", "reviewer"] as const) { + const join = dockerCalls.find((call) => call.args.includes("join") && call.args.join(" ").includes(role)); + expect(join?.env?.SHAREDNET_JOIN_IDEMPOTENCY_KEY).toBe(`run-${role}`); + } + }); + + it.each([ + ["room_created", ["/rooms"]], + ["goal_started", ["/rooms", "/rooms/rom_Lifecycle/goal"]], + ["roles_joined", ["/rooms", "/rooms/rom_Lifecycle/goal", "/rooms/rom_Lifecycle/invites"]], + ["listeners_ready", ["/rooms", "/rooms/rom_Lifecycle/goal", "/rooms/rom_Lifecycle/invites"]], + ["kickoff_posted", ["/rooms", "/rooms/rom_Lifecycle/goal", "/rooms/rom_Lifecycle/invites", "/rooms/rom_Lifecycle/owner-messages"]], + ] as const)("resumes exactly after the %s checkpoint", async (checkpoint, forbiddenPaths) => { + const fixture = await lifecycleFixture(checkpoint, "2026-10-18T04:00:00.000Z"); + const requests: string[] = []; + const result = await resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker: lifecycleDocker(fixture.team, []), plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ client: { request: async (_method: string, path: string) => { + requests.push(path); + if (path.endsWith("/goal")) return { message: { sequence: 1 } }; + if (path.endsWith("/invites")) return { token: `rit_${"t".repeat(43)}` }; + if (path.endsWith("/owner-messages")) return { message: { sequence: 2 } }; + if (path === "/rooms/rom_Lifecycle") return { room: { state: "closed" } }; + return {}; + } } as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), + sleep: async () => undefined, + }); + + expect(result).toMatchObject({ startup_checkpoint: "kickoff_posted", outcome: "failed" }); + for (const path of forbiddenPaths) expect(requests).not.toContain(path); + }); + + it("lets cancel own a dead monitor, closes once, and finishes the run", async () => { + const fixture = await lifecycleFixture("kickoff_posted"); + const first = await claimMonitor(fixture.paths, fixture.run.run_id, fixture.probe); + fixture.processState.identity = "replacement-process-start"; + const requests: string[] = []; + const result = await cancelPrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker: lifecycleDocker(fixture.team, []), plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ client: { request: async (_method: string, path: string) => { requests.push(path); return {}; } } as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), + }); + + expect(result).toMatchObject({ cancelled: true, recovered: true, outcome: "cancelled" }); + expect(requests).toEqual(["/rooms/rom_Lifecycle/close"]); + expect((await readActiveRun(fixture.paths))).toBeNull(); + expect(first.start_identity).toBe("test-process-start"); + }); + + it("records an unexpected Room closure as failed and clears the active slot", async () => { + const fixture = await lifecycleFixture("kickoff_posted"); + const requests: string[] = []; + const result = await resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker: lifecycleDocker(fixture.team, []), plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ client: { request: async (_method: string, path: string) => { + requests.push(path); + if (path === "/rooms/rom_Lifecycle") return { room: { state: "closed" } }; + return {}; + } } as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), + sleep: async () => undefined, + }); + + expect(result).toMatchObject({ outcome: "failed", reason: expect.stringContaining("closed") }); + expect(await readActiveRun(fixture.paths)).toBeNull(); + expect(requests[0]).toBe("/rooms/rom_Lifecycle"); + }); + + it("reconciles a success close that committed before the terminal record", async () => { + const fixture = await lifecycleFixture("kickoff_posted"); + const revision = "b".repeat(64); + await writeRun(fixture.paths, { + ...fixture.run, + candidate: { compiled_digest: "a".repeat(64), work_id: "work-1", result_ref: "evt-1", revision_sha256: revision, observed_at: "2026-10-18T00:01:00.000Z" }, + updated_at: "2026-10-18T00:01:00.000Z", + }); + const result = await resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker: lifecycleDocker(fixture.team, []), plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ client: { request: async (_method: string, path: string) => { + if (path === "/rooms/rom_Lifecycle") return { room: { state: "closed", goal: { ended_by: { trigger: "prometheus_success", detail: revision } } } }; + throw new Error(`unexpected request ${path}`); + } } as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), + }); + + expect(result).toMatchObject({ outcome: "succeeded", candidate: { revision_sha256: revision }, reason: expect.stringContaining("Recovered") }); + expect(await readActiveRun(fixture.paths)).toBeNull(); + }); + + it("records an irrecoverable monitor protocol error as failed and closes the Room", async () => { + const fixture = await lifecycleFixture("kickoff_posted"); + const requests: string[] = []; + const result = await resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker: lifecycleDocker(fixture.team, []), plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ client: { request: async (_method: string, path: string) => { + requests.push(path); + if (path === "/rooms/rom_Lifecycle") return { room: { state: "open" } }; + if (path.endsWith("/state")) return { protocol_version: "rac/1", sequence: 0, digest: "empty", projection: { work: {} }, obligations: "none", events: [] }; + if (path.includes("/messages?")) return { items: "malformed", has_more: false }; + return {}; + } } as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), + sleep: async () => undefined, + }); + + expect(result).toMatchObject({ outcome: "failed", reason: expect.stringContaining("prometheus_history_incomplete") }); + expect(requests).toContain("/rooms/rom_Lifecycle/close"); + expect(await readActiveRun(fixture.paths)).toBeNull(); + }); + + it("leaves a transient transport failure resumable instead of closing the Room", async () => { + const fixture = await lifecycleFixture("kickoff_posted"); + const requests: string[] = []; + await expect(resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker: lifecycleDocker(fixture.team, []), plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ client: { request: async (_method: string, path: string) => { + requests.push(path); + throw new Error("network details that must not escape"); + } } as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), + sleep: async () => undefined, + })).rejects.toMatchObject({ code: "service_unavailable", message: "SharedNet could not complete the request." }); + + expect(requests).toEqual(["/rooms/rom_Lifecycle"]); + expect(await readActiveRun(fixture.paths)).toMatchObject({ outcome: "running", startup_checkpoint: "kickoff_posted" }); + }); + + it("does not post kickoff while a resumed pre-kickoff seat is retrying", async () => { + const fixture = await lifecycleFixture("roles_joined", "2026-10-18T04:00:00.000Z"); + const requests: string[] = []; + const docker: DockerRunner = async (args) => { + const role = (["coordinator", "builder", "reviewer"] as const).find((candidate) => args.join(" ").includes(candidate)) ?? "coordinator"; + if (args[0] === "container" && args[1] === "inspect") return { code: 0, stdout: inspectJson(fixture.team, role), stderr: "" }; + if (args[0] === "exec" && args.includes("cat")) return { code: 0, stdout: `${fixture.team.roles[role].thread_id}\n`, stderr: "" }; + if (args.includes("--status")) return { code: 0, stdout: JSON.stringify({ running: true, wake_protocol: "wake-plugins-v1", seats: [{ room_id: "rom_Lifecycle", member_id: `i_${role}`, state: role === "reviewer" ? "retrying" : "listening" }] }), stderr: "" }; + return { code: 0, stdout: "", stderr: "" }; + }; + + await expect(resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker, plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ client: { request: async (_method: string, path: string) => { requests.push(path); return {}; } } as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), + sleep: async () => undefined, + })).rejects.toMatchObject({ code: "serve_not_listening" }); + + expect(requests).not.toContain("/rooms/rom_Lifecycle/owner-messages"); + expect(await readActiveRun(fixture.paths)).toMatchObject({ startup_checkpoint: "roles_joined", outcome: "starting" }); + }); + + it("terminalizes an irreparable saved-container identity mismatch", async () => { + const fixture = await lifecycleFixture("reserved", "2026-10-18T04:00:00.000Z"); + const docker: DockerRunner = async (args) => { + if (args[0] === "container" && args[1] === "inspect") { + return { code: 0, stdout: JSON.stringify([{ Config: { Image: "unexpected/image", User: "0:0", Cmd: ["sleep", "infinity"], Labels: {} }, Mounts: [] }]), stderr: "" }; + } + return { code: 0, stdout: "", stderr: "" }; + }; + const result = await resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker, plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => { throw new Error("must not connect before a Room exists"); }, + }); + + expect(result).toMatchObject({ outcome: "failed", reason: expect.stringContaining("prometheus_team_invalid") }); + expect(await readActiveRun(fixture.paths)).toBeNull(); + }); + + it("reports live monitor, Room, runtime, and barrier truth when dependencies are available", async () => { + const fixture = await lifecycleFixture("kickoff_posted"); + const holder = await claimMonitor(fixture.paths, fixture.run.run_id, fixture.probe); + fixture.processState.identity = "replacement-process-start"; + const status = await prometheusStatus(fixture.workspace, fixture.env, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker: lifecycleDocker(fixture.team, [], true), plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ client: { request: async (_method: string, path: string) => { + if (path === "/rooms/rom_Lifecycle") return { room: { state: "open" } }; + if (path.endsWith("/state")) return { protocol_version: "rac/1", sequence: 0, digest: "empty", projection: { work: {} }, obligations: "none", events: [] }; + if (path.includes("/messages?")) return { items: [], has_more: false }; + return {}; + } } as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), + }); + + expect(status.monitor).toMatchObject({ state: "reused", pid: holder.pid }); + expect(status.live).toMatchObject({ available: true, room_state: "open", runtime: { healthy: false, roles: { builder: { state: "working" }, reviewer: { unanswered: [{ through: 7 }] } } }, barrier: { ready: false } }); + }); +}); diff --git a/packages/cli/src/prometheus.ts b/packages/cli/src/prometheus.ts new file mode 100644 index 0000000..f1ba9ad --- /dev/null +++ b/packages/cli/src/prometheus.ts @@ -0,0 +1,609 @@ +import { createHash, randomUUID } from "node:crypto"; +import { readFile, stat } from "node:fs/promises"; +import { basename, join, resolve } from "node:path"; + +import type { ApiClient } from "./api-client.ts"; +import type { CompiledState } from "./compiled.ts"; +import { asCliError, localError } from "./errors.ts"; +import type { DockerRunner, RunPlan } from "./goal-run.ts"; +import { + PROMETHEUS_ROLES, + assertPrometheusGoalUnaddressed, + parsePrometheusCandidate, + parsePrometheusPolicy, + parseTerminalReceipts, + type CandidateResult, + type PrometheusMessage, + type PrometheusPolicy, + type TerminalReceiptResult, +} from "./prometheus-protocol.ts"; +import { + attachPrometheusRoom, + ensurePrometheusListeners, + ensurePrometheusTeam, + inspectPrometheusRuntime, + runPrometheusCheck, + verifyPrometheusTeamIdentity, + type PrometheusRuntimeStatus, +} from "./prometheus-runtime.ts"; +import { + claimMonitor, + finishRun, + prometheusPaths, + readActiveRun, + readTeam, + requestCancel, + reserveActiveRun, + writeRun, + writeTeam, + type PrometheusPaths, + type PrometheusRoster, + type PrometheusRunState, + type PrometheusTeamState, +} from "./prometheus-state.ts"; +import { inspectProcessIdentity, systemProcessProbe, type ProcessProbe } from "./process-identity.ts"; + +const MIN_TIMEOUT_MS = 60_000; +const MAX_TIMEOUT_MS = 86_400_000; + +export type PrometheusBarrierResult = + | { ready: false; reasons: string[]; revisionSha256?: string } + | { ready: true; candidate: Extract["candidate"]; receipts: TerminalReceiptResult["receipts"] }; + +export function evaluatePrometheusBarrier(input: { + candidate: CandidateResult; + receipts: TerminalReceiptResult; + runtime: PrometheusRuntimeStatus; +}): PrometheusBarrierResult { + if (!input.candidate.ok) return { ready: false, reasons: input.candidate.reasons }; + const revisionSha256 = input.candidate.candidate.revisionSha256; + const reasons: string[] = []; + if (!input.runtime.healthy) reasons.push("runtime status is unhealthy"); + if (!input.receipts.ok) { + for (const role of input.receipts.missingRoles) reasons.push(`${role} terminal receipt is missing`); + } + for (const role of PROMETHEUS_ROLES) { + const roleStatus = input.runtime.roles[role]; + if (roleStatus.state !== "listening") reasons.push(`${role} is ${roleStatus.state}`); + if (roleStatus.unanswered.length > 0) reasons.push(`${role} has unsettled handoff work`); + } + return reasons.length > 0 + ? { ready: false, reasons, revisionSha256 } + : { ready: true, candidate: input.candidate.candidate, receipts: input.receipts.receipts }; +} + +export function prometheusKickoff(input: { + runId: string; + goalSequence: number; + roster: PrometheusRoster; + policy: PrometheusPolicy; + goalSha256: string; + check: string; + deadlineAt: string; +}): string { + const roster = PROMETHEUS_ROLES.map((role) => `${role}=${input.roster[role].member_id}`).join(", "); + return [ + `@${input.roster.coordinator.member_id} Start Prometheus run ${input.runId} for Goal #${input.goalSequence}.`, + `Exact roster: ${roster}. Address every request and handoff to one exact member id.`, + `PROMETHEUS.md sha256=${input.policy.sha256}; GOAL.md sha256=${input.goalSha256}; deadline=${input.deadlineAt}.`, + "The complete current policy supersedes earlier policy:", + input.policy.raw, + `Create compiled work for ${input.roster.builder.member_id} requiring deliverable submission and verified_by ${input.roster.reviewer.member_id}. The builder publishes work.result. The reviewer downloads the exact artifact, checks its SHA-256, uploads review evidence, then publishes verification.pass bound to the current result_ref. Resolve only the verified head; defects require a successor using supersedes.`, + `Semantic check after the evidence barrier: ${input.check}`, + `After the final head is resolved, request one receipt from builder and reviewer, then post yours. Exact line: [sharednet-prometheus:v1] {"type":"terminal.receipt","run_id":"${input.runId}","role":"","revision_sha256":""}`, + ].join("\n\n"); +} + +export interface PrometheusConnection { + client: ApiClient; + memberToken: string; + apiKey: string; + baseUrl: string; +} + +export interface PrometheusDependencies { + env: Record; + cwd: string; + now: () => Date; + docker: DockerRunner; + plan?: RunPlan; + preparePlan?: () => Promise; + connect: () => Promise; + processProbe?: ProcessProbe; + sleep?: (milliseconds: number) => Promise; +} + +export interface StartPrometheusInput { + workspace: string; + goalPath: string; + model: string; + check: string; + timeoutMs: number; +} + +function hash(raw: string): string { + return createHash("sha256").update(raw).digest("hex"); +} + +function timestampAfter(previous: string, now: Date): string { + return new Date(Math.max(now.getTime(), Date.parse(previous) + 1)).toISOString(); +} + +async function readInputs(input: StartPrometheusInput): Promise<{ workspace: string; policy: PrometheusPolicy; goal: string; goalPath: string }> { + if (!input.model || !/^[A-Za-z0-9._/-]{1,100}$/.test(input.model)) throw localError("invalid_arguments", "--model names an invalid model."); + if (!input.check.trim()) throw localError("invalid_arguments", "--check must not be empty."); + if (!Number.isSafeInteger(input.timeoutMs) || input.timeoutMs < MIN_TIMEOUT_MS || input.timeoutMs > MAX_TIMEOUT_MS) throw localError("invalid_arguments", "--timeout must be between 1m and 24h."); + const workspace = resolve(input.workspace); + if (!(await stat(workspace).then((entry) => entry.isDirectory(), () => false))) throw localError("invalid_arguments", `--workspace ${workspace} is not a directory.`); + let policyRaw: string; + let goal: string; + const goalPath = resolve(workspace, input.goalPath); + try { policyRaw = await readFile(join(workspace, "PROMETHEUS.md"), "utf8"); } catch { throw localError("invalid_arguments", "The workspace needs PROMETHEUS.md."); } + try { goal = await readFile(goalPath, "utf8"); } catch { throw localError("invalid_arguments", `The goal file could not be read: ${input.goalPath}`); } + if (!goal.trim()) throw localError("invalid_arguments", "The goal file is empty."); + const policy = parsePrometheusPolicy(policyRaw); + assertPrometheusGoalUnaddressed(goal); + return { workspace, policy, goal, goalPath }; +} + +function newRun(paths: PrometheusPaths, input: StartPrometheusInput, policy: PrometheusPolicy, goal: string, goalPath: string, now: Date): PrometheusRunState { + const runId = `run_${now.toISOString().replace(/[-:.TZ]/g, "")}_${randomUUID().slice(0, 8)}`; + const createdAt = now.toISOString(); + return { + schema_version: 1, + run_id: runId, + workspace_key: paths.workspaceKey, + workspace_path: paths.workspacePath, + configuration: { model: input.model, goal_path: goalPath, goal_sha256: hash(goal), policy_sha256: policy.sha256, check_command: input.check }, + startup_checkpoint: "reserved", + room: null, + roster: null, + idempotency_keys: { room: `${runId}-room`, goal: `${runId}-goal`, roles: { coordinator: `${runId}-coordinator`, builder: `${runId}-builder`, reviewer: `${runId}-reviewer` }, kickoff: `${runId}-kickoff` }, + deadline_at: new Date(now.getTime() + input.timeoutMs).toISOString(), + monitor: null, + cancel_requested: false, + outcome: "starting", + candidate: null, + checks: [], + created_at: createdAt, + updated_at: createdAt, + finished_at: null, + reason: null, + }; +} + +async function saveCheckpoint(paths: PrometheusPaths, run: PrometheusRunState, changes: Partial, now: Date): Promise { + const next = { ...run, ...changes, updated_at: timestampAfter(run.updated_at, now) }; + await writeRun(paths, next); + return next; +} + +async function readSavedInputs(run: PrometheusRunState): Promise<{ policy: PrometheusPolicy; goal: string }> { + let policyRaw: string; + let goal: string; + try { policyRaw = await readFile(join(run.workspace_path, "PROMETHEUS.md"), "utf8"); } catch { throw localError("prometheus_policy_changed", "The saved run's PROMETHEUS.md is unavailable."); } + try { goal = await readFile(run.configuration.goal_path, "utf8"); } catch { throw localError("prometheus_goal_changed", "The saved run's goal file is unavailable."); } + const policy = parsePrometheusPolicy(policyRaw); + assertPrometheusGoalUnaddressed(goal); + if (policy.sha256 !== run.configuration.policy_sha256) throw localError("prometheus_policy_changed", "PROMETHEUS.md changed after this run was reserved."); + if (hash(goal) !== run.configuration.goal_sha256) throw localError("prometheus_goal_changed", "The goal file changed after this run was reserved."); + return { policy, goal }; +} + +/** Reserve, create and seed one fresh compiled Room. Monitoring continues in the same process. */ +export async function startPrometheusRun(input: StartPrometheusInput, dependencies: PrometheusDependencies): Promise { + // Local policy and goal validation deliberately precede credentials, Docker and HTTP. + const loaded = await readInputs(input); + const paths = await prometheusPaths(dependencies.env, loaded.workspace); + const existing = await readTeam(paths); + if (existing && existing.model !== input.model) throw localError("prometheus_model_mismatch", `This workspace's Prometheus team uses ${existing.model}; local v1 cannot change it.`); + const plan = dependencies.plan ?? await dependencies.preparePlan?.(); + if (!plan) throw localError("prometheus_runtime_unavailable", "The Prometheus Docker runtime was not prepared."); + const run = newRun(paths, input, loaded.policy, loaded.goal, loaded.goalPath, dependencies.now()); + await reserveActiveRun(paths, run, dependencies.processProbe ?? systemProcessProbe); + return continuePrometheusRun(run, loaded, existing, { ...dependencies, plan, paths }); +} + +interface MonitorDependencies extends PrometheusDependencies { + plan: RunPlan; + paths: PrometheusPaths; + team: PrometheusTeamState; + connection: PrometheusConnection; +} + +interface StartupDependencies extends PrometheusDependencies { + plan: RunPlan; + paths: PrometheusPaths; +} + +async function finishStoppedRun( + run: PrometheusRunState, + lease: NonNullable, + dependencies: Pick & { connection?: PrometheusConnection }, +): Promise { + const outcome = run.cancel_requested ? "cancelled" : "timed_out"; + if (run.room && dependencies.connection) { + await dependencies.connection.client.request( + "POST", + `/rooms/${encodeURIComponent(run.room.room_id)}/close`, + dependencies.connection.apiKey, + { trigger: `prometheus_${outcome}`, detail: run.run_id }, + { "idempotency-key": `${run.run_id}-close-${outcome}` }, + ); + } + const finishedAt = timestampAfter(run.updated_at, dependencies.now()); + const final: PrometheusRunState = { + ...run, + monitor: lease, + outcome, + updated_at: finishedAt, + finished_at: finishedAt, + reason: outcome === "cancelled" ? "Cancellation requested." : "The original deadline passed.", + }; + await finishRun(dependencies.paths, final, lease); + return final; +} + +async function finishFailedRun( + run: PrometheusRunState, + lease: NonNullable, + failure: ReturnType, + dependencies: Pick & { connection?: PrometheusConnection }, +): Promise { + if (run.room && dependencies.connection && failure.code !== "prometheus_room_closed") { + await dependencies.connection.client.request( + "POST", `/rooms/${encodeURIComponent(run.room.room_id)}/close`, dependencies.connection.apiKey, + { trigger: "prometheus_failed", detail: run.run_id }, + { "idempotency-key": `${run.run_id}-close-failed` }, + ).catch(() => undefined); + } + const finishedAt = timestampAfter(run.updated_at, dependencies.now()); + const final: PrometheusRunState = { + ...run, + monitor: lease, + outcome: "failed", + updated_at: finishedAt, + finished_at: finishedAt, + reason: `${failure.code}: ${failure.message}`.slice(0, 4_000), + }; + await finishRun(dependencies.paths, final, lease); + return final; +} + +async function continuePrometheusRun( + initial: PrometheusRunState, + loaded: { policy: PrometheusPolicy; goal: string }, + savedTeam: PrometheusTeamState | null, + dependencies: StartupDependencies, +): Promise { + const lease = await claimMonitor(dependencies.paths, initial.run_id, dependencies.processProbe ?? systemProcessProbe); + let run = await readActiveRun(dependencies.paths); + if (!run || run.run_id !== initial.run_id) throw localError("prometheus_monitor_lease_lost", `Prometheus run ${initial.run_id} is no longer active.`); + + if (run.cancel_requested || dependencies.now().getTime() >= Date.parse(run.deadline_at)) { + const connection = run.room ? await dependencies.connect() : undefined; + return finishStoppedRun(run, lease, { paths: dependencies.paths, now: dependencies.now, ...(connection ? { connection } : {}) }); + } + + const existing = savedTeam ?? await readTeam(dependencies.paths); + let team: PrometheusTeamState; + try { + if (run.startup_checkpoint !== "reserved" && !existing) { + throw localError("prometheus_team_missing", "The active Prometheus run has no saved team."); + } + team = await ensurePrometheusTeam({ + workspaceKey: dependencies.paths.workspaceKey, + workspacePath: dependencies.paths.workspacePath, + model: run.configuration.model, + policySha256: loaded.policy.sha256, + policy: loaded.policy.raw, + plan: dependencies.plan, + existing, + }, { docker: dependencies.docker, now: dependencies.now, sleep: dependencies.sleep }); + } catch (error) { + const failure = asCliError(error); + if (!["prometheus_team_invalid", "prometheus_team_missing", "prometheus_team_mismatch"].includes(failure.code)) throw error; + const connection = run.room ? await dependencies.connect().catch(() => undefined) : undefined; + return finishFailedRun(run, lease, failure, { paths: dependencies.paths, now: dependencies.now, ...(connection ? { connection } : {}) }); + } + await writeTeam(dependencies.paths, team); + const connection = await dependencies.connect(); + + if (run.startup_checkpoint === "reserved") { + const created = await connection.client.request<{ room?: { id?: string } }>( + "POST", "/rooms", connection.memberToken, + { name: `Prometheus ${basename(dependencies.paths.workspacePath)}`, type: "compiled" }, + { "idempotency-key": run.idempotency_keys.room }, + ); + const roomId = created.room?.id; + if (!roomId) throw localError("invalid_server_response", "The compiled Prometheus Room id did not come back."); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "room_created", room: { room_id: roomId, goal_sequence: null, kickoff_sequence: null } }, dependencies.now()); + } + + if (run.startup_checkpoint === "room_created") { + const remaining = Date.parse(run.deadline_at) - dependencies.now().getTime(); + const started = await connection.client.request<{ message?: { sequence?: number } }>( + "POST", `/rooms/${encodeURIComponent(run.room!.room_id)}/goal`, connection.apiKey, + { content: loaded.goal, until: [`after ${Math.max(1, Math.ceil(remaining / 1000))}s`] }, + { "idempotency-key": run.idempotency_keys.goal }, + ); + const goalSequence = started.message?.sequence; + if (!Number.isSafeInteger(goalSequence) || goalSequence! < 1) throw localError("invalid_server_response", "The Prometheus Goal sequence did not come back."); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "goal_started", room: { ...run.room!, goal_sequence: goalSequence! } }, dependencies.now()); + } + + if (run.startup_checkpoint === "goal_started") { + const invite = await connection.client.request<{ token?: string }>( + "POST", `/rooms/${encodeURIComponent(run.room!.room_id)}/invites`, connection.memberToken, undefined, + { "idempotency-key": `${run.idempotency_keys.room}-invite` }, + ); + if (!invite.token) throw localError("invalid_server_response", "The Prometheus Room invite did not come back."); + const roster = await attachPrometheusRoom(team, { + roomId: run.room!.room_id, + inviteToken: invite.token, + baseUrl: connection.baseUrl, + idempotencyKeys: run.idempotency_keys.roles, + }, { docker: dependencies.docker, now: dependencies.now, sleep: dependencies.sleep }); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "roles_joined", roster }, dependencies.now()); + } + + if (run.startup_checkpoint === "roles_joined") { + await ensurePrometheusListeners(team, run.room!.room_id, run.roster!, connection.baseUrl, { docker: dependencies.docker, now: dependencies.now, sleep: dependencies.sleep }); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "listeners_ready" }, dependencies.now()); + } + + if (run.startup_checkpoint === "listeners_ready") { + const content = prometheusKickoff({ + runId: run.run_id, + goalSequence: run.room!.goal_sequence!, + roster: run.roster!, + policy: loaded.policy, + goalSha256: run.configuration.goal_sha256, + check: run.configuration.check_command, + deadlineAt: run.deadline_at, + }); + const kicked = await connection.client.request<{ message?: { sequence?: number } }>( + "POST", `/rooms/${encodeURIComponent(run.room!.room_id)}/owner-messages`, connection.apiKey, + { content, as: "runner" }, { "idempotency-key": run.idempotency_keys.kickoff }, + ); + const kickoffSequence = kicked.message?.sequence; + if (!Number.isSafeInteger(kickoffSequence) || kickoffSequence! <= run.room!.goal_sequence!) throw localError("invalid_server_response", "The Prometheus kickoff sequence did not come back."); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "kickoff_posted", outcome: "running", room: { ...run.room!, kickoff_sequence: kickoffSequence! } }, dependencies.now()); + } + + return monitorPrometheusRun(run, { ...dependencies, team, connection }); +} + +/** Read the complete ordered Room log; completion receipts may be later than the first page. */ +export async function readPrometheusMessages( + client: ApiClient, + roomId: string, + memberToken: string, +): Promise { + const messages: PrometheusMessage[] = []; + let after = 0; + for (;;) { + const page = await client.request<{ items?: PrometheusMessage[]; has_more?: boolean }>( + "GET", + `/rooms/${encodeURIComponent(roomId)}/messages?after=${after}&order=asc&limit=100`, + memberToken, + ); + if (!Array.isArray(page.items) || typeof page.has_more !== "boolean") { + throw localError("prometheus_history_incomplete", "The Prometheus Room history is incomplete."); + } + let cursor = after; + for (const message of page.items) { + if (!Number.isSafeInteger(message.sequence) || message.sequence <= cursor) { + throw localError("prometheus_history_incomplete", "The Prometheus Room history is malformed or out of order."); + } + cursor = message.sequence; + } + messages.push(...page.items); + if (!page.has_more) return messages; + if (cursor <= after) throw localError("prometheus_history_incomplete", "Prometheus Room pagination did not advance."); + after = cursor; + } +} + +interface RoomLifecycle { + state: string; + endedBy: { trigger: string; detail: string | null } | null; +} + +async function roomLifecycle(run: PrometheusRunState, dependencies: Pick): Promise { + const roomId = run.room!.room_id; + const response = await dependencies.connection.client.request<{ room?: { state?: unknown; goal?: { ended_by?: { trigger?: unknown; detail?: unknown } | null } | null } }>("GET", `/rooms/${encodeURIComponent(roomId)}`, dependencies.connection.memberToken); + const state = response.room?.state; + const ended = response.room?.goal?.ended_by; + if (typeof state !== "string") throw localError("invalid_server_response", "The Prometheus Room state did not come back."); + const endedBy = ended && typeof ended.trigger === "string" && (typeof ended.detail === "string" || ended.detail === null) + ? { trigger: ended.trigger, detail: ended.detail } + : null; + return { state, endedBy }; +} + +async function snapshot(run: PrometheusRunState, dependencies: MonitorDependencies) { + const roomId = run.room!.room_id; + const state = await dependencies.connection.client.request("GET", `/rooms/${encodeURIComponent(roomId)}/state`, dependencies.connection.memberToken); + const messages = await readPrometheusMessages(dependencies.connection.client, roomId, dependencies.connection.memberToken); + const protocolRoster = { + coordinator: { memberId: run.roster!.coordinator.member_id }, + builder: { memberId: run.roster!.builder.member_id }, + reviewer: { memberId: run.roster!.reviewer.member_id }, + }; + const candidate = parsePrometheusCandidate(state, protocolRoster); + const receipts = candidate.ok ? parseTerminalReceipts(messages, protocolRoster, run.run_id, candidate.candidate.revisionSha256) : { ok: false as const, receipts: {}, missingRoles: [...PROMETHEUS_ROLES] }; + const runtime = await inspectPrometheusRuntime(dependencies.team, roomId, run.roster!, { docker: dependencies.docker, baseUrl: dependencies.connection.baseUrl }); + return { state, candidate, receipts, runtime, barrier: evaluatePrometheusBarrier({ candidate, receipts, runtime }) }; +} + +function sameReady(left: PrometheusBarrierResult, right: PrometheusBarrierResult): boolean { + return left.ready && right.ready && left.candidate.stateDigest === right.candidate.stateDigest && left.candidate.revisionSha256 === right.candidate.revisionSha256 && JSON.stringify(left.receipts) === JSON.stringify(right.receipts); +} + +export async function monitorPrometheusRun(run: PrometheusRunState, dependencies: MonitorDependencies): Promise { + const lease = await claimMonitor(dependencies.paths, run.run_id, dependencies.processProbe ?? systemProcessProbe); + const sleep = dependencies.sleep ?? ((milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds))); + const feedbackAttempted = new Set(); + try { + for (;;) { + const current = (await readActiveRun(dependencies.paths)) ?? run; + const now = dependencies.now(); + if (current.cancel_requested || now.getTime() >= Date.parse(current.deadline_at)) { + return finishStoppedRun(current, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + } + const lifecycle = await roomLifecycle(current, dependencies); + if (lifecycle.state !== "open") { + if (current.candidate && lifecycle.endedBy?.trigger === "prometheus_success" && lifecycle.endedBy.detail === current.candidate.revision_sha256) { + const finishedAt = timestampAfter(current.updated_at, dependencies.now()); + const final: PrometheusRunState = { + ...current, + monitor: lease, + outcome: "succeeded", + updated_at: finishedAt, + finished_at: finishedAt, + reason: `Recovered committed Prometheus success for ${current.candidate.revision_sha256}.`, + }; + await finishRun(dependencies.paths, final, lease); + return final; + } + throw localError("prometheus_room_closed", "The Prometheus Room closed before the completion barrier passed."); + } + const observed = await snapshot(current, dependencies); + if (observed.barrier.ready) { + const revision = observed.barrier.candidate.revisionSha256; + const prior = current.checks.find((entry) => entry.revision_sha256 === revision); + let passingCheck = prior?.exit_code === 0; + if (!prior) { + const checked = await runPrometheusCheck({ image: dependencies.plan.image, workspace: current.workspace_path }, current.configuration.check_command, { + SHAREDNET_PROMETHEUS_RUN_ID: current.run_id, + SHAREDNET_PROMETHEUS_WORK_ID: observed.barrier.candidate.workId, + SHAREDNET_PROMETHEUS_RESULT_REF: observed.barrier.candidate.resultRef, + SHAREDNET_PROMETHEUS_REVISION_SHA256: revision, + }, { docker: dependencies.docker, now: dependencies.now }); + if (checked.exitCode === 125) throw localError("prometheus_check_launch_failed", "The Prometheus semantic-check container could not be launched."); + const summary = { revision_sha256: revision, started_at: checked.startedAt, finished_at: checked.finishedAt, exit_code: Math.max(0, checked.exitCode), output: checked.output, output_sha256: checked.outputSha256 }; + run = await saveCheckpoint(dependencies.paths, current, { monitor: lease, checks: [...current.checks, summary] }, dependencies.now()); + if (checked.exitCode !== 0) { + const feedback = `@${current.roster!.coordinator.member_id} Semantic check failed for revision ${revision}:\n${checked.output.slice(-2_000)}`; + await dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/owner-messages`, dependencies.connection.apiKey, { content: feedback, as: "runner" }, { "idempotency-key": `${current.run_id}-check-${revision}` }); + feedbackAttempted.add(revision); + } else { + passingCheck = true; + } + } else if (prior.exit_code !== 0 && !feedbackAttempted.has(revision)) { + const feedback = `@${current.roster!.coordinator.member_id} Semantic check failed for revision ${revision}:\n${prior.output.slice(-2_000)}`; + await dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/owner-messages`, dependencies.connection.apiKey, { content: feedback, as: "runner" }, { "idempotency-key": `${current.run_id}-check-${revision}` }); + feedbackAttempted.add(revision); + } + if (passingCheck) { + const after = await snapshot(run, dependencies); + if (sameReady(observed.barrier, after.barrier)) { + const observedAt = timestampAfter(run.updated_at, dependencies.now()); + run = await saveCheckpoint(dependencies.paths, run, { + monitor: lease, + candidate: { compiled_digest: observed.barrier.candidate.stateDigest, work_id: observed.barrier.candidate.workId, result_ref: observed.barrier.candidate.resultRef, revision_sha256: revision, observed_at: observedAt }, + }, dependencies.now()); + await dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/close`, dependencies.connection.apiKey, { trigger: "prometheus_success", detail: revision }, { "idempotency-key": `${current.run_id}-close-success` }); + const finishedAt = timestampAfter(run.updated_at, dependencies.now()); + const final: PrometheusRunState = { ...run, monitor: lease, outcome: "succeeded", updated_at: finishedAt, finished_at: finishedAt, reason: `Prometheus completion barrier passed for ${revision}.` }; + await finishRun(dependencies.paths, final, lease); + return final; + } + } + } + await sleep(1_000); + } + } catch (error) { + const failure = asCliError(error); + if (failure.code === "prometheus_monitor_active" || failure.code === "prometheus_monitor_lease_lost") throw error; + // Transport and service failures do not prove the Room or run is invalid. + // Keep the durable run active so a later `prometheus resume` can reclaim it. + if (failure.exitCode === 5) throw failure; + const current = (await readActiveRun(dependencies.paths)) ?? run; + return finishFailedRun(current, lease, failure, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + } +} + +type PrometheusStatusDependencies = Pick; + +export async function prometheusStatus( + workspace: string, + env: Record, + dependencies?: PrometheusStatusDependencies, +) { + const paths = await prometheusPaths(env, workspace); + const [team, run] = await Promise.all([readTeam(paths), readActiveRun(paths)]); + const base = { workspace: paths.workspacePath, team: team ? { model: team.model, roles: PROMETHEUS_ROLES } : null, run }; + if (!run) return base; + const monitor = run.monitor === null + ? { state: "none" as const, pid: null, claimed_at: null, renewed_at: null } + : { + state: inspectProcessIdentity(dependencies?.processProbe ?? systemProcessProbe, run.monitor), + pid: run.monitor.pid, + claimed_at: run.monitor.claimed_at, + renewed_at: run.monitor.renewed_at, + }; + if (!dependencies) { + return { ...base, monitor, live: { available: false, reason: "Live inspection was not attempted." } }; + } + if (!team || !dependencies.plan || !run.room || !run.roster || run.startup_checkpoint !== "kickoff_posted") { + return { ...base, monitor, live: { available: false, reason: "The run has not reached a live inspectable checkpoint." } }; + } + try { + await verifyPrometheusTeamIdentity({ + workspaceKey: paths.workspaceKey, + workspacePath: paths.workspacePath, + model: team.model, + policySha256: team.last_policy_sha256, + policy: "", + plan: dependencies.plan, + existing: team, + }, { docker: dependencies.docker }); + const connection = await dependencies.connect(); + const monitorDependencies = { ...dependencies, env, cwd: paths.workspacePath, now: () => new Date(), plan: dependencies.plan, paths, team, connection }; + const lifecycle = await roomLifecycle(run, monitorDependencies); + if (lifecycle.state !== "open") return { ...base, monitor, live: { available: true, room_state: lifecycle.state, ended_by: lifecycle.endedBy } }; + const observed = await snapshot(run, monitorDependencies); + return { ...base, monitor, live: { available: true, room_state: lifecycle.state, runtime: observed.runtime, barrier: observed.barrier } }; + } catch (error) { + const failure = asCliError(error); + return { ...base, monitor, live: { available: false, error: { code: failure.code, message: failure.message } } }; + } +} + +type PrometheusCancelDependencies = Pick; + +export async function cancelPrometheusRun(workspace: string, dependencies: PrometheusCancelDependencies) { + const paths = await prometheusPaths(dependencies.env, workspace); + const active = await readActiveRun(paths); + if (!active) return { cancelled: false, active: false }; + await requestCancel(paths, active.run_id); + try { + const lease = await claimMonitor(paths, active.run_id, dependencies.processProbe ?? systemProcessProbe); + const current = await readActiveRun(paths); + if (!current) return { cancelled: true, active: false, recovered: false, run_id: active.run_id }; + const connection = current.room ? await dependencies.connect() : undefined; + const final = await finishStoppedRun(current, lease, { paths, now: dependencies.now, ...(connection ? { connection } : {}) }); + return { cancelled: true, active: false, recovered: true, run_id: active.run_id, outcome: final.outcome }; + } catch (error) { + const failure = asCliError(error); + if (failure.code === "prometheus_run_not_active" && (await readActiveRun(paths)) === null) { + return { cancelled: true, active: false, recovered: false, run_id: active.run_id }; + } + if (failure.code !== "prometheus_monitor_active") throw error; + return { cancelled: true, active: true, recovered: false, run_id: active.run_id }; + } +} + +/** Take over a fully seeded run without recreating its Room, seats, threads, goal or kickoff. */ +export async function resumePrometheusRun(workspace: string, dependencies: PrometheusDependencies): Promise { + const paths = await prometheusPaths(dependencies.env, workspace); + const run = await readActiveRun(paths); + if (!run) throw localError("prometheus_run_not_active", "This workspace has no active Prometheus run to resume."); + const team = await readTeam(paths); + const plan = dependencies.plan ?? await dependencies.preparePlan?.(); + if (!plan) throw localError("prometheus_runtime_unavailable", "The Prometheus Docker runtime was not prepared."); + const loaded = await readSavedInputs(run); + return continuePrometheusRun(run, loaded, team, { ...dependencies, plan, paths }); +} diff --git a/packages/cli/src/storage.test.ts b/packages/cli/src/storage.test.ts index 759a386..339528d 100644 --- a/packages/cli/src/storage.test.ts +++ b/packages/cli/src/storage.test.ts @@ -1,6 +1,6 @@ // @vitest-environment node -import { mkdtemp, readFile, stat, symlink, writeFile } from "node:fs/promises"; +import { chmod, mkdtemp, readFile, stat, symlink, writeFile } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join } from "node:path"; @@ -8,10 +8,13 @@ import { afterEach, describe, expect, it } from "vitest"; import { backupStoredApiCredential, + ensureOwnerOnlyDirectory, getOrCreateInstallationSecret, getStoragePaths, listSessions, + readOwnerOnlyFile, readStoredApiCredential, + writeOwnerOnlyJson, writeSession, writeStoredApiCredential, type StoredApiCredential, @@ -31,6 +34,25 @@ async function fixtureRoot() { } describe("secure local state", () => { + it("exports the owner-only primitives used by non-credential state", async () => { + const root = await fixtureRoot(); + const directory = join(root, "state", "nested"); + const file = join(directory, "state.json"); + + await ensureOwnerOnlyDirectory(directory); + await writeOwnerOnlyJson(file, { generation: 1 }); + await writeOwnerOnlyJson(file, { generation: 2 }); + + expect((await stat(directory)).mode & 0o777).toBe(0o700); + expect((await stat(file)).mode & 0o777).toBe(0o600); + expect(await readOwnerOnlyFile(file)).toBe('{"generation":2}\n'); + + await chmod(file, 0o640); + await expect(readOwnerOnlyFile(file)).rejects.toMatchObject({ + code: "unsafe_credential_storage", + }); + }); + it("creates session directories as 0700 and files as 0600", async () => { const root = await fixtureRoot(); const paths = getStoragePaths({ diff --git a/packages/cli/src/storage.ts b/packages/cli/src/storage.ts index c6101e4..683720c 100644 --- a/packages/cli/src/storage.ts +++ b/packages/cli/src/storage.ts @@ -153,10 +153,16 @@ export function getStoragePaths(env: Environment = process.env): StoragePaths { }; } -async function ensureSecureDirectory(path: string): Promise { +/** Validate an existing owner-only directory without following a symlink. */ +export async function assertOwnerOnlyDirectory(path: string): Promise { if (process.platform === "win32") unsafeStorage(); - await mkdir(path, { recursive: true, mode: 0o700 }); - const info = await lstat(path); + let info; + try { + info = await lstat(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return false; + throw error; + } if ( info.isSymbolicLink() || !info.isDirectory() || @@ -165,9 +171,18 @@ async function ensureSecureDirectory(path: string): Promise { ) { unsafeStorage(); } + return true; +} + +/** Ensure a directory is private to the current OS owner and is not a symlink. */ +export async function ensureOwnerOnlyDirectory(path: string): Promise { + if (process.platform === "win32") unsafeStorage(); + await mkdir(path, { recursive: true, mode: 0o700 }); + if (!(await assertOwnerOnlyDirectory(path))) unsafeStorage(); } -async function readSecureFile(path: string): Promise { +/** Read an owner-only regular file without following a symlink. */ +export async function readOwnerOnlyFile(path: string): Promise { let info; try { info = await lstat(path); @@ -205,12 +220,13 @@ async function readSecureFile(path: string): Promise { } } -async function writeSecureJson(path: string, value: unknown): Promise { +/** Atomically replace an owner-only JSON file and durably sync its parent directory. */ +export async function writeOwnerOnlyJson(path: string, value: unknown): Promise { const parent = dirname(path); - await ensureSecureDirectory(parent); + await ensureOwnerOnlyDirectory(parent); // Validate an existing target before an atomic replacement. Missing is fine. - await readSecureFile(path); + await readOwnerOnlyFile(path); const temp = join(parent, `.sharednet-${randomBytes(12).toString("hex")}.tmp`); let handle; try { @@ -234,13 +250,14 @@ async function writeSecureJson(path: string, value: unknown): Promise { } } -async function createSecureJsonIfAbsent(path: string, value: unknown): Promise { +/** Atomically create a complete owner-only JSON file, returning false when it already exists. */ +export async function createOwnerOnlyJsonIfAbsent(path: string, value: unknown): Promise { return createSecureTextIfAbsent(path, `${JSON.stringify(value)}\n`); } async function createSecureTextIfAbsent(path: string, text: string): Promise { const parent = dirname(path); - await ensureSecureDirectory(parent); + await ensureOwnerOnlyDirectory(parent); const temp = join(parent, `.sharednet-${randomBytes(12).toString("hex")}.tmp`); let handle; try { @@ -332,7 +349,7 @@ function parseSession(raw: string): StoredSession { export async function readStoredApiCredential( paths: StoragePaths, ): Promise { - const raw = await readSecureFile(paths.credentialsFile); + const raw = await readOwnerOnlyFile(paths.credentialsFile); return raw === null ? null : parseCredential(raw); } @@ -341,7 +358,7 @@ export async function writeStoredApiCredential( paths: StoragePaths, credential: StoredApiCredential, ): Promise { - await writeSecureJson(paths.credentialsFile, credential); + await writeOwnerOnlyJson(paths.credentialsFile, credential); } /** @@ -353,7 +370,7 @@ export async function backupStoredApiCredential( paths: StoragePaths, now: Date, ): Promise { - const raw = await readSecureFile(paths.credentialsFile); + const raw = await readOwnerOnlyFile(paths.credentialsFile); if (raw === null) return null; const stamp = now.toISOString().replace(/[:.]/g, "-"); for (let attempt = 0; attempt < 100; attempt += 1) { @@ -369,7 +386,7 @@ export async function getOrCreateInstallationSecret( const credential = await readStoredApiCredential(paths); if (credential) return credential.installation_secret; - const raw = await readSecureFile(paths.installationFile); + const raw = await readOwnerOnlyFile(paths.installationFile); if (raw !== null) { const value = parseJsonObject(raw, "The installation file"); if (value.schema_version !== 1) { @@ -382,11 +399,11 @@ export async function getOrCreateInstallationSecret( schema_version: 1, installation_secret: randomBytes(32).toString("base64url"), }; - if (await createSecureJsonIfAbsent(paths.installationFile, installation)) { + if (await createOwnerOnlyJsonIfAbsent(paths.installationFile, installation)) { return installation.installation_secret; } - const winnerRaw = await readSecureFile(paths.installationFile); + const winnerRaw = await readOwnerOnlyFile(paths.installationFile); if (winnerRaw === null) { throw localError("invalid_local_state", "The installation secret could not be created."); } @@ -404,13 +421,13 @@ export async function writeSession( if (!/^i_[A-Za-z0-9_-]+$/.test(session.instance_id)) { throw localError("invalid_local_state", "The Instance ID is invalid."); } - await writeSecureJson(join(paths.sessionsDir, `${session.instance_id}.json`), session); + await writeOwnerOnlyJson(join(paths.sessionsDir, `${session.instance_id}.json`), session); } export async function listSessions(paths: StoragePaths): Promise { let entries; try { - await ensureSecureDirectory(paths.sessionsDir); + await ensureOwnerOnlyDirectory(paths.sessionsDir); entries = await readdir(paths.sessionsDir, { withFileTypes: true }); } catch (error) { if ((error as NodeJS.ErrnoException).code === "ENOENT") return []; @@ -420,7 +437,7 @@ export async function listSessions(paths: StoragePaths): Promise a.name.localeCompare(b.name))) { if (!entry.name.endsWith(".json")) continue; if (!entry.isFile() || entry.isSymbolicLink()) unsafeStorage(); - const raw = await readSecureFile(join(paths.sessionsDir, entry.name)); + const raw = await readOwnerOnlyFile(join(paths.sessionsDir, entry.name)); if (raw !== null) sessions.push(parseSession(raw)); } return sessions; @@ -433,14 +450,14 @@ export async function readSessionById( if (!/^i_[A-Za-z0-9_-]+$/.test(instanceId)) { throw localError("invalid_session_id", "The Instance ID is invalid."); } - const raw = await readSecureFile(join(paths.sessionsDir, `${instanceId}.json`)); + const raw = await readOwnerOnlyFile(join(paths.sessionsDir, `${instanceId}.json`)); return raw === null ? null : parseSession(raw); } export async function deleteSession(paths: StoragePaths, instanceId: string): Promise { if (!/^i_[A-Za-z0-9_-]+$/.test(instanceId)) return; const path = join(paths.sessionsDir, `${instanceId}.json`); - const raw = await readSecureFile(path); + const raw = await readOwnerOnlyFile(path); if (raw === null) return; await unlink(path); } @@ -488,7 +505,7 @@ export async function writeRoomCredential( paths: StoragePaths, credential: StoredRoomCredential, ): Promise { - await writeSecureJson( + await writeOwnerOnlyJson( roomCredentialFile(paths, credential.room_id, credential.member_id), credential, ); @@ -499,7 +516,7 @@ export async function readRoomCredential( roomId: string, memberId: string, ): Promise { - const raw = await readSecureFile(roomCredentialFile(paths, roomId, memberId)); + const raw = await readOwnerOnlyFile(roomCredentialFile(paths, roomId, memberId)); return raw === null ? null : parseRoomCredential(raw); } diff --git a/scripts/export-manifest.json b/scripts/export-manifest.json index 8df831d..a8249c9 100644 --- a/scripts/export-manifest.json +++ b/scripts/export-manifest.json @@ -354,10 +354,21 @@ "packages/cli/src/serve-service.test.ts", "docs/harness-modes.md", "docs/superpowers/plans/2026-10-16-harness-mode-registry.md", + "docs/plans/2026-10-18-prometheus-local-design.md", + "docs/superpowers/plans/2026-10-18-prometheus-local-v1.md", "packages/cli/src/harness-modes.ts", "packages/cli/src/harness-modes.test.ts", "packages/cli/src/goal-serve-demo.ts", "packages/cli/src/goal-serve-demo.test.ts", - "docs/superpowers/plans/2026-10-14-serve-goal-demo.md" + "docs/superpowers/plans/2026-10-14-serve-goal-demo.md", + "docs/prometheus-local.md", + "packages/cli/src/prometheus-protocol.ts", + "packages/cli/src/prometheus-protocol.test.ts", + "packages/cli/src/prometheus-state.ts", + "packages/cli/src/prometheus-state.test.ts", + "packages/cli/src/prometheus-runtime.ts", + "packages/cli/src/prometheus-runtime.test.ts", + "packages/cli/src/prometheus.ts", + "packages/cli/src/prometheus.test.ts" ] } diff --git a/scripts/package-smoke.mjs b/scripts/package-smoke.mjs index af4eced..c4bfe9e 100644 --- a/scripts/package-smoke.mjs +++ b/scripts/package-smoke.mjs @@ -32,7 +32,14 @@ try { const [pack] = JSON.parse(packed.stdout); assert.equal(pack.name, "sharednet"); const files = pack.files.map(({ path }) => path); - for (const path of ["bin/sharednet.js", "dist/main.js", "dist/cli.js", "dist/cli.d.ts", "LICENSE", "package.json"]) { + for (const path of [ + "bin/sharednet.js", "dist/main.js", "dist/cli.js", "dist/cli.d.ts", + "dist/prometheus.js", "dist/prometheus.d.ts", + "dist/prometheus-protocol.js", "dist/prometheus-protocol.d.ts", + "dist/prometheus-runtime.js", "dist/prometheus-runtime.d.ts", + "dist/prometheus-state.js", "dist/prometheus-state.d.ts", + "LICENSE", "package.json", + ]) { assert(files.includes(path), `Missing packaged entry: ${path}`); } assert(files.every((path) => /^(?:bin\/sharednet\.js|dist\/[\w-]+\.(?:js|d\.ts)|package\.json|README\.md|LICENSE)$/.test(path)), "Unexpected tarball contents"); @@ -68,6 +75,17 @@ try { assert.equal(failure?.code, 2, "Installed executable must return its usage exit code"); assert.equal(failure.stdout, ""); assert.equal(JSON.parse(failure.stderr).error.code, "unknown_command"); + const prometheus = await exec(join(consumer, "node_modules/.bin/sharednet"), [ + "prometheus", "status", "--workspace", consumer, "--json", + ], { + cwd: consumer, + env: { ...env, NODE_OPTIONS: "--no-experimental-strip-types" }, + timeout: 10_000, + }); + assert.deepEqual(JSON.parse(prometheus.stdout), { workspace: consumer, team: null, run: null }); + assert.equal(prometheus.stderr, ""); + const installedFiles = files.join("\n"); + assert.doesNotMatch(installedFiles, /(?:credential|auth\.json|\.sharednet|prometheus\/local-v1)/i); const sha256 = createHash("sha256").update(await readFile(tarball)).digest("hex"); await writeFile(join(artifacts, "cli-smoke.json"), JSON.stringify({ name: pack.name, version: pack.version, tarball: pack.filename, sha256, files }, null, 2) + "\n"); console.log(`Installed-package smoke passed: ${pack.filename}; sha256=${sha256}`); From 59c5a090644247c2fbcc934e85a5ec8a3de02a33 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 19 Oct 2026 02:28:30 +0000 Subject: [PATCH 2/9] fix(cli): tolerate Prometheus state removal races --- .../cli/src/prometheus-state-race.test.ts | 157 ++++++++++++++++++ packages/cli/src/prometheus-state.ts | 11 +- packages/cli/src/storage.ts | 9 +- 3 files changed, 174 insertions(+), 3 deletions(-) create mode 100644 packages/cli/src/prometheus-state-race.test.ts diff --git a/packages/cli/src/prometheus-state-race.test.ts b/packages/cli/src/prometheus-state-race.test.ts new file mode 100644 index 0000000..a6bc103 --- /dev/null +++ b/packages/cli/src/prometheus-state-race.test.ts @@ -0,0 +1,157 @@ +// @vitest-environment node + +import { mkdtemp, mkdir, readFile, rm, writeFile } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +import { afterEach, describe, expect, it, vi } from "vitest"; + +const races = vi.hoisted(() => ({ + disappearBeforeOpen: null as string | null, + disappearBeforeUnlink: null as string | null, + openedAfterRemoval: false, + unlinkedAfterRemoval: false, +})); + +vi.mock("node:fs/promises", async (importOriginal) => { + const actual = await importOriginal(); + return { + ...actual, + open: async (...args: Parameters) => { + const path = String(args[0]); + if (races.disappearBeforeOpen === path) { + races.disappearBeforeOpen = null; + await actual.unlink(args[0]); + races.openedAfterRemoval = true; + } + return actual.open(...args); + }, + unlink: async (...args: Parameters) => { + const path = String(args[0]); + if (races.disappearBeforeUnlink === path) { + races.disappearBeforeUnlink = null; + await actual.unlink(...args); + races.unlinkedAfterRemoval = true; + } + return actual.unlink(...args); + }, + }; +}); + +import type { ProcessProbe } from "./process-identity.ts"; +import { + claimMonitor, + finishRun, + prometheusPaths, + readRun, + reserveActiveRun, + type PrometheusRunState, +} from "./prometheus-state.ts"; +import { readOwnerOnlyFile } from "./storage.ts"; + +const cleanup: string[] = []; + +afterEach(async () => { + races.disappearBeforeOpen = null; + races.disappearBeforeUnlink = null; + races.openedAfterRemoval = false; + races.unlinkedAfterRemoval = false; + await Promise.all(cleanup.splice(0).map((path) => rm(path, { recursive: true, force: true }))); +}); + +async function fixture() { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-race-")); + cleanup.push(root); + const workspace = join(root, "workspace"); + await mkdir(workspace); + const paths = await prometheusPaths( + { HOME: root, XDG_CONFIG_HOME: join(root, "config"), XDG_STATE_HOME: join(root, "state") }, + workspace, + ); + return { root, paths }; +} + +function fakeProbe(): ProcessProbe { + return { + supported: true, + alive: (pid) => pid === process.pid, + identity: (pid) => pid === process.pid ? "race-test-process" : null, + argv: () => null, + }; +} + +function run( + paths: Awaited>, +): PrometheusRunState { + return { + schema_version: 1, + run_id: "run_20261019_race", + workspace_key: paths.workspaceKey, + workspace_path: paths.workspacePath, + configuration: { + model: "gpt-6-sol", + goal_path: join(paths.workspacePath, "GOAL.md"), + goal_sha256: "a".repeat(64), + policy_sha256: "b".repeat(64), + check_command: "pnpm test", + }, + startup_checkpoint: "reserved", + room: null, + roster: null, + idempotency_keys: { + room: "run_20261019_race-room", + goal: "run_20261019_race-goal", + roles: { + coordinator: "run_20261019_race-coordinator", + builder: "run_20261019_race-builder", + reviewer: "run_20261019_race-reviewer", + }, + kickoff: "run_20261019_race-kickoff", + }, + deadline_at: "2026-10-19T04:00:00.000Z", + monitor: null, + cancel_requested: false, + outcome: "starting", + candidate: null, + checks: [], + created_at: "2026-10-19T02:00:00.000Z", + updated_at: "2026-10-19T02:01:00.000Z", + finished_at: null, + reason: null, + }; +} + +describe("Prometheus state filesystem races", () => { + it("treats a file removed between lstat and no-follow open as absent", async () => { + const { root } = await fixture(); + const path = join(root, "owner-only.json"); + await writeFile(path, "{}\n", { mode: 0o600 }); + races.disappearBeforeOpen = path; + + await expect(readOwnerOnlyFile(path)).resolves.toBeNull(); + expect(races.openedAfterRemoval).toBe(true); + }); + + it("finishes a run when another cleanup wins the active-file unlink", async () => { + const { paths } = await fixture(); + const initial = run(paths); + const probe = fakeProbe(); + await reserveActiveRun(paths, initial, probe); + const lease = await claimMonitor(paths, initial.run_id, probe); + const current = (await readRun(paths, initial.run_id))!; + const finishedAt = new Date(Date.parse(current.updated_at) + 1_000).toISOString(); + const finalState: PrometheusRunState = { + ...current, + outcome: "failed", + updated_at: finishedAt, + finished_at: finishedAt, + reason: "Synthetic terminal result for a deletion-race regression.", + }; + races.disappearBeforeUnlink = paths.activeFile; + + await expect(finishRun(paths, finalState, lease)).resolves.toBeUndefined(); + expect(races.unlinkedAfterRemoval).toBe(true); + expect(await readRun(paths, initial.run_id)).toEqual(finalState); + await expect(readFile(paths.activeFile, "utf8")).rejects.toMatchObject({ code: "ENOENT" }); + }); +}); diff --git a/packages/cli/src/prometheus-state.ts b/packages/cli/src/prometheus-state.ts index bc8adee..bce941c 100644 --- a/packages/cli/src/prometheus-state.ts +++ b/packages/cli/src/prometheus-state.ts @@ -768,8 +768,15 @@ export async function readActiveRun(paths: PrometheusPaths): Promise { - if ((await readOwnerOnlyFile(path)) === null) return; - await unlink(path); + try { + // Unlinking a symlink removes the link itself, so deletion does not need + // the read-side no-follow validation. Avoiding a read before unlink also + // removes the check/read/unlink window used by competing state cleaners. + await unlink(path); + } catch (error) { + if ((error as NodeJS.ErrnoException).code === "ENOENT") return; + throw error; + } const directory = await open(dirname(path), "r"); try { await directory.sync(); diff --git a/packages/cli/src/storage.ts b/packages/cli/src/storage.ts index 683720c..76f6b0d 100644 --- a/packages/cli/src/storage.ts +++ b/packages/cli/src/storage.ts @@ -203,7 +203,14 @@ export async function readOwnerOnlyFile(path: string): Promise { let handle; try { const noFollow = "O_NOFOLLOW" in constants ? constants.O_NOFOLLOW : 0; - handle = await open(path, constants.O_RDONLY | noFollow); + try { + handle = await open(path, constants.O_RDONLY | noFollow); + } catch (error) { + // A concurrent owner may remove the file after lstat. That is the same + // observable state as a file which was absent at the initial lookup. + if ((error as NodeJS.ErrnoException).code === "ENOENT") return null; + throw error; + } const openedInfo = await handle.stat(); if ( !openedInfo.isFile() || From eaf3bfb80bb216cd93344cabc3469f33ae31dab8 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 19 Oct 2026 02:30:14 +0000 Subject: [PATCH 3/9] fix(cli): export Prometheus race regression --- scripts/export-manifest.json | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/export-manifest.json b/scripts/export-manifest.json index a8249c9..4b00c83 100644 --- a/scripts/export-manifest.json +++ b/scripts/export-manifest.json @@ -365,6 +365,7 @@ "packages/cli/src/prometheus-protocol.ts", "packages/cli/src/prometheus-protocol.test.ts", "packages/cli/src/prometheus-state.ts", + "packages/cli/src/prometheus-state-race.test.ts", "packages/cli/src/prometheus-state.test.ts", "packages/cli/src/prometheus-runtime.ts", "packages/cli/src/prometheus-runtime.test.ts", From c9a2b1c0bb895d050b406a394820ae5706b47810 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 19 Oct 2026 02:41:57 +0000 Subject: [PATCH 4/9] fix(cli): stop Prometheus work at authoritative deadline Make Docker operations abortable, cap bootstrap and semantic checks by the run's absolute deadline, and poll durable cancellation while they execute. Re-read active state at startup and completion boundaries, recover a remotely committed success before applying a later stop, and require the close response to confirm the exact revision before recording local success. Co-authored-by: OpenAI Codex --- packages/cli/src/goal-run.ts | 22 ++- packages/cli/src/prometheus-runtime.test.ts | 27 +++ packages/cli/src/prometheus-runtime.ts | 26 ++- packages/cli/src/prometheus.test.ts | 38 +++- packages/cli/src/prometheus.ts | 189 ++++++++++++++++---- 5 files changed, 262 insertions(+), 40 deletions(-) diff --git a/packages/cli/src/goal-run.ts b/packages/cli/src/goal-run.ts index 8e16900..148b1ab 100644 --- a/packages/cli/src/goal-run.ts +++ b/packages/cli/src/goal-run.ts @@ -71,7 +71,7 @@ export interface DockerResult { */ export type DockerRunner = ( args: readonly string[], - options?: { input?: string; env?: Record; onLine?: (line: string) => void }, + options?: { input?: string; env?: Record; onLine?: (line: string) => void; signal?: AbortSignal }, ) => Promise; export function dockerRunner(baseEnv: Record): DockerRunner { @@ -80,6 +80,15 @@ export function dockerRunner(baseEnv: Record): Docke return (args, options = {}) => new Promise((resolveRun) => { const child = spawn("docker", [...args], { env: { ...clientEnv, ...options.env } as NodeJS.ProcessEnv, stdio: ["pipe", "pipe", "pipe"] }); + let settled = false; + let killTimer: ReturnType | null = null; + const abort = () => { + child.kill("SIGTERM"); + killTimer = setTimeout(() => child.kill("SIGKILL"), 5_000); + killTimer.unref?.(); + }; + options.signal?.addEventListener("abort", abort, { once: true }); + if (options.signal?.aborted) abort(); let stdout = ""; let stderr = ""; let partial = ""; @@ -99,10 +108,17 @@ export function dockerRunner(baseEnv: Record): Docke child.stderr.on("data", (chunk: string) => { stderr = (stderr + chunk).slice(-64_000); }); - child.on("error", () => resolveRun({ code: 127, stdout, stderr: "docker could not be started" })); + const finish = (result: DockerResult) => { + if (settled) return; + settled = true; + options.signal?.removeEventListener("abort", abort); + if (killTimer) clearTimeout(killTimer); + resolveRun(result); + }; + child.on("error", () => finish({ code: 127, stdout, stderr: "docker could not be started" })); child.on("close", (code) => { if (options.onLine && partial) options.onLine(partial); - resolveRun({ code: code ?? 1, stdout, stderr }); + finish({ code: code ?? (options.signal?.aborted ? 143 : 1), stdout, stderr }); }); child.stdin.end(options.input ?? ""); }); diff --git a/packages/cli/src/prometheus-runtime.test.ts b/packages/cli/src/prometheus-runtime.test.ts index 370de2d..027bffb 100644 --- a/packages/cli/src/prometheus-runtime.test.ts +++ b/packages/cli/src/prometheus-runtime.test.ts @@ -203,11 +203,38 @@ describe("persistent Prometheus runtime", () => { SHAREDNET_PROMETHEUS_REVISION_SHA256: "b".repeat(64), }); expect(seen[0]!.args.join(" ")).not.toContain("run-1"); + expect(seen[0]!.args).toContain("timeout"); + expect(seen[0]!.args).toContain("120.000s"); expect(result.exitCode).toBe(7); expect(Buffer.byteLength(result.output)).toBeLessThanOrEqual(64_000); expect(result.outputSha256).toMatch(/^[0-9a-f]{64}$/); }); + it("passes cancellation into an attached semantic-check container", async () => { + const controller = new AbortController(); + let sawSignal = false; + const docker: DockerRunner = async (_args, options) => new Promise((resolve) => { + sawSignal = options?.signal === controller.signal; + options?.signal?.addEventListener("abort", () => resolve({ code: 143, stdout: "", stderr: "terminated" }), { once: true }); + }); + const pending = runPrometheusCheck( + { image: DEFAULT_IMAGE, workspace: plan.workspace }, + "pnpm test", + { + SHAREDNET_PROMETHEUS_RUN_ID: "run-1", + SHAREDNET_PROMETHEUS_WORK_ID: "work-1", + SHAREDNET_PROMETHEUS_RESULT_REF: "evt-1", + SHAREDNET_PROMETHEUS_REVISION_SHA256: "b".repeat(64), + }, + { docker, timeoutMs: 30_000, signal: controller.signal }, + ); + + controller.abort(); + + await expect(pending).resolves.toMatchObject({ exitCode: 143 }); + expect(sawSignal).toBe(true); + }); + it("restarts unfiltered listeners for saved seats without joining again", async () => { const calls: string[][] = []; const team: PrometheusTeamState = { diff --git a/packages/cli/src/prometheus-runtime.ts b/packages/cli/src/prometheus-runtime.ts index fa16ecf..88b0519 100644 --- a/packages/cli/src/prometheus-runtime.ts +++ b/packages/cli/src/prometheus-runtime.ts @@ -28,6 +28,8 @@ export interface EnsureTeamInput { policySha256: string; policy: string; plan: RunPlan; + /** Absolute run deadline. Bootstrap turns must never outlive it. */ + deadlineAt?: string; existing?: PrometheusTeamState | null; } @@ -209,6 +211,16 @@ function bootstrapPrompt(role: PrometheusRole, policy: string): string { ].join("\n\n"); } +function bootstrapTimeoutSeconds(input: EnsureTeamInput, dependencies: PrometheusRuntimeDependencies): string { + const maximumMs = 120_000; + if (!input.deadlineAt) return "120"; + const remainingMs = Date.parse(input.deadlineAt) - (dependencies.now ?? (() => new Date()))().getTime(); + if (!Number.isFinite(remainingMs) || remainingMs <= 0) { + throw localError("prometheus_deadline_reached", "The Prometheus deadline passed before agent bootstrap completed."); + } + return `${(Math.min(maximumMs, remainingMs) / 1_000).toFixed(3)}s`; +} + /** Create the fixed local team once; subsequent calls verify and reuse its exact containers and threads. */ export async function ensurePrometheusTeam( input: EnsureTeamInput, @@ -273,7 +285,7 @@ export async function ensurePrometheusTeam( "exec", "--user", user, "--workdir", CONTAINER_WORKSPACE, "-e", `HOME=${CONTAINER_HOME}`, "-e", `CODEX_HOME=${CONTAINER_HOME}/.codex`, ...(input.plan.codexAuth?.kind === "api-key" ? ["-e", "OPENAI_API_KEY"] : []), - container, "timeout", "120", "codex", "exec", "--json", "--skip-git-repo-check", "--dangerously-bypass-approvals-and-sandbox", "-m", input.model, bootstrapPrompt(role, input.policy), + container, "timeout", "--signal=TERM", "--kill-after=5s", bootstrapTimeoutSeconds(input, dependencies), "codex", "exec", "--json", "--skip-git-repo-check", "--dangerously-bypass-approvals-and-sandbox", "-m", input.model, bootstrapPrompt(role, input.policy), ]); const thread = threadId(bootstrapped.stdout); if (bootstrapped.code !== 0 || thread === null) throw localError("agent_bootstrap_failed", `The ${role} persistent Codex thread could not be created.`); @@ -398,16 +410,22 @@ export async function runPrometheusCheck( runtime: { image: string; workspace: string }, check: string, environment: Record<"SHAREDNET_PROMETHEUS_RUN_ID" | "SHAREDNET_PROMETHEUS_WORK_ID" | "SHAREDNET_PROMETHEUS_RESULT_REF" | "SHAREDNET_PROMETHEUS_REVISION_SHA256", string>, - dependencies: Pick, + dependencies: Pick & { timeoutMs?: number; signal?: AbortSignal }, ): Promise { + const timeoutMs = dependencies.timeoutMs ?? 120_000; + if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { + throw localError("prometheus_deadline_reached", "The Prometheus deadline passed before the semantic check could start."); + } const now = dependencies.now ?? (() => new Date()); const startedAt = now().toISOString(); const result = await dependencies.docker([ "run", "--rm", "--volume", `${runtime.workspace}:${CONTAINER_WORKSPACE}`, "--workdir", CONTAINER_WORKSPACE, ...Object.keys(environment).flatMap((name) => ["-e", name]), - runtime.image, "sh", "-c", check, - ], { env: environment }); + runtime.image, + "timeout", "--signal=TERM", "--kill-after=5s", `${(timeoutMs / 1_000).toFixed(3)}s`, + "sh", "-c", check, + ], { env: environment, signal: dependencies.signal }); const output = bounded(`${result.stdout}${result.stderr}`); return { startedAt, diff --git a/packages/cli/src/prometheus.test.ts b/packages/cli/src/prometheus.test.ts index 1f8bef3..627f999 100644 --- a/packages/cli/src/prometheus.test.ts +++ b/packages/cli/src/prometheus.test.ts @@ -9,7 +9,7 @@ import { afterEach, describe, expect, it } from "vitest"; import { DEFAULT_IMAGE, type DockerRunner, type RunPlan } from "./goal-run.ts"; import { cancelPrometheusRun, evaluatePrometheusBarrier, prometheusKickoff, prometheusStatus, readPrometheusMessages, resumePrometheusRun } from "./prometheus.ts"; -import { claimMonitor, prometheusPaths, readActiveRun, reserveActiveRun, writeRun, writeTeam, type PrometheusRunState, type PrometheusTeamState } from "./prometheus-state.ts"; +import { claimMonitor, prometheusPaths, readActiveRun, requestCancel, reserveActiveRun, writeRun, writeTeam, type PrometheusRunState, type PrometheusTeamState } from "./prometheus-state.ts"; import type { ProcessProbe } from "./process-identity.ts"; const candidate = { @@ -293,13 +293,14 @@ describe("Prometheus lifecycle recovery", () => { }); it("reconciles a success close that committed before the terminal record", async () => { - const fixture = await lifecycleFixture("kickoff_posted"); + const fixture = await lifecycleFixture("kickoff_posted", "2026-10-18T00:30:00.000Z"); const revision = "b".repeat(64); await writeRun(fixture.paths, { ...fixture.run, candidate: { compiled_digest: "a".repeat(64), work_id: "work-1", result_ref: "evt-1", revision_sha256: revision, observed_at: "2026-10-18T00:01:00.000Z" }, updated_at: "2026-10-18T00:01:00.000Z", }); + await requestCancel(fixture.paths, fixture.run.run_id); const result = await resumePrometheusRun(fixture.workspace, { env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), docker: lifecycleDocker(fixture.team, []), plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, @@ -309,7 +310,38 @@ describe("Prometheus lifecycle recovery", () => { } } as never, memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai" }), }); - expect(result).toMatchObject({ outcome: "succeeded", candidate: { revision_sha256: revision }, reason: expect.stringContaining("Recovered") }); + expect(result).toMatchObject({ outcome: "succeeded", cancel_requested: true, candidate: { revision_sha256: revision }, reason: expect.stringContaining("Recovered") }); + expect(await readActiveRun(fixture.paths)).toBeNull(); + }); + + it("does not create a Room when cancellation lands during reusable-team bootstrap", async () => { + const fixture = await lifecycleFixture("reserved", "2026-10-18T04:00:00.000Z"); + const calls: string[][] = []; + const baseDocker = lifecycleDocker(fixture.team, []); + let cancelled = false; + const docker: DockerRunner = async (args, options) => { + calls.push([...args]); + if (!cancelled && args[0] === "start") { + cancelled = true; + await requestCancel(fixture.paths, fixture.run.run_id); + await new Promise((resolve) => { + if (options?.signal?.aborted) resolve(); + else options?.signal?.addEventListener("abort", () => resolve(), { once: true }); + }); + expect(options?.signal?.aborted).toBe(true); + } + return baseDocker(args, options); + }; + let connected = false; + + const result = await resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker, plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => { connected = true; throw new Error("must not connect after cancellation"); }, + }); + + expect(result).toMatchObject({ outcome: "cancelled", startup_checkpoint: "reserved" }); + expect(connected).toBe(false); expect(await readActiveRun(fixture.paths)).toBeNull(); }); diff --git a/packages/cli/src/prometheus.ts b/packages/cli/src/prometheus.ts index f1ba9ad..db07a7c 100644 --- a/packages/cli/src/prometheus.ts +++ b/packages/cli/src/prometheus.ts @@ -175,7 +175,11 @@ function newRun(paths: PrometheusPaths, input: StartPrometheusInput, policy: Pro async function saveCheckpoint(paths: PrometheusPaths, run: PrometheusRunState, changes: Partial, now: Date): Promise { const next = { ...run, ...changes, updated_at: timestampAfter(run.updated_at, now) }; await writeRun(paths, next); - return next; + const authoritative = await readActiveRun(paths); + if (!authoritative || authoritative.run_id !== run.run_id) { + throw localError("prometheus_monitor_lease_lost", `Prometheus run ${run.run_id} is no longer active.`); + } + return authoritative; } async function readSavedInputs(run: PrometheusRunState): Promise<{ policy: PrometheusPolicy; goal: string }> { @@ -270,6 +274,75 @@ async function finishFailedRun( return final; } +async function finishRecoveredSuccess( + run: PrometheusRunState, + lease: NonNullable, + dependencies: Pick, +): Promise { + const revision = run.candidate!.revision_sha256; + const finishedAt = timestampAfter(run.updated_at, dependencies.now()); + const final: PrometheusRunState = { + ...run, + monitor: lease, + outcome: "succeeded", + updated_at: finishedAt, + finished_at: finishedAt, + reason: `Recovered committed Prometheus success for ${revision}.`, + }; + await finishRun(dependencies.paths, final, lease); + return final; +} + +async function reloadActive( + runId: string, + dependencies: Pick, +): Promise { + const current = await readActiveRun(dependencies.paths); + if (!current || current.run_id !== runId) { + throw localError("prometheus_monitor_lease_lost", `Prometheus run ${runId} is no longer active.`); + } + return current; +} + +function watchRunStop( + run: PrometheusRunState, + paths: PrometheusPaths, + now: () => Date, +): { signal: AbortSignal; dispose: () => void } { + const controller = new AbortController(); + const remainingMs = Math.max(0, Date.parse(run.deadline_at) - now().getTime()); + const deadline = setTimeout(() => controller.abort(), remainingMs); + deadline.unref?.(); + let reading = false; + const poll = setInterval(() => { + if (reading || controller.signal.aborted) return; + reading = true; + void readActiveRun(paths).then((current) => { + if (!current || current.run_id !== run.run_id || current.cancel_requested) controller.abort(); + }).catch(() => undefined).finally(() => { reading = false; }); + }, 50); + poll.unref?.(); + return { + signal: controller.signal, + dispose: () => { clearTimeout(deadline); clearInterval(poll); }, + }; +} + +async function stopOrRecover( + run: PrometheusRunState, + lease: NonNullable, + dependencies: Pick & { connection?: PrometheusConnection }, +): Promise { + if (!run.cancel_requested && dependencies.now().getTime() < Date.parse(run.deadline_at)) return null; + if (run.room && run.candidate && dependencies.connection) { + const lifecycle = await roomLifecycle(run, { connection: dependencies.connection }); + if (lifecycle.state !== "open" && lifecycle.endedBy?.trigger === "prometheus_success" && lifecycle.endedBy.detail === run.candidate.revision_sha256) { + return finishRecoveredSuccess(run, lease, dependencies); + } + } + return finishStoppedRun(run, lease, dependencies); +} + async function continuePrometheusRun( initial: PrometheusRunState, loaded: { policy: PrometheusPolicy; goal: string }, @@ -282,11 +355,12 @@ async function continuePrometheusRun( if (run.cancel_requested || dependencies.now().getTime() >= Date.parse(run.deadline_at)) { const connection = run.room ? await dependencies.connect() : undefined; - return finishStoppedRun(run, lease, { paths: dependencies.paths, now: dependencies.now, ...(connection ? { connection } : {}) }); + return (await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, ...(connection ? { connection } : {}) }))!; } const existing = savedTeam ?? await readTeam(dependencies.paths); let team: PrometheusTeamState; + const bootstrapStop = watchRunStop(run, dependencies.paths, dependencies.now); try { if (run.startup_checkpoint !== "reserved" && !existing) { throw localError("prometheus_team_missing", "The active Prometheus run has no saved team."); @@ -298,16 +372,32 @@ async function continuePrometheusRun( policySha256: loaded.policy.sha256, policy: loaded.policy.raw, plan: dependencies.plan, + deadlineAt: run.deadline_at, existing, - }, { docker: dependencies.docker, now: dependencies.now, sleep: dependencies.sleep }); + }, { + docker: (args, options) => dependencies.docker(args, { ...options, signal: bootstrapStop.signal }), + now: dependencies.now, + sleep: dependencies.sleep, + }); } catch (error) { + const current = await reloadActive(run.run_id, dependencies); + const stopped = await stopOrRecover(current, lease, { paths: dependencies.paths, now: dependencies.now }); + if (stopped) return stopped; const failure = asCliError(error); if (!["prometheus_team_invalid", "prometheus_team_missing", "prometheus_team_mismatch"].includes(failure.code)) throw error; const connection = run.room ? await dependencies.connect().catch(() => undefined) : undefined; return finishFailedRun(run, lease, failure, { paths: dependencies.paths, now: dependencies.now, ...(connection ? { connection } : {}) }); + } finally { + bootstrapStop.dispose(); } await writeTeam(dependencies.paths, team); + run = await reloadActive(run.run_id, dependencies); + const stoppedAfterBootstrap = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now }); + if (stoppedAfterBootstrap) return stoppedAfterBootstrap; const connection = await dependencies.connect(); + run = await reloadActive(run.run_id, dependencies); + const stoppedAfterConnect = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stoppedAfterConnect) return stoppedAfterConnect; if (run.startup_checkpoint === "reserved") { const created = await connection.client.request<{ room?: { id?: string } }>( @@ -318,6 +408,8 @@ async function continuePrometheusRun( const roomId = created.room?.id; if (!roomId) throw localError("invalid_server_response", "The compiled Prometheus Room id did not come back."); run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "room_created", room: { room_id: roomId, goal_sequence: null, kickoff_sequence: null } }, dependencies.now()); + const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stopped) return stopped; } if (run.startup_checkpoint === "room_created") { @@ -330,6 +422,8 @@ async function continuePrometheusRun( const goalSequence = started.message?.sequence; if (!Number.isSafeInteger(goalSequence) || goalSequence! < 1) throw localError("invalid_server_response", "The Prometheus Goal sequence did not come back."); run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "goal_started", room: { ...run.room!, goal_sequence: goalSequence! } }, dependencies.now()); + const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stopped) return stopped; } if (run.startup_checkpoint === "goal_started") { @@ -338,6 +432,9 @@ async function continuePrometheusRun( { "idempotency-key": `${run.idempotency_keys.room}-invite` }, ); if (!invite.token) throw localError("invalid_server_response", "The Prometheus Room invite did not come back."); + run = await reloadActive(run.run_id, dependencies); + const stoppedAfterInvite = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stoppedAfterInvite) return stoppedAfterInvite; const roster = await attachPrometheusRoom(team, { roomId: run.room!.room_id, inviteToken: invite.token, @@ -345,11 +442,15 @@ async function continuePrometheusRun( idempotencyKeys: run.idempotency_keys.roles, }, { docker: dependencies.docker, now: dependencies.now, sleep: dependencies.sleep }); run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "roles_joined", roster }, dependencies.now()); + const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stopped) return stopped; } if (run.startup_checkpoint === "roles_joined") { await ensurePrometheusListeners(team, run.room!.room_id, run.roster!, connection.baseUrl, { docker: dependencies.docker, now: dependencies.now, sleep: dependencies.sleep }); run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "listeners_ready" }, dependencies.now()); + const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stopped) return stopped; } if (run.startup_checkpoint === "listeners_ready") { @@ -447,43 +548,53 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies const feedbackAttempted = new Set(); try { for (;;) { - const current = (await readActiveRun(dependencies.paths)) ?? run; + const current = await reloadActive(run.run_id, dependencies); const now = dependencies.now(); if (current.cancel_requested || now.getTime() >= Date.parse(current.deadline_at)) { - return finishStoppedRun(current, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + return (await stopOrRecover(current, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }))!; } const lifecycle = await roomLifecycle(current, dependencies); if (lifecycle.state !== "open") { if (current.candidate && lifecycle.endedBy?.trigger === "prometheus_success" && lifecycle.endedBy.detail === current.candidate.revision_sha256) { - const finishedAt = timestampAfter(current.updated_at, dependencies.now()); - const final: PrometheusRunState = { - ...current, - monitor: lease, - outcome: "succeeded", - updated_at: finishedAt, - finished_at: finishedAt, - reason: `Recovered committed Prometheus success for ${current.candidate.revision_sha256}.`, - }; - await finishRun(dependencies.paths, final, lease); - return final; + return finishRecoveredSuccess(current, lease, dependencies); } throw localError("prometheus_room_closed", "The Prometheus Room closed before the completion barrier passed."); } - const observed = await snapshot(current, dependencies); + const afterLifecycle = await reloadActive(current.run_id, dependencies); + const stoppedAfterLifecycle = await stopOrRecover(afterLifecycle, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + if (stoppedAfterLifecycle) return stoppedAfterLifecycle; + const observed = await snapshot(afterLifecycle, dependencies); + const afterSnapshot = await reloadActive(current.run_id, dependencies); + const stoppedAfterSnapshot = await stopOrRecover(afterSnapshot, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + if (stoppedAfterSnapshot) return stoppedAfterSnapshot; if (observed.barrier.ready) { const revision = observed.barrier.candidate.revisionSha256; - const prior = current.checks.find((entry) => entry.revision_sha256 === revision); + const prior = afterSnapshot.checks.find((entry) => entry.revision_sha256 === revision); let passingCheck = prior?.exit_code === 0; if (!prior) { - const checked = await runPrometheusCheck({ image: dependencies.plan.image, workspace: current.workspace_path }, current.configuration.check_command, { - SHAREDNET_PROMETHEUS_RUN_ID: current.run_id, - SHAREDNET_PROMETHEUS_WORK_ID: observed.barrier.candidate.workId, - SHAREDNET_PROMETHEUS_RESULT_REF: observed.barrier.candidate.resultRef, - SHAREDNET_PROMETHEUS_REVISION_SHA256: revision, - }, { docker: dependencies.docker, now: dependencies.now }); + const checkStop = watchRunStop(afterSnapshot, dependencies.paths, dependencies.now); + let checked: Awaited>; + try { + checked = await runPrometheusCheck({ image: dependencies.plan.image, workspace: current.workspace_path }, current.configuration.check_command, { + SHAREDNET_PROMETHEUS_RUN_ID: current.run_id, + SHAREDNET_PROMETHEUS_WORK_ID: observed.barrier.candidate.workId, + SHAREDNET_PROMETHEUS_RESULT_REF: observed.barrier.candidate.resultRef, + SHAREDNET_PROMETHEUS_REVISION_SHA256: revision, + }, { + docker: dependencies.docker, + now: dependencies.now, + timeoutMs: Date.parse(afterSnapshot.deadline_at) - dependencies.now().getTime(), + signal: checkStop.signal, + }); + } finally { + checkStop.dispose(); + } + const afterCheck = await reloadActive(current.run_id, dependencies); + const stoppedAfterCheck = await stopOrRecover(afterCheck, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + if (stoppedAfterCheck) return stoppedAfterCheck; if (checked.exitCode === 125) throw localError("prometheus_check_launch_failed", "The Prometheus semantic-check container could not be launched."); const summary = { revision_sha256: revision, started_at: checked.startedAt, finished_at: checked.finishedAt, exit_code: Math.max(0, checked.exitCode), output: checked.output, output_sha256: checked.outputSha256 }; - run = await saveCheckpoint(dependencies.paths, current, { monitor: lease, checks: [...current.checks, summary] }, dependencies.now()); + run = await saveCheckpoint(dependencies.paths, afterCheck, { monitor: lease, checks: [...afterCheck.checks, summary] }, dependencies.now()); if (checked.exitCode !== 0) { const feedback = `@${current.roster!.coordinator.member_id} Semantic check failed for revision ${revision}:\n${checked.output.slice(-2_000)}`; await dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/owner-messages`, dependencies.connection.apiKey, { content: feedback, as: "runner" }, { "idempotency-key": `${current.run_id}-check-${revision}` }); @@ -498,13 +609,28 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies } if (passingCheck) { const after = await snapshot(run, dependencies); + const beforeSuccess = await reloadActive(current.run_id, dependencies); + const stoppedBeforeSuccess = await stopOrRecover(beforeSuccess, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + if (stoppedBeforeSuccess) return stoppedBeforeSuccess; if (sameReady(observed.barrier, after.barrier)) { - const observedAt = timestampAfter(run.updated_at, dependencies.now()); - run = await saveCheckpoint(dependencies.paths, run, { + const observedAt = timestampAfter(beforeSuccess.updated_at, dependencies.now()); + run = await saveCheckpoint(dependencies.paths, beforeSuccess, { monitor: lease, candidate: { compiled_digest: observed.barrier.candidate.stateDigest, work_id: observed.barrier.candidate.workId, result_ref: observed.barrier.candidate.resultRef, revision_sha256: revision, observed_at: observedAt }, }, dependencies.now()); - await dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/close`, dependencies.connection.apiKey, { trigger: "prometheus_success", detail: revision }, { "idempotency-key": `${current.run_id}-close-success` }); + run = await reloadActive(run.run_id, dependencies); + const stoppedImmediatelyBeforeSuccess = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + if (stoppedImmediatelyBeforeSuccess) return stoppedImmediatelyBeforeSuccess; + const closed = await dependencies.connection.client.request<{ room?: { state?: unknown; goal?: { ended_by?: { trigger?: unknown; detail?: unknown } | null } | null } }>( + "POST", + `/rooms/${encodeURIComponent(current.room!.room_id)}/close`, + dependencies.connection.apiKey, + { trigger: "prometheus_success", detail: revision }, + { "idempotency-key": `${current.run_id}-close-success` }, + ); + if (closed.room?.state !== "closed" || closed.room.goal?.ended_by?.trigger !== "prometheus_success" || closed.room.goal.ended_by.detail !== revision) { + throw localError("prometheus_success_unconfirmed", "SharedNet did not confirm the exact Prometheus success close."); + } const finishedAt = timestampAfter(run.updated_at, dependencies.now()); const final: PrometheusRunState = { ...run, monitor: lease, outcome: "succeeded", updated_at: finishedAt, finished_at: finishedAt, reason: `Prometheus completion barrier passed for ${revision}.` }; await finishRun(dependencies.paths, final, lease); @@ -517,10 +643,13 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies } catch (error) { const failure = asCliError(error); if (failure.code === "prometheus_monitor_active" || failure.code === "prometheus_monitor_lease_lost") throw error; + // The close may have committed even when its response was lost or malformed. Keep the + // candidate durable so resume can reconcile authoritative Room ended_by before any stop. + if (failure.code === "prometheus_success_unconfirmed") throw error; // Transport and service failures do not prove the Room or run is invalid. // Keep the durable run active so a later `prometheus resume` can reclaim it. if (failure.exitCode === 5) throw failure; - const current = (await readActiveRun(dependencies.paths)) ?? run; + const current = await reloadActive(run.run_id, dependencies); return finishFailedRun(current, lease, failure, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); } } @@ -584,7 +713,7 @@ export async function cancelPrometheusRun(workspace: string, dependencies: Prome const current = await readActiveRun(paths); if (!current) return { cancelled: true, active: false, recovered: false, run_id: active.run_id }; const connection = current.room ? await dependencies.connect() : undefined; - const final = await finishStoppedRun(current, lease, { paths, now: dependencies.now, ...(connection ? { connection } : {}) }); + const final = (await stopOrRecover(current, lease, { paths, now: dependencies.now, ...(connection ? { connection } : {}) }))!; return { cancelled: true, active: false, recovered: true, run_id: active.run_id, outcome: final.outcome }; } catch (error) { const failure = asCliError(error); From bf5da043b507f0dbefc43c001d85f2dafee45940 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 19 Oct 2026 02:47:34 +0000 Subject: [PATCH 5/9] fix(cli): bind Prometheus checks to reviewed artifacts Announce runner-derived candidate revisions, pin and verify the exact submission archive, and run semantic checks only against a sandboxed extracted snapshot. Persist the snapshot digest so resumed checks cannot drift back to the mutable workspace. Co-authored-by: OpenAI Codex --- .../2026-10-18-prometheus-local-design.md | 19 ++- docs/prometheus-local.md | 24 +++- packages/cli/src/prometheus-protocol.test.ts | 38 +++--- packages/cli/src/prometheus-protocol.ts | 6 + packages/cli/src/prometheus-runtime.test.ts | 92 +++++++++++++- packages/cli/src/prometheus-runtime.ts | 119 +++++++++++++++++- packages/cli/src/prometheus-state.test.ts | 2 + packages/cli/src/prometheus-state.ts | 4 +- packages/cli/src/prometheus.test.ts | 12 +- packages/cli/src/prometheus.ts | 63 ++++++++-- 10 files changed, 336 insertions(+), 43 deletions(-) diff --git a/docs/plans/2026-10-18-prometheus-local-design.md b/docs/plans/2026-10-18-prometheus-local-design.md index 1501797..59cf2ff 100644 --- a/docs/plans/2026-10-18-prometheus-local-design.md +++ b/docs/plans/2026-10-18-prometheus-local-design.md @@ -85,7 +85,7 @@ Every workspace has exactly these roles: | Role | Stable seat name | Responsibility | | --- | --- | --- | | Coordinator | `coordinator` | Decompose the goal, issue typed work, route exact handoffs, resolve the final work, and synthesize the result. | -| Builder | `builder` | Accept the typed work, change the shared workspace, upload the final `submission` artifact, and report its result reference and digest. | +| Builder | `builder` | Accept the typed work, change the shared workspace, upload exactly one workspace-rooted tar artifact named `submission`, and report its result reference and digest. | | Reviewer | `reviewer` | Independently download and inspect the exact submitted artifact, upload review evidence, and issue `verification.pass` for the current result. | All three roles use Codex in v1. One model is shared by the team. The number of @@ -221,19 +221,28 @@ all of these conditions in order: 6. A post-check read returns the same compiled digest and candidate revision, the same three receipts, and the same idle/no-handoff state. -The semantic check runs only after conditions 1 through 4 hold. It runs in the -fresh checker container inherited from PR 48, with the workspace mounted at -`/workspace` and these non-secret environment variables: +Local v1 accepts exactly one deliverable named `submission`, encoded as a tar +archive rooted at the workspace to check. The runner downloads that exact +artifact and verifies its bytes against the compiled artifact receipt. It announces the candidate +revision and the deterministic artifact-snapshot digest so each role has an +actionable receipt value. The semantic check runs only after conditions 1 +through 4 hold. It runs in a fresh, network-disabled checker container with a +read-only root: the tar is mounted read-only and extracted into disposable +`/workspace` tmpfs, while a separate writable `/tmp` tmpfs supports ordinary +test and build tools. The snapshot manifest remains in the read-only +`/prometheus-input` mount. The mutable shared workspace is never mounted. The +checker receives these non-secret environment variables: ```text SHAREDNET_PROMETHEUS_RUN_ID SHAREDNET_PROMETHEUS_WORK_ID SHAREDNET_PROMETHEUS_RESULT_REF SHAREDNET_PROMETHEUS_REVISION_SHA256 +SHAREDNET_PROMETHEUS_ARTIFACT_SNAPSHOT_SHA256 ``` The run record stores its start and finish times, exit code, bounded output, -output SHA-256, and candidate revision. A nonzero check blocks that exact +output SHA-256, candidate revision, and artifact-snapshot SHA-256. A nonzero check blocks that exact revision and produces one bounded, exact-`@coordinator` feedback message. It is not retried until compiled work produces a different revision. A passing check is discarded if the post-check read sees a different revision or any new diff --git a/docs/prometheus-local.md b/docs/prometheus-local.md index ff859c3..08f18b5 100644 --- a/docs/prometheus-local.md +++ b/docs/prometheus-local.md @@ -103,18 +103,38 @@ Timeout and cancellation do not delete the stable team containers or volumes. ## Evidence and completion -The coordinator assigns typed compiled work to the builder. The builder -publishes a result artifact. The reviewer must independently download and +The coordinator assigns typed compiled work to the builder. Local v1 requires +exactly one deliverable named `submission`; it must be a tar archive whose root +is the workspace the semantic check should inspect. The builder publishes that +artifact. The reviewer must independently download and inspect that exact artifact, publish separate review evidence, and issue a `verification.pass` bound to the current result reference. A valid candidate has no open work or unmet verification requirement. +Once that candidate exists, the runner downloads every declared deliverable, +verifies its bytes against the compiled artifact receipt, and announces the +exact candidate revision and artifact-snapshot digest to all three roles. The +roles copy that runner-announced revision into their terminal receipts; they do +not have to reproduce the runner's canonical hash calculation. + Plain chat remains nonbinding. In particular, `DONE` has no effect. Completion also requires one revision-bound terminal receipt from every role, all three listeners idle and healthy, no unsettled handoff, a passing semantic `--check`, and an unchanged second read of the compiled candidate after the check. Any new compiled revision makes older receipts stale. +The semantic check does not mount the mutable shared workspace. It mounts the +downloaded snapshot directory read-only at `/prometheus-input`, extracts its +`submission.tar` into a disposable `/workspace` tmpfs, and supplies a separate +writable `/tmp` tmpfs for ordinary test and build tools. The checker otherwise +runs with a read-only container root, no network, no added capabilities, and +`no-new-privileges`. The pinned manifest stays under `/prometheus-input`; it +maps the deliverable and artifact id to its SHA-256 and records +`SHAREDNET_PROMETHEUS_ARTIFACT_SNAPSHOT_SHA256`. Source, fixtures, and a +relative checker such as `python3 check.py` therefore come from the reviewed +tar. Local v1 verifies the saved bytes again on resume before reusing a cached +check result. + ## Local state and credentials Metadata is keyed by the workspace's real absolute path and stored under: diff --git a/packages/cli/src/prometheus-protocol.test.ts b/packages/cli/src/prometheus-protocol.test.ts index 8cc4fcb..fc73262 100644 --- a/packages/cli/src/prometheus-protocol.test.ts +++ b/packages/cli/src/prometheus-protocol.test.ts @@ -290,6 +290,25 @@ describe("Prometheus candidate revisions", () => { }); }); + it.each([ + ["renamed", ["bundle"], { bundle: SUBMISSION }], + ["multiple", ["submission", "report"], { submission: SUBMISSION, report: "art_Report0001" }], + ])("rejects a %s local-v1 deliverable contract", (_case, deliverables, artifacts) => { + const state = candidateFixture(); + state.projection.work.W1!.deliverables = deliverables; + state.projection.work.W1!.artifacts = artifacts; + state.events[4]!.refs.artifacts = artifacts; + if (deliverables.length > 1) { + (state.events[4]!.refs.artifact_receipts as Record)["art_Report0001"] = { + id: "art_Report0001", + sha256: "c".repeat(64), + }; + } + const parsed = parsePrometheusCandidate(state, ROSTER); + expect(parsed.ok).toBe(false); + if (!parsed.ok) expect(parsed.reasons).toContain("invalid-submission-artifact"); + }); + it.each([ ["nonterminal-work", (state: TestCompiledState) => { state.projection.work.W0!.state = "OPEN"; }], ["ambiguous-head", (state: TestCompiledState) => { @@ -324,22 +343,13 @@ describe("Prometheus candidate revisions", () => { if (!parsed.ok) expect(parsed.reasons).toContain(reason); }); - it("sorts work artifacts and verification evidence before hashing the revision", () => { + it("sorts verification evidence before hashing the revision", () => { const first = candidateFixture(); const head = first.projection.work.W1!; const result = first.events[4]!; - const report = "art_Report0001"; - const reportSha256 = "c".repeat(64); const secondRequirement = "verify:secondary"; const secondEvidence = "art_Proof00001"; const secondEvidenceSha256 = "e".repeat(64); - head.deliverables = ["report", "submission"]; - head.artifacts = { report, submission: SUBMISSION }; - result.refs.artifacts = { report, submission: SUBMISSION }; - result.refs.artifact_receipts = { - [report]: { id: report, sha256: reportSha256 }, - [SUBMISSION]: { id: SUBMISSION, sha256: SUBMISSION_SHA256 }, - }; head.requirements = { [secondRequirement]: { kind: "verify", eligible: [REVIEWER], q: 1, predicate: null, @@ -361,16 +371,10 @@ describe("Prometheus candidate revisions", () => { const second = structuredClone(first); const secondHead = second.projection.work.W1!; - secondHead.artifacts = { submission: SUBMISSION, report }; secondHead.requirements = { [REQUIREMENT]: secondHead.requirements[REQUIREMENT]!, [secondRequirement]: secondHead.requirements[secondRequirement]!, }; - second.events[4]!.refs.artifacts = { submission: SUBMISSION, report }; - second.events[4]!.refs.artifact_receipts = { - [SUBMISSION]: { id: SUBMISSION, sha256: SUBMISSION_SHA256 }, - [report]: { id: report, sha256: reportSha256 }, - }; const one = parsePrometheusCandidate(first, ROSTER); const two = parsePrometheusCandidate(second, ROSTER); @@ -378,7 +382,7 @@ describe("Prometheus candidate revisions", () => { expect(one.ok).toBe(true); expect(two.ok).toBe(true); if (one.ok && two.ok) { - expect(one.candidate.artifacts.map((artifact) => artifact.name)).toEqual(["report", "submission"]); + expect(one.candidate.artifacts.map((artifact) => artifact.name)).toEqual(["submission"]); expect(one.candidate.verifications.map((verification) => verification.eventHash)).toEqual([ VERIFICATION_HASH, `sha256:${"7".repeat(24)}`, diff --git a/packages/cli/src/prometheus-protocol.ts b/packages/cli/src/prometheus-protocol.ts index d08ad38..4d4e3c0 100644 --- a/packages/cli/src/prometheus-protocol.ts +++ b/packages/cli/src/prometheus-protocol.ts @@ -354,6 +354,7 @@ const CANDIDATE_REASON_ORDER = [ "ambiguous-head", "rejected-head", "missing-result-event", + "invalid-submission-artifact", "missing-artifact-receipt", "bad-artifact-sha256", "stale-result-ref", @@ -464,6 +465,11 @@ export function parsePrometheusCandidate(state: CompiledState, roster: Prometheu const resultReceipts = resultEvent.refs.artifact_receipts; const artifacts: CandidateArtifact[] = []; const resultArtifactIds = new Set(); + if (head.deliverables.length !== 1 || head.deliverables[0] !== "submission" || + !validNamedArtifacts(resultArtifacts) || Object.keys(resultArtifacts).length !== 1 || + !Object.hasOwn(resultArtifacts, "submission")) { + reasons.add("invalid-submission-artifact"); + } if (!validNamedArtifacts(resultArtifacts) || !validReceipts(resultReceipts) || head.deliverables.length === 0 || !head.deliverables.every((name) => Object.hasOwn(resultArtifacts, name)) || !sameRecord(head.artifacts, resultArtifacts)) { diff --git a/packages/cli/src/prometheus-runtime.test.ts b/packages/cli/src/prometheus-runtime.test.ts index 027bffb..168000f 100644 --- a/packages/cli/src/prometheus-runtime.test.ts +++ b/packages/cli/src/prometheus-runtime.test.ts @@ -1,9 +1,14 @@ // @vitest-environment node +import { createHash } from "node:crypto"; +import { mkdtemp, readFile, rm, stat } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + import { describe, expect, it } from "vitest"; import { DEFAULT_IMAGE, type DockerRunner, type RunPlan } from "./goal-run.ts"; -import { ensurePrometheusListeners, ensurePrometheusTeam, inspectPrometheusRuntime, runPrometheusCheck } from "./prometheus-runtime.ts"; +import { ensurePrometheusListeners, ensurePrometheusTeam, inspectPrometheusRuntime, materializePrometheusArtifactSnapshot, runPrometheusCheck } from "./prometheus-runtime.ts"; import type { PrometheusTeamState } from "./prometheus-state.ts"; const plan: RunPlan = { @@ -186,13 +191,14 @@ describe("persistent Prometheus runtime", () => { return { code: 7, stdout: "x".repeat(70_000), stderr: "tail" }; }; const result = await runPrometheusCheck( - { image: DEFAULT_IMAGE, workspace: plan.workspace }, + { image: DEFAULT_IMAGE, snapshot: "/state/pinned-artifacts", snapshotSha256: "c".repeat(64) }, "pnpm test", { SHAREDNET_PROMETHEUS_RUN_ID: "run-1", SHAREDNET_PROMETHEUS_WORK_ID: "work-1", SHAREDNET_PROMETHEUS_RESULT_REF: "evt-1", SHAREDNET_PROMETHEUS_REVISION_SHA256: "b".repeat(64), + SHAREDNET_PROMETHEUS_ARTIFACT_SNAPSHOT_SHA256: "c".repeat(64), }, { docker, now: () => new Date("2026-10-18T00:00:00.000Z") }, ); @@ -201,7 +207,14 @@ describe("persistent Prometheus runtime", () => { SHAREDNET_PROMETHEUS_WORK_ID: "work-1", SHAREDNET_PROMETHEUS_RESULT_REF: "evt-1", SHAREDNET_PROMETHEUS_REVISION_SHA256: "b".repeat(64), + SHAREDNET_PROMETHEUS_ARTIFACT_SNAPSHOT_SHA256: "c".repeat(64), }); + expect(seen[0]!.args).toContain("/state/pinned-artifacts:/prometheus-input:ro"); + expect(seen[0]!.args).toEqual(expect.arrayContaining(["--read-only", "--network", "none", "--cap-drop", "ALL", "--tmpfs"])); + expect(seen[0]!.args).toContain("/workspace:rw,nosuid,nodev,size=1g"); + expect(seen[0]!.args).toContain("/tmp:rw,nosuid,nodev,size=256m"); + expect(seen[0]!.args.join(" ")).toContain("tar -xf /prometheus-input/submission.tar"); + expect(seen[0]!.args.join(" ")).not.toContain(`${plan.workspace}:`); expect(seen[0]!.args.join(" ")).not.toContain("run-1"); expect(seen[0]!.args).toContain("timeout"); expect(seen[0]!.args).toContain("120.000s"); @@ -218,13 +231,14 @@ describe("persistent Prometheus runtime", () => { options?.signal?.addEventListener("abort", () => resolve({ code: 143, stdout: "", stderr: "terminated" }), { once: true }); }); const pending = runPrometheusCheck( - { image: DEFAULT_IMAGE, workspace: plan.workspace }, + { image: DEFAULT_IMAGE, snapshot: "/state/pinned-artifacts", snapshotSha256: "c".repeat(64) }, "pnpm test", { SHAREDNET_PROMETHEUS_RUN_ID: "run-1", SHAREDNET_PROMETHEUS_WORK_ID: "work-1", SHAREDNET_PROMETHEUS_RESULT_REF: "evt-1", SHAREDNET_PROMETHEUS_REVISION_SHA256: "b".repeat(64), + SHAREDNET_PROMETHEUS_ARTIFACT_SNAPSHOT_SHA256: "c".repeat(64), }, { docker, timeoutMs: 30_000, signal: controller.signal }, ); @@ -235,6 +249,78 @@ describe("persistent Prometheus runtime", () => { expect(sawSignal).toBe(true); }); + it("refuses to launch a check when its announced snapshot digest differs from the mounted snapshot", async () => { + let launches = 0; + await expect(runPrometheusCheck( + { image: DEFAULT_IMAGE, snapshot: "/state/pinned-artifacts", snapshotSha256: "c".repeat(64) }, + "pnpm test", + { + SHAREDNET_PROMETHEUS_RUN_ID: "run-1", + SHAREDNET_PROMETHEUS_WORK_ID: "work-1", + SHAREDNET_PROMETHEUS_RESULT_REF: "evt-1", + SHAREDNET_PROMETHEUS_REVISION_SHA256: "b".repeat(64), + SHAREDNET_PROMETHEUS_ARTIFACT_SNAPSHOT_SHA256: "d".repeat(64), + }, + { docker: async () => { launches += 1; return { code: 0, stdout: "", stderr: "" }; } }, + )).rejects.toMatchObject({ code: "prometheus_artifact_snapshot_invalid" }); + expect(launches).toBe(0); + }); + + it("materializes and re-verifies an immutable snapshot of the exact reviewed artifact bytes", async () => { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-artifacts-")); + const snapshot = join(root, "snapshot"); + const bytes = new TextEncoder().encode("reviewed artifact\n"); + const sha256 = createHash("sha256").update(bytes).digest("hex"); + let downloads = 0; + const client = { + requestBytes: async (path: string) => { + downloads += 1; + expect(path).toBe("/artifacts/art_Submit0001/content"); + return { bytes, filename: "submission.tar", sha256 }; + }, + }; + try { + const first = await materializePrometheusArtifactSnapshot( + [{ name: "submission", artifactId: "art_Submit0001", sha256 }], + client as never, + "member", + snapshot, + ); + const second = await materializePrometheusArtifactSnapshot( + [{ name: "submission", artifactId: "art_Submit0001", sha256 }], + client as never, + "member", + snapshot, + ); + + expect(downloads).toBe(1); + expect(second).toEqual(first); + expect(await readFile(join(snapshot, "submission.tar"), "utf8")).toBe("reviewed artifact\n"); + expect((await stat(snapshot)).mode & 0o777).toBe(0o555); + expect((await stat(join(snapshot, "submission.tar"))).mode & 0o777).toBe(0o444); + expect(first.sha256).toMatch(/^[0-9a-f]{64}$/); + } finally { + await import("node:fs/promises").then(({ chmod }) => chmod(snapshot, 0o700)).catch(() => undefined); + await rm(root, { recursive: true, force: true }); + } + }); + + it("rejects artifact content that does not match the compiled receipt", async () => { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-artifacts-bad-")); + const snapshot = join(root, "snapshot"); + try { + await expect(materializePrometheusArtifactSnapshot( + [{ name: "submission", artifactId: "art_Submit0001", sha256: "a".repeat(64) }], + { requestBytes: async () => ({ bytes: new TextEncoder().encode("changed"), filename: null, sha256: null }) } as never, + "member", + snapshot, + )).rejects.toMatchObject({ code: "prometheus_artifact_changed" }); + } finally { + await import("node:fs/promises").then(({ chmod }) => chmod(snapshot, 0o700)).catch(() => undefined); + await rm(root, { recursive: true, force: true }); + } + }); + it("restarts unfiltered listeners for saved seats without joining again", async () => { const calls: string[][] = []; const team: PrometheusTeamState = { diff --git a/packages/cli/src/prometheus-runtime.ts b/packages/cli/src/prometheus-runtime.ts index 88b0519..125292f 100644 --- a/packages/cli/src/prometheus-runtime.ts +++ b/packages/cli/src/prometheus-runtime.ts @@ -1,8 +1,11 @@ -import { createHash } from "node:crypto"; +import { createHash, randomUUID } from "node:crypto"; +import { chmod, mkdir, readFile, rename, rm, writeFile } from "node:fs/promises"; +import { dirname } from "node:path"; +import type { ApiClient } from "./api-client.ts"; import { localError } from "./errors.ts"; import { type DockerRunner, type RunPlan } from "./goal-run.ts"; -import { PROMETHEUS_ROLES, type PrometheusRole } from "./prometheus-protocol.ts"; +import { PROMETHEUS_ROLES, type CandidateArtifact, type PrometheusRole } from "./prometheus-protocol.ts"; import type { PrometheusRoster, PrometheusTeamState } from "./prometheus-state.ts"; const CONTAINER_WORKSPACE = "/workspace"; @@ -12,6 +15,7 @@ const CONTAINER_AUTH = "/run/codex/auth.json"; const THREAD_MARKER = `${CONTAINER_HOME}/.prometheus-thread-id`; const MAX_OUTPUT_BYTES = 64_000; const PROMETHEUS_LABEL = "io.sharednet.prometheus"; +const ARTIFACT_MANIFEST = ".sharednet-artifacts.json"; export interface PrometheusRuntimeDependencies { docker: DockerRunner; @@ -406,25 +410,128 @@ function bounded(raw: string): string { return bytes.byteLength <= MAX_OUTPUT_BYTES ? raw : bytes.subarray(bytes.byteLength - MAX_OUTPUT_BYTES).toString("utf8"); } +interface ArtifactSnapshotManifest { + schema_version: 1; + artifacts: Array<{ name: string; artifact_id: string; sha256: string; file: string }>; + snapshot_sha256: string; +} + +export interface PrometheusArtifactSnapshot { + path: string; + sha256: string; + artifacts: ArtifactSnapshotManifest["artifacts"]; +} + +function snapshotDigest(artifacts: ArtifactSnapshotManifest["artifacts"]): string { + return createHash("sha256").update(JSON.stringify({ schema_version: 1, artifacts }), "utf8").digest("hex"); +} + +async function verifyArtifactSnapshot(target: string, expected: readonly CandidateArtifact[]): Promise { + let parsed: ArtifactSnapshotManifest; + try { + parsed = JSON.parse(await readFile(`${target}/${ARTIFACT_MANIFEST}`, "utf8")) as ArtifactSnapshotManifest; + } catch { + throw localError("prometheus_artifact_snapshot_invalid", "The saved Prometheus artifact snapshot is missing or invalid."); + } + if (parsed.schema_version !== 1 || !Array.isArray(parsed.artifacts) || !/^[0-9a-f]{64}$/.test(parsed.snapshot_sha256)) { + throw localError("prometheus_artifact_snapshot_invalid", "The saved Prometheus artifact snapshot is invalid."); + } + const wanted = expected.map((artifact) => ({ name: artifact.name, artifact_id: artifact.artifactId, sha256: artifact.sha256 })); + const actual = parsed.artifacts.map(({ name, artifact_id, sha256 }) => ({ name, artifact_id, sha256 })); + if (JSON.stringify(actual) !== JSON.stringify(wanted) || snapshotDigest(parsed.artifacts) !== parsed.snapshot_sha256) { + throw localError("prometheus_artifact_snapshot_invalid", "The saved Prometheus artifact snapshot does not match the current reviewed artifacts."); + } + for (const artifact of parsed.artifacts) { + if (artifact.file !== "submission.tar") { + throw localError("prometheus_artifact_snapshot_invalid", "The saved Prometheus artifact snapshot contains an unsafe filename."); + } + const bytes = await readFile(`${target}/${artifact.file}`).catch(() => null); + if (bytes === null || createHash("sha256").update(bytes).digest("hex") !== artifact.sha256) { + throw localError("prometheus_artifact_snapshot_invalid", "The saved Prometheus artifact bytes no longer match their reviewed digest."); + } + } + return { path: target, sha256: parsed.snapshot_sha256, artifacts: parsed.artifacts }; +} + +/** Download the service-attested deliverables once and expose only those immutable bytes to the semantic checker. */ +export async function materializePrometheusArtifactSnapshot( + artifacts: readonly CandidateArtifact[], + client: Pick, + memberToken: string, + target: string, +): Promise { + if (artifacts.length !== 1 || artifacts[0]?.name !== "submission") { + throw localError("prometheus_artifact_snapshot_invalid", "Prometheus local v1 requires exactly one submission tar artifact."); + } + try { + return await verifyArtifactSnapshot(target, artifacts); + } catch (error) { + if ((error as { code?: string }).code !== "prometheus_artifact_snapshot_invalid") throw error; + const exists = await readFile(`${target}/${ARTIFACT_MANIFEST}`).then(() => true, () => false); + if (exists) throw error; + } + + await mkdir(dirname(target), { recursive: true, mode: 0o700 }); + const temporary = `${target}.tmp-${randomUUID()}`; + await mkdir(temporary, { mode: 0o700 }); + try { + const manifestArtifacts: ArtifactSnapshotManifest["artifacts"] = []; + for (const artifact of artifacts) { + const downloaded = await client.requestBytes(`/artifacts/${encodeURIComponent(artifact.artifactId)}/content`, memberToken); + const digest = createHash("sha256").update(downloaded.bytes).digest("hex"); + if (digest !== artifact.sha256 || (downloaded.sha256 !== null && downloaded.sha256 !== artifact.sha256)) { + throw localError("prometheus_artifact_changed", `Artifact ${artifact.artifactId} does not match its compiled receipt.`); + } + const file = "submission.tar"; + await writeFile(`${temporary}/${file}`, downloaded.bytes, { flag: "wx", mode: 0o444 }); + manifestArtifacts.push({ name: artifact.name, artifact_id: artifact.artifactId, sha256: artifact.sha256, file }); + } + const sha256 = snapshotDigest(manifestArtifacts); + const manifest: ArtifactSnapshotManifest = { schema_version: 1, artifacts: manifestArtifacts, snapshot_sha256: sha256 }; + await writeFile(`${temporary}/${ARTIFACT_MANIFEST}`, `${JSON.stringify(manifest, null, 2)}\n`, { flag: "wx", mode: 0o444 }); + await chmod(temporary, 0o555); + try { + await rename(temporary, target); + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST" && (error as NodeJS.ErrnoException).code !== "ENOTEMPTY") throw error; + await chmod(temporary, 0o700); + await rm(temporary, { recursive: true, force: true }); + } + return verifyArtifactSnapshot(target, artifacts); + } catch (error) { + await chmod(temporary, 0o700).catch(() => undefined); + await rm(temporary, { recursive: true, force: true }); + throw error; + } +} + export async function runPrometheusCheck( - runtime: { image: string; workspace: string }, + runtime: { image: string; snapshot: string; snapshotSha256: string }, check: string, - environment: Record<"SHAREDNET_PROMETHEUS_RUN_ID" | "SHAREDNET_PROMETHEUS_WORK_ID" | "SHAREDNET_PROMETHEUS_RESULT_REF" | "SHAREDNET_PROMETHEUS_REVISION_SHA256", string>, + environment: Record<"SHAREDNET_PROMETHEUS_RUN_ID" | "SHAREDNET_PROMETHEUS_WORK_ID" | "SHAREDNET_PROMETHEUS_RESULT_REF" | "SHAREDNET_PROMETHEUS_REVISION_SHA256" | "SHAREDNET_PROMETHEUS_ARTIFACT_SNAPSHOT_SHA256", string>, dependencies: Pick & { timeoutMs?: number; signal?: AbortSignal }, ): Promise { const timeoutMs = dependencies.timeoutMs ?? 120_000; if (!Number.isFinite(timeoutMs) || timeoutMs <= 0) { throw localError("prometheus_deadline_reached", "The Prometheus deadline passed before the semantic check could start."); } + if (runtime.snapshotSha256 !== environment.SHAREDNET_PROMETHEUS_ARTIFACT_SNAPSHOT_SHA256) { + throw localError("prometheus_artifact_snapshot_invalid", "The semantic check metadata does not match the mounted artifact snapshot."); + } const now = dependencies.now ?? (() => new Date()); const startedAt = now().toISOString(); const result = await dependencies.docker([ - "run", "--rm", "--volume", `${runtime.workspace}:${CONTAINER_WORKSPACE}`, + "run", "--rm", "--read-only", "--network", "none", "--cap-drop", "ALL", + "--security-opt", "no-new-privileges", "--volume", `${runtime.snapshot}:/prometheus-input:ro`, + "--tmpfs", `${CONTAINER_WORKSPACE}:rw,nosuid,nodev,size=1g`, + "--tmpfs", "/tmp:rw,nosuid,nodev,size=256m", "--workdir", CONTAINER_WORKSPACE, ...Object.keys(environment).flatMap((name) => ["-e", name]), runtime.image, "timeout", "--signal=TERM", "--kill-after=5s", `${(timeoutMs / 1_000).toFixed(3)}s`, - "sh", "-c", check, + "sh", "-c", + "set -eu; tar -xf /prometheus-input/submission.tar -C /workspace --no-same-owner --no-same-permissions; exec sh -c \"$1\"", + "prometheus-check", check, ], { env: environment, signal: dependencies.signal }); const output = bounded(`${result.stdout}${result.stderr}`); return { diff --git a/packages/cli/src/prometheus-state.test.ts b/packages/cli/src/prometheus-state.test.ts index 9ad2da0..88e2f2c 100644 --- a/packages/cli/src/prometheus-state.test.ts +++ b/packages/cli/src/prometheus-state.test.ts @@ -370,6 +370,7 @@ describe("Prometheus owner-only state", () => { checks: [ { revision_sha256: SHA_A, + artifact_snapshot_sha256: SHA_B, started_at: CREATED_AT, finished_at: UPDATED_AT, exit_code: 1, @@ -393,6 +394,7 @@ describe("Prometheus owner-only state", () => { checks: [ { revision_sha256: SHA_A, + artifact_snapshot_sha256: SHA_B, started_at: CREATED_AT, finished_at: UPDATED_AT, exit_code: 1, diff --git a/packages/cli/src/prometheus-state.ts b/packages/cli/src/prometheus-state.ts index bce941c..2434676 100644 --- a/packages/cli/src/prometheus-state.ts +++ b/packages/cli/src/prometheus-state.ts @@ -128,6 +128,7 @@ export interface PrometheusCandidateSummary { export interface PrometheusCheckSummary { revision_sha256: string; + artifact_snapshot_sha256: string; started_at: string; finished_at: string; exit_code: number; @@ -398,7 +399,7 @@ function parseCandidate(value: unknown, source: string): PrometheusCandidateSumm function parseCheck(value: unknown, source: string): PrometheusCheckSummary { const object = record( value, - ["revision_sha256", "started_at", "finished_at", "exit_code", "output", "output_sha256"], + ["revision_sha256", "artifact_snapshot_sha256", "started_at", "finished_at", "exit_code", "output", "output_sha256"], source, ); const startedAt = timestamp(object.started_at, source); @@ -406,6 +407,7 @@ function parseCheck(value: unknown, source: string): PrometheusCheckSummary { if (Date.parse(finishedAt) < Date.parse(startedAt)) invalidState(source); return { revision_sha256: pattern(object.revision_sha256, HEX_SHA256, source), + artifact_snapshot_sha256: pattern(object.artifact_snapshot_sha256, HEX_SHA256, source), started_at: startedAt, finished_at: finishedAt, exit_code: safeInteger(object.exit_code, source), diff --git a/packages/cli/src/prometheus.test.ts b/packages/cli/src/prometheus.test.ts index 627f999..e03a5c9 100644 --- a/packages/cli/src/prometheus.test.ts +++ b/packages/cli/src/prometheus.test.ts @@ -8,7 +8,7 @@ import { join } from "node:path"; import { afterEach, describe, expect, it } from "vitest"; import { DEFAULT_IMAGE, type DockerRunner, type RunPlan } from "./goal-run.ts"; -import { cancelPrometheusRun, evaluatePrometheusBarrier, prometheusKickoff, prometheusStatus, readPrometheusMessages, resumePrometheusRun } from "./prometheus.ts"; +import { cancelPrometheusRun, evaluatePrometheusBarrier, prometheusCandidateHandoff, prometheusKickoff, prometheusStatus, readPrometheusMessages, resumePrometheusRun } from "./prometheus.ts"; import { claimMonitor, prometheusPaths, readActiveRun, requestCancel, reserveActiveRun, writeRun, writeTeam, type PrometheusRunState, type PrometheusTeamState } from "./prometheus-state.ts"; import type { ProcessProbe } from "./process-identity.ts"; @@ -190,9 +190,19 @@ describe("Prometheus completion barrier", () => { expect(kickoff).toContain("i_build"); expect(kickoff).toContain("i_review"); expect(kickoff).toContain("verification.pass"); + expect(kickoff).toContain("runner will announce the exact candidate revision"); expect(kickoff).not.toContain("@all"); expect(kickoff).not.toContain("@builder"); }); + + it("builds an exact-member handoff containing the runner-derived revision and pinned snapshot", () => { + const roster = { coordinator: { member_id: "i_coord", thread_id: "t1", container_name: "c1" }, builder: { member_id: "i_build", thread_id: "t2", container_name: "c2" }, reviewer: { member_id: "i_review", thread_id: "t3", container_name: "c3" } }; + const handoff = prometheusCandidateHandoff("run-1", roster, candidate, "c".repeat(64)); + expect(handoff).toContain("@i_coord @i_build @i_review"); + expect(handoff).toContain(candidate.revisionSha256); + expect(handoff).toContain(`artifact_snapshot_sha256=${"c".repeat(64)}`); + expect(handoff).toContain('"run_id":"run-1"'); + }); }); describe("Prometheus lifecycle recovery", () => { diff --git a/packages/cli/src/prometheus.ts b/packages/cli/src/prometheus.ts index db07a7c..84b9e72 100644 --- a/packages/cli/src/prometheus.ts +++ b/packages/cli/src/prometheus.ts @@ -13,6 +13,7 @@ import { parsePrometheusPolicy, parseTerminalReceipts, type CandidateResult, + type PrometheusCandidate, type PrometheusMessage, type PrometheusPolicy, type TerminalReceiptResult, @@ -22,6 +23,7 @@ import { ensurePrometheusListeners, ensurePrometheusTeam, inspectPrometheusRuntime, + materializePrometheusArtifactSnapshot, runPrometheusCheck, verifyPrometheusTeamIdentity, type PrometheusRuntimeStatus, @@ -88,12 +90,22 @@ export function prometheusKickoff(input: { `PROMETHEUS.md sha256=${input.policy.sha256}; GOAL.md sha256=${input.goalSha256}; deadline=${input.deadlineAt}.`, "The complete current policy supersedes earlier policy:", input.policy.raw, - `Create compiled work for ${input.roster.builder.member_id} requiring deliverable submission and verified_by ${input.roster.reviewer.member_id}. The builder publishes work.result. The reviewer downloads the exact artifact, checks its SHA-256, uploads review evidence, then publishes verification.pass bound to the current result_ref. Resolve only the verified head; defects require a successor using supersedes.`, + `Create compiled work for ${input.roster.builder.member_id} requiring exactly one deliverable named submission and verified_by ${input.roster.reviewer.member_id}. The submission must be a workspace-rooted tar archive for the semantic checker. The builder publishes work.result. The reviewer downloads the exact artifact, checks its SHA-256, uploads review evidence, then publishes verification.pass bound to the current result_ref. Resolve only the verified head; defects require a successor using supersedes.`, `Semantic check after the evidence barrier: ${input.check}`, - `After the final head is resolved, request one receipt from builder and reviewer, then post yours. Exact line: [sharednet-prometheus:v1] {"type":"terminal.receipt","run_id":"${input.runId}","role":"","revision_sha256":""}`, + `After the final head is resolved and reviewed, wait: the runner will announce the exact candidate revision and pinned artifact snapshot digest. Then request one receipt from builder and reviewer and post yours. Exact line: [sharednet-prometheus:v1] {"type":"terminal.receipt","run_id":"${input.runId}","role":"","revision_sha256":""}`, ].join("\n\n"); } +export function prometheusCandidateHandoff( + runId: string, + roster: PrometheusRoster, + candidate: PrometheusCandidate, + artifactSnapshotSha256: string, +): string { + const recipients = PROMETHEUS_ROLES.map((role) => `@${roster[role].member_id}`).join(" "); + return `${recipients} Runner pinned candidate revision ${candidate.revisionSha256}; artifact_snapshot_sha256=${artifactSnapshotSha256}. Send exactly one authenticated receipt using [sharednet-prometheus:v1] {"type":"terminal.receipt","run_id":"${runId}","role":"","revision_sha256":"${candidate.revisionSha256}"}`; +} + export interface PrometheusConnection { client: ApiClient; memberToken: string; @@ -546,6 +558,7 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies const lease = await claimMonitor(dependencies.paths, run.run_id, dependencies.processProbe ?? systemProcessProbe); const sleep = dependencies.sleep ?? ((milliseconds: number) => new Promise((resolve) => setTimeout(resolve, milliseconds))); const feedbackAttempted = new Set(); + const candidateAnnounced = new Set(); try { for (;;) { const current = await reloadActive(run.run_id, dependencies); @@ -567,23 +580,55 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies const afterSnapshot = await reloadActive(current.run_id, dependencies); const stoppedAfterSnapshot = await stopOrRecover(afterSnapshot, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); if (stoppedAfterSnapshot) return stoppedAfterSnapshot; + const pinned = observed.candidate.ok + ? await materializePrometheusArtifactSnapshot( + observed.candidate.candidate.artifacts, + dependencies.connection.client, + dependencies.connection.memberToken, + join(dependencies.paths.runDir(afterSnapshot.run_id), "artifact-snapshots", observed.candidate.candidate.revisionSha256), + ) + : null; + let active = await reloadActive(current.run_id, dependencies); + const stoppedAfterPin = await stopOrRecover(active, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + if (stoppedAfterPin) return stoppedAfterPin; + run = active; + if (observed.candidate.ok && !observed.receipts.ok && !candidateAnnounced.has(observed.candidate.candidate.revisionSha256)) { + const revision = observed.candidate.candidate.revisionSha256; + const content = prometheusCandidateHandoff(active.run_id, active.roster!, observed.candidate.candidate, pinned!.sha256); + await dependencies.connection.client.request( + "POST", + `/rooms/${encodeURIComponent(active.room!.room_id)}/owner-messages`, + dependencies.connection.apiKey, + { content, as: "runner" }, + { "idempotency-key": `${active.run_id}-candidate-${revision}` }, + ); + candidateAnnounced.add(revision); + active = await reloadActive(current.run_id, dependencies); + const stoppedAfterAnnouncement = await stopOrRecover(active, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + if (stoppedAfterAnnouncement) return stoppedAfterAnnouncement; + } if (observed.barrier.ready) { const revision = observed.barrier.candidate.revisionSha256; - const prior = afterSnapshot.checks.find((entry) => entry.revision_sha256 === revision); + const prior = active.checks.find((entry) => entry.revision_sha256 === revision); + if (pinned === null) throw localError("prometheus_artifact_snapshot_invalid", "The ready candidate has no artifact snapshot."); + if (prior && prior.artifact_snapshot_sha256 !== pinned.sha256) { + throw localError("prometheus_artifact_snapshot_invalid", "The saved semantic check names a different artifact snapshot."); + } let passingCheck = prior?.exit_code === 0; if (!prior) { - const checkStop = watchRunStop(afterSnapshot, dependencies.paths, dependencies.now); + const checkStop = watchRunStop(active, dependencies.paths, dependencies.now); let checked: Awaited>; try { - checked = await runPrometheusCheck({ image: dependencies.plan.image, workspace: current.workspace_path }, current.configuration.check_command, { - SHAREDNET_PROMETHEUS_RUN_ID: current.run_id, + checked = await runPrometheusCheck({ image: dependencies.plan.image, snapshot: pinned.path, snapshotSha256: pinned.sha256 }, active.configuration.check_command, { + SHAREDNET_PROMETHEUS_RUN_ID: active.run_id, SHAREDNET_PROMETHEUS_WORK_ID: observed.barrier.candidate.workId, SHAREDNET_PROMETHEUS_RESULT_REF: observed.barrier.candidate.resultRef, SHAREDNET_PROMETHEUS_REVISION_SHA256: revision, + SHAREDNET_PROMETHEUS_ARTIFACT_SNAPSHOT_SHA256: pinned.sha256, }, { docker: dependencies.docker, now: dependencies.now, - timeoutMs: Date.parse(afterSnapshot.deadline_at) - dependencies.now().getTime(), + timeoutMs: Math.min(120_000, Date.parse(active.deadline_at) - dependencies.now().getTime()), signal: checkStop.signal, }); } finally { @@ -593,8 +638,10 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies const stoppedAfterCheck = await stopOrRecover(afterCheck, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); if (stoppedAfterCheck) return stoppedAfterCheck; if (checked.exitCode === 125) throw localError("prometheus_check_launch_failed", "The Prometheus semantic-check container could not be launched."); - const summary = { revision_sha256: revision, started_at: checked.startedAt, finished_at: checked.finishedAt, exit_code: Math.max(0, checked.exitCode), output: checked.output, output_sha256: checked.outputSha256 }; + const summary = { revision_sha256: revision, artifact_snapshot_sha256: pinned.sha256, started_at: checked.startedAt, finished_at: checked.finishedAt, exit_code: Math.max(0, checked.exitCode), output: checked.output, output_sha256: checked.outputSha256 }; run = await saveCheckpoint(dependencies.paths, afterCheck, { monitor: lease, checks: [...afterCheck.checks, summary] }, dependencies.now()); + const stoppedAfterCheckCheckpoint = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + if (stoppedAfterCheckCheckpoint) return stoppedAfterCheckCheckpoint; if (checked.exitCode !== 0) { const feedback = `@${current.roster!.coordinator.member_id} Semantic check failed for revision ${revision}:\n${checked.output.slice(-2_000)}`; await dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/owner-messages`, dependencies.connection.apiKey, { content: feedback, as: "runner" }, { "idempotency-key": `${current.run_id}-check-${revision}` }); From 8c4b3712e1280f858b6e0f930c78aa83bcbec66d Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 19 Oct 2026 02:48:33 +0000 Subject: [PATCH 6/9] fix(cli): close stopped Prometheus startup rooms Reconnect before terminalizing cancellation or deadline during reusable-team bootstrap, recheck the authoritative stop bit after persisting semantic-check evidence, and cap each check attempt at two minutes even when the run has a longer budget. The hosted close route still needs a server-side state-digest CAS to remove the remaining remote TOCTOU window. Co-authored-by: OpenAI Codex --- packages/cli/src/prometheus.test.ts | 31 +++++++++++++++++++++++++++++ packages/cli/src/prometheus.ts | 16 ++++++++++++--- 2 files changed, 44 insertions(+), 3 deletions(-) diff --git a/packages/cli/src/prometheus.test.ts b/packages/cli/src/prometheus.test.ts index e03a5c9..1aa37a6 100644 --- a/packages/cli/src/prometheus.test.ts +++ b/packages/cli/src/prometheus.test.ts @@ -355,6 +355,37 @@ describe("Prometheus lifecycle recovery", () => { expect(await readActiveRun(fixture.paths)).toBeNull(); }); + it("closes an already-created Room when cancellation lands during reusable-team bootstrap", async () => { + const fixture = await lifecycleFixture("room_created", "2026-10-18T04:00:00.000Z"); + const baseDocker = lifecycleDocker(fixture.team, []); + let cancelled = false; + const docker: DockerRunner = async (args, options) => { + if (!cancelled && args[0] === "start") { + cancelled = true; + await requestCancel(fixture.paths, fixture.run.run_id); + await new Promise((resolve) => { + if (options?.signal?.aborted) resolve(); + else options?.signal?.addEventListener("abort", () => resolve(), { once: true }); + }); + } + return baseDocker(args, options); + }; + const requests: string[] = []; + + const result = await resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker, plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ + client: { request: async (_method: string, path: string) => { requests.push(path); return {}; } } as never, + memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai", + }), + }); + + expect(result).toMatchObject({ outcome: "cancelled", startup_checkpoint: "room_created" }); + expect(requests).toEqual(["/rooms/rom_Lifecycle/close"]); + expect(await readActiveRun(fixture.paths)).toBeNull(); + }); + it("records an irrecoverable monitor protocol error as failed and closes the Room", async () => { const fixture = await lifecycleFixture("kickoff_posted"); const requests: string[] = []; diff --git a/packages/cli/src/prometheus.ts b/packages/cli/src/prometheus.ts index 84b9e72..75c4ef1 100644 --- a/packages/cli/src/prometheus.ts +++ b/packages/cli/src/prometheus.ts @@ -393,7 +393,12 @@ async function continuePrometheusRun( }); } catch (error) { const current = await reloadActive(run.run_id, dependencies); - const stopped = await stopOrRecover(current, lease, { paths: dependencies.paths, now: dependencies.now }); + const stopConnection = current.room ? await dependencies.connect() : undefined; + const stopped = await stopOrRecover(current, lease, { + paths: dependencies.paths, + now: dependencies.now, + ...(stopConnection ? { connection: stopConnection } : {}), + }); if (stopped) return stopped; const failure = asCliError(error); if (!["prometheus_team_invalid", "prometheus_team_missing", "prometheus_team_mismatch"].includes(failure.code)) throw error; @@ -404,9 +409,14 @@ async function continuePrometheusRun( } await writeTeam(dependencies.paths, team); run = await reloadActive(run.run_id, dependencies); - const stoppedAfterBootstrap = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now }); + const bootstrapConnection = run.room ? await dependencies.connect() : undefined; + const stoppedAfterBootstrap = await stopOrRecover(run, lease, { + paths: dependencies.paths, + now: dependencies.now, + ...(bootstrapConnection ? { connection: bootstrapConnection } : {}), + }); if (stoppedAfterBootstrap) return stoppedAfterBootstrap; - const connection = await dependencies.connect(); + const connection = bootstrapConnection ?? await dependencies.connect(); run = await reloadActive(run.run_id, dependencies); const stoppedAfterConnect = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); if (stoppedAfterConnect) return stoppedAfterConnect; From 8d90c66a8665e2875612d5c94156aa86b60261a1 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 19 Oct 2026 02:55:26 +0000 Subject: [PATCH 7/9] docs: mark Prometheus hosted contract gate --- .../2026-10-18-prometheus-local-design.md | 10 ++++++---- docs/prometheus-local.md | 19 ++++++++++++++----- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/docs/plans/2026-10-18-prometheus-local-design.md b/docs/plans/2026-10-18-prometheus-local-design.md index 59cf2ff..2249ec7 100644 --- a/docs/plans/2026-10-18-prometheus-local-design.md +++ b/docs/plans/2026-10-18-prometheus-local-design.md @@ -368,8 +368,10 @@ Local v1 deliberately defers: - automatic team reset, model migration, container-volume garbage collection, or import/export of a team; - making either reserved `runtime/*@1` harness mode available; and -- new server routes or changes to the compiled Room protocol. +- a server-owned scheduler beyond the minimal replay-safe write and + conditional-close contract required by local v1. -Those features require their own versioned design. Local v1 proves the fixed -three-role lifecycle and completion barrier using existing public service -contracts. +Those features require their own versioned design. Local v1 implements and +locally verifies the fixed three-role lifecycle and completion barrier. It is +not production-runnable until the coordinated server write/CAS contract and a +no-model hosted smoke test pass. diff --git a/docs/prometheus-local.md b/docs/prometheus-local.md index 08f18b5..c54e6ca 100644 --- a/docs/prometheus-local.md +++ b/docs/prometheus-local.md @@ -8,11 +8,19 @@ every run gets a fresh compiled SharedNet Room and fresh Room member ids. This is a local CLI runtime. It is not the hosted Prometheus control plane, an elastic agent pool, or either reserved `runtime/*@1` research harness. +> **Availability gate:** this Preview is not compatible with the currently +> deployed SharedNet write contract. Do not run it against production until a +> coordinated server release provides replay-safe Goal, owner-message, invite, +> role-join, and conditional-close operations. PR #57 remains a Draft until +> that contract and a no-model hosted smoke test pass. + ## Prerequisites - Node.js 22.18 or newer and this build of the `sharednet` CLI; - Docker, with permission to create containers and named volumes; - a SharedNet login for creating the Room and Goal; +- a SharedNet server release that advertises the Prometheus write/CAS + contract described above; - a working Codex login or `OPENAI_API_KEY` for the three role containers; and - a real workspace directory containing `PROMETHEUS.md` and the Goal file. @@ -45,8 +53,9 @@ member id. Prometheus sends one exact kickoff to the new Room's coordinator. ## Preview limitations -Local v1 records every startup checkpoint and replays unfinished external -writes with the run's original idempotency keys. It also verifies the exact +Local v1 records every startup checkpoint and is designed to replay unfinished +external writes with the run's original idempotency keys once the required +server contract is deployed. It also verifies the exact saved image, user, labels, mounts, container names, volumes, and thread markers before reuse. A verified container left before bootstrap completion is resumed from bootstrap; an identity mismatch terminates the run instead of adopting an @@ -57,9 +66,9 @@ The remaining preview boundaries are operational: - This remains a foreground controller. A crash needs an explicit `resume`; it does not launch a host service automatically. - A transient Docker, listener, authentication, or SharedNet transport error - leaves the checkpoint active for repair and `resume`. Stable external-write - keys prevent a repaired resume from creating another Room, Goal, role seat, - or kickoff. + leaves the checkpoint active for repair and `resume`. Replay safety depends + on the gated server contract; the currently deployed service cannot provide + the Room, Goal, role-seat, message, and close guarantees this client needs. - Partially created named volumes are retained. Verified containers and saved thread markers are reconciled, but unused volumes may require later cleanup. - `status` always returns durable state. For an active fully seeded run it also From 03b7e5b47166c7cbe82f89cc012fa4fa636acafd Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 19 Oct 2026 03:08:20 +0000 Subject: [PATCH 8/9] fix(cli): bound Prometheus operations to run lifetime --- packages/cli/src/api-client.ts | 4 + packages/cli/src/prometheus-runtime.test.ts | 24 ++ packages/cli/src/prometheus-runtime.ts | 3 +- packages/cli/src/prometheus-state.test.ts | 33 ++- packages/cli/src/prometheus-state.ts | 5 +- packages/cli/src/prometheus.test.ts | 39 ++- packages/cli/src/prometheus.ts | 249 ++++++++++++-------- 7 files changed, 251 insertions(+), 106 deletions(-) diff --git a/packages/cli/src/api-client.ts b/packages/cli/src/api-client.ts index 6c48e12..4392444 100644 --- a/packages/cli/src/api-client.ts +++ b/packages/cli/src/api-client.ts @@ -79,12 +79,14 @@ export class ApiClient { async requestBytes( path: string, credential: string, + signal?: AbortSignal, ): Promise<{ bytes: Uint8Array; filename: string | null; sha256: string | null }> { let response: Response; try { response = await this.fetch(`${this.baseUrl}/api/v1${path}`, { method: "GET", headers: credential ? { authorization: `Bearer ${credential}` } : {}, + ...(signal ? { signal } : {}), }); } catch { throw new CliError("service_unavailable", "The SharedNet service could not be reached.", 5); @@ -117,6 +119,7 @@ export class ApiClient { credential: string, body?: unknown, requestHeaders: Record = {}, + signal?: AbortSignal, ): Promise { let response: Response; // Bytes go up as they are; anything else is JSON. @@ -131,6 +134,7 @@ export class ApiClient { ...requestHeaders, }, ...(body === undefined ? {} : { body: raw ? (body as unknown as BodyInit) : JSON.stringify(body) }), + ...(signal ? { signal } : {}), }); } catch { throw new CliError( diff --git a/packages/cli/src/prometheus-runtime.test.ts b/packages/cli/src/prometheus-runtime.test.ts index 168000f..91438cc 100644 --- a/packages/cli/src/prometheus-runtime.test.ts +++ b/packages/cli/src/prometheus-runtime.test.ts @@ -321,6 +321,30 @@ describe("persistent Prometheus runtime", () => { } }); + it("passes the run cancellation signal into artifact downloads", async () => { + const root = await mkdtemp(join(tmpdir(), "sharednet-prometheus-artifacts-cancel-")); + const snapshot = join(root, "snapshot"); + const controller = new AbortController(); + let sawSignal = false; + try { + await expect(materializePrometheusArtifactSnapshot( + [{ name: "submission", artifactId: "art_Submit0001", sha256: "a".repeat(64) }], + { + requestBytes: async (_path: string, _credential: string, signal?: AbortSignal) => { + sawSignal = signal === controller.signal; + throw new Error("stopped"); + }, + } as never, + "member", + snapshot, + controller.signal, + )).rejects.toThrow("stopped"); + expect(sawSignal).toBe(true); + } finally { + await rm(root, { recursive: true, force: true }); + } + }); + it("restarts unfiltered listeners for saved seats without joining again", async () => { const calls: string[][] = []; const team: PrometheusTeamState = { diff --git a/packages/cli/src/prometheus-runtime.ts b/packages/cli/src/prometheus-runtime.ts index 125292f..923360d 100644 --- a/packages/cli/src/prometheus-runtime.ts +++ b/packages/cli/src/prometheus-runtime.ts @@ -459,6 +459,7 @@ export async function materializePrometheusArtifactSnapshot( client: Pick, memberToken: string, target: string, + signal?: AbortSignal, ): Promise { if (artifacts.length !== 1 || artifacts[0]?.name !== "submission") { throw localError("prometheus_artifact_snapshot_invalid", "Prometheus local v1 requires exactly one submission tar artifact."); @@ -477,7 +478,7 @@ export async function materializePrometheusArtifactSnapshot( try { const manifestArtifacts: ArtifactSnapshotManifest["artifacts"] = []; for (const artifact of artifacts) { - const downloaded = await client.requestBytes(`/artifacts/${encodeURIComponent(artifact.artifactId)}/content`, memberToken); + const downloaded = await client.requestBytes(`/artifacts/${encodeURIComponent(artifact.artifactId)}/content`, memberToken, signal); const digest = createHash("sha256").update(downloaded.bytes).digest("hex"); if (digest !== artifact.sha256 || (downloaded.sha256 !== null && downloaded.sha256 !== artifact.sha256)) { throw localError("prometheus_artifact_changed", `Artifact ${artifact.artifactId} does not match its compiled receipt.`); diff --git a/packages/cli/src/prometheus-state.test.ts b/packages/cli/src/prometheus-state.test.ts index 88e2f2c..c07acbe 100644 --- a/packages/cli/src/prometheus-state.test.ts +++ b/packages/cli/src/prometheus-state.test.ts @@ -321,21 +321,16 @@ describe("Prometheus owner-only state", () => { const { paths } = await fixture(); const state = run(paths); const syntheticCapability = ["sn", "k_visibly-synthetic-test-only"].join(""); - const secretNames = [ + const credentialPrefixes = [ ["sn", "k_"].join(""), ["sn", "i_"].join(""), ["ri", "t_"].join(""), - "member_token", - "api_key", - "invite", - "OPENAI_API_KEY", - "auth.json", ]; await writeTeam(paths, team(paths)); await reserveActiveRun(paths, state, fakeProbe().probe); const serialized = `${await readFile(paths.teamFile, "utf8")}\n${await readFile(paths.runFile(state.run_id), "utf8")}`; - for (const forbidden of secretNames) expect(serialized).not.toContain(forbidden); + for (const forbidden of credentialPrefixes) expect(serialized).not.toContain(forbidden); await expect( writeTeam(paths, { ...team(paths), api_key: syntheticCapability } as PrometheusTeamState), @@ -348,6 +343,29 @@ describe("Prometheus owner-only state", () => { ).rejects.toMatchObject({ code: "invalid_local_state" }); }); + it("persists ordinary checker output that names credential concepts", async () => { + const { paths } = await fixture(); + const output = "invite flow failed while reading api_key from auth.json fixture"; + const state = { + ...run(paths), + checks: [ + { + revision_sha256: SHA_A, + artifact_snapshot_sha256: SHA_B, + started_at: CREATED_AT, + finished_at: UPDATED_AT, + exit_code: 1, + output, + output_sha256: createHash("sha256").update(output).digest("hex"), + }, + ], + }; + + await reserveActiveRun(paths, run(paths), fakeProbe().probe); + await writeRun(paths, state); + expect((await readRun(paths, state.run_id)).checks[0]?.output).toBe(output); + }); + it("rejects synthetic OpenAI and Codex credentials embedded in free text", async () => { const openAiKey = ["s", "k-proj-visibly-synthetic-test-credential"].join(""); const codexAuth = JSON.stringify({ @@ -407,6 +425,7 @@ describe("Prometheus owner-only state", () => { for (const makeState of cases) { const { paths } = await fixture(); + await reserveActiveRun(paths, run(paths), fakeProbe().probe); await expect(writeRun(paths, makeState(run(paths)))).rejects.toMatchObject({ code: "invalid_local_state", }); diff --git a/packages/cli/src/prometheus-state.ts b/packages/cli/src/prometheus-state.ts index 2434676..4f6af48 100644 --- a/packages/cli/src/prometheus-state.ts +++ b/packages/cli/src/prometheus-state.ts @@ -37,13 +37,14 @@ const TERMINAL_OUTCOMES = new Set([ "failed", ]); const HEX_SHA256 = /^[0-9a-f]{64}$/; +const COMPILED_STATE_DIGEST = /^sha256:[0-9a-f]{24}$/; const WORKSPACE_KEY = /^[0-9a-f]{24}$/; const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; const ROOM_ID = /^rom_[A-Za-z0-9]+$/; const MEMBER_ID = /^(?:i|mem)_[A-Za-z0-9]+$/; const DOCKER_NAME = /^[A-Za-z0-9][A-Za-z0-9_.-]{0,127}$/; const MODEL_NAME = /^[A-Za-z0-9._/-]{1,100}$/; -const FORBIDDEN_STATE_TEXT = /(?:snk_|sni_|rit_|member_token|api_key|invite|OPENAI_API_KEY|auth\.json|access_token|refresh_token|id_token|client_secret|\bsk-(?:proj-|svcacct-)?[A-Za-z0-9_-]{16,})/i; +const FORBIDDEN_STATE_TEXT = /(?:\b(?:snk|sni|rit|clp|afk)_[A-Za-z0-9_-]{16,}\b|\bsk-(?:proj-|svcacct-)?[A-Za-z0-9_-]{16,}|(?:\\?")(?:access_token|refresh_token|id_token|client_secret|api_key|member_token)(?:\\?")\s*:\s*(?:\\?")[^"\\\r\n]{16,})/i; const FORBIDDEN_JWT = /(?:\bBearer\s+[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}|\beyJ[A-Za-z0-9_-]{5,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,})/i; export type PrometheusStartupCheckpoint = (typeof STARTUP_CHECKPOINTS)[number]; @@ -388,7 +389,7 @@ function parseCandidate(value: unknown, source: string): PrometheusCandidateSumm source, ); return { - compiled_digest: pattern(object.compiled_digest, HEX_SHA256, source), + compiled_digest: pattern(object.compiled_digest, COMPILED_STATE_DIGEST, source), work_id: pattern(object.work_id, SAFE_ID, source), result_ref: pattern(object.result_ref, SAFE_ID, source), revision_sha256: pattern(object.revision_sha256, HEX_SHA256, source), diff --git a/packages/cli/src/prometheus.test.ts b/packages/cli/src/prometheus.test.ts index 1aa37a6..5d3bf72 100644 --- a/packages/cli/src/prometheus.test.ts +++ b/packages/cli/src/prometheus.test.ts @@ -13,7 +13,7 @@ import { claimMonitor, prometheusPaths, readActiveRun, requestCancel, reserveAct import type { ProcessProbe } from "./process-identity.ts"; const candidate = { - stateDigest: "a".repeat(64), + stateDigest: `sha256:${"a".repeat(24)}`, workId: "work-1", resultRef: "evt-1", artifacts: [], @@ -125,9 +125,12 @@ function lifecycleDocker(team: PrometheusTeamState, calls: Array<{ args: readonl describe("Prometheus completion barrier", () => { it("reads every message page so late terminal receipts remain visible", async () => { const paths: string[] = []; + const controller = new AbortController(); + const signals: Array = []; const client = { - request: async (_method: string, path: string) => { + request: async (_method: string, path: string, _credential: string, _body?: unknown, _headers?: Record, signal?: AbortSignal) => { paths.push(path); + signals.push(signal); if (path.includes("after=0")) { return { items: Array.from({ length: 100 }, (_, index) => ({ sequence: index + 1, content: `message ${index + 1}` })), @@ -141,7 +144,7 @@ describe("Prometheus completion barrier", () => { }, }; - const messages = await readPrometheusMessages(client as never, "rom_123", "token"); + const messages = await readPrometheusMessages(client as never, "rom_123", "token", controller.signal); expect(messages).toHaveLength(101); expect(messages.at(-1)?.sequence).toBe(101); @@ -149,6 +152,7 @@ describe("Prometheus completion barrier", () => { "/rooms/rom_123/messages?after=0&order=asc&limit=100", "/rooms/rom_123/messages?after=100&order=asc&limit=100", ]); + expect(signals).toEqual([controller.signal, controller.signal]); }); it("requires candidate, all authenticated receipts, listening seats, and no handed work", () => { @@ -307,7 +311,7 @@ describe("Prometheus lifecycle recovery", () => { const revision = "b".repeat(64); await writeRun(fixture.paths, { ...fixture.run, - candidate: { compiled_digest: "a".repeat(64), work_id: "work-1", result_ref: "evt-1", revision_sha256: revision, observed_at: "2026-10-18T00:01:00.000Z" }, + candidate: { compiled_digest: `sha256:${"a".repeat(24)}`, work_id: "work-1", result_ref: "evt-1", revision_sha256: revision, observed_at: "2026-10-18T00:01:00.000Z" }, updated_at: "2026-10-18T00:01:00.000Z", }); await requestCancel(fixture.paths, fixture.run.run_id); @@ -355,6 +359,33 @@ describe("Prometheus lifecycle recovery", () => { expect(await readActiveRun(fixture.paths)).toBeNull(); }); + it("aborts an in-flight Room request when cancellation is persisted", async () => { + const fixture = await lifecycleFixture("reserved", "2026-10-18T04:00:00.000Z"); + let sawSignal = false; + + const result = await resumePrometheusRun(fixture.workspace, { + env: fixture.env, cwd: fixture.workspace, now: () => new Date("2026-10-18T01:00:00.000Z"), + docker: lifecycleDocker(fixture.team, []), plan: { ...plan, workspace: fixture.workspace }, processProbe: fixture.probe, + connect: async () => ({ + client: { request: async (_method: string, path: string, _credential: string, _body?: unknown, _headers?: Record, signal?: AbortSignal) => { + if (path !== "/rooms") throw new Error(`unexpected request ${path}`); + sawSignal = signal instanceof AbortSignal; + await requestCancel(fixture.paths, fixture.run.run_id); + await new Promise((resolve) => { + if (signal?.aborted) resolve(); + else signal?.addEventListener("abort", () => resolve(), { once: true }); + }); + throw new Error("request aborted"); + } } as never, + memberToken: "member", apiKey: "api", baseUrl: "https://www.sharednet.ai", + }), + }); + + expect(sawSignal).toBe(true); + expect(result).toMatchObject({ outcome: "cancelled", startup_checkpoint: "reserved" }); + expect(await readActiveRun(fixture.paths)).toBeNull(); + }); + it("closes an already-created Room when cancellation lands during reusable-team bootstrap", async () => { const fixture = await lifecycleFixture("room_created", "2026-10-18T04:00:00.000Z"); const baseDocker = lifecycleDocker(fixture.team, []); diff --git a/packages/cli/src/prometheus.ts b/packages/cli/src/prometheus.ts index 75c4ef1..bc354f8 100644 --- a/packages/cli/src/prometheus.ts +++ b/packages/cli/src/prometheus.ts @@ -340,6 +340,23 @@ function watchRunStop( }; } +async function whileRunActive( + run: PrometheusRunState, + dependencies: Pick, + operation: (signal: AbortSignal) => Promise, +): Promise { + const stop = watchRunStop(run, dependencies.paths, dependencies.now); + try { + return await operation(stop.signal); + } finally { + stop.dispose(); + } +} + +function dockerWithSignal(docker: DockerRunner, signal: AbortSignal): DockerRunner { + return (args, options) => docker(args, { ...options, signal }); +} + async function stopOrRecover( run: PrometheusRunState, lease: NonNullable, @@ -362,8 +379,9 @@ async function continuePrometheusRun( dependencies: StartupDependencies, ): Promise { const lease = await claimMonitor(dependencies.paths, initial.run_id, dependencies.processProbe ?? systemProcessProbe); - let run = await readActiveRun(dependencies.paths); - if (!run || run.run_id !== initial.run_id) throw localError("prometheus_monitor_lease_lost", `Prometheus run ${initial.run_id} is no longer active.`); + const persistedRun = await readActiveRun(dependencies.paths); + if (!persistedRun || persistedRun.run_id !== initial.run_id) throw localError("prometheus_monitor_lease_lost", `Prometheus run ${initial.run_id} is no longer active.`); + let run: PrometheusRunState = persistedRun; if (run.cancel_requested || dependencies.now().getTime() >= Date.parse(run.deadline_at)) { const connection = run.room ? await dependencies.connect() : undefined; @@ -421,77 +439,97 @@ async function continuePrometheusRun( const stoppedAfterConnect = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); if (stoppedAfterConnect) return stoppedAfterConnect; - if (run.startup_checkpoint === "reserved") { - const created = await connection.client.request<{ room?: { id?: string } }>( - "POST", "/rooms", connection.memberToken, - { name: `Prometheus ${basename(dependencies.paths.workspacePath)}`, type: "compiled" }, - { "idempotency-key": run.idempotency_keys.room }, - ); - const roomId = created.room?.id; - if (!roomId) throw localError("invalid_server_response", "The compiled Prometheus Room id did not come back."); - run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "room_created", room: { room_id: roomId, goal_sequence: null, kickoff_sequence: null } }, dependencies.now()); - const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); - if (stopped) return stopped; - } + try { + if (run.startup_checkpoint === "reserved") { + const created = await whileRunActive(run, dependencies, (signal) => + connection.client.request<{ room?: { id?: string } }>( + "POST", "/rooms", connection.memberToken, + { name: `Prometheus ${basename(dependencies.paths.workspacePath)}`, type: "compiled" }, + { "idempotency-key": run.idempotency_keys.room }, + signal, + )); + const roomId = created.room?.id; + if (!roomId) throw localError("invalid_server_response", "The compiled Prometheus Room id did not come back."); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "room_created", room: { room_id: roomId, goal_sequence: null, kickoff_sequence: null } }, dependencies.now()); + const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stopped) return stopped; + } - if (run.startup_checkpoint === "room_created") { - const remaining = Date.parse(run.deadline_at) - dependencies.now().getTime(); - const started = await connection.client.request<{ message?: { sequence?: number } }>( - "POST", `/rooms/${encodeURIComponent(run.room!.room_id)}/goal`, connection.apiKey, - { content: loaded.goal, until: [`after ${Math.max(1, Math.ceil(remaining / 1000))}s`] }, - { "idempotency-key": run.idempotency_keys.goal }, - ); - const goalSequence = started.message?.sequence; - if (!Number.isSafeInteger(goalSequence) || goalSequence! < 1) throw localError("invalid_server_response", "The Prometheus Goal sequence did not come back."); - run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "goal_started", room: { ...run.room!, goal_sequence: goalSequence! } }, dependencies.now()); - const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); - if (stopped) return stopped; - } + if (run.startup_checkpoint === "room_created") { + const remaining = Date.parse(run.deadline_at) - dependencies.now().getTime(); + const started = await whileRunActive(run, dependencies, (signal) => + connection.client.request<{ message?: { sequence?: number } }>( + "POST", `/rooms/${encodeURIComponent(run.room!.room_id)}/goal`, connection.apiKey, + { content: loaded.goal, until: [`after ${Math.max(1, Math.ceil(remaining / 1000))}s`] }, + { "idempotency-key": run.idempotency_keys.goal }, + signal, + )); + const goalSequence = started.message?.sequence; + if (!Number.isSafeInteger(goalSequence) || goalSequence! < 1) throw localError("invalid_server_response", "The Prometheus Goal sequence did not come back."); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "goal_started", room: { ...run.room!, goal_sequence: goalSequence! } }, dependencies.now()); + const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stopped) return stopped; + } - if (run.startup_checkpoint === "goal_started") { - const invite = await connection.client.request<{ token?: string }>( - "POST", `/rooms/${encodeURIComponent(run.room!.room_id)}/invites`, connection.memberToken, undefined, - { "idempotency-key": `${run.idempotency_keys.room}-invite` }, - ); - if (!invite.token) throw localError("invalid_server_response", "The Prometheus Room invite did not come back."); - run = await reloadActive(run.run_id, dependencies); - const stoppedAfterInvite = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); - if (stoppedAfterInvite) return stoppedAfterInvite; - const roster = await attachPrometheusRoom(team, { - roomId: run.room!.room_id, - inviteToken: invite.token, - baseUrl: connection.baseUrl, - idempotencyKeys: run.idempotency_keys.roles, - }, { docker: dependencies.docker, now: dependencies.now, sleep: dependencies.sleep }); - run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "roles_joined", roster }, dependencies.now()); - const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); - if (stopped) return stopped; - } + if (run.startup_checkpoint === "goal_started") { + const invite = await whileRunActive(run, dependencies, (signal) => + connection.client.request<{ token?: string }>( + "POST", `/rooms/${encodeURIComponent(run.room!.room_id)}/invites`, connection.memberToken, undefined, + { "idempotency-key": `${run.idempotency_keys.room}-invite` }, + signal, + )); + if (!invite.token) throw localError("invalid_server_response", "The Prometheus Room invite did not come back."); + const inviteToken = invite.token; + run = await reloadActive(run.run_id, dependencies); + const stoppedAfterInvite = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stoppedAfterInvite) return stoppedAfterInvite; + const roster = await whileRunActive(run, dependencies, (signal) => attachPrometheusRoom(team, { + roomId: run.room!.room_id, + inviteToken, + baseUrl: connection.baseUrl, + idempotencyKeys: run.idempotency_keys.roles, + }, { docker: dockerWithSignal(dependencies.docker, signal), now: dependencies.now, sleep: dependencies.sleep })); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "roles_joined", roster }, dependencies.now()); + const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stopped) return stopped; + } - if (run.startup_checkpoint === "roles_joined") { - await ensurePrometheusListeners(team, run.room!.room_id, run.roster!, connection.baseUrl, { docker: dependencies.docker, now: dependencies.now, sleep: dependencies.sleep }); - run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "listeners_ready" }, dependencies.now()); - const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); - if (stopped) return stopped; - } + if (run.startup_checkpoint === "roles_joined") { + await whileRunActive(run, dependencies, (signal) => ensurePrometheusListeners(team, run.room!.room_id, run.roster!, connection.baseUrl, { docker: dockerWithSignal(dependencies.docker, signal), now: dependencies.now, sleep: dependencies.sleep })); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "listeners_ready" }, dependencies.now()); + const stopped = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection }); + if (stopped) return stopped; + } - if (run.startup_checkpoint === "listeners_ready") { - const content = prometheusKickoff({ - runId: run.run_id, - goalSequence: run.room!.goal_sequence!, - roster: run.roster!, - policy: loaded.policy, - goalSha256: run.configuration.goal_sha256, - check: run.configuration.check_command, - deadlineAt: run.deadline_at, + if (run.startup_checkpoint === "listeners_ready") { + const content = prometheusKickoff({ + runId: run.run_id, + goalSequence: run.room!.goal_sequence!, + roster: run.roster!, + policy: loaded.policy, + goalSha256: run.configuration.goal_sha256, + check: run.configuration.check_command, + deadlineAt: run.deadline_at, + }); + const kicked = await whileRunActive(run, dependencies, (signal) => + connection.client.request<{ message?: { sequence?: number } }>( + "POST", `/rooms/${encodeURIComponent(run.room!.room_id)}/owner-messages`, connection.apiKey, + { content, as: "runner" }, { "idempotency-key": run.idempotency_keys.kickoff }, + signal, + )); + const kickoffSequence = kicked.message?.sequence; + if (!Number.isSafeInteger(kickoffSequence) || kickoffSequence! <= run.room!.goal_sequence!) throw localError("invalid_server_response", "The Prometheus kickoff sequence did not come back."); + run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "kickoff_posted", outcome: "running", room: { ...run.room!, kickoff_sequence: kickoffSequence! } }, dependencies.now()); + } + } catch (error) { + const current = await reloadActive(run.run_id, dependencies); + const stopped = await stopOrRecover(current, lease, { + paths: dependencies.paths, + now: dependencies.now, + connection, }); - const kicked = await connection.client.request<{ message?: { sequence?: number } }>( - "POST", `/rooms/${encodeURIComponent(run.room!.room_id)}/owner-messages`, connection.apiKey, - { content, as: "runner" }, { "idempotency-key": run.idempotency_keys.kickoff }, - ); - const kickoffSequence = kicked.message?.sequence; - if (!Number.isSafeInteger(kickoffSequence) || kickoffSequence! <= run.room!.goal_sequence!) throw localError("invalid_server_response", "The Prometheus kickoff sequence did not come back."); - run = await saveCheckpoint(dependencies.paths, run, { startup_checkpoint: "kickoff_posted", outcome: "running", room: { ...run.room!, kickoff_sequence: kickoffSequence! } }, dependencies.now()); + if (stopped) return stopped; + throw error; } return monitorPrometheusRun(run, { ...dependencies, team, connection }); @@ -502,6 +540,7 @@ export async function readPrometheusMessages( client: ApiClient, roomId: string, memberToken: string, + signal?: AbortSignal, ): Promise { const messages: PrometheusMessage[] = []; let after = 0; @@ -510,6 +549,9 @@ export async function readPrometheusMessages( "GET", `/rooms/${encodeURIComponent(roomId)}/messages?after=${after}&order=asc&limit=100`, memberToken, + undefined, + {}, + signal, ); if (!Array.isArray(page.items) || typeof page.has_more !== "boolean") { throw localError("prometheus_history_incomplete", "The Prometheus Room history is incomplete."); @@ -533,9 +575,19 @@ interface RoomLifecycle { endedBy: { trigger: string; detail: string | null } | null; } -async function roomLifecycle(run: PrometheusRunState, dependencies: Pick): Promise { +async function roomLifecycle( + run: PrometheusRunState, + dependencies: Pick & { signal?: AbortSignal }, +): Promise { const roomId = run.room!.room_id; - const response = await dependencies.connection.client.request<{ room?: { state?: unknown; goal?: { ended_by?: { trigger?: unknown; detail?: unknown } | null } | null } }>("GET", `/rooms/${encodeURIComponent(roomId)}`, dependencies.connection.memberToken); + const response = await dependencies.connection.client.request<{ room?: { state?: unknown; goal?: { ended_by?: { trigger?: unknown; detail?: unknown } | null } | null } }>( + "GET", + `/rooms/${encodeURIComponent(roomId)}`, + dependencies.connection.memberToken, + undefined, + {}, + dependencies.signal, + ); const state = response.room?.state; const ended = response.room?.goal?.ended_by; if (typeof state !== "string") throw localError("invalid_server_response", "The Prometheus Room state did not come back."); @@ -545,10 +597,17 @@ async function roomLifecycle(run: PrometheusRunState, dependencies: Pick("GET", `/rooms/${encodeURIComponent(roomId)}/state`, dependencies.connection.memberToken); - const messages = await readPrometheusMessages(dependencies.connection.client, roomId, dependencies.connection.memberToken); + const state = await dependencies.connection.client.request( + "GET", + `/rooms/${encodeURIComponent(roomId)}/state`, + dependencies.connection.memberToken, + undefined, + {}, + signal, + ); + const messages = await readPrometheusMessages(dependencies.connection.client, roomId, dependencies.connection.memberToken, signal); const protocolRoster = { coordinator: { memberId: run.roster!.coordinator.member_id }, builder: { memberId: run.roster!.builder.member_id }, @@ -556,7 +615,8 @@ async function snapshot(run: PrometheusRunState, dependencies: MonitorDependenci }; const candidate = parsePrometheusCandidate(state, protocolRoster); const receipts = candidate.ok ? parseTerminalReceipts(messages, protocolRoster, run.run_id, candidate.candidate.revisionSha256) : { ok: false as const, receipts: {}, missingRoles: [...PROMETHEUS_ROLES] }; - const runtime = await inspectPrometheusRuntime(dependencies.team, roomId, run.roster!, { docker: dependencies.docker, baseUrl: dependencies.connection.baseUrl }); + const docker = signal ? dockerWithSignal(dependencies.docker, signal) : dependencies.docker; + const runtime = await inspectPrometheusRuntime(dependencies.team, roomId, run.roster!, { docker, baseUrl: dependencies.connection.baseUrl }); return { state, candidate, receipts, runtime, barrier: evaluatePrometheusBarrier({ candidate, receipts, runtime }) }; } @@ -576,7 +636,7 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies if (current.cancel_requested || now.getTime() >= Date.parse(current.deadline_at)) { return (await stopOrRecover(current, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }))!; } - const lifecycle = await roomLifecycle(current, dependencies); + const lifecycle = await whileRunActive(current, dependencies, (signal) => roomLifecycle(current, { connection: dependencies.connection, signal })); if (lifecycle.state !== "open") { if (current.candidate && lifecycle.endedBy?.trigger === "prometheus_success" && lifecycle.endedBy.detail === current.candidate.revision_sha256) { return finishRecoveredSuccess(current, lease, dependencies); @@ -586,18 +646,19 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies const afterLifecycle = await reloadActive(current.run_id, dependencies); const stoppedAfterLifecycle = await stopOrRecover(afterLifecycle, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); if (stoppedAfterLifecycle) return stoppedAfterLifecycle; - const observed = await snapshot(afterLifecycle, dependencies); + const observed = await whileRunActive(afterLifecycle, dependencies, (signal) => snapshot(afterLifecycle, dependencies, signal)); const afterSnapshot = await reloadActive(current.run_id, dependencies); const stoppedAfterSnapshot = await stopOrRecover(afterSnapshot, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); if (stoppedAfterSnapshot) return stoppedAfterSnapshot; - const pinned = observed.candidate.ok - ? await materializePrometheusArtifactSnapshot( - observed.candidate.candidate.artifacts, - dependencies.connection.client, - dependencies.connection.memberToken, - join(dependencies.paths.runDir(afterSnapshot.run_id), "artifact-snapshots", observed.candidate.candidate.revisionSha256), - ) - : null; + const pinned = await whileRunActive(afterSnapshot, dependencies, async (signal) => observed.candidate.ok + ? await materializePrometheusArtifactSnapshot( + observed.candidate.candidate.artifacts, + dependencies.connection.client, + dependencies.connection.memberToken, + join(dependencies.paths.runDir(afterSnapshot.run_id), "artifact-snapshots", observed.candidate.candidate.revisionSha256), + signal, + ) + : null); let active = await reloadActive(current.run_id, dependencies); const stoppedAfterPin = await stopOrRecover(active, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); if (stoppedAfterPin) return stoppedAfterPin; @@ -605,13 +666,14 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies if (observed.candidate.ok && !observed.receipts.ok && !candidateAnnounced.has(observed.candidate.candidate.revisionSha256)) { const revision = observed.candidate.candidate.revisionSha256; const content = prometheusCandidateHandoff(active.run_id, active.roster!, observed.candidate.candidate, pinned!.sha256); - await dependencies.connection.client.request( + await whileRunActive(active, dependencies, (signal) => dependencies.connection.client.request( "POST", `/rooms/${encodeURIComponent(active.room!.room_id)}/owner-messages`, dependencies.connection.apiKey, { content, as: "runner" }, { "idempotency-key": `${active.run_id}-candidate-${revision}` }, - ); + signal, + )); candidateAnnounced.add(revision); active = await reloadActive(current.run_id, dependencies); const stoppedAfterAnnouncement = await stopOrRecover(active, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); @@ -654,18 +716,18 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies if (stoppedAfterCheckCheckpoint) return stoppedAfterCheckCheckpoint; if (checked.exitCode !== 0) { const feedback = `@${current.roster!.coordinator.member_id} Semantic check failed for revision ${revision}:\n${checked.output.slice(-2_000)}`; - await dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/owner-messages`, dependencies.connection.apiKey, { content: feedback, as: "runner" }, { "idempotency-key": `${current.run_id}-check-${revision}` }); + await whileRunActive(run, dependencies, (signal) => dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/owner-messages`, dependencies.connection.apiKey, { content: feedback, as: "runner" }, { "idempotency-key": `${current.run_id}-check-${revision}` }, signal)); feedbackAttempted.add(revision); } else { passingCheck = true; } } else if (prior.exit_code !== 0 && !feedbackAttempted.has(revision)) { const feedback = `@${current.roster!.coordinator.member_id} Semantic check failed for revision ${revision}:\n${prior.output.slice(-2_000)}`; - await dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/owner-messages`, dependencies.connection.apiKey, { content: feedback, as: "runner" }, { "idempotency-key": `${current.run_id}-check-${revision}` }); + await whileRunActive(active, dependencies, (signal) => dependencies.connection.client.request("POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/owner-messages`, dependencies.connection.apiKey, { content: feedback, as: "runner" }, { "idempotency-key": `${current.run_id}-check-${revision}` }, signal)); feedbackAttempted.add(revision); } if (passingCheck) { - const after = await snapshot(run, dependencies); + const after = await whileRunActive(run, dependencies, (signal) => snapshot(run, dependencies, signal)); const beforeSuccess = await reloadActive(current.run_id, dependencies); const stoppedBeforeSuccess = await stopOrRecover(beforeSuccess, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); if (stoppedBeforeSuccess) return stoppedBeforeSuccess; @@ -678,13 +740,14 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies run = await reloadActive(run.run_id, dependencies); const stoppedImmediatelyBeforeSuccess = await stopOrRecover(run, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); if (stoppedImmediatelyBeforeSuccess) return stoppedImmediatelyBeforeSuccess; - const closed = await dependencies.connection.client.request<{ room?: { state?: unknown; goal?: { ended_by?: { trigger?: unknown; detail?: unknown } | null } | null } }>( + const closed = await whileRunActive(run, dependencies, (signal) => dependencies.connection.client.request<{ room?: { state?: unknown; goal?: { ended_by?: { trigger?: unknown; detail?: unknown } | null } | null } }>( "POST", `/rooms/${encodeURIComponent(current.room!.room_id)}/close`, dependencies.connection.apiKey, { trigger: "prometheus_success", detail: revision }, { "idempotency-key": `${current.run_id}-close-success` }, - ); + signal, + )); if (closed.room?.state !== "closed" || closed.room.goal?.ended_by?.trigger !== "prometheus_success" || closed.room.goal.ended_by.detail !== revision) { throw localError("prometheus_success_unconfirmed", "SharedNet did not confirm the exact Prometheus success close."); } @@ -703,10 +766,12 @@ export async function monitorPrometheusRun(run: PrometheusRunState, dependencies // The close may have committed even when its response was lost or malformed. Keep the // candidate durable so resume can reconcile authoritative Room ended_by before any stop. if (failure.code === "prometheus_success_unconfirmed") throw error; + const current = await reloadActive(run.run_id, dependencies); + const stopped = await stopOrRecover(current, lease, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); + if (stopped) return stopped; // Transport and service failures do not prove the Room or run is invalid. // Keep the durable run active so a later `prometheus resume` can reclaim it. if (failure.exitCode === 5) throw failure; - const current = await reloadActive(run.run_id, dependencies); return finishFailedRun(current, lease, failure, { paths: dependencies.paths, now: dependencies.now, connection: dependencies.connection }); } } From 3020aabd91eab3569747dd1aaa6884bcc59517b3 Mon Sep 17 00:00:00 2001 From: Xisen-Wang Date: Mon, 19 Oct 2026 03:14:12 +0000 Subject: [PATCH 9/9] fix(cli): persist protocol candidate identifiers --- packages/cli/src/prometheus-state.ts | 5 +++-- packages/cli/src/prometheus.test.ts | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/packages/cli/src/prometheus-state.ts b/packages/cli/src/prometheus-state.ts index 4f6af48..32a7ebe 100644 --- a/packages/cli/src/prometheus-state.ts +++ b/packages/cli/src/prometheus-state.ts @@ -38,6 +38,7 @@ const TERMINAL_OUTCOMES = new Set([ ]); const HEX_SHA256 = /^[0-9a-f]{64}$/; const COMPILED_STATE_DIGEST = /^sha256:[0-9a-f]{24}$/; +const PROMETHEUS_WORK_ID = /^[A-Za-z0-9_.:-]{1,64}$/; const WORKSPACE_KEY = /^[0-9a-f]{24}$/; const SAFE_ID = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/; const ROOM_ID = /^rom_[A-Za-z0-9]+$/; @@ -390,8 +391,8 @@ function parseCandidate(value: unknown, source: string): PrometheusCandidateSumm ); return { compiled_digest: pattern(object.compiled_digest, COMPILED_STATE_DIGEST, source), - work_id: pattern(object.work_id, SAFE_ID, source), - result_ref: pattern(object.result_ref, SAFE_ID, source), + work_id: pattern(object.work_id, PROMETHEUS_WORK_ID, source), + result_ref: pattern(object.result_ref, COMPILED_STATE_DIGEST, source), revision_sha256: pattern(object.revision_sha256, HEX_SHA256, source), observed_at: timestamp(object.observed_at, source), }; diff --git a/packages/cli/src/prometheus.test.ts b/packages/cli/src/prometheus.test.ts index 5d3bf72..097f20d 100644 --- a/packages/cli/src/prometheus.test.ts +++ b/packages/cli/src/prometheus.test.ts @@ -14,8 +14,8 @@ import type { ProcessProbe } from "./process-identity.ts"; const candidate = { stateDigest: `sha256:${"a".repeat(24)}`, - workId: "work-1", - resultRef: "evt-1", + workId: "work:1", + resultRef: `sha256:${"5".repeat(24)}`, artifacts: [], verifications: [], revisionSha256: "b".repeat(64), @@ -311,7 +311,7 @@ describe("Prometheus lifecycle recovery", () => { const revision = "b".repeat(64); await writeRun(fixture.paths, { ...fixture.run, - candidate: { compiled_digest: `sha256:${"a".repeat(24)}`, work_id: "work-1", result_ref: "evt-1", revision_sha256: revision, observed_at: "2026-10-18T00:01:00.000Z" }, + candidate: { compiled_digest: `sha256:${"a".repeat(24)}`, work_id: "work:1", result_ref: `sha256:${"5".repeat(24)}`, revision_sha256: revision, observed_at: "2026-10-18T00:01:00.000Z" }, updated_at: "2026-10-18T00:01:00.000Z", }); await requestCancel(fixture.paths, fixture.run.run_id);