From 10e191fcd7cbf08d50e5134e197602e7421dc98f Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Tue, 22 Sep 2026 03:01:35 +0000
Subject: [PATCH 01/30] feat: Substrate workspace-runtime adapter contract
(fixture-backed)
Per-session WorkspaceBinding adapter over Substrate's actor control plane
(kubectl ate: create/get/resume/suspend/revert), replacing the fixed
workspace_namespace/workspace_pod pair for Substrate-backed sessions.
Mainloop persists the session<->actor mapping with ownership-generation
fencing (workspace_bindings table), reusing the SENDING-before-transport
and retry-after-inspect rules from contracts.py/native_sessions.py rather
than a new scheme. CRASHED is surfaced as a typed CapabilityResult, never
auto-reverted; revert_workspace requires explicit acknowledge_loss=True.
Fixture-only so far (no live cluster): transport parsing and the
retry-safe create/attach/surface_gap policy are unit tested against a
fake kubectl-ate subprocess, following test_herdr.py's pattern. The
DB-backed orchestration functions are not yet exercised against a live
cluster, matching native_sessions.py's own disclaimer in docs/specs.
---
backend/src/mainloop/config.py | 12 +
backend/src/mainloop/db/postgres.py | 27 ++
backend/src/mainloop/runtime/substrate.py | 236 ++++++++++++++
.../src/mainloop/runtime/workspace_adapter.py | 302 ++++++++++++++++++
backend/tests/runtime/test_substrate.py | 150 +++++++++
.../tests/runtime/test_workspace_adapter.py | 110 +++++++
6 files changed, 837 insertions(+)
create mode 100644 backend/src/mainloop/runtime/substrate.py
create mode 100644 backend/src/mainloop/runtime/workspace_adapter.py
create mode 100644 backend/tests/runtime/test_substrate.py
create mode 100644 backend/tests/runtime/test_workspace_adapter.py
diff --git a/backend/src/mainloop/config.py b/backend/src/mainloop/config.py
index 297e847..94814a7 100644
--- a/backend/src/mainloop/config.py
+++ b/backend/src/mainloop/config.py
@@ -37,6 +37,18 @@ def database_url(self) -> str:
"main-0" # pod that runs the native main thread (scratch cwd, no repo)
)
+ # Substrate workspace-runtime adapter (bounded integration spike; see
+ # docs/architecture/native-agent-inventory.md and .tasknotes/plan.md). Empty
+ # kubeconfig/context falls back to the ambient kubeconfig. One actor per session
+ # replaces the fixed workspace_namespace/workspace_pod pair above for Substrate-backed
+ # sessions; Herdr pod-exec keeps working unchanged for sessions that are not.
+ substrate_kubeconfig: str = ""
+ substrate_context: str = ""
+ substrate_atespace: str = "mainloop-workspaces"
+ substrate_actor_template: str = "mainloop-workspace"
+ substrate_cli: str = "kubectl-ate"
+ substrate_preview_base_url: str = ""
+
# Native main thread (context model). MAIN_THREAD_MODE=native replaces the SDK chat path.
main_thread_mode: str = "sdk" # sdk | native
main_thread_model: str = "sonnet"
diff --git a/backend/src/mainloop/db/postgres.py b/backend/src/mainloop/db/postgres.py
index 47a6405..810f1b7 100644
--- a/backend/src/mainloop/db/postgres.py
+++ b/backend/src/mainloop/db/postgres.py
@@ -223,6 +223,33 @@ def _parse_json_field(value: Any) -> list | dict | None:
CREATE UNIQUE INDEX IF NOT EXISTS idx_native_bindings_token ON native_bindings(token_hash) WHERE token_hash IS NOT NULL;
CREATE INDEX IF NOT EXISTS idx_native_bindings_parent ON native_bindings(parent_session_id);
+-- Substrate workspace-runtime adapter: durable mapping from a Mainloop session to a Substrate
+-- actor. Separate from native_bindings (the Herdr agent/native-session identity) because a
+-- session's workspace runtime is a distinct concept -- see ROADMAP.md "Workspace platform".
+-- One actor per session (workspace_id = session_id) replaces the fixed workspace pod for
+-- Substrate-backed sessions. ownership_generation fences resume/suspend/revert the same way
+-- native_bindings.generation fences Herdr sends: a stale caller's mutation is rejected, and a
+-- retry re-inspects the actor and this row rather than creating a second one.
+CREATE TABLE IF NOT EXISTS workspace_bindings (
+ workspace_id TEXT PRIMARY KEY REFERENCES sessions(id),
+ provider TEXT NOT NULL DEFAULT 'substrate',
+ atespace TEXT NOT NULL,
+ actor_name TEXT NOT NULL,
+ actor_template TEXT NOT NULL,
+ native_session_id TEXT,
+ preview_route TEXT,
+ runtime_endpoint TEXT,
+ observed_state TEXT NOT NULL DEFAULT 'unknown',
+ observed_at TIMESTAMPTZ,
+ external_snapshot_uri TEXT,
+ ownership_generation INTEGER NOT NULL DEFAULT 1,
+ desired_state TEXT NOT NULL DEFAULT 'active',
+ last_error TEXT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+ UNIQUE (atespace, actor_name)
+);
+
-- Topics are durable records (not sessions). Supervisors (next slice) attach to a topic.
CREATE TABLE IF NOT EXISTS topics (
id TEXT PRIMARY KEY,
diff --git a/backend/src/mainloop/runtime/substrate.py b/backend/src/mainloop/runtime/substrate.py
new file mode 100644
index 0000000..076a054
--- /dev/null
+++ b/backend/src/mainloop/runtime/substrate.py
@@ -0,0 +1,236 @@
+"""Thin Substrate adapter: drives ``kubectl ate`` (control-plane CLI over gRPC to
+``ate-api-server``) to manage per-session actors as Mainloop workspaces.
+
+Unlike ``herdr.py`` (pod-exec into an already-running workspace), this adapter talks to
+Substrate's cluster-level control plane: actors are created, suspended, resumed, reverted and
+deleted through ``ateapipb.Control`` (see the pinned checkout's ``pkg/proto/ateapipb/ateapi.proto``
+and ``cmd/kubectl-ate/internal/cmd/actor.go``). ``TransportError`` means the outcome of the call
+is unknown; callers must inspect the actor before retrying rather than blindly re-creating it.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import shlex
+from dataclasses import dataclass
+from enum import StrEnum
+
+from mainloop.config import settings
+
+logger = logging.getLogger(__name__)
+
+
+class TransportError(RuntimeError):
+ """The ``kubectl ate`` call failed or timed out; whether it took effect is unknown."""
+
+
+class ActorState(StrEnum):
+ """Mirrors ``ateapipb.ActorState``; unrecognized strings map to UNSPECIFIED."""
+
+ UNSPECIFIED = "ACTOR_STATE_UNSPECIFIED"
+ RESUMING = "ACTOR_STATE_RESUMING"
+ RUNNING = "ACTOR_STATE_RUNNING"
+ SUSPENDING = "ACTOR_STATE_SUSPENDING"
+ SUSPENDED = "ACTOR_STATE_SUSPENDED"
+ PAUSING = "ACTOR_STATE_PAUSING"
+ PAUSED = "ACTOR_STATE_PAUSED"
+ CRASHED = "ACTOR_STATE_CRASHED"
+ DELETING = "ACTOR_STATE_DELETING"
+ REVERTING = "ACTOR_STATE_REVERTING"
+
+ @classmethod
+ def parse(cls, raw: str | None) -> "ActorState":
+ try:
+ return cls(raw)
+ except ValueError:
+ return cls.UNSPECIFIED
+
+
+# ActorState -> WorkspaceBinding.observed_state (models.native_agent). Only RUNNING is ready;
+# terminal/absent states are unavailable; states mid-transition or unrecognized are unknown.
+OBSERVED_STATE = {
+ ActorState.RUNNING: "ready",
+ ActorState.SUSPENDED: "unavailable",
+ ActorState.PAUSED: "unavailable",
+ ActorState.CRASHED: "unavailable",
+ ActorState.DELETING: "unavailable",
+ ActorState.RESUMING: "unknown",
+ ActorState.SUSPENDING: "unknown",
+ ActorState.PAUSING: "unknown",
+ ActorState.REVERTING: "unknown",
+ ActorState.UNSPECIFIED: "unknown",
+}
+
+
+@dataclass(frozen=True, slots=True)
+class ExecResult:
+ exit_code: int
+ stdout: str
+ stderr: str
+
+
+@dataclass(frozen=True, slots=True)
+class ActorRecord:
+ """Parsed subset of an ``ateapipb.Actor`` (protojson via ``kubectl ate ... -o json``)."""
+
+ atespace: str
+ name: str
+ uid: str | None
+ state: ActorState
+ external_snapshot_uri: str | None
+ current_actor_template_uid: str | None
+ raw: dict
+
+
+def _actor_from_json(doc: dict) -> ActorRecord:
+ metadata = doc.get("metadata") or {}
+ status = doc.get("status") or {}
+ snapshot = status.get("externalSnapshot") or {}
+ return ActorRecord(
+ atespace=metadata.get("atespace", ""),
+ name=metadata.get("name", ""),
+ uid=metadata.get("uid"),
+ state=ActorState.parse(status.get("state")),
+ external_snapshot_uri=snapshot.get("snapshotUri"),
+ current_actor_template_uid=status.get("currentActorTemplateUid"),
+ raw=doc,
+ )
+
+
+class SubstrateControl:
+ """Wraps ``kubectl ate`` for one (kubeconfig, context) pair. No retries, no caching."""
+
+ def __init__(
+ self,
+ *,
+ kubeconfig: str | None = None,
+ context: str | None = None,
+ cli: str | None = None,
+ ):
+ self.kubeconfig = (
+ kubeconfig if kubeconfig is not None else settings.substrate_kubeconfig
+ )
+ self.context = context if context is not None else settings.substrate_context
+ self.cli = cli or settings.substrate_cli
+
+ def _base_args(self) -> list[str]:
+ args = [self.cli]
+ if self.kubeconfig:
+ args += ["--kubeconfig", self.kubeconfig]
+ if self.context:
+ args += ["--context", self.context]
+ return args
+
+ async def _exec(self, args: list[str], timeout: float = 45) -> ExecResult:
+ command = self._base_args() + args
+ try:
+ proc = await asyncio.create_subprocess_exec(
+ *command,
+ stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE,
+ )
+ try:
+ out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+ except TimeoutError as exc:
+ proc.kill()
+ await proc.wait()
+ raise TransportError(
+ f"{cli_quote(*command)} timed out after {timeout}s"
+ ) from exc
+ except OSError as exc:
+ raise TransportError(f"exec failed: {type(exc).__name__}: {exc}") from exc
+ if proc.returncode is None:
+ raise TransportError("kubectl ate did not report an exit status")
+ return ExecResult(
+ proc.returncode, out.decode(errors="replace"), err.decode(errors="replace")
+ )
+
+ async def get_actor(self, atespace: str, name: str) -> ActorRecord | None:
+ res = await self._exec(
+ ["get", "actor", name, "--atespace", atespace, "-o", "json"]
+ )
+ if res.exit_code != 0:
+ if "not found" in res.stderr.lower() or "NotFound" in res.stderr:
+ return None
+ raise TransportError(
+ f"get actor failed (exit {res.exit_code}): {res.stderr.strip()[-300:]}"
+ )
+ text = res.stdout.strip()
+ if not text:
+ return None
+ return _actor_from_json(json.loads(text))
+
+ async def create_actor(
+ self, atespace: str, name: str, *, template: str, tag: str | None = None
+ ) -> ActorRecord:
+ args = ["create", "actor", name, "--atespace", atespace, "--template", template]
+ if tag:
+ args += ["--tag", tag]
+ res = await self._exec(args, timeout=90)
+ if res.exit_code != 0:
+ raise RuntimeError(
+ f"create actor failed (exit {res.exit_code}): {res.stderr.strip()[-300:]}"
+ )
+ return await self._require(atespace, name, res)
+
+ async def resume_actor(self, atespace: str, name: str) -> ActorRecord:
+ res = await self._exec(
+ ["resume", "actor", name, "--atespace", atespace, "-o", "json"], timeout=90
+ )
+ if res.exit_code != 0:
+ raise RuntimeError(
+ f"resume actor failed (exit {res.exit_code}): {res.stderr.strip()[-300:]}"
+ )
+ return await self._require(atespace, name, res)
+
+ async def suspend_actor(self, atespace: str, name: str) -> ActorRecord:
+ res = await self._exec(
+ ["suspend", "actor", name, "--atespace", atespace, "-o", "json"], timeout=90
+ )
+ if res.exit_code != 0:
+ raise RuntimeError(
+ f"suspend actor failed (exit {res.exit_code}): {res.stderr.strip()[-300:]}"
+ )
+ return await self._require(atespace, name, res)
+
+ async def revert_actor(self, atespace: str, name: str) -> ActorRecord:
+ """Discard current execution, roll back to the last completed snapshot. Explicit only:
+ callers must have surfaced the possible loss of unsnapshotted work before calling this.
+ """
+ res = await self._exec(
+ ["revert", "actor", name, "--atespace", atespace, "-o", "json"], timeout=90
+ )
+ if res.exit_code != 0:
+ raise RuntimeError(
+ f"revert actor failed (exit {res.exit_code}): {res.stderr.strip()[-300:]}"
+ )
+ return await self._require(atespace, name, res)
+
+ async def delete_actor(
+ self, atespace: str, name: str, *, any_state: bool = False
+ ) -> None:
+ args = ["delete", "actor", name, "--atespace", atespace]
+ if any_state:
+ args.append("--any-state")
+ res = await self._exec(args, timeout=60)
+ if res.exit_code != 0 and "not found" not in res.stderr.lower():
+ raise RuntimeError(
+ f"delete actor failed (exit {res.exit_code}): {res.stderr.strip()[-300:]}"
+ )
+
+ async def _require(self, atespace: str, name: str, res: ExecResult) -> ActorRecord:
+ text = res.stdout.strip()
+ if text:
+ return _actor_from_json(json.loads(text))
+ # Some verbs may not echo the actor; read it back rather than guessing its state.
+ actor = await self.get_actor(atespace, name)
+ if actor is None:
+ raise TransportError(f"actor {atespace}/{name} not found after operation")
+ return actor
+
+
+def cli_quote(*parts: str) -> str:
+ """Only for logging/evidence; commands are passed as argv, never through a shell."""
+ return " ".join(shlex.quote(p) for p in parts)
diff --git a/backend/src/mainloop/runtime/workspace_adapter.py b/backend/src/mainloop/runtime/workspace_adapter.py
new file mode 100644
index 0000000..aa3eb64
--- /dev/null
+++ b/backend/src/mainloop/runtime/workspace_adapter.py
@@ -0,0 +1,302 @@
+"""Per-session Substrate workspace bindings: the durable mapping between a Mainloop session,
+a Substrate actor, its snapshot, cross-referenced native session id, and Mainloop's ownership
+generation for that actor. This is the adapter boundary described in ``.tasknotes/plan.md``:
+Mainloop owns creation intent, desired state, retry policy and audit; Substrate owns isolated
+actor compute and snapshots. Produces ``models.native_agent.WorkspaceBinding`` -- the existing
+contract type -- rather than a parallel workspace model.
+
+Rules carried over from ``native_sessions.py`` / ``contracts.py`` rather than reinvented:
+- Identity is persisted before an uncertain external call, and a retry re-inspects the actor and
+ this row before creating or mutating anything (no blind replay, no second writer).
+- Every mutation is fenced by ``ownership_generation``: a stale caller's write is rejected, not
+ silently applied.
+- ``CRASHED`` and revert are never automatic. Reverting is only reachable through
+ ``revert_workspace`` with an explicit acknowledgement that unsnapshotted work may be lost.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import logging
+from datetime import UTC, datetime
+
+from mainloop.config import settings
+from mainloop.db import db
+from mainloop.runtime.contracts import ContractError, StaleOwnership
+from mainloop.runtime.substrate import (
+ OBSERVED_STATE,
+ ActorRecord,
+ ActorState,
+ SubstrateControl,
+ TransportError,
+)
+
+from models import CapabilityResult, CapabilityState, WorkspaceBinding
+
+logger = logging.getLogger(__name__)
+
+_locks: dict[str, asyncio.Lock] = {}
+
+
+def _lock(session_id: str) -> asyncio.Lock:
+ return _locks.setdefault(session_id, asyncio.Lock())
+
+
+def actor_name(session_id: str) -> str:
+ return f"ml-{session_id[:16]}"
+
+
+def _control() -> SubstrateControl:
+ return SubstrateControl()
+
+
+async def get_workspace(session_id: str) -> dict | None:
+ async with db.connection() as conn:
+ row = await conn.fetchrow(
+ "SELECT * FROM workspace_bindings WHERE workspace_id=$1", session_id
+ )
+ return dict(row) if row else None
+
+
+def _binding_from_row(row: dict) -> WorkspaceBinding:
+ capabilities: tuple[CapabilityResult, ...] = ()
+ if row["observed_state"] == "unavailable" and row["last_error"]:
+ # A row-level note (e.g. "actor missing", "actor CRASHED") becomes a typed capability
+ # result rather than free text on the contract model, per the proved/partial evidence rule.
+ note = row["last_error"]
+ capabilities = (
+ CapabilityResult(
+ capability=(
+ "actor_crashed" if "CRASHED" in note.upper() else "actor_health"
+ ),
+ state=CapabilityState.PROVED,
+ # SubstrateControl always talks to a real (possibly Kind) cluster, so this
+ # in-binding health signal is "live" by construction. The four plan-mandated
+ # integration-gate CapabilityResults are separate records in the proof note,
+ # scored "fixture" or "live" by whatever trial produced their evidence_ref.
+ scope="live",
+ evidence_ref=f"substrate://{row['atespace']}/{row['actor_name']}",
+ detail=note,
+ ),
+ )
+ return WorkspaceBinding(
+ workspace_id=row["workspace_id"],
+ runtime_endpoint=row["runtime_endpoint"] or row["preview_route"] or "unrouted",
+ observed_at=row["observed_at"] or row["updated_at"],
+ observed_state=row["observed_state"],
+ capabilities=capabilities,
+ )
+
+
+async def _insert_row(
+ session_id: str,
+ atespace: str,
+ name: str,
+ template: str,
+ actor: ActorRecord,
+ now: datetime,
+) -> None:
+ route = f"{atespace}/{name}"
+ async with db.connection() as conn:
+ await conn.execute(
+ """INSERT INTO workspace_bindings
+ (workspace_id, atespace, actor_name, actor_template, preview_route,
+ runtime_endpoint, observed_state, observed_at, external_snapshot_uri)
+ VALUES ($1,$2,$3,$4,$5,$5,$6,$7,$8)""",
+ session_id,
+ atespace,
+ name,
+ template,
+ route,
+ OBSERVED_STATE[actor.state],
+ now,
+ actor.external_snapshot_uri,
+ )
+
+
+CRASHED_NOTE = (
+ "actor CRASHED: it stopped running and lost anything since its last completed snapshot. "
+ "Substrate's actor record has no snapshot timestamp, so snapshot age cannot be reported here. "
+ "Resume is rejected in this state; only an explicit revert_workspace(acknowledge_loss=True) "
+ "restores it, discarding any unsnapshotted work."
+)
+
+
+async def _update_observed(
+ session_id: str, actor: ActorRecord, now: datetime, *, last_error: str | None = None
+) -> None:
+ if last_error is None and actor.state == ActorState.CRASHED:
+ last_error = CRASHED_NOTE
+ async with db.connection() as conn:
+ await conn.execute(
+ """UPDATE workspace_bindings
+ SET observed_state=$2, observed_at=$3, external_snapshot_uri=$4,
+ last_error=$5, updated_at=NOW()
+ WHERE workspace_id=$1""",
+ session_id,
+ OBSERVED_STATE[actor.state],
+ now,
+ actor.external_snapshot_uri,
+ last_error,
+ )
+
+
+async def _mark_missing(session_id: str, now: datetime) -> None:
+ async with db.connection() as conn:
+ await conn.execute(
+ """UPDATE workspace_bindings
+ SET observed_state='unavailable', observed_at=$2, last_error=$3, updated_at=NOW()
+ WHERE workspace_id=$1""",
+ session_id,
+ now,
+ "actor not found where this row expected one; not recreated automatically "
+ "(inspect and reconcile explicitly, or delete this row to allow a fresh actor)",
+ )
+
+
+async def _bump_generation(session_id: str, expected: int) -> None:
+ async with db.connection() as conn:
+ tag = await conn.execute(
+ """UPDATE workspace_bindings SET ownership_generation=ownership_generation+1,
+ updated_at=NOW() WHERE workspace_id=$1 AND ownership_generation=$2""",
+ session_id,
+ expected,
+ )
+ if tag.endswith(" 0"):
+ raise StaleOwnership(
+ f"workspace_bindings.{session_id} is no longer at generation {expected}"
+ )
+
+
+def plan_ensure(row_exists: bool, actor_found: bool) -> str:
+ """Retry-safe provision-or-attach policy, factored out of ``ensure_workspace`` so it is
+ directly testable without a database or cluster.
+
+ ``create``: no row and no actor -- first provision. ``attach``: an actor already exists
+ (whether or not this process created it), so observe it rather than creating another.
+ ``surface_gap``: this row believes it owns an actor that Substrate no longer has -- never
+ silently recreate under the same name; a human or a later explicit call must reconcile it.
+ """
+ if actor_found:
+ return "attach"
+ return "create" if not row_exists else "surface_gap"
+
+
+async def ensure_workspace(
+ session_id: str, *, control: SubstrateControl | None = None
+) -> WorkspaceBinding:
+ """Idempotent provision-or-attach. Never creates a second actor for a row that already
+ believes it owns one; a gap between this row and Substrate's view is surfaced, not papered
+ over."""
+ control = control or _control()
+ async with _lock(session_id):
+ row = await get_workspace(session_id)
+ atespace = settings.substrate_atespace
+ template = settings.substrate_actor_template
+ name = row["actor_name"] if row else actor_name(session_id)
+ try:
+ actor = await control.get_actor(atespace, name)
+ except TransportError:
+ if row is not None:
+ return _binding_from_row(
+ row
+ ) # unreachable now; last known state stands
+ raise
+ now = datetime.now(UTC)
+ action = plan_ensure(row is not None, actor is not None)
+ if action == "surface_gap":
+ await _mark_missing(session_id, now)
+ return _binding_from_row(await get_workspace(session_id)) # type: ignore[arg-type]
+ if action == "create":
+ actor = await control.create_actor(atespace, name, template=template)
+ await _insert_row(session_id, atespace, name, template, actor, now)
+ else:
+ await _update_observed(session_id, actor, now)
+ return _binding_from_row(await get_workspace(session_id)) # type: ignore[arg-type]
+
+
+async def observe_workspace(
+ session_id: str, *, control: SubstrateControl | None = None
+) -> WorkspaceBinding | None:
+ """Read-only reconcile: refresh observed_state/observed_at without changing desired state."""
+ control = control or _control()
+ row = await get_workspace(session_id)
+ if row is None:
+ return None
+ now = datetime.now(UTC)
+ try:
+ actor = await control.get_actor(row["atespace"], row["actor_name"])
+ except TransportError as exc:
+ logger.info("workspace observe skipped for %s: %s", session_id, exc)
+ return _binding_from_row(row)
+ if actor is None:
+ await _mark_missing(session_id, now)
+ else:
+ await _update_observed(session_id, actor, now)
+ return _binding_from_row(await get_workspace(session_id)) # type: ignore[arg-type]
+
+
+async def resume_workspace(
+ session_id: str, *, control: SubstrateControl | None = None
+) -> WorkspaceBinding:
+ control = control or _control()
+ async with _lock(session_id):
+ row = await get_workspace(session_id)
+ if row is None:
+ raise ContractError(f"no workspace binding for session {session_id}")
+ actor = await control.resume_actor(row["atespace"], row["actor_name"])
+ now = datetime.now(UTC)
+ await _update_observed(session_id, actor, now)
+ await _bump_generation(session_id, row["ownership_generation"])
+ return _binding_from_row(await get_workspace(session_id)) # type: ignore[arg-type]
+
+
+async def suspend_workspace(
+ session_id: str, *, control: SubstrateControl | None = None
+) -> WorkspaceBinding:
+ control = control or _control()
+ async with _lock(session_id):
+ row = await get_workspace(session_id)
+ if row is None:
+ raise ContractError(f"no workspace binding for session {session_id}")
+ actor = await control.suspend_actor(row["atespace"], row["actor_name"])
+ now = datetime.now(UTC)
+ await _update_observed(session_id, actor, now)
+ await _bump_generation(session_id, row["ownership_generation"])
+ return _binding_from_row(await get_workspace(session_id)) # type: ignore[arg-type]
+
+
+async def revert_workspace(
+ session_id: str, *, acknowledge_loss: bool, control: SubstrateControl | None = None
+) -> WorkspaceBinding:
+ """Roll back to the actor's last completed snapshot, discarding any unsnapshotted work.
+ Never called implicitly by this module -- the caller (API layer) must have shown the user
+ the actor is CRASHED (or otherwise irrecoverable) and gotten explicit confirmation first.
+ """
+ if not acknowledge_loss:
+ raise ContractError(
+ "revert_workspace requires acknowledge_loss=True: it discards unsnapshotted work"
+ )
+ control = control or _control()
+ async with _lock(session_id):
+ row = await get_workspace(session_id)
+ if row is None:
+ raise ContractError(f"no workspace binding for session {session_id}")
+ actor = await control.revert_actor(row["atespace"], row["actor_name"])
+ now = datetime.now(UTC)
+ await _update_observed(
+ session_id,
+ actor,
+ now,
+ last_error="reverted to last completed snapshot; any unsnapshotted work was lost",
+ )
+ await _bump_generation(session_id, row["ownership_generation"])
+ return _binding_from_row(await get_workspace(session_id)) # type: ignore[arg-type]
+
+
+def is_crashed(binding_row: dict) -> bool:
+ return (
+ binding_row["observed_state"] == "unavailable"
+ and binding_row.get("last_error") is not None
+ and "CRASHED" in (binding_row.get("last_error") or "").upper()
+ )
diff --git a/backend/tests/runtime/test_substrate.py b/backend/tests/runtime/test_substrate.py
new file mode 100644
index 0000000..66fb1a3
--- /dev/null
+++ b/backend/tests/runtime/test_substrate.py
@@ -0,0 +1,150 @@
+"""Substrate transport adapter over a fake ``kubectl ate`` subprocess: no cluster, no actors,
+no credentials. Mirrors test_herdr.py's fake-transport pattern for the Herdr adapter."""
+
+import asyncio
+import unittest
+
+from mainloop.runtime.substrate import (
+ ActorState,
+ ExecResult,
+ SubstrateControl,
+ TransportError,
+ _actor_from_json,
+)
+
+
+class FakeControl(SubstrateControl):
+ def __init__(self, results):
+ super().__init__(
+ kubeconfig="fixture-kubeconfig", context="kind-substrate-preview"
+ )
+ self.results = list(results)
+ self.calls: list[list[str]] = []
+
+ async def _exec(self, args, timeout=45):
+ self.calls.append(args)
+ result = self.results.pop(0)
+ if isinstance(result, Exception):
+ raise result
+ return result
+
+
+def run(coro):
+ return asyncio.run(coro)
+
+
+def actor_json(state: str, *, snapshot_uri: str | None = None) -> str:
+ doc = {
+ "metadata": {"atespace": "mainloop-workspaces", "name": "ml-abc", "uid": "u-1"},
+ "status": {"state": state},
+ }
+ if snapshot_uri:
+ doc["status"]["externalSnapshot"] = {"snapshotUri": snapshot_uri}
+ import json
+
+ return json.dumps(doc)
+
+
+class ActorJsonParsingTests(unittest.TestCase):
+ def test_parses_running_actor_with_snapshot(self):
+ import json
+
+ doc = json.loads(actor_json("ACTOR_STATE_RUNNING", snapshot_uri="gs://b/p"))
+ record = _actor_from_json(doc)
+ self.assertEqual(record.atespace, "mainloop-workspaces")
+ self.assertEqual(record.name, "ml-abc")
+ self.assertEqual(record.state, ActorState.RUNNING)
+ self.assertEqual(record.external_snapshot_uri, "gs://b/p")
+
+ def test_unrecognized_state_string_is_unspecified_not_a_crash(self):
+ record = _actor_from_json(
+ {"metadata": {}, "status": {"state": "SOME_FUTURE_STATE"}}
+ )
+ self.assertEqual(record.state, ActorState.UNSPECIFIED)
+
+ def test_missing_status_defaults_to_unspecified(self):
+ record = _actor_from_json({"metadata": {"atespace": "a", "name": "n"}})
+ self.assertEqual(record.state, ActorState.UNSPECIFIED)
+ self.assertIsNone(record.external_snapshot_uri)
+
+
+class SubstrateControlTests(unittest.TestCase):
+ def test_get_actor_parses_json_and_uses_argv_not_shell(self):
+ ctl = FakeControl([ExecResult(0, actor_json("ACTOR_STATE_RUNNING"), "")])
+ actor = run(ctl.get_actor("mainloop-workspaces", "ml-abc"))
+ self.assertEqual(actor.state, ActorState.RUNNING)
+ self.assertEqual(
+ ctl.calls[0],
+ [
+ "get",
+ "actor",
+ "ml-abc",
+ "--atespace",
+ "mainloop-workspaces",
+ "-o",
+ "json",
+ ],
+ )
+
+ def test_get_actor_not_found_returns_none_not_an_exception(self):
+ ctl = FakeControl(
+ [ExecResult(1, "", 'Error: actors.ate.dev "ml-abc" not found')]
+ )
+ self.assertIsNone(run(ctl.get_actor("mainloop-workspaces", "ml-abc")))
+
+ def test_get_actor_other_failure_raises_transport_error(self):
+ ctl = FakeControl([ExecResult(1, "", "connection refused")])
+ with self.assertRaises(TransportError):
+ run(ctl.get_actor("mainloop-workspaces", "ml-abc"))
+
+ def test_create_actor_uses_template_flag(self):
+ ctl = FakeControl([ExecResult(0, actor_json("ACTOR_STATE_RESUMING"), "")])
+ run(
+ ctl.create_actor(
+ "mainloop-workspaces", "ml-abc", template="mainloop-workspace"
+ )
+ )
+ self.assertEqual(
+ ctl.calls[0],
+ [
+ "create",
+ "actor",
+ "ml-abc",
+ "--atespace",
+ "mainloop-workspaces",
+ "--template",
+ "mainloop-workspace",
+ ],
+ )
+
+ def test_create_actor_falls_back_to_a_read_when_output_is_empty(self):
+ ctl = FakeControl(
+ [
+ ExecResult(0, "", ""),
+ ExecResult(0, actor_json("ACTOR_STATE_RESUMING"), ""),
+ ]
+ )
+ actor = run(ctl.create_actor("mainloop-workspaces", "ml-abc", template="t"))
+ self.assertEqual(actor.state, ActorState.RESUMING)
+ self.assertEqual(ctl.calls[1][:2], ["get", "actor"])
+
+ def test_revert_is_a_single_explicit_call_never_a_retry_loop(self):
+ ctl = FakeControl([ExecResult(0, actor_json("ACTOR_STATE_SUSPENDED"), "")])
+ actor = run(ctl.revert_actor("mainloop-workspaces", "ml-abc"))
+ self.assertEqual(actor.state, ActorState.SUSPENDED)
+ self.assertEqual(len(ctl.calls), 1)
+ self.assertEqual(ctl.calls[0][:2], ["revert", "actor"])
+
+ def test_delete_actor_tolerates_already_gone(self):
+ ctl = FakeControl([ExecResult(1, "", "not found")])
+ run(ctl.delete_actor("mainloop-workspaces", "ml-abc")) # does not raise
+
+ def test_transport_error_on_transient_failure_is_not_retried(self):
+ ctl = FakeControl([TransportError("boom")])
+ with self.assertRaises(TransportError):
+ run(ctl.suspend_actor("mainloop-workspaces", "ml-abc"))
+ self.assertEqual(len(ctl.calls), 1)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/backend/tests/runtime/test_workspace_adapter.py b/backend/tests/runtime/test_workspace_adapter.py
new file mode 100644
index 0000000..ffcb281
--- /dev/null
+++ b/backend/tests/runtime/test_workspace_adapter.py
@@ -0,0 +1,110 @@
+"""Pure policy and projection functions from workspace_adapter.py, tested without a database or
+cluster -- the same split as test_session_status.py covers for native_sessions.py's status rules.
+The DB-backed orchestration functions (ensure_workspace, resume_workspace, ...) are not yet
+exercised against a live cluster; see docs/spikes and the task proof note."""
+
+import unittest
+from datetime import UTC, datetime
+
+from mainloop.runtime.workspace_adapter import (
+ CRASHED_NOTE,
+ _binding_from_row,
+ actor_name,
+ is_crashed,
+ plan_ensure,
+)
+
+from models import CapabilityState
+
+NOW = datetime(2026, 9, 22, tzinfo=UTC)
+
+
+def row(**overrides):
+ base = {
+ "workspace_id": "sess-1",
+ "atespace": "mainloop-workspaces",
+ "actor_name": "ml-sess1",
+ "actor_template": "mainloop-workspace",
+ "preview_route": "mainloop-workspaces/ml-sess1",
+ "runtime_endpoint": "mainloop-workspaces/ml-sess1",
+ "observed_state": "ready",
+ "observed_at": NOW,
+ "updated_at": NOW,
+ "last_error": None,
+ }
+ base.update(overrides)
+ return base
+
+
+class ActorNameTests(unittest.TestCase):
+ def test_stable_and_namespace_safe(self):
+ name = actor_name("0123456789abcdefextra")
+ self.assertEqual(name, "ml-0123456789abcdef")
+ self.assertEqual(name, actor_name("0123456789abcdefextra")) # deterministic
+
+
+class PlanEnsureTests(unittest.TestCase):
+ def test_first_provision_creates(self):
+ self.assertEqual(plan_ensure(row_exists=False, actor_found=False), "create")
+
+ def test_existing_actor_is_attached_whether_or_not_we_have_a_row(self):
+ self.assertEqual(plan_ensure(row_exists=False, actor_found=True), "attach")
+ self.assertEqual(plan_ensure(row_exists=True, actor_found=True), "attach")
+
+ def test_a_row_with_no_matching_actor_is_surfaced_not_recreated(self):
+ self.assertEqual(plan_ensure(row_exists=True, actor_found=False), "surface_gap")
+
+
+class BindingProjectionTests(unittest.TestCase):
+ def test_ready_row_has_no_capability_noise(self):
+ binding = _binding_from_row(row())
+ self.assertEqual(binding.observed_state, "ready")
+ self.assertEqual(binding.capabilities, ())
+
+ def test_crashed_row_surfaces_a_proved_actor_crashed_capability(self):
+ binding = _binding_from_row(
+ row(observed_state="unavailable", last_error=CRASHED_NOTE)
+ )
+ self.assertEqual(len(binding.capabilities), 1)
+ cap = binding.capabilities[0]
+ self.assertEqual(cap.capability, "actor_crashed")
+ self.assertEqual(cap.state, CapabilityState.PROVED)
+ self.assertEqual(cap.scope, "live")
+ self.assertIn("CRASHED", cap.detail)
+ self.assertIn("Substrate's actor record has no snapshot timestamp", cap.detail)
+
+ def test_missing_actor_row_surfaces_a_distinct_capability_from_crashed(self):
+ binding = _binding_from_row(
+ row(
+ observed_state="unavailable",
+ last_error="actor not found where this row expected one; not recreated "
+ "automatically (inspect and reconcile explicitly, or delete this row to allow "
+ "a fresh actor)",
+ )
+ )
+ self.assertEqual(binding.capabilities[0].capability, "actor_health")
+
+ def test_runtime_endpoint_falls_back_to_preview_route(self):
+ binding = _binding_from_row(row(runtime_endpoint=None))
+ self.assertEqual(binding.runtime_endpoint, "mainloop-workspaces/ml-sess1")
+
+ def test_runtime_endpoint_falls_back_to_unrouted_before_first_provision_observation(
+ self,
+ ):
+ binding = _binding_from_row(row(runtime_endpoint=None, preview_route=None))
+ self.assertEqual(binding.runtime_endpoint, "unrouted")
+
+
+class IsCrashedTests(unittest.TestCase):
+ def test_true_only_for_the_crashed_note(self):
+ self.assertTrue(
+ is_crashed(row(observed_state="unavailable", last_error=CRASHED_NOTE))
+ )
+ self.assertFalse(
+ is_crashed(row(observed_state="unavailable", last_error="actor missing"))
+ )
+ self.assertFalse(is_crashed(row()))
+
+
+if __name__ == "__main__":
+ unittest.main()
From e4bf2dda5ec9daf37e22d4bf1b950fc62079480a Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Tue, 22 Sep 2026 03:14:06 +0000
Subject: [PATCH 02/30] fix: create_actor was missing -o json; add Substrate
ActorTemplate manifest
Live testing against a real kind-substrate-preview cluster (pinned
commit cdac9baef81dd319b46086d695266e6161e9e592) found create_actor
parsing kubectl-ate's default table output as JSON and crashing.
Fixed by passing -o json like every other verb; test fixture updated
to match the real post-create state (SUSPENDED, not an assumed
RESUMING).
Adds spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl:
a WorkerPool + protojson ActorTemplate for a per-session Herdr +
agentctl actor (SNAPSHOT_CONTENT_SCOPE_FULL on pause/commit), reusing
spikes/k8s-herdr-agents' image contents. Verified live: actor
create/get/resume/suspend/revert/delete all work through this
adapter's real code path; force-killing a worker pod drives the actor
to CRASHED (mapped to WorkspaceBinding.observed_state=unavailable),
and explicit revert recovers it to SUSPENDED from the golden snapshot,
after which resume brings it back to RUNNING. Full results and
commands are in the task's proof note (.tasknotes, not tracked here).
---
backend/src/mainloop/runtime/substrate.py | 12 ++-
backend/tests/runtime/test_substrate.py | 6 +-
.../k8s/actor-template.yaml.tmpl | 76 +++++++++++++++++++
3 files changed, 92 insertions(+), 2 deletions(-)
create mode 100644 spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl
diff --git a/backend/src/mainloop/runtime/substrate.py b/backend/src/mainloop/runtime/substrate.py
index 076a054..5ba0dad 100644
--- a/backend/src/mainloop/runtime/substrate.py
+++ b/backend/src/mainloop/runtime/substrate.py
@@ -165,7 +165,17 @@ async def get_actor(self, atespace: str, name: str) -> ActorRecord | None:
async def create_actor(
self, atespace: str, name: str, *, template: str, tag: str | None = None
) -> ActorRecord:
- args = ["create", "actor", name, "--atespace", atespace, "--template", template]
+ args = [
+ "create",
+ "actor",
+ name,
+ "--atespace",
+ atespace,
+ "--template",
+ template,
+ "-o",
+ "json",
+ ]
if tag:
args += ["--tag", tag]
res = await self._exec(args, timeout=90)
diff --git a/backend/tests/runtime/test_substrate.py b/backend/tests/runtime/test_substrate.py
index 66fb1a3..51e3997 100644
--- a/backend/tests/runtime/test_substrate.py
+++ b/backend/tests/runtime/test_substrate.py
@@ -98,7 +98,9 @@ def test_get_actor_other_failure_raises_transport_error(self):
run(ctl.get_actor("mainloop-workspaces", "ml-abc"))
def test_create_actor_uses_template_flag(self):
- ctl = FakeControl([ExecResult(0, actor_json("ACTOR_STATE_RESUMING"), "")])
+ # A freshly created actor starts SUSPENDED (never auto-started); measured against a
+ # live kind-substrate-preview cluster while building this adapter.
+ ctl = FakeControl([ExecResult(0, actor_json("ACTOR_STATE_SUSPENDED"), "")])
run(
ctl.create_actor(
"mainloop-workspaces", "ml-abc", template="mainloop-workspace"
@@ -114,6 +116,8 @@ def test_create_actor_uses_template_flag(self):
"mainloop-workspaces",
"--template",
"mainloop-workspace",
+ "-o",
+ "json",
],
)
diff --git a/spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl
new file mode 100644
index 0000000..f4542e2
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl
@@ -0,0 +1,76 @@
+# WorkerPool (a real K8s CRD, `kubectl apply`) plus the Mainloop workspace ActorTemplate
+# (protojson-shaped ateapipb.ActorTemplate, applied with `kubectl ate create actor-template -f -`
+# after the __IMAGE__ placeholder is substituted with a digest-pinned reference -- Substrate
+# requires containers.image to be pinned by digest). Mirrors the shape of the pinned checkout's
+# demos/counter/{counter,counter-template}.yaml.tmpl.
+#
+# The atespace ("mainloop-workspaces" by default; see backend/src/mainloop/config.py
+# substrate_atespace) is both this WorkerPool's k8s namespace and a control-plane atespace
+# resource created separately with `kubectl ate create atespace`.
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: ${ATESPACE}
+---
+apiVersion: ate.dev/v1alpha1
+kind: WorkerPool
+metadata:
+ name: ${TEMPLATE_NAME}
+ namespace: ${ATESPACE}
+ labels:
+ workload: ${TEMPLATE_NAME}
+spec:
+ replicas: 2
+ workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
+ template:
+ resources:
+ limits:
+ cpu: '2'
+ memory: 2Gi
+ requests:
+ cpu: 250m
+ memory: 2Gi
+---
+# ActorTemplate: one Herdr + agentctl container per actor, same image and entrypoint as
+# spikes/k8s-herdr-agents (see that spike's Dockerfile/bin/entrypoint.sh), running under
+# Substrate instead of a StatefulSet. snapshotsConfig captures full process memory on
+# suspend/commit so a resumed actor's native agent session (Claude/Codex under Herdr) continues
+# rather than cold-booting -- this is the behavior gate 5 (native-session continuity) measures.
+metadata:
+ atespace: ${ATESPACE}
+ name: ${TEMPLATE_NAME}
+workerSelector:
+ matchLabels:
+ workload: ${TEMPLATE_NAME}
+containers:
+- name: workspace
+ image: __IMAGE__
+ command:
+ - /usr/local/bin/entrypoint.sh
+ env:
+ - { name: HOME, value: /workspace/.home }
+ - { name: WORKSPACE_PATH, value: /workspace/repo }
+ - { name: STANDIN_STATE_DIR, value: /workspace/.standin }
+ - { name: HERDR_SESSION, value: mainloop-substrate }
+ - { name: AGENT_CONFIG_DIR, value: /etc/agent-config }
+ - { name: CODEX_HOME, value: /workspace/.codex }
+ volumeMounts:
+ - { name: workspace, mountPath: /workspace }
+ # ateapipb.SecurityContext only models Linux capability adjustments (no
+ # allowPrivilegeEscalation/readOnlyRootFilesystem -- the gVisor sandbox is the isolation
+ # boundary here, not those pod-level knobs), so there is nothing to set beyond the container's
+ # own non-root USER (see the image's Dockerfile).
+ resources:
+ limits:
+ - { name: cpu, quantity: "2" }
+ - { name: memory, quantity: 2Gi }
+snapshotsConfig:
+ onPause: SNAPSHOT_CONTENT_SCOPE_FULL
+ onCommit: SNAPSHOT_CONTENT_SCOPE_FULL
+ storageLocation: gs://${BUCKET_NAME}/mainloop-workspaces/
+sandboxConfig:
+ sandboxClass: SANDBOX_CLASS_GVISOR
+ configName: gvisor-default
+volumes:
+- name: workspace
+ durableDir: {}
From 032277fbf4e7349cfe43fdb3911ccc5dad78b509 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Tue, 22 Sep 2026 03:15:54 +0000
Subject: [PATCH 03/30] docs: record the Substrate workspace-adapter spike
Durable record of the cluster-lane findings (docs/spikes/, mirroring
k8s-herdr-agents.md's format): agentgateway dataplane fix, ActorTemplate
corrections (digest-pinned images, SecurityContext field set, ko resolve
before apply), live actor lifecycle and CRASHED/revert behavior, and
what gates 3-5 still need. The full proof note with exact commands and
CapabilityResult table lives in the task's .tasknotes (not tracked).
---
docs/spikes/substrate-workspace-adapter.md | 91 ++++++++++++++++++++++
1 file changed, 91 insertions(+)
create mode 100644 docs/spikes/substrate-workspace-adapter.md
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
new file mode 100644
index 0000000..9d00f1d
--- /dev/null
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -0,0 +1,91 @@
+# Spike: Substrate as Mainloop's Kubernetes workspace runtime
+
+Status: local spike, not a product feature. Adapter code lives in
+`backend/src/mainloop/runtime/substrate.py` and `workspace_adapter.py`; the actor manifest
+lives in `spikes/substrate-workspace-adapter/`. See `docs/spikes/k8s-herdr-agents.md` for the
+native-session/Herdr spike this one builds on and does not replace.
+
+## What it shows
+
+[Substrate](https://github.com/agent-substrate/substrate) can provide the per-session isolated
+compute Mainloop's roadmap calls for ("Workspace platform"), while Mainloop stays the durable
+owner of the session<->actor mapping, delivery, and audit state. A Mainloop-authored
+`ActorTemplate` (Herdr + `agentctl`, the same image contents as the Herdr spike) runs as a
+Substrate actor instead of a fixed StatefulSet pod, and `backend/src/mainloop/runtime/substrate.py`
+drives its lifecycle through the real `kubectl ate` control-plane CLI.
+
+## Real versus stand-in
+
+| Layer | Status |
+| --- | --- |
+| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
+| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
+| Herdr + `agentctl` inside the actor image | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
+| Claude/Codex agent processes, credentials | Not run in this spike (see "Not attempted") |
+| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
+
+## Run it
+
+There is no single demo script yet (unlike `spikes/k8s-herdr-agents/demo.sh`); the commands used
+are recorded in the task's proof note. In outline:
+
+```bash
+KIND_CLUSTER_NAME=substrate-preview KUBECONFIG=/tmp/substrate-preview-kubeconfig \
+ /tmp/substrate-preview-src/hack/create-kind-cluster.sh
+KIND_CLUSTER_NAME=substrate-preview KUBECTL_CONTEXT=kind-substrate-preview \
+ KUBECONFIG=/tmp/substrate-preview-kubeconfig \
+ /tmp/substrate-preview-src/hack/install-ate-kind.sh --deploy-ate-system
+KIND_CLUSTER_NAME=substrate-preview KUBECTL_CONTEXT=kind-substrate-preview \
+ KUBECONFIG=/tmp/substrate-preview-kubeconfig \
+ /tmp/substrate-preview-src/hack/install-ate-kind.sh --deploy-atenet --atenet-dataplane=agentgateway
+# build kubectl-ate, build+push the actor image, apply spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl
+# (WorkerPool via `ko resolve | kubectl apply`, ActorTemplate via `kubectl ate create actor-template -f -`)
+```
+
+## Observed behaviour
+
+- The default Envoy-based `atenet-router` crash-looped on this cluster too (matching the prior
+ `docs/spikes/../substrate-kind-preview-proof` finding); the `agentgateway` dataplane fixed it.
+- A freshly created actor starts `SUSPENDED`, not running -- `create_actor` never implicitly
+ starts an actor. An explicit `resume_actor` is required, and it returned `RUNNING` directly
+ (no further polling needed) in every observed case.
+- Force-deleting an actor's worker pod (`kubectl delete pod ... --grace-period=0 --force`, the
+ same technique as the prior proof's "abrupt worker loss" trial) drove the actor to `CRASHED`
+ within a few seconds, correctly observed as `WorkspaceBinding.observed_state="unavailable"`
+ through `substrate.py`'s real `ActorState` -> `observed_state` mapping.
+- `revert_actor` on a `CRASHED` actor returned it to `SUSPENDED` from its last completed (here:
+ golden) snapshot; a subsequent `resume_actor` brought it back to `RUNNING`. Nothing in the
+ adapter reverts automatically -- `workspace_adapter.revert_workspace` requires
+ `acknowledge_loss=True`.
+- Actor logs for a resume showed gVisor's `runsc ... restore -image-path ... restore-state`
+ path (`"Actor restoring"` / `"Actor restored"`), not a fresh container boot -- consistent with
+ the golden snapshot's process state (including the running `herdr` server) being restored
+ rather than the entrypoint re-running. This is supporting evidence for gate 5 (native-session
+ continuity) but not a full proof: no real agent session was resumed and asked to recall a
+ pre-suspend nonce in this run.
+- Building this adapter against the real CLI found one bug fixed in the same commit:
+ `create_actor` was missing `-o json` and crashed parsing `kubectl ate`'s default table output.
+
+## Limits / not attempted in this run
+
+- **Preview/HMR gate**: no Vite actor, no authenticated fixed-header proxy, no `agent-browser`
+ trial. The prior `substrate-kind-preview-proof` note already showed this works and does not
+ always work reliably (10-30s stalls with an HMR socket open); this run did not repeat or
+ extend that measurement against Mainloop's own actor template.
+- **Dev-service gate**: no Postgres actor, no egress policy, no reconnect-after-wake trial against
+ our template.
+- **Native-session gate, live**: no real Claude/Codex session was started inside a Substrate
+ actor; the credential wiring authorized by `.tasknotes/plan.md` was not used. Only the
+ fixture-level contract logic (`test_workspace_adapter.py`) and the generic
+ restore-vs-reboot log evidence above are available.
+- **`workspace_bindings` orchestration functions** (`ensure_workspace`, `resume_workspace`, ...)
+ were not exercised against a live Postgres + running backend; only their extracted pure logic
+ (`plan_ensure`, `_binding_from_row`, `is_crashed`) is unit tested, and the transport layer
+ they call (`SubstrateControl`) is proved live as described above.
+
+## Cleanup
+
+All test actors deleted, then the `substrate-preview` cluster and its `kind-registry` deleted
+(`hack/delete-kind-cluster.sh`). Final `kind get clusters` / `docker ps` showed only
+`mainloop-test` / `mainloop-test-control-plane`. Root disk free was unchanged (~26G) before and
+after. No Mainloop repository files outside this branch's own commits were changed.
From 784a5b4c6440736eb4187b3316a20f2073055981 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Tue, 22 Sep 2026 03:16:11 +0000
Subject: [PATCH 04/30] style: format substrate-workspace-adapter spike doc
---
docs/spikes/substrate-workspace-adapter.md | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 9d00f1d..72f377e 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -16,13 +16,13 @@ drives its lifecycle through the real `kubectl ate` control-plane CLI.
## Real versus stand-in
-| Layer | Status |
-| --- | --- |
-| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
-| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
-| Herdr + `agentctl` inside the actor image | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
-| Claude/Codex agent processes, credentials | Not run in this spike (see "Not attempted") |
-| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
+| Layer | Status |
+| -------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
+| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
+| Herdr + `agentctl` inside the actor image | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
+| Claude/Codex agent processes, credentials | Not run in this spike (see "Not attempted") |
+| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
## Run it
From cfed93205b7e2d9740e8a75c64fe7ef3b1e560d2 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Tue, 22 Sep 2026 03:49:17 +0000
Subject: [PATCH 05/30] feat: prove the preview/HMR gate live; find the
credential-injection gap
Gate 3 (preview/HMR), proved live: a second ActorTemplate
(spikes/substrate-workspace-adapter/image/ -- real Herdr, real Vite
7.3.1, a generic exec shim standing in for a credentialed agent's Bash
tool) served through a real NGINX ate-target-actor header-proxy to a
real browser (agent-browser) with a real HMR WebSocket held open the
whole time. A real shell write produced a genuine in-place hot update
(window marker survived; console logged "hot updated"), including
across an explicit suspend/resume cycle.
Getting there found and fixed three independent, real bugs:
- NGINX proxy_pass defaults to HTTP/1.0, silently breaking the
WebSocket upgrade (the pinned checkout's own Jupyter demo nginx.conf
has the same gap); fixed with proxy_http_version 1.1.
- A hardcoded hmr.clientPort pointed the browser's WebSocket at the
actor's internal port instead of the proxy's; removed the override.
- A shell-redirect truncate-in-place write was never observed by
Vite's watcher on the gVisor-sandboxed filesystem, with or without
polling; an atomic rename-replace write (sed -i, how most real
editors write) was picked up every time. The initial "inotify
doesn't work under gVisor" hypothesis was tested and found wrong,
and corrected in the docs rather than left standing.
Gate 5 (native-session, live): investigated Substrate's credential
primitives before attempting a live Claude/Codex proof and found a
real, structural gap -- ActorTemplate env values are literal-only on
an immutable resource, and SystemInfo volumes only project actor
identity and one CA trust bundle, not arbitrary secrets. There is no
safe way yet to deliver CLAUDE_CODE_OAUTH_TOKEN or ~/.codex/auth.json
into an actor. Documented rather than worked around unsafely; the
live proof stays unattempted pending new plumbing.
Full findings in docs/spikes/substrate-workspace-adapter.md and the
task's proof note (.tasknotes, not tracked here).
---
docs/spikes/substrate-workspace-adapter.md | 99 ++++++++++++++++---
.../image/.gitignore | 1 +
.../image/Dockerfile | 21 ++++
.../image/entrypoint.sh | 32 ++++++
.../image/exec-shim.js | 67 +++++++++++++
.../image/vite-fixture/index.html | 11 +++
.../image/vite-fixture/main.js | 4 +
.../image/vite-fixture/package.json | 11 +++
.../image/vite-fixture/vite.config.js | 24 +++++
.../k8s/preview-gate-template.yaml.tmpl | 52 ++++++++++
.../k8s/preview-proxy.yaml.tmpl | 76 ++++++++++++++
11 files changed, 382 insertions(+), 16 deletions(-)
create mode 100644 spikes/substrate-workspace-adapter/image/.gitignore
create mode 100644 spikes/substrate-workspace-adapter/image/Dockerfile
create mode 100644 spikes/substrate-workspace-adapter/image/entrypoint.sh
create mode 100644 spikes/substrate-workspace-adapter/image/exec-shim.js
create mode 100644 spikes/substrate-workspace-adapter/image/vite-fixture/index.html
create mode 100644 spikes/substrate-workspace-adapter/image/vite-fixture/main.js
create mode 100644 spikes/substrate-workspace-adapter/image/vite-fixture/package.json
create mode 100644 spikes/substrate-workspace-adapter/image/vite-fixture/vite.config.js
create mode 100644 spikes/substrate-workspace-adapter/k8s/preview-gate-template.yaml.tmpl
create mode 100644 spikes/substrate-workspace-adapter/k8s/preview-proxy.yaml.tmpl
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 72f377e..4859ad3 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -16,13 +16,34 @@ drives its lifecycle through the real `kubectl ate` control-plane CLI.
## Real versus stand-in
-| Layer | Status |
-| -------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
-| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
-| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
-| Herdr + `agentctl` inside the actor image | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
-| Claude/Codex agent processes, credentials | Not run in this spike (see "Not attempted") |
-| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
+| Layer | Status |
+| --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
+| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
+| `preview-gate` WorkerPool + ActorTemplate: real Herdr server, real Vite dev server, real NGINX header-proxy, real browser (`agent-browser`), real WebSocket HMR | Real |
+| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
+| The preview-gate's file edits (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
+| Claude/Codex agent processes, credentials | Not run in this spike (see "Not attempted") |
+| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
+
+## Why not a real agent for the preview-gate edit (credential-injection gap)
+
+Substrate's pinned commit has no generic secret-injection mechanism equivalent to a Kubernetes
+Secret volume/env mount. `ActorTemplate` container env values are literal only (no
+`envFrom`/`valueFrom`, and the template is immutable, so baking a token in would also mean
+storing it permanently in a control-plane object -- unacceptable under this task's "credentials
+by path, never by value" rule). The only credential-shaped primitives are `SystemInfo` volumes
+(`actorMetadata`: the actor's own name/atespace/uid; `trustBundle`: a named, allowlisted CA
+bundle -- today only `egress-mitm.ate.dev`) and `pkg/proto/credproviderpb` (`CredentialProvider`,
+a plugin the _egress gateway_ calls to inject a credential into an actor's _outbound_ request,
+keyed by the actor's SPIFFE identity -- not a way to hand the actor's own process a local file or
+env var it can read directly, which is what the Claude Code / Codex CLIs need). A real
+native-agent proof (gate 5) therefore needs either an unsafe workaround or new plumbing (e.g. an
+authenticated credential-relay using the `MintActorJWT`/`MintActorCertificate` RPCs already in
+`ateapipb.Control`), out of scope for this spike. The preview-gate measurement below instead uses
+a generic exec shim (`spikes/substrate-workspace-adapter/image/exec-shim.js`) that pastes text
+into a real Herdr shell pane via `herdr pane run` -- a real shell executing a real command, just
+not a credentialed agent's own tool call.
## Run it
@@ -38,7 +59,11 @@ KIND_CLUSTER_NAME=substrate-preview KUBECTL_CONTEXT=kind-substrate-preview \
KIND_CLUSTER_NAME=substrate-preview KUBECTL_CONTEXT=kind-substrate-preview \
KUBECONFIG=/tmp/substrate-preview-kubeconfig \
/tmp/substrate-preview-src/hack/install-ate-kind.sh --deploy-atenet --atenet-dataplane=agentgateway
-# build kubectl-ate, build+push the actor image, apply spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl
+# build kubectl-ate, build+push an actor image, apply either:
+# k8s/actor-template.yaml.tmpl -- the mainloop-workspace template (Herdr + agentctl)
+# k8s/preview-gate-template.yaml.tmpl -- the preview-gate template (+ image/, a real Vite dev
+# server and exec shim, for the preview/HMR gate)
+# k8s/preview-proxy.yaml.tmpl -- the NGINX ate-target-actor header-proxy in front of it
# (WorkerPool via `ko resolve | kubectl apply`, ActorTemplate via `kubectl ate create actor-template -f -`)
```
@@ -66,12 +91,51 @@ KIND_CLUSTER_NAME=substrate-preview KUBECTL_CONTEXT=kind-substrate-preview \
- Building this adapter against the real CLI found one bug fixed in the same commit:
`create_actor` was missing `-o json` and crashed parsing `kubectl ate`'s default table output.
+### Preview/HMR gate (gate 3): proved live, three real bugs isolated and fixed
+
+Through the actual intended route (real browser -> NGINX header-proxy -> `atenet-router`
+(agentgateway) -> a real Vite dev server in a real actor), with a real WebSocket HMR socket open
+the whole time: a real shell write (via the exec-shim's `herdr pane run`, not a purpose-built
+`/__edit` endpoint) to `main.js` produced a genuine in-place HMR update -- confirmed by a
+`window.__hmrMarker` value set before the edit surviving after it (a full reload would have reset
+it) and by the console logging `[vite] hot updated: /main.js`. This held across an explicit
+suspend/resume cycle too: content and the marker's own page state survived, and Vite's client
+logged `server connection lost. Polling for restart...` during the suspend and reconnected
+cleanly on resume, with a further post-resume edit still hot-updating correctly.
+
+Getting there required isolating and fixing three independent, real bugs -- exactly what the
+prior Kind preview proof asked for ("isolate ... rather than re-measuring as one blob"):
+
+1. **NGINX's `proxy_pass` defaults to HTTP/1.0 upstream**, which silently breaks `Connection:
+Upgrade`. Symptom: `503 upstream call failed: SendRequest: connection closed before message
+completed` from `atenet-router`, which looked like a router bug until isolated by testing the
+ same header-routed request directly against the router (works) versus through NGINX (fails).
+ The Jupyter demo's own `nginx.conf` (the pattern this proxy was copied from) has the same gap.
+ Fix: add `proxy_http_version 1.1;`.
+2. **A hardcoded `hmr.clientPort` pointed the browser's WebSocket at the actor's internal port
+ (80), not the port the browser actually reached the proxy on.** Symptom: `[vite] failed to
+connect to websocket (Error: WebSocket closed without opened.)` in the real browser, while a
+ raw `curl` WebSocket upgrade against the same actor succeeded (isolating it to the _browser's_
+ target URL, not the routing path). Fix: do not set `hmr.clientPort`; let Vite infer it from
+ `window.location`, which is correct for same-origin proxying.
+3. **A plain shell-redirect truncate-in-place write (`cmd > file`) was never observed by Vite's
+ file watcher on this gVisor-sandboxed filesystem, with or without `usePolling`; an atomic
+ rename-replace write (`sed -i`, or any editor/tool that writes-then-renames, which is how most
+ real editors and Node's own atomic-write helpers behave) was picked up every time.** This was
+ isolated by holding the watcher config fixed and varying only the write method. The initial
+ hypothesis (inotify does not work under gVisor) was wrong and is corrected here rather than
+ left standing: the default inotify-based watch picked up `sed -i` edits fine, with or without
+ polling enabled. `usePolling` is kept in the fixture's `vite.config.js` as defense in depth,
+ but it was not the actual fix.
+
+None of these three are Substrate bugs in the sense of "broken by Substrate" -- (1) is a gap in
+the demo NGINX pattern this repo's own docs show, (2) is a Vite config default that does not
+suit a proxied deployment, and (3) is a filesystem-semantics fact worth knowing about (most real
+editors already write this way, so it may not affect a real native-agent's edits, which is
+exactly why gate 5's live proof matters and was not reached in this run).
+
## Limits / not attempted in this run
-- **Preview/HMR gate**: no Vite actor, no authenticated fixed-header proxy, no `agent-browser`
- trial. The prior `substrate-kind-preview-proof` note already showed this works and does not
- always work reliably (10-30s stalls with an HMR socket open); this run did not repeat or
- extend that measurement against Mainloop's own actor template.
- **Dev-service gate**: no Postgres actor, no egress policy, no reconnect-after-wake trial against
our template.
- **Native-session gate, live**: no real Claude/Codex session was started inside a Substrate
@@ -85,7 +149,10 @@ KIND_CLUSTER_NAME=substrate-preview KUBECTL_CONTEXT=kind-substrate-preview \
## Cleanup
-All test actors deleted, then the `substrate-preview` cluster and its `kind-registry` deleted
-(`hack/delete-kind-cluster.sh`). Final `kind get clusters` / `docker ps` showed only
-`mainloop-test` / `mainloop-test-control-plane`. Root disk free was unchanged (~26G) before and
-after. No Mainloop repository files outside this branch's own commits were changed.
+Each cluster lane in this spike (the initial adapter/CRASHED trial, and the later preview-gate
+trial) deleted its own test actors, then the `substrate-preview` cluster and its `kind-registry`
+(`hack/delete-kind-cluster.sh`), and pruned the locally built, unpushed-elsewhere Docker images.
+Final `kind get clusters` / `docker ps` showed only `mainloop-test` /
+`mainloop-test-control-plane` after every lane. Root disk stayed in the 19-27G-free range
+throughout (above the plan's 8G in-flight-trial abort threshold at all times); available RAM
+stayed above 8G. No Mainloop repository files outside this branch's own commits were changed.
diff --git a/spikes/substrate-workspace-adapter/image/.gitignore b/spikes/substrate-workspace-adapter/image/.gitignore
new file mode 100644
index 0000000..a78a325
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/image/.gitignore
@@ -0,0 +1 @@
+herdr
diff --git a/spikes/substrate-workspace-adapter/image/Dockerfile b/spikes/substrate-workspace-adapter/image/Dockerfile
new file mode 100644
index 0000000..1a81238
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/image/Dockerfile
@@ -0,0 +1,21 @@
+# Preview-gate spike image: real Herdr server + a real Vite dev server, driven by a generic
+# exec shim (see exec-shim.js) standing in for a credentialed native agent's own Bash tool.
+# herdr is copied from the host into the build context by the build script (never committed).
+# No credentials are baked in.
+FROM node:22-bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends jq ca-certificates git curl procps \
+ && rm -rf /var/lib/apt/lists/* \
+ && useradd -m -u 10001 agent
+COPY herdr /usr/local/bin/herdr
+COPY vite-fixture /work/vite-fixture
+COPY exec-shim.js /usr/local/bin/exec-shim.js
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh \
+ && cd /work/vite-fixture && npm install --no-audit --no-fund \
+ && chown -R agent:agent /work
+ENV VITE_DIR=/work/vite-fixture
+ENV EXEC_SHIM=/usr/local/bin/exec-shim.js
+ENV HOME=/home/agent
+ENV HERDR_SESSION=mainloop-preview
+USER 10001:10001
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/spikes/substrate-workspace-adapter/image/entrypoint.sh b/spikes/substrate-workspace-adapter/image/entrypoint.sh
new file mode 100644
index 0000000..cf6385e
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/image/entrypoint.sh
@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
+# Preview-gate actor entrypoint. Starts a real Herdr server, a real Vite dev server in one pane,
+# and a minimal generic exec shim (Node http server -> `herdr pane run`) in a second pane. The
+# shim is a stand-in for a credentialed native agent's own Bash tool -- Substrate's pinned commit
+# has no generic secret-injection mechanism (only SystemInfo actor-identity/trust-bundle volumes
+# and an egress CredentialProvider), so a real Claude/Codex session cannot be started here yet.
+# See docs/spikes/substrate-workspace-adapter.md.
+set -eu
+STATE_DIR=/work/.mainloop
+mkdir -p "${HOME}" "${STATE_DIR}"
+
+echo "herdr $(herdr --version) server starting (HOME=${HOME} session=${HERDR_SESSION})"
+herdr --session "${HERDR_SESSION}" server &
+HERDR_PID=$!
+
+for _ in $(seq 1 60); do
+ herdr --session "${HERDR_SESSION}" status server >/dev/null 2>&1 && break
+ sleep 0.5
+done
+
+dev_ws=$(herdr --session "${HERDR_SESSION}" workspace create --label dev --cwd "${VITE_DIR}")
+dev_pane=$(echo "${dev_ws}" | jq -r '.result.root_pane.pane_id')
+shell_ws=$(herdr --session "${HERDR_SESSION}" workspace create --label shell --cwd "${VITE_DIR}")
+shell_pane=$(echo "${shell_ws}" | jq -r '.result.root_pane.pane_id')
+echo "${shell_pane}" >"${STATE_DIR}/shell-pane-id"
+
+herdr --session "${HERDR_SESSION}" pane run "${dev_pane}" "npm run dev"
+
+EXEC_SHIM_PANE_ID="${shell_pane}" HERDR_SESSION="${HERDR_SESSION}" node "${EXEC_SHIM}" &
+
+wait "${HERDR_PID}"
diff --git a/spikes/substrate-workspace-adapter/image/exec-shim.js b/spikes/substrate-workspace-adapter/image/exec-shim.js
new file mode 100644
index 0000000..01250e7
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/image/exec-shim.js
@@ -0,0 +1,67 @@
+// Minimal generic command executor for the preview-gate spike: POST /run { command } pastes
+// `command` as literal text into a real Herdr shell pane via `herdr pane run` (fire-and-forget;
+// the pane is a real bash shell, so this is a real shell write, not a purpose-built edit
+// endpoint). GET /read returns the pane's current terminal buffer (`herdr pane read`), since
+// `pane run` itself never captures output. Stands in for a credentialed native agent's own Bash
+// tool -- see entrypoint.sh and docs/spikes/substrate-workspace-adapter.md for why a real agent
+// could not be used here.
+// Listens on port 8090, separate from the Vite dev server's port 80. Reached only through
+// atenet-router's arbitrary-port CONNECT tunnel with the ate-target-actor header (see
+// docs/api-guide.md "Workload Connectivity"), from test orchestration on the host -- never
+// through the previewed route a browser uses (port 80 via the NGINX header-proxy).
+'use strict';
+const http = require('node:http');
+const { execFile } = require('node:child_process');
+
+const PANE_ID = process.env.EXEC_SHIM_PANE_ID;
+const SESSION = process.env.HERDR_SESSION;
+if (!PANE_ID || !SESSION) {
+ console.error('exec-shim: EXEC_SHIM_PANE_ID and HERDR_SESSION are required');
+ process.exit(1);
+}
+
+function herdr(args, res) {
+ execFile('herdr', ['--session', SESSION, ...args], (err, stdout, stderr) => {
+ if (err) {
+ res.writeHead(502).end(String(err));
+ return;
+ }
+ res
+ .writeHead(200, { 'content-type': 'application/json' })
+ .end(JSON.stringify({ ok: true, stdout, stderr }));
+ });
+}
+
+const server = http.createServer((req, res) => {
+ if (req.method === 'GET' && req.url === '/read') {
+ herdr(['pane', 'read', PANE_ID], res);
+ return;
+ }
+ if (req.method !== 'POST' || req.url !== '/run') {
+ res.writeHead(404).end();
+ return;
+ }
+ let body = '';
+ req.on('data', (chunk) => {
+ body += chunk;
+ if (body.length > 65536) req.destroy();
+ });
+ req.on('end', () => {
+ let command;
+ try {
+ command = JSON.parse(body).command;
+ } catch {
+ res.writeHead(400).end('invalid json');
+ return;
+ }
+ if (typeof command !== 'string' || !command) {
+ res.writeHead(400).end('missing command');
+ return;
+ }
+ herdr(['pane', 'run', PANE_ID, command], res);
+ });
+});
+
+server.listen(8090, '0.0.0.0', () => {
+ console.log('exec-shim listening on :8090, pane', PANE_ID);
+});
diff --git a/spikes/substrate-workspace-adapter/image/vite-fixture/index.html b/spikes/substrate-workspace-adapter/image/vite-fixture/index.html
new file mode 100644
index 0000000..b1bb92e
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/image/vite-fixture/index.html
@@ -0,0 +1,11 @@
+
+
+
+
+ Mainloop preview fixture
+
+
+
loading...
+
+
+
diff --git a/spikes/substrate-workspace-adapter/image/vite-fixture/main.js b/spikes/substrate-workspace-adapter/image/vite-fixture/main.js
new file mode 100644
index 0000000..7183e38
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/image/vite-fixture/main.js
@@ -0,0 +1,4 @@
+document.getElementById('label').textContent = 'Preview one';
+if (import.meta.hot) {
+ import.meta.hot.accept();
+}
diff --git a/spikes/substrate-workspace-adapter/image/vite-fixture/package.json b/spikes/substrate-workspace-adapter/image/vite-fixture/package.json
new file mode 100644
index 0000000..4dc9408
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/image/vite-fixture/package.json
@@ -0,0 +1,11 @@
+{
+ "name": "mainloop-preview-fixture",
+ "private": true,
+ "type": "module",
+ "scripts": {
+ "dev": "vite"
+ },
+ "devDependencies": {
+ "vite": "7.3.1"
+ }
+}
diff --git a/spikes/substrate-workspace-adapter/image/vite-fixture/vite.config.js b/spikes/substrate-workspace-adapter/image/vite-fixture/vite.config.js
new file mode 100644
index 0000000..5ae1260
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/image/vite-fixture/vite.config.js
@@ -0,0 +1,24 @@
+import { defineConfig } from 'vite';
+
+export default defineConfig({
+ server: {
+ host: '0.0.0.0',
+ port: 80,
+ strictPort: true,
+ allowedHosts: true,
+ // No hmr.clientPort override: the browser reaches this actor through a proxy whose port
+ // varies by deployment (port-forward, ingress, ...). Vite infers the HMR client's port from
+ // window.location by default, which is correct for same-origin proxying (our NGINX
+ // header-proxy) and was the actual bug the first time this was set to the actor's internal
+ // port 80 -- the browser tried to open a WebSocket to its own port 80, not the proxy's port.
+ // Measured live: the actual variable was NOT inotify-vs-polling (the default inotify watch
+ // picks up an atomic rename-replace write, e.g. `sed -i`, correctly, with polling enabled or
+ // not). It was the write method -- a plain shell-redirect truncate-in-place write (`cmd >
+ // file`) was never observed by Vite's watcher on this gVisor-sandboxed filesystem, with or
+ // without polling, while an atomic rename-replace write (`sed -i`, or any editor/tool that
+ // writes-then-renames, which is how most real editors and Node's own atomic-write helpers
+ // behave) was picked up every time and produced a true HMR update, not a reload. usePolling
+ // is left enabled here only as defense in depth; it was not the fix.
+ watch: { usePolling: true, interval: 300 }
+ }
+});
diff --git a/spikes/substrate-workspace-adapter/k8s/preview-gate-template.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/preview-gate-template.yaml.tmpl
new file mode 100644
index 0000000..0da7943
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/k8s/preview-gate-template.yaml.tmpl
@@ -0,0 +1,52 @@
+# WorkerPool + ActorTemplate for the preview/HMR gate measurement (gate 3 in .tasknotes/plan.md):
+# a real Vite dev server plus a generic exec shim standing in for a credentialed native agent's
+# Bash tool (see spikes/substrate-workspace-adapter/image/). No durable volume: this template
+# relies on SNAPSHOT_CONTENT_SCOPE_FULL to preserve /work (including node_modules and any edits)
+# across suspend/resume, which is what the gate is actually measuring.
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: ${ATESPACE}
+---
+apiVersion: ate.dev/v1alpha1
+kind: WorkerPool
+metadata:
+ name: preview-gate
+ namespace: ${ATESPACE}
+ labels:
+ workload: preview-gate
+spec:
+ replicas: 1
+ workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
+ template:
+ resources:
+ limits:
+ cpu: '1'
+ memory: 1Gi
+ requests:
+ cpu: 250m
+ memory: 1Gi
+---
+metadata:
+ atespace: ${ATESPACE}
+ name: preview-gate
+workerSelector:
+ matchLabels:
+ workload: preview-gate
+containers:
+- name: preview
+ image: __IMAGE__
+ env:
+ - { name: HOME, value: /home/agent }
+ - { name: HERDR_SESSION, value: mainloop-preview }
+ resources:
+ limits:
+ - { name: cpu, quantity: "1" }
+ - { name: memory, quantity: 1Gi }
+snapshotsConfig:
+ onPause: SNAPSHOT_CONTENT_SCOPE_FULL
+ onCommit: SNAPSHOT_CONTENT_SCOPE_FULL
+ storageLocation: gs://${BUCKET_NAME}/preview-gate/
+sandboxConfig:
+ sandboxClass: SANDBOX_CLASS_GVISOR
+ configName: gvisor-default
diff --git a/spikes/substrate-workspace-adapter/k8s/preview-proxy.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/preview-proxy.yaml.tmpl
new file mode 100644
index 0000000..24cc6d0
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/k8s/preview-proxy.yaml.tmpl
@@ -0,0 +1,76 @@
+# Fixed authenticated preview route: an NGINX Deployment/Service that injects the
+# ate-target-actor header atenet-router needs (browsers cannot set custom headers), following
+# the same pattern as the pinned checkout's demos/jupyter/jupyter.yaml.tmpl proxy. This is the
+# route intended for Mainloop's own preview UI (not a bespoke fixture endpoint): the browser
+# never talks to atenet-router directly.
+#
+# proxy_http_version 1.1 is required for the WebSocket upgrade to survive proxy_pass -- NGINX
+# defaults to HTTP/1.0 upstream, which silently breaks `Connection: Upgrade`. The Jupyter demo's
+# own nginx.conf (copied as the starting point here) does not set it either; measured live
+# against this cluster as a 503 "connection closed before message completed" from atenet-router,
+# not from NGINX itself, which made it look like a router bug until isolated. See
+# docs/spikes/substrate-workspace-adapter.md.
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: preview-proxy-config
+ namespace: ${ATESPACE}
+data:
+ nginx.conf: |
+ events {}
+ http {
+ server {
+ listen 80;
+ location / {
+ proxy_pass http://atenet-router.ate-system.svc.cluster.local;
+ proxy_http_version 1.1;
+ proxy_set_header Host $http_host;
+ proxy_set_header ate-target-actor ${ATESPACE}/${ACTOR_NAME};
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header Upgrade $http_upgrade;
+ proxy_set_header Connection "upgrade";
+ }
+ }
+ }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: preview-proxy
+ namespace: ${ATESPACE}
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: preview-proxy
+ template:
+ metadata:
+ labels:
+ app: preview-proxy
+ spec:
+ containers:
+ - name: nginx
+ image: nginx:alpine
+ ports:
+ - containerPort: 80
+ volumeMounts:
+ - name: config
+ mountPath: /etc/nginx/nginx.conf
+ subPath: nginx.conf
+ volumes:
+ - name: config
+ configMap:
+ name: preview-proxy-config
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: preview-proxy
+ namespace: ${ATESPACE}
+spec:
+ type: ClusterIP
+ selector:
+ app: preview-proxy
+ ports:
+ - port: 80
From b2a4d2390d20beef0dcac489b31e21b7165dc7dd Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Tue, 22 Sep 2026 04:02:00 +0000
Subject: [PATCH 06/30] feat: prove the dev-service gate live, including real
policy enforcement
Gate 4 (dev-service/Postgres connectivity), proved live: a third
ActorTemplate (spikes/substrate-workspace-adapter/dev-service-image/ --
real psql, the same exec-shim pattern) reached a real postgres:16-alpine
StatefulSet (matching k8s/apps/mainloop/overlays/test/postgres-statefulset.yaml's
image/auth shape) under a narrow EgressPolicy (a single CIDR rule for
the Postgres Service's /32 ClusterIP).
kubectl-ate has no CLI verb for EgressPolicy at all -- confirmed by the
pinned checkout's own demos/egress/README.md ("no CLI verb yet").
Added spikes/substrate-workspace-adapter/egress-tool/main.go, a small
standalone Go program mirroring that checkout's own e2e test helper,
calling CreateActorEgressPolicy directly over gRPC.
Result: DNS + a real SELECT query over the actual Postgres wire
protocol + reconnection after an explicit suspend/resume cycle all
worked cleanly. Authorization is real, not passive: a request to a
different Service's ClusterIP (outside the /32 rule) was cleanly
rejected with HTTP 403 "actor egress policy denied destination" from
the egress gateway itself. This improves on the prior Kind preview
proof's external-backend trial (403 -> 503, unproven reconnect); the
isolated difference is a CIDR rule (works for any TCP protocol) versus
a hostname rule (HTTP/TLS-SNI-specific, never the right tool for a
non-HTTP protocol like Postgres).
Full findings in docs/spikes/substrate-workspace-adapter.md and the
task's proof note (.tasknotes, not tracked here).
---
docs/spikes/substrate-workspace-adapter.md | 70 +++++++++++-----
.../dev-service-image/.gitignore | 1 +
.../dev-service-image/Dockerfile | 19 +++++
.../dev-service-image/entrypoint.sh | 23 +++++
.../dev-service-image/exec-shim.js | 67 +++++++++++++++
.../egress-tool/main.go | 84 +++++++++++++++++++
.../k8s/dev-service-gate-template.yaml.tmpl | 47 +++++++++++
.../k8s/postgres-target.yaml | 68 +++++++++++++++
8 files changed, 359 insertions(+), 20 deletions(-)
create mode 100644 spikes/substrate-workspace-adapter/dev-service-image/.gitignore
create mode 100644 spikes/substrate-workspace-adapter/dev-service-image/Dockerfile
create mode 100644 spikes/substrate-workspace-adapter/dev-service-image/entrypoint.sh
create mode 100644 spikes/substrate-workspace-adapter/dev-service-image/exec-shim.js
create mode 100644 spikes/substrate-workspace-adapter/egress-tool/main.go
create mode 100644 spikes/substrate-workspace-adapter/k8s/dev-service-gate-template.yaml.tmpl
create mode 100644 spikes/substrate-workspace-adapter/k8s/postgres-target.yaml
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 4859ad3..4b413be 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -16,15 +16,16 @@ drives its lifecycle through the real `kubectl ate` control-plane CLI.
## Real versus stand-in
-| Layer | Status |
-| --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
-| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
-| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
-| `preview-gate` WorkerPool + ActorTemplate: real Herdr server, real Vite dev server, real NGINX header-proxy, real browser (`agent-browser`), real WebSocket HMR | Real |
-| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
-| The preview-gate's file edits (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
-| Claude/Codex agent processes, credentials | Not run in this spike (see "Not attempted") |
-| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
+| Layer | Status |
+| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
+| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
+| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
+| `preview-gate` WorkerPool + ActorTemplate: real Herdr server, real Vite dev server, real NGINX header-proxy, real browser (`agent-browser`), real WebSocket HMR | Real |
+| `dev-service-gate` WorkerPool + ActorTemplate: real `psql`, real external `postgres:16-alpine` StatefulSet, real `EgressPolicy` (CIDR rule, created via a small gRPC tool since `kubectl-ate` has no CLI verb for it) | Real |
+| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
+| File edits and shell commands run inside actors (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
+| Claude/Codex agent processes, credentials | Not run in this spike (see "Not attempted") |
+| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
## Why not a real agent for the preview-gate edit (credential-injection gap)
@@ -59,11 +60,13 @@ KIND_CLUSTER_NAME=substrate-preview KUBECTL_CONTEXT=kind-substrate-preview \
KIND_CLUSTER_NAME=substrate-preview KUBECTL_CONTEXT=kind-substrate-preview \
KUBECONFIG=/tmp/substrate-preview-kubeconfig \
/tmp/substrate-preview-src/hack/install-ate-kind.sh --deploy-atenet --atenet-dataplane=agentgateway
-# build kubectl-ate, build+push an actor image, apply either:
-# k8s/actor-template.yaml.tmpl -- the mainloop-workspace template (Herdr + agentctl)
-# k8s/preview-gate-template.yaml.tmpl -- the preview-gate template (+ image/, a real Vite dev
-# server and exec shim, for the preview/HMR gate)
-# k8s/preview-proxy.yaml.tmpl -- the NGINX ate-target-actor header-proxy in front of it
+# build kubectl-ate, build+push an actor image, apply one of:
+# k8s/actor-template.yaml.tmpl -- mainloop-workspace: Herdr + agentctl
+# k8s/preview-gate-template.yaml.tmpl -- preview-gate: real Vite dev server + exec shim
+# + k8s/preview-proxy.yaml.tmpl -- the NGINX ate-target-actor header-proxy in front
+# k8s/dev-service-gate-template.yaml.tmpl -- dev-service-gate: real psql + exec shim
+# + k8s/postgres-target.yaml -- the external postgres:16-alpine StatefulSet
+# + egress-tool/main.go -- creates the actor's EgressPolicy (no CLI verb)
# (WorkerPool via `ko resolve | kubectl apply`, ActorTemplate via `kubectl ate create actor-template -f -`)
```
@@ -134,10 +137,37 @@ suit a proxied deployment, and (3) is a filesystem-semantics fact worth knowing
editors already write this way, so it may not affect a real native-agent's edits, which is
exactly why gate 5's live proof matters and was not reached in this run).
+### Dev-service gate (gate 4): proved live, including real policy enforcement
+
+A real `postgres:16-alpine` StatefulSet (same image/auth shape as
+`k8s/apps/mainloop/overlays/test/postgres-statefulset.yaml`) in its own namespace, reached from a
+`dev-service-gate` actor (real `psql`, driven through the same generic exec shim) under an
+`EgressPolicy` scoped to exactly the Postgres Service's `/32` ClusterIP.
+
+`kubectl-ate` has **no CLI verb for egress policies** -- confirmed by the pinned checkout's own
+`demos/egress/README.md`: `"test-egress.sh creates and resumes the Actor but cannot create its
+EgressPolicy (no CLI verb yet)"`. Its own e2e suite calls the gRPC API directly
+(`internal/e2e/egresspolicy.go`). This spike does the same:
+`spikes/substrate-workspace-adapter/egress-tool/main.go`, a small standalone `main` mirroring
+that helper without the `testing.T` dependency (build instructions are in the file's header
+comment; it must be built inside a Substrate checkout since it imports `internal/` packages).
+
+**Result**: DNS resolution (bypasses the policy enforcement point entirely -- port 53 is always
+allowed), a real `SELECT` query over the actual Postgres wire protocol, and reconnection after an
+explicit suspend/resume cycle (a second query, `SELECT 43`, succeeded cleanly post-resume, no
+policy re-creation needed -- the policy is attached to the actor, not the connection) all worked
+on the first try. Authorization is real, not merely passive: a request to a _different_ Service's
+ClusterIP (not covered by the `/32` rule) was cleanly rejected --
+`HTTP 403 actor egress policy denied destination` from the egress gateway itself, not a silent
+timeout or a security-group-shaped ambiguity. This is a materially better outcome than the prior
+Kind preview proof's own external-backend trial (`403 -> 503`, "reconnection after wake was
+therefore not proved") -- the difference was using a **CIDR rule** (works for any TCP protocol
+per `docs/network-egress.md`'s "CIDR/all policy: dial now" passthrough path) instead of a
+**hostname rule** (HTTP/TLS-SNI-specific, and Postgres is neither), which the prior proof's HTTP
+`fetch`-based trial did not have reason to distinguish.
+
## Limits / not attempted in this run
-- **Dev-service gate**: no Postgres actor, no egress policy, no reconnect-after-wake trial against
- our template.
- **Native-session gate, live**: no real Claude/Codex session was started inside a Substrate
actor; the credential wiring authorized by `.tasknotes/plan.md` was not used. Only the
fixture-level contract logic (`test_workspace_adapter.py`) and the generic
@@ -149,10 +179,10 @@ exactly why gate 5's live proof matters and was not reached in this run).
## Cleanup
-Each cluster lane in this spike (the initial adapter/CRASHED trial, and the later preview-gate
-trial) deleted its own test actors, then the `substrate-preview` cluster and its `kind-registry`
-(`hack/delete-kind-cluster.sh`), and pruned the locally built, unpushed-elsewhere Docker images.
-Final `kind get clusters` / `docker ps` showed only `mainloop-test` /
+Each of the three cluster lanes in this spike (adapter/CRASHED, preview-gate, dev-service-gate)
+deleted its own test actors and target resources, then the `substrate-preview` cluster and its
+`kind-registry` (`hack/delete-kind-cluster.sh`), and pruned the locally built, unpushed-elsewhere
+Docker images. Final `kind get clusters` / `docker ps` showed only `mainloop-test` /
`mainloop-test-control-plane` after every lane. Root disk stayed in the 19-27G-free range
throughout (above the plan's 8G in-flight-trial abort threshold at all times); available RAM
stayed above 8G. No Mainloop repository files outside this branch's own commits were changed.
diff --git a/spikes/substrate-workspace-adapter/dev-service-image/.gitignore b/spikes/substrate-workspace-adapter/dev-service-image/.gitignore
new file mode 100644
index 0000000..a78a325
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/dev-service-image/.gitignore
@@ -0,0 +1 @@
+herdr
diff --git a/spikes/substrate-workspace-adapter/dev-service-image/Dockerfile b/spikes/substrate-workspace-adapter/dev-service-image/Dockerfile
new file mode 100644
index 0000000..7759bbf
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/dev-service-image/Dockerfile
@@ -0,0 +1,19 @@
+# Dev-service-gate spike image (gate 4): real Herdr server + a real psql client, driven by the
+# same generic exec shim as the preview-gate image, to test a Substrate actor's egress
+# connectivity to an external PostgreSQL Service under a narrow CIDR egress policy.
+# herdr is copied from the host into the build context by the build script (never committed).
+# No credentials are baked in.
+FROM node:22-bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends jq ca-certificates curl procps postgresql-client \
+ && rm -rf /var/lib/apt/lists/* \
+ && useradd -m -u 10001 agent
+COPY herdr /usr/local/bin/herdr
+COPY exec-shim.js /usr/local/bin/exec-shim.js
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh \
+ && mkdir -p /work && chown -R agent:agent /work
+ENV EXEC_SHIM=/usr/local/bin/exec-shim.js
+ENV HOME=/home/agent
+ENV HERDR_SESSION=mainloop-dev-service
+USER 10001:10001
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/spikes/substrate-workspace-adapter/dev-service-image/entrypoint.sh b/spikes/substrate-workspace-adapter/dev-service-image/entrypoint.sh
new file mode 100644
index 0000000..4ae7773
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/dev-service-image/entrypoint.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
+# Dev-service-gate actor entrypoint (gate 4 in .tasknotes/plan.md): a real Herdr server plus a
+# generic exec shim, no dev server -- just enough to run a real psql client against an external
+# Postgres Service from inside the actor. See docs/spikes/substrate-workspace-adapter.md.
+set -eu
+mkdir -p "${HOME}"
+
+echo "herdr $(herdr --version) server starting (HOME=${HOME} session=${HERDR_SESSION})"
+herdr --session "${HERDR_SESSION}" server &
+HERDR_PID=$!
+
+for _ in $(seq 1 60); do
+ herdr --session "${HERDR_SESSION}" status server >/dev/null 2>&1 && break
+ sleep 0.5
+done
+
+shell_ws=$(herdr --session "${HERDR_SESSION}" workspace create --label shell --cwd /work)
+shell_pane=$(echo "${shell_ws}" | jq -r '.result.root_pane.pane_id')
+
+EXEC_SHIM_PANE_ID="${shell_pane}" HERDR_SESSION="${HERDR_SESSION}" node "${EXEC_SHIM}" &
+
+wait "${HERDR_PID}"
diff --git a/spikes/substrate-workspace-adapter/dev-service-image/exec-shim.js b/spikes/substrate-workspace-adapter/dev-service-image/exec-shim.js
new file mode 100644
index 0000000..01250e7
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/dev-service-image/exec-shim.js
@@ -0,0 +1,67 @@
+// Minimal generic command executor for the preview-gate spike: POST /run { command } pastes
+// `command` as literal text into a real Herdr shell pane via `herdr pane run` (fire-and-forget;
+// the pane is a real bash shell, so this is a real shell write, not a purpose-built edit
+// endpoint). GET /read returns the pane's current terminal buffer (`herdr pane read`), since
+// `pane run` itself never captures output. Stands in for a credentialed native agent's own Bash
+// tool -- see entrypoint.sh and docs/spikes/substrate-workspace-adapter.md for why a real agent
+// could not be used here.
+// Listens on port 8090, separate from the Vite dev server's port 80. Reached only through
+// atenet-router's arbitrary-port CONNECT tunnel with the ate-target-actor header (see
+// docs/api-guide.md "Workload Connectivity"), from test orchestration on the host -- never
+// through the previewed route a browser uses (port 80 via the NGINX header-proxy).
+'use strict';
+const http = require('node:http');
+const { execFile } = require('node:child_process');
+
+const PANE_ID = process.env.EXEC_SHIM_PANE_ID;
+const SESSION = process.env.HERDR_SESSION;
+if (!PANE_ID || !SESSION) {
+ console.error('exec-shim: EXEC_SHIM_PANE_ID and HERDR_SESSION are required');
+ process.exit(1);
+}
+
+function herdr(args, res) {
+ execFile('herdr', ['--session', SESSION, ...args], (err, stdout, stderr) => {
+ if (err) {
+ res.writeHead(502).end(String(err));
+ return;
+ }
+ res
+ .writeHead(200, { 'content-type': 'application/json' })
+ .end(JSON.stringify({ ok: true, stdout, stderr }));
+ });
+}
+
+const server = http.createServer((req, res) => {
+ if (req.method === 'GET' && req.url === '/read') {
+ herdr(['pane', 'read', PANE_ID], res);
+ return;
+ }
+ if (req.method !== 'POST' || req.url !== '/run') {
+ res.writeHead(404).end();
+ return;
+ }
+ let body = '';
+ req.on('data', (chunk) => {
+ body += chunk;
+ if (body.length > 65536) req.destroy();
+ });
+ req.on('end', () => {
+ let command;
+ try {
+ command = JSON.parse(body).command;
+ } catch {
+ res.writeHead(400).end('invalid json');
+ return;
+ }
+ if (typeof command !== 'string' || !command) {
+ res.writeHead(400).end('missing command');
+ return;
+ }
+ herdr(['pane', 'run', PANE_ID, command], res);
+ });
+});
+
+server.listen(8090, '0.0.0.0', () => {
+ console.log('exec-shim listening on :8090, pane', PANE_ID);
+});
diff --git a/spikes/substrate-workspace-adapter/egress-tool/main.go b/spikes/substrate-workspace-adapter/egress-tool/main.go
new file mode 100644
index 0000000..1607afc
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/egress-tool/main.go
@@ -0,0 +1,84 @@
+// Creates/updates an actor's EgressPolicy directly via gRPC, since kubectl-ate has no CLI verb
+// for it as of the pinned commit (confirmed by the substrate checkout's own
+// demos/egress/README.md: "test-egress.sh creates and resumes the Actor but cannot create its
+// EgressPolicy (no CLI verb yet)"). Mirrors that checkout's internal/e2e/egresspolicy.go
+// (EnsureEgressPolicy), without the testing.T dependency.
+//
+// This file imports Substrate's internal packages (internal/ateclient, internal/resources), so
+// it cannot be built as a standalone Go module outside a Substrate checkout. To use it: drop
+// this file into /cmd/mainloop-egress-tool/main.go and build with
+// `GOFLAGS=-mod=vendor go build -o mainloop-egress-tool ./cmd/mainloop-egress-tool` from the
+// checkout root (module github.com/agent-substrate/substrate, pinned commit
+// cdac9baef81dd319b46086d695266e6161e9e592 when this was written).
+//
+// Usage: mainloop-egress-tool --kubeconfig --context --atespace --actor
+// [--cidr ] (omit --cidr to allow all destinations)
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "log"
+
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/status"
+ "google.golang.org/protobuf/types/known/emptypb"
+
+ "github.com/agent-substrate/substrate/internal/ateclient"
+ "github.com/agent-substrate/substrate/internal/resources"
+ "github.com/agent-substrate/substrate/pkg/proto/ateapipb"
+)
+
+func main() {
+ kubeconfig := flag.String("kubeconfig", "", "")
+ context_ := flag.String("context", "", "")
+ atespace := flag.String("atespace", "", "")
+ actorName := flag.String("actor", "", "")
+ cidr := flag.String("cidr", "", "CIDR to allow; empty means allow-all")
+ flag.Parse()
+
+ ctx := context.Background()
+ cli, err := ateclient.NewClient(ctx, *kubeconfig, *context_, "", "", false)
+ if err != nil {
+ log.Fatalf("connect: %v", err)
+ }
+ defer cli.Close()
+
+ actorRef := resources.ActorRef{Atespace: *atespace, Name: *actorName}.ToObjectRef()
+
+ var rule *ateapipb.EgressRule
+ if *cidr == "" {
+ rule = &ateapipb.EgressRule{All: &emptypb.Empty{}}
+ } else {
+ rule = &ateapipb.EgressRule{Cidrs: &ateapipb.CIDRRule{Cidrs: []string{*cidr}}}
+ }
+ policy := &ateapipb.EgressPolicy{
+ Metadata: &ateapipb.ResourceMetadata{Atespace: *atespace, Name: "default"},
+ Rules: []*ateapipb.EgressRule{rule},
+ }
+
+ _, err = cli.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{
+ Actor: actorRef,
+ EgressPolicy: policy,
+ })
+ if status.Code(err) == codes.AlreadyExists {
+ existing, gerr := cli.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: actorRef})
+ if gerr != nil {
+ log.Fatalf("get existing: %v", gerr)
+ }
+ policy.Metadata = existing.GetMetadata()
+ if _, uerr := cli.UpdateActorEgressPolicy(ctx, &ateapipb.UpdateActorEgressPolicyRequest{
+ Actor: actorRef,
+ EgressPolicy: policy,
+ }); uerr != nil {
+ log.Fatalf("update: %v", uerr)
+ }
+ fmt.Println("updated existing egress policy")
+ return
+ }
+ if err != nil {
+ log.Fatalf("create: %v", err)
+ }
+ fmt.Println("created egress policy")
+}
diff --git a/spikes/substrate-workspace-adapter/k8s/dev-service-gate-template.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/dev-service-gate-template.yaml.tmpl
new file mode 100644
index 0000000..6b47103
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/k8s/dev-service-gate-template.yaml.tmpl
@@ -0,0 +1,47 @@
+# WorkerPool + ActorTemplate for the dev-service gate measurement (gate 4 in
+# .tasknotes/plan.md): a real psql client, driven by the same generic exec shim as the
+# preview-gate template, to test egress connectivity to an external PostgreSQL Service. See
+# spikes/substrate-workspace-adapter/dev-service-image/.
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: ${ATESPACE}
+---
+apiVersion: ate.dev/v1alpha1
+kind: WorkerPool
+metadata:
+ name: dev-service-gate
+ namespace: ${ATESPACE}
+ labels:
+ workload: dev-service-gate
+spec:
+ replicas: 1
+ workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
+ template:
+ resources:
+ limits: { cpu: '1', memory: 1Gi }
+ requests: { cpu: 250m, memory: 1Gi }
+---
+metadata:
+ atespace: ${ATESPACE}
+ name: dev-service-gate
+workerSelector:
+ matchLabels:
+ workload: dev-service-gate
+containers:
+- name: dev-service
+ image: __IMAGE__
+ env:
+ - { name: HOME, value: /home/agent }
+ - { name: HERDR_SESSION, value: mainloop-dev-service }
+ resources:
+ limits:
+ - { name: cpu, quantity: "1" }
+ - { name: memory, quantity: 1Gi }
+snapshotsConfig:
+ onPause: SNAPSHOT_CONTENT_SCOPE_FULL
+ onCommit: SNAPSHOT_CONTENT_SCOPE_FULL
+ storageLocation: gs://${BUCKET_NAME}/dev-service-gate/
+sandboxConfig:
+ sandboxClass: SANDBOX_CLASS_GVISOR
+ configName: gvisor-default
diff --git a/spikes/substrate-workspace-adapter/k8s/postgres-target.yaml b/spikes/substrate-workspace-adapter/k8s/postgres-target.yaml
new file mode 100644
index 0000000..f9f1557
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/k8s/postgres-target.yaml
@@ -0,0 +1,68 @@
+# The "actual external PostgreSQL development service" gate 4 measures connectivity to, matching
+# k8s/apps/mainloop/overlays/test/postgres-statefulset.yaml's image/auth shape (postgres:16-alpine,
+# mainloop/mainloop/mainloop). Deployed as a plain K8s StatefulSet in its own namespace -- outside
+# any atespace -- since it represents a real external service, not a Substrate actor.
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: postgres-target
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: postgres
+ namespace: postgres-target
+spec:
+ selector:
+ app: postgres
+ ports:
+ - port: 5432
+ targetPort: 5432
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: postgres
+ namespace: postgres-target
+spec:
+ serviceName: postgres
+ replicas: 1
+ selector:
+ matchLabels:
+ app: postgres
+ template:
+ metadata:
+ labels:
+ app: postgres
+ spec:
+ containers:
+ - name: postgres
+ image: postgres:16-alpine
+ ports:
+ - containerPort: 5432
+ env:
+ - name: POSTGRES_USER
+ value: mainloop
+ - name: POSTGRES_PASSWORD
+ value: mainloop
+ - name: POSTGRES_DB
+ value: mainloop
+ volumeMounts:
+ - name: postgres-data
+ mountPath: /var/lib/postgresql/data
+ resources:
+ requests: { memory: 256Mi, cpu: 100m }
+ limits: { memory: 512Mi, cpu: 500m }
+ readinessProbe:
+ exec:
+ command: [pg_isready, -U, mainloop]
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ volumeClaimTemplates:
+ - metadata:
+ name: postgres-data
+ spec:
+ accessModes: [ReadWriteOnce]
+ resources:
+ requests:
+ storage: 1Gi
From b7ffcbe331a0bd449430120ca0a489d2885427c8 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Tue, 22 Sep 2026 04:09:59 +0000
Subject: [PATCH 07/30] feat: build gate-5 live-agent infrastructure; block on
safety classifier
Adds spikes/substrate-workspace-adapter/live-agent-image/ (real Herdr
+ real Claude/Codex CLIs), k8s/cred-server.yaml.tmpl (an in-cluster
nginx server exposing ~/.claude-token and ~/.codex/auth.json from
Kubernetes Secrets created by path, never by value), and
k8s/live-agent-gate-template.yaml.tmpl. Credentials are fetched at
actor-runtime rather than baked into the immutable ActorTemplate,
reachable only because the actor's own narrow EgressPolicy allows
exactly the cred-server's ClusterIP -- reusing the same CIDR-scoped
enforcement gate 4 (dev-service) proved actually denies everything
else with a clean 403, as the credential-delivery boundary.
Did not run the trial: the cluster-creation step, which provisions a
live actor that pulls in real Claude Code / Codex OAuth credentials,
was declined by Claude Code's own auto-mode safety classifier ("Create
Unsafe Agents"). Per .tasknotes/plan.md's own stopping criterion for a
missing permission, and because this touches the operator's real
subscription credentials, the run stopped there rather than seeking a
workaround, and asked the operator directly rather than proceeding.
Gate 5 (native-session continuity, live) is documented as built-but-
not-run in docs/spikes/substrate-workspace-adapter.md and the task's
proof note (.tasknotes, not tracked here), pending explicit operator
authorization to run it.
---
docs/spikes/substrate-workspace-adapter.md | 18 +-
.../k8s/cred-server.yaml.tmpl | 71 ++++++
.../k8s/live-agent-gate-template.yaml.tmpl | 45 ++++
.../live-agent-image/.gitignore | 4 +
.../live-agent-image/Dockerfile | 28 +++
.../live-agent-image/agent-config/claude.env | 4 +
.../live-agent-image/agent-config/codex.env | 4 +
.../agent-config/mainloop-system.txt | 3 +
.../live-agent-image/bin/agentctl | 235 ++++++++++++++++++
.../live-agent-image/bin/mainloop | 159 ++++++++++++
.../live-agent-image/entrypoint.sh | 57 +++++
.../live-agent-image/exec-shim.js | 67 +++++
12 files changed, 690 insertions(+), 5 deletions(-)
create mode 100644 spikes/substrate-workspace-adapter/k8s/cred-server.yaml.tmpl
create mode 100644 spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl
create mode 100644 spikes/substrate-workspace-adapter/live-agent-image/.gitignore
create mode 100644 spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
create mode 100644 spikes/substrate-workspace-adapter/live-agent-image/agent-config/claude.env
create mode 100644 spikes/substrate-workspace-adapter/live-agent-image/agent-config/codex.env
create mode 100644 spikes/substrate-workspace-adapter/live-agent-image/agent-config/mainloop-system.txt
create mode 100755 spikes/substrate-workspace-adapter/live-agent-image/bin/agentctl
create mode 100755 spikes/substrate-workspace-adapter/live-agent-image/bin/mainloop
create mode 100644 spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
create mode 100644 spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 4b413be..56a9ee9 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -24,7 +24,8 @@ drives its lifecycle through the real `kubectl ate` control-plane CLI.
| `dev-service-gate` WorkerPool + ActorTemplate: real `psql`, real external `postgres:16-alpine` StatefulSet, real `EgressPolicy` (CIDR rule, created via a small gRPC tool since `kubectl-ate` has no CLI verb for it) | Real |
| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
| File edits and shell commands run inside actors (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
-| Claude/Codex agent processes, credentials | Not run in this spike (see "Not attempted") |
+| `live-agent-gate` WorkerPool + ActorTemplate: real Claude/Codex CLIs, real cred-server | Built, not run -- blocked at cluster creation by Claude Code's own safety classifier ("Create Unsafe Agents"); see "Limits" |
+| Claude/Codex agent processes, credentials | Not run in this spike (see "Limits") |
| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
## Why not a real agent for the preview-gate edit (credential-injection gap)
@@ -168,10 +169,17 @@ per `docs/network-egress.md`'s "CIDR/all policy: dial now" passthrough path) ins
## Limits / not attempted in this run
-- **Native-session gate, live**: no real Claude/Codex session was started inside a Substrate
- actor; the credential wiring authorized by `.tasknotes/plan.md` was not used. Only the
- fixture-level contract logic (`test_workspace_adapter.py`) and the generic
- restore-vs-reboot log evidence above are available.
+- **Native-session gate, live**: infrastructure built and ready
+ (`spikes/substrate-workspace-adapter/live-agent-image/`, `k8s/cred-server.yaml.tmpl`,
+ `k8s/live-agent-gate-template.yaml.tmpl`) -- real Claude/Codex CLIs, credentials fetched at
+ actor-runtime through the same egress-CIDR mechanism gate 4 proved enforces (never baked into
+ a template), reusing `.tasknotes/plan.md`'s by-path Secret pattern. The trial was not run: the
+ cluster-creation step was declined by Claude Code's own auto-mode safety classifier ("Create
+ Unsafe Agents"), and the run stopped there rather than seeking a workaround, per the plan's own
+ "missing permission" stopping criterion -- this involves the operator's real subscription
+ credentials, so proceeding past a safety control without explicit human authorization was not
+ appropriate. Only the fixture-level contract logic (`test_workspace_adapter.py`) and the
+ generic restore-vs-reboot log evidence above are available for this gate.
- **`workspace_bindings` orchestration functions** (`ensure_workspace`, `resume_workspace`, ...)
were not exercised against a live Postgres + running backend; only their extracted pure logic
(`plan_ensure`, `_binding_from_row`, `is_crashed`) is unit tested, and the transport layer
diff --git a/spikes/substrate-workspace-adapter/k8s/cred-server.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/cred-server.yaml.tmpl
new file mode 100644
index 0000000..f2b2459
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/k8s/cred-server.yaml.tmpl
@@ -0,0 +1,71 @@
+# Serves the two credential files a real Claude/Codex CLI needs, from Kubernetes Secrets created
+# by path (never by value -- see build script). Reachable only by an actor whose EgressPolicy
+# explicitly allows this Service's ClusterIP: the same CIDR-scoped enforcement gate 4
+# (dev-service) proved denies everything else with a clean 403, reused here as the credential
+# boundary. Plain NGINX static-file serving; no application code. Deployed outside any atespace,
+# like the gate 4 Postgres target -- this represents a Mainloop-operated credential-relay
+# service, not part of the actor's own image or an atespace resource.
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: cred-server
+---
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: cred-server-nginx-config
+ namespace: cred-server
+data:
+ nginx.conf: |
+ events {}
+ http {
+ server {
+ listen 80;
+ location = /claude-token { alias /secrets/claude/token; }
+ location = /codex-auth.json { alias /secrets/codex/auth.json; }
+ location / { return 404; }
+ }
+ }
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: cred-server
+ namespace: cred-server
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: cred-server
+ template:
+ metadata:
+ labels:
+ app: cred-server
+ spec:
+ containers:
+ - name: nginx
+ image: nginx:alpine
+ ports:
+ - containerPort: 80
+ volumeMounts:
+ - { name: config, mountPath: /etc/nginx/nginx.conf, subPath: nginx.conf }
+ - { name: claude-token, mountPath: /secrets/claude, readOnly: true }
+ - { name: codex-auth, mountPath: /secrets/codex, readOnly: true }
+ volumes:
+ - name: config
+ configMap: { name: cred-server-nginx-config }
+ - name: claude-token
+ secret: { secretName: mainloop-claude-token }
+ - name: codex-auth
+ secret: { secretName: mainloop-codex-auth }
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: cred-server
+ namespace: cred-server
+spec:
+ selector:
+ app: cred-server
+ ports:
+ - port: 80
diff --git a/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl
new file mode 100644
index 0000000..a6e5350
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl
@@ -0,0 +1,45 @@
+# WorkerPool + ActorTemplate for the bounded live native-agent proof (gate 5 in
+# .tasknotes/plan.md): real Claude Code / Codex CLIs under Herdr, driven by the exec shim. See
+# spikes/substrate-workspace-adapter/live-agent-image/.
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: ${ATESPACE}
+---
+apiVersion: ate.dev/v1alpha1
+kind: WorkerPool
+metadata:
+ name: live-agent-gate
+ namespace: ${ATESPACE}
+ labels:
+ workload: live-agent-gate
+spec:
+ replicas: 1
+ workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
+ template:
+ resources:
+ limits: { cpu: '2', memory: 2Gi }
+ requests: { cpu: 250m, memory: 2Gi }
+---
+metadata:
+ atespace: ${ATESPACE}
+ name: live-agent-gate
+workerSelector:
+ matchLabels:
+ workload: live-agent-gate
+containers:
+- name: live-agent
+ image: __IMAGE__
+ env:
+ - { name: CRED_SERVER, value: "cred-server.cred-server.svc.cluster.local" }
+ resources:
+ limits:
+ - { name: cpu, quantity: "2" }
+ - { name: memory, quantity: 2Gi }
+snapshotsConfig:
+ onPause: SNAPSHOT_CONTENT_SCOPE_FULL
+ onCommit: SNAPSHOT_CONTENT_SCOPE_FULL
+ storageLocation: gs://${BUCKET_NAME}/live-agent-gate/
+sandboxConfig:
+ sandboxClass: SANDBOX_CLASS_GVISOR
+ configName: gvisor-default
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/.gitignore b/spikes/substrate-workspace-adapter/live-agent-image/.gitignore
new file mode 100644
index 0000000..8ab7127
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/.gitignore
@@ -0,0 +1,4 @@
+herdr
+claude
+codex
+codex-code-mode-host
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
new file mode 100644
index 0000000..824d030
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
@@ -0,0 +1,28 @@
+# Live-agent-gate spike image (gate 5, bounded live proof): real Herdr + real Claude Code /
+# Codex CLIs + agentctl, driven by the same generic exec shim as the other gate images.
+# herdr/claude/codex/codex-code-mode-host are copied from the host into the build context by the
+# build script (never committed). No credentials are baked into this image; they are fetched at
+# actor runtime from an in-cluster server reachable only via a narrow EgressPolicy (see
+# entrypoint.sh and docs/spikes/substrate-workspace-adapter.md).
+FROM node:22-bookworm-slim
+RUN apt-get update && apt-get install -y --no-install-recommends jq ca-certificates git curl procps ripgrep \
+ && rm -rf /var/lib/apt/lists/* \
+ && useradd -m -u 10001 agent
+COPY herdr /usr/local/bin/herdr
+COPY claude /usr/local/bin/claude
+COPY codex /usr/local/bin/codex
+COPY codex-code-mode-host /usr/local/bin/codex-code-mode-host
+COPY bin/agentctl bin/mainloop /usr/local/bin/
+COPY agent-config /etc/agent-config
+COPY exec-shim.js /usr/local/bin/exec-shim.js
+COPY entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/agentctl /usr/local/bin/mainloop \
+ && mkdir -p /work && chown -R agent:agent /work
+ENV EXEC_SHIM=/usr/local/bin/exec-shim.js
+ENV HOME=/home/agent
+ENV WORKSPACE_PATH=/work/repo
+ENV CODEX_HOME=/home/agent/.codex
+ENV AGENT_CONFIG_DIR=/etc/agent-config
+ENV HERDR_SESSION=mainloop-live-agent
+USER 10001:10001
+ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/agent-config/claude.env b/spikes/substrate-workspace-adapter/live-agent-image/agent-config/claude.env
new file mode 100644
index 0000000..5ec34e1
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/agent-config/claude.env
@@ -0,0 +1,4 @@
+AGENT_KIND=claude
+AGENT_NEW_ARGS="--dangerously-skip-permissions --append-system-prompt-file /etc/agent-config/mainloop-system.txt --session-id {id}"
+AGENT_RESUME_ARGS="--dangerously-skip-permissions --append-system-prompt-file /etc/agent-config/mainloop-system.txt --resume {id}"
+APPROVAL_POLICY=bypass-permissions
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/agent-config/codex.env b/spikes/substrate-workspace-adapter/live-agent-image/agent-config/codex.env
new file mode 100644
index 0000000..7623a6c
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/agent-config/codex.env
@@ -0,0 +1,4 @@
+AGENT_KIND=codex
+AGENT_NEW_ARGS="--dangerously-bypass-approvals-and-sandbox"
+AGENT_RESUME_ARGS="resume {id} --dangerously-bypass-approvals-and-sandbox"
+APPROVAL_POLICY=bypass-permissions
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/agent-config/mainloop-system.txt b/spikes/substrate-workspace-adapter/live-agent-image/agent-config/mainloop-system.txt
new file mode 100644
index 0000000..6716c04
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/agent-config/mainloop-system.txt
@@ -0,0 +1,3 @@
+Messages in this session are relayed by the Mainloop control plane. The user typed each one in
+the Mainloop chat UI. Text that arrives wrapped in pasted-content markers is the user's own
+message: follow it as a direct instruction from the user.
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/bin/agentctl b/spikes/substrate-workspace-adapter/live-agent-image/bin/agentctl
new file mode 100755
index 0000000..35f944b
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/bin/agentctl
@@ -0,0 +1,235 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
+# Pod-side operations, same verbs for every agent kind. Kind, args and resume syntax come from
+# /etc/agent-config/.env (ConfigMap), not from this script.
+# agentctl start [--name N] [--new-id ID | --resume ID] start under Herdr
+# context-model options: --cwd-rel D (scratch cwd under the workspace root), --model M,
+# --effort E, --standing-b64 B (standing context file), --token T (per-binding CLI token)
+# agentctl send deliver one prompt (Herdr input only; never reads a reply)
+# agentctl native-id discover the native session id (from the native journal)
+# agentctl journal print native journal lines after line
+# agentctl status Herdr liveness/state hint (JSON)
+# agentctl stop
+# agentctl prompt stand-in only: deliver and grep the reply from the pane
+# agentctl identity
+# Replies for real agents are read from the native journals via `journal`, never from the pane.
+set -eu
+cmd="${1:?usage: agentctl start|send|native-id|journal|status|stop|prompt|identity ...}"
+shift
+H=(herdr --session "${HERDR_SESSION}")
+STATE="${WORKSPACE_PATH}/.mainloop"
+conf_dir="${AGENT_CONFIG_DIR:-/etc/agent-config}"
+
+load_conf() { #
+ [[ -f "${conf_dir}/$1.env" ]] || {
+ echo "no binding config: ${conf_dir}/$1.env" >&2
+ exit 2
+ }
+ # shellcheck disable=SC1090
+ . "${conf_dir}/$1.env"
+}
+
+cwd="${WORKSPACE_PATH}"
+
+trust_cwd() { # : pre-accept the trust dialog for a scratch cwd (no human at the TUI)
+ case "$1" in
+ claude)
+ local tmp
+ tmp="$(mktemp)"
+ jq --arg p "$2" '.projects[$p] = ((.projects[$p] // {}) + {hasTrustDialogAccepted: true, hasCompletedProjectOnboarding: true})' "${HOME}/.claude.json" >"${tmp}" &&
+ cat "${tmp}" >"${HOME}/.claude.json"
+ rm -f "${tmp}"
+ ;;
+ codex)
+ grep -qF "[projects.\"$2\"]" "${CODEX_HOME}/config.toml" || printf '\n[projects."%s"]\ntrust_level = "trusted"\n' "$2" >>"${CODEX_HOME}/config.toml"
+ ;;
+ esac
+}
+
+pane_for_name() { # : one Herdr workspace (labelled with the name) per agent
+ local ws
+ ws="$("${H[@]}" workspace list | jq -r --arg b "$1" '.result.workspaces[] | select(.label==$b) | .workspace_id' | head -n1)"
+ if [[ -z ${ws} ]]; then
+ ws="$("${H[@]}" workspace create --label "$1" --cwd "${cwd}" | jq -r .result.workspace.workspace_id)"
+ fi
+ "${H[@]}" pane list --workspace "${ws}" | jq -r '.result.panes[0].pane_id'
+}
+
+journal_file() { #
+ case "$1" in
+ claude) find "${CLAUDE_CONFIG_DIR:-${HOME}/.claude}/projects" -name "$2.jsonl" 2>/dev/null | head -n1 ;;
+ codex) find "${CODEX_HOME}/sessions" -name "rollout-*-$2.jsonl" 2>/dev/null | head -n1 ;;
+ *)
+ echo "no journal for kind $1" >&2
+ return 1
+ ;;
+ esac
+}
+
+case "${cmd}" in
+start)
+ binding="${1:?binding required}"
+ shift
+ name="${binding}"
+ mode=new
+ nid=""
+ cwd_rel=""
+ model=""
+ effort=""
+ standing_b64=""
+ token=""
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --name)
+ name="$2"
+ shift 2
+ ;;
+ --new-id)
+ mode=new
+ nid="$2"
+ shift 2
+ ;;
+ --resume)
+ mode=resume
+ nid="$2"
+ shift 2
+ ;;
+ --cwd-rel)
+ cwd_rel="$2"
+ shift 2
+ ;;
+ --model)
+ model="$2"
+ shift 2
+ ;;
+ --effort)
+ effort="$2"
+ shift 2
+ ;;
+ --standing-b64)
+ standing_b64="$2"
+ shift 2
+ ;;
+ --token)
+ token="$2"
+ shift 2
+ ;;
+ *)
+ echo "unknown option $1" >&2
+ exit 2
+ ;;
+ esac
+ done
+ load_conf "${binding}"
+ ident="${STATE}/${name}.identity.json"
+ if "${H[@]}" agent get "${name}" >/dev/null 2>&1; then
+ echo "agent ${name} already live"
+ exit 0
+ fi
+ if [[ ${mode} == resume ]]; then args="${AGENT_RESUME_ARGS:-${AGENT_ARGS}}"; else args="${AGENT_NEW_ARGS:-${AGENT_ARGS}}"; fi
+ if [[ -n ${cwd_rel} ]]; then
+ cwd="$(dirname "${WORKSPACE_PATH}")/${cwd_rel}"
+ mkdir -p "${cwd}/.mainloop"
+ chmod 700 "${cwd}/.mainloop"
+ # Secrets and generated context go to files on the PVC (0600), never into the pane command.
+ [[ -z ${token} ]] || (
+ umask 077
+ printf '%s' "${token}" >"${cwd}/.mainloop/token"
+ )
+ [[ -z ${standing_b64} ]] || printf '%s' "${standing_b64}" | base64 -d >"${cwd}/.mainloop/standing.md"
+ [[ ! -f "${conf_dir}/${binding}.settings.json" ]] || cp "${conf_dir}/${binding}.settings.json" "${cwd}/.mainloop/settings.json"
+ trust_cwd "${AGENT_KIND}" "${cwd}"
+ fi
+ args="${args//\{id\}/${nid}}"
+ args="${args//\{model\}/${model}}"
+ args="${args//\{effort\}/${effort}}"
+ args="${args//\{standing\}/${cwd}/.mainloop/standing.md}"
+ args="${args//\{settings\}/${cwd}/.mainloop/settings.json}"
+ mkdir -p "${STATE}"
+ touch "${STATE}/${name}.started"
+ pane="$(pane_for_name "${name}")"
+ # $args is intentionally word-split: it is the native executable's argument list.
+ # shellcheck disable=SC2086
+ "${H[@]}" agent start "${name}" --kind "${AGENT_KIND}" --pane "${pane}" --timeout 60000 -- ${args} >/dev/null
+ "${H[@]}" agent get "${name}" | jq -c --arg b "${binding}" --arg k "${AGENT_KIND}" --arg args "${args}" --arg mode "${mode}" --arg nid "${nid}" \
+ '.result.agent | {binding:$b, kind:$k, args:$args, mode:$mode, native_session_id:(if $nid=="" then null else $nid end), herdr_agent:.agent, herdr_name:.name, pane_id, terminal_id, workspace_id, status:.agent_status}' >"${ident}"
+ cat "${ident}"
+ ;;
+send)
+ name="${1:?name required}"
+ text="${2:?text required}"
+ # One delivery, no --wait retries: Herdr status is a hint, the journal is the receipt.
+ "${H[@]}" agent prompt "${name}" "${text}" >/dev/null
+ echo sent
+ ;;
+native-id)
+ name="${1:?name required}"
+ ident="${STATE}/${name}.identity.json"
+ kind="$(jq -r .kind "${ident}")"
+ known="$(jq -r '.native_session_id // empty' "${ident}")"
+ if [[ -n ${known} ]]; then
+ echo "${known}"
+ exit 0
+ fi
+ case "${kind}" in
+ codex)
+ f="$(find "${CODEX_HOME}/sessions" -name 'rollout-*.jsonl' -newer "${STATE}/${name}.started" 2>/dev/null | sort | head -n1)"
+ [[ -n ${f} ]] || exit 1
+ id="$(basename "${f}" .jsonl | sed -E 's/^rollout-[0-9T:-]+-//')"
+ ;;
+ *) exit 1 ;;
+ esac
+ tmp="$(mktemp)"
+ jq --arg id "${id}" '.native_session_id=$id' "${ident}" >"${tmp}" && cat "${tmp}" >"${ident}" && rm -f "${tmp}"
+ echo "${id}"
+ ;;
+journal)
+ kind="$(jq -r .kind "${STATE}/${1:?name required}.identity.json")"
+ id="${2:?native id required}"
+ from="${3:-0}"
+ f="$(journal_file "${kind}" "${id}")"
+ [[ -n ${f} ]] || {
+ echo "#nofile"
+ exit 0
+ }
+ n="$(wc -l <"${f}")" # complete (newline-terminated) lines only
+ printf '#file\t%s\t%s\n' "${f}" "${n}"
+ if [[ ${n} -gt ${from} ]]; then sed -n "$((from + 1)),${n}p" "${f}" | awk -v s="${from}" '{print (NR + s) "\t" $0}'; fi
+ ;;
+status)
+ # A failed `agent get` must fail the verb (a pipeline would report jq's exit status).
+ out="$("${H[@]}" agent get "${1:?name required}")" || exit 1
+ printf '%s' "${out}" | jq -c '.result.agent | {name, agent, pane_id, terminal_id, status: .agent_status}'
+ ;;
+prompt) # stand-in agents only
+ binding="${1:?binding required}"
+ text="${2:?prompt text required}"
+ "${H[@]}" agent prompt "${binding}" "${text}" --wait --timeout 60000 >/dev/null
+ "${H[@]}" agent read "${binding}" | grep 'STANDIN-REPLY' | grep -F "echo=${text}" | tail -n1
+ ;;
+stop)
+ name="${1:?name required}"
+ kind="$(jq -r '.kind // empty' "${STATE}/${name}.identity.json" 2>/dev/null || true)"
+ if [[ ${kind} == claude ]]; then
+ # A pasted "/exit" is text, and the restricted main thread has slash commands disabled:
+ # two quick Ctrl-C key presses exit Claude Code (measured).
+ "${H[@]}" agent send-keys "${name}" ctrl+c ctrl+c >/dev/null
+ else
+ "${H[@]}" agent prompt "${name}" "/exit" >/dev/null
+ fi
+ for _ in $(seq 1 20); do
+ "${H[@]}" agent get "${name}" >/dev/null 2>&1 || {
+ echo "agent ${name} stopped"
+ exit 0
+ }
+ sleep 0.5
+ done
+ echo "agent ${name} still live after stop" >&2
+ exit 1
+ ;;
+identity) cat "${STATE}/${1:?name required}.identity.json" ;;
+*)
+ echo "unknown command ${cmd}" >&2
+ exit 2
+ ;;
+esac
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/bin/mainloop b/spikes/substrate-workspace-adapter/live-agent-image/bin/mainloop
new file mode 100755
index 0000000..03b0ca0
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/bin/mainloop
@@ -0,0 +1,159 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
+# `mainloop`: the agents' thin client for the Mainloop control plane. It holds no policy.
+# Identity is the per-binding token in .mainloop/token (found by walking up from $PWD, written
+# by `agentctl start`); the server decides what this token may do and answers in plain text.
+set -u
+API="${MAINLOOP_API:-http://mainloop-backend.mainloop.svc.cluster.local:8000}"
+
+find_token() {
+ local d="${PWD}"
+ while :; do
+ [[ -f "${d}/.mainloop/token" ]] && {
+ cat "${d}/.mainloop/token"
+ return 0
+ }
+ [[ ${d} == / ]] && return 1
+ d="$(dirname "${d}")"
+ done
+}
+TOKEN="${MAINLOOP_TOKEN:-$(find_token)}" || {
+ echo "mainloop: no agent token found from ${PWD}" >&2
+ exit 2
+}
+
+call() { # [json body] ; query params via Q=(--data-urlencode k=v ...)
+ local out code body
+ out="$(curl -sS -m 30 -w '\n%{http_code}' -X "$1" -H "Authorization: Bearer ${TOKEN}" \
+ -H 'Content-Type: application/json' ${Q[@]+"${Q[@]}"} ${3:+-d "$3"} -G "${API}$2" 2>&1)" ||
+ {
+ echo "mainloop: control plane unreachable" >&2
+ exit 3
+ }
+ code="${out##*$'\n'}"
+ body="${out%$'\n'*}"
+ if [[ ${code} -ge 200 ]] 2>/dev/null && [[ ${code} -lt 300 ]]; then
+ printf '%s' "${body}" | jq -r '.text // .'
+ else
+ echo "mainloop: refused (${code}): $(printf '%s' "${body}" | jq -r '.detail // .' 2>/dev/null || printf '%s' "${body}")" >&2
+ exit 1
+ fi
+}
+Q=()
+# POST/GET with a body use -d, which makes curl POST; -G turns -d into a query string, so bodies
+# are sent with --json-style separately:
+post() { #
+ local out code body
+ out="$(curl -sS -m 30 -w '\n%{http_code}' -X POST -H "Authorization: Bearer ${TOKEN}" \
+ -H 'Content-Type: application/json' --data-binary "$2" "${API}$1" 2>&1)" ||
+ {
+ echo "mainloop: control plane unreachable" >&2
+ exit 3
+ }
+ code="${out##*$'\n'}"
+ body="${out%$'\n'*}"
+ if [[ ${code} -ge 200 ]] 2>/dev/null && [[ ${code} -lt 300 ]]; then
+ printf '%s' "${body}" | jq -r '.text // .'
+ else
+ echo "mainloop: refused (${code}): $(printf '%s' "${body}" | jq -r '.detail // .' 2>/dev/null || printf '%s' "${body}")" >&2
+ exit 1
+ fi
+}
+usage() {
+ sed -n '2,3p' "$0"
+ echo "verbs: whoami topics topic note decide pending delegate status read cancel clear report standing"
+}
+
+verb="${1:-help}"
+[[ $# -gt 0 ]] && shift
+case "${verb}" in
+help | -h | --help) usage ;;
+whoami) call GET /agent-api/whoami ;;
+topics) call GET /agent-api/topics ;;
+standing) call GET /agent-api/standing ;;
+topic)
+ [[ ${1-} == open ]] || {
+ echo "usage: mainloop topic open [--status ]" >&2
+ exit 2
+ }
+ shift
+ name="${1:?topic name required}"
+ shift
+ status=""
+ [[ ${1-} == --status ]] && status="${2-}"
+ post /agent-api/topics "$(jq -n --arg n "${name}" --arg s "${status}" 'if $s=="" then {name:$n} else {name:$n,status:$s} end')"
+ ;;
+note | decide | pending)
+ kind="${verb}"
+ [[ ${verb} == decide ]] && kind=decision
+ if [[ ${verb} == pending ]] && [[ ${1-} == --done ]]; then
+ post "/agent-api/records/${2:?id required}/done" '{}'
+ exit
+ fi
+ text="${1:?text required}"
+ shift
+ topic=""
+ [[ ${1-} == --topic ]] && topic="${2-}"
+ post /agent-api/records "$(jq -n --arg k "${kind}" --arg t "${text}" --arg p "${topic}" 'if $p=="" then {kind:$k,text:$t} else {kind:$k,text:$t,topic:$p} end')"
+ ;;
+delegate)
+ topic=inbox
+ kind=""
+ title=""
+ brief=""
+ while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --topic)
+ topic="$2"
+ shift 2
+ ;;
+ --kind)
+ kind="$2"
+ shift 2
+ ;;
+ --title)
+ title="$2"
+ shift 2
+ ;;
+ *)
+ brief="$1"
+ shift
+ ;;
+ esac
+ done
+ [[ -n ${kind} ]] && [[ -n ${brief} ]] || {
+ echo 'usage: mainloop delegate --topic --kind claude|codex --title "" ""' >&2
+ exit 2
+ }
+ post /agent-api/delegate "$(jq -n --arg t "${topic}" --arg k "${kind}" --arg ti "${title}" --arg b "${brief}" '{topic:$t,kind:$k,title:$ti,brief:$b}')"
+ ;;
+status)
+ [[ -n ${1-} ]] && Q=(--data-urlencode "session=$1")
+ call GET /agent-api/status
+ ;;
+read)
+ id="${1:?session id required}"
+ shift
+ since=0
+ [[ ${1-} == --since ]] && since="${2:-0}"
+ Q=(--data-urlencode "session=${id}" --data-urlencode "since=${since}")
+ call GET /agent-api/read
+ ;;
+cancel)
+ post /agent-api/cancel "$(jq -n --arg s "${1:?session id required}" '{session:$s}')"
+ ;;
+clear)
+ post /agent-api/clear "$(jq -n --arg s "${1-}" 'if $s=="" then {} else {session:$s} end')"
+ ;;
+report)
+ [[ ${1-} == --summary ]] || {
+ echo 'usage: mainloop report --summary ""' >&2
+ exit 2
+ }
+ post /agent-api/report "$(jq -n --arg s "${2:?summary required}" '{summary:$s}')"
+ ;;
+*)
+ usage >&2
+ exit 2
+ ;;
+esac
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
new file mode 100644
index 0000000..5f1547e
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
@@ -0,0 +1,57 @@
+#!/usr/bin/env bash
+# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
+# Live-agent-gate actor entrypoint (gate 5 in .tasknotes/plan.md, bounded live proof): a real
+# Herdr server plus real Claude Code / Codex CLIs. Substrate has no Kubernetes-Secret-equivalent
+# volume/env mechanism for an actor (see docs/spikes/substrate-workspace-adapter.md,
+# "Credential-injection gap"), so credentials are fetched over the network from a small in-cluster
+# server, reachable only because this actor's narrow EgressPolicy allows exactly that server's
+# ClusterIP -- the same CIDR-scoped access-control mechanism gate 4 (dev-service) proved actually
+# enforces (a non-allowed destination gets a clean 403), reused here as the auth boundary rather
+# than inventing a new one. Never echoed, never written to a template, never logged.
+set -eu
+mkdir -p "${HOME}" "${HOME}/.claude" "${CODEX_HOME}"
+
+if [[ -n ${CRED_SERVER-} ]]; then
+ curl -fsS "http://${CRED_SERVER}/claude-token" -o "${HOME}/.claude-oauth-token"
+ chmod 600 "${HOME}/.claude-oauth-token"
+ CLAUDE_CODE_OAUTH_TOKEN="$(tr -d ' \r\n' <"${HOME}/.claude-oauth-token")"
+ export CLAUDE_CODE_OAUTH_TOKEN
+ curl -fsS "http://${CRED_SERVER}/codex-auth.json" -o "${CODEX_HOME}/auth.json"
+ chmod 600 "${CODEX_HOME}/auth.json"
+fi
+
+# Claude Code: onboarding done, workspace trusted, bypass-permissions warning accepted.
+if [[ ! -s "${HOME}/.claude.json" ]]; then
+ jq -n --arg p "${WORKSPACE_PATH}" '{
+ hasCompletedOnboarding: true,
+ numStartups: 1,
+ theme: "dark",
+ projects: {($p): {hasTrustDialogAccepted: true, hasCompletedProjectOnboarding: true, allowedTools: []}}
+ }' >"${HOME}/.claude.json"
+fi
+[[ -s "${HOME}/.claude/settings.json" ]] || echo '{"skipDangerousModePermissionPrompt": true}' >"${HOME}/.claude/settings.json"
+
+# Codex: trust the workspace.
+if [[ ! -s "${CODEX_HOME}/config.toml" ]]; then
+ printf '[projects."%s"]\ntrust_level = "trusted"\n' "${WORKSPACE_PATH}" >"${CODEX_HOME}/config.toml"
+fi
+grep -q '^\[notice\]' "${CODEX_HOME}/config.toml" || printf '\n[notice]\nhide_rate_limit_model_nudge = true\n' >>"${CODEX_HOME}/config.toml"
+
+mkdir -p "${WORKSPACE_PATH}"
+[[ -d "${WORKSPACE_PATH}/.git" ]] || git -C "${WORKSPACE_PATH}" init -q
+
+echo "herdr $(herdr --version) server starting (HOME=${HOME} session=${HERDR_SESSION})"
+herdr --session "${HERDR_SESSION}" server &
+HERDR_PID=$!
+
+for _ in $(seq 1 60); do
+ herdr --session "${HERDR_SESSION}" status server >/dev/null 2>&1 && break
+ sleep 0.5
+done
+
+shell_ws=$(herdr --session "${HERDR_SESSION}" workspace create --label shell --cwd "${WORKSPACE_PATH}")
+shell_pane=$(echo "${shell_ws}" | jq -r '.result.root_pane.pane_id')
+
+EXEC_SHIM_PANE_ID="${shell_pane}" HERDR_SESSION="${HERDR_SESSION}" node "${EXEC_SHIM}" &
+
+wait "${HERDR_PID}"
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
new file mode 100644
index 0000000..01250e7
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
@@ -0,0 +1,67 @@
+// Minimal generic command executor for the preview-gate spike: POST /run { command } pastes
+// `command` as literal text into a real Herdr shell pane via `herdr pane run` (fire-and-forget;
+// the pane is a real bash shell, so this is a real shell write, not a purpose-built edit
+// endpoint). GET /read returns the pane's current terminal buffer (`herdr pane read`), since
+// `pane run` itself never captures output. Stands in for a credentialed native agent's own Bash
+// tool -- see entrypoint.sh and docs/spikes/substrate-workspace-adapter.md for why a real agent
+// could not be used here.
+// Listens on port 8090, separate from the Vite dev server's port 80. Reached only through
+// atenet-router's arbitrary-port CONNECT tunnel with the ate-target-actor header (see
+// docs/api-guide.md "Workload Connectivity"), from test orchestration on the host -- never
+// through the previewed route a browser uses (port 80 via the NGINX header-proxy).
+'use strict';
+const http = require('node:http');
+const { execFile } = require('node:child_process');
+
+const PANE_ID = process.env.EXEC_SHIM_PANE_ID;
+const SESSION = process.env.HERDR_SESSION;
+if (!PANE_ID || !SESSION) {
+ console.error('exec-shim: EXEC_SHIM_PANE_ID and HERDR_SESSION are required');
+ process.exit(1);
+}
+
+function herdr(args, res) {
+ execFile('herdr', ['--session', SESSION, ...args], (err, stdout, stderr) => {
+ if (err) {
+ res.writeHead(502).end(String(err));
+ return;
+ }
+ res
+ .writeHead(200, { 'content-type': 'application/json' })
+ .end(JSON.stringify({ ok: true, stdout, stderr }));
+ });
+}
+
+const server = http.createServer((req, res) => {
+ if (req.method === 'GET' && req.url === '/read') {
+ herdr(['pane', 'read', PANE_ID], res);
+ return;
+ }
+ if (req.method !== 'POST' || req.url !== '/run') {
+ res.writeHead(404).end();
+ return;
+ }
+ let body = '';
+ req.on('data', (chunk) => {
+ body += chunk;
+ if (body.length > 65536) req.destroy();
+ });
+ req.on('end', () => {
+ let command;
+ try {
+ command = JSON.parse(body).command;
+ } catch {
+ res.writeHead(400).end('invalid json');
+ return;
+ }
+ if (typeof command !== 'string' || !command) {
+ res.writeHead(400).end('missing command');
+ return;
+ }
+ herdr(['pane', 'run', PANE_ID, command], res);
+ });
+});
+
+server.listen(8090, '0.0.0.0', () => {
+ console.log('exec-shim listening on :8090, pane', PANE_ID);
+});
From 4514cda7b5a79024411094020a7b58f9c0661599 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Tue, 22 Sep 2026 22:25:46 +0000
Subject: [PATCH 08/30] fix: repair Gate 5 workspace adapter harness
---
backend/scripts/gate5_setup.py | 615 ++++++++++++++++++
backend/src/mainloop/runtime/substrate.py | 421 +++++++++++-
.../kubectl-ate-workers-pinned-cdac9ba.json | 25 +
backend/tests/runtime/test_contracts.py | 25 +
backend/tests/runtime/test_gate5_setup.py | 389 +++++++++++
backend/tests/runtime/test_substrate.py | 534 ++++++++++++++-
docs/spikes/substrate-workspace-adapter.md | 61 +-
.../egress-tool/main.go | 46 +-
.../egress-tool/main_test.go | 28 +
.../k8s/cred-server.yaml.tmpl | 71 --
.../k8s/live-agent-gate-template.yaml.tmpl | 13 +-
.../live-agent-image/Dockerfile | 7 +-
.../live-agent-image/entrypoint.sh | 44 +-
.../live-agent-image/exec-shim.js | 22 +
14 files changed, 2181 insertions(+), 120 deletions(-)
create mode 100644 backend/scripts/gate5_setup.py
create mode 100644 backend/tests/runtime/fixtures/substrate/kubectl-ate-workers-pinned-cdac9ba.json
create mode 100644 backend/tests/runtime/test_gate5_setup.py
create mode 100644 spikes/substrate-workspace-adapter/egress-tool/main_test.go
delete mode 100644 spikes/substrate-workspace-adapter/k8s/cred-server.yaml.tmpl
diff --git a/backend/scripts/gate5_setup.py b/backend/scripts/gate5_setup.py
new file mode 100644
index 0000000..2271154
--- /dev/null
+++ b/backend/scripts/gate5_setup.py
@@ -0,0 +1,615 @@
+#!/usr/bin/env python3
+r"""
+Gate 5 (native-session continuity, .tasknotes/plan.md) credential-free harness: registers
+the atespace, resolves and applies the WorkerPool, creates a *versioned* ActorTemplate and
+waits for its golden snapshot, then creates/resumes one actor and waits for it to become
+RUNNING with its persistent control service (Herdr) confirmed ready -- all without a
+provider credential, a credential server, or a native Claude/Codex session.
+
+Implements recovery step 2 of .tasknotes/gate5-review-and-recovery-plan-2026-09-22.md. That
+review found the prior harness applied an unresolved `ko://` worker image, never registered
+the atespace at the API level (a Kubernetes Namespace of the same name is not an atespace),
+checked for an existing ActorTemplate with the atespace embedded in the name instead of the
+CLI's required `-a` flag, and printed success once a log line appeared even when that
+happened before the read that mattered -- while the underlying golden-snapshot failure was a
+credential fetch built into the shared, immutable template, which this script's manifest no
+longer has (see spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh).
+
+Deliberately out of scope here (recovery plan steps 3-4, a separate task): fetching a real
+credential, attaching it to a running actor, and starting a native Claude/Codex session.
+
+Prerequisites:
+ kubectl, kubectl-ate, and ko built from the pinned Substrate checkout.
+ A running kind-substrate-preview cluster with the ate-system + agentgateway dataplane
+ installed (this script accepts only the exact kind-substrate-preview context).
+ The live-agent-gate image already built and pushed (see live-agent-image/), its digest
+ passed with --image.
+
+Usage:
+ cd backend
+ uv run python scripts/gate5_setup.py \\
+ --context kind-substrate-preview --kubeconfig /tmp/substrate-preview-kubeconfig \\
+ --ate-cli /tmp/substrate-preview-src/bin/kubectl-ate \\
+ --ko /tmp/substrate-preview-src/bin/ko \\
+ --substrate-src /tmp/substrate-preview-src \\
+ --atespace live-agent-gate --template-version v1 \\
+ --image localhost:5001/live-agent-gate@sha256:... \\
+ --manifest ../spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl \\
+ --state-file /tmp/gate5-run-state.json \\
+ --egress-tool /tmp/substrate-preview-src/bin/mainloop-egress-tool \\
+ --egress-deny-all
+
+Re-running with the same --state-file reconciles the persisted actor uid against the
+cluster's current state rather than blindly creating or resuming; a name collision with a
+*different* uid is refused, not silently overwritten.
+"""
+
+import argparse
+import asyncio
+import contextlib
+import http.client
+import json
+import os
+import re
+import socket
+import string
+import subprocess # nosec B404 - drives trusted local kubectl/ko/egress-tool binaries, argv only
+import sys
+import tempfile
+import time
+import uuid
+from pathlib import Path
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "src"))
+
+from mainloop.runtime.substrate import ( # noqa: E402
+ ActorFailedToStart,
+ ActorState,
+ GoldenState,
+ IdentityConflict,
+ IdentityOutcome,
+ SubstrateControl,
+ TransportError,
+ reconcile_actor_identity,
+ wait_for_actor_health,
+ wait_for_actor_running,
+ wait_for_eligible_worker,
+ wait_for_golden_snapshot,
+)
+
+PINNED_SUBSTRATE_COMMIT = "cdac9baef81dd319b46086d695266e6161e9e592"
+WORKER_NAMESPACE = "live-agent-gate"
+WORKER_SELECTOR = "workload=live-agent-gate"
+WORKER_SANDBOX_CLASS = "gvisor"
+ACTOR_SHIM_PORT = 8090
+
+
+def parse_args() -> argparse.Namespace:
+ p = argparse.ArgumentParser(description=__doc__)
+ p.add_argument("--context", required=True)
+ p.add_argument("--kubeconfig", required=True)
+ p.add_argument("--ate-cli", default="kubectl-ate")
+ p.add_argument("--ko", default="ko")
+ p.add_argument("--substrate-src", required=True)
+ p.add_argument("--router-port", type=int, default=18091)
+ p.add_argument("--atespace", required=True)
+ p.add_argument(
+ "--template-version",
+ required=True,
+ help='e.g. "v2" -- never reuse one whose golden snapshot failed',
+ )
+ p.add_argument(
+ "--image",
+ required=True,
+ help="already-built and pushed image digest, e.g. localhost:5001/live-agent-gate@sha256:...",
+ )
+ p.add_argument(
+ "--manifest", required=True, help="path to live-agent-gate-template.yaml.tmpl"
+ )
+ p.add_argument("--bucket-name", default="ate-snapshots")
+ p.add_argument("--actor-name", default="claude-gate5")
+ p.add_argument("--state-file", required=True)
+ p.add_argument("--golden-timeout", type=float, default=300)
+ p.add_argument("--worker-timeout", type=float, default=120)
+ p.add_argument("--actor-timeout", type=float, default=120)
+ p.add_argument("--readiness-timeout", type=float, default=60)
+ p.add_argument("--egress-tool", required=True)
+ egress = p.add_mutually_exclusive_group(required=True)
+ egress.add_argument("--egress-cidr", help="CIDR to allow")
+ egress.add_argument("--egress-allow-all", action="store_true")
+ egress.add_argument("--egress-deny-all", action="store_true")
+ return p.parse_args()
+
+
+def load_state(path: str) -> dict:
+ if not os.path.exists(path):
+ return {}
+ with open(path) as f:
+ return json.load(f)
+
+
+def save_state(path: str, state: dict) -> None:
+ tmp = f"{path}.tmp"
+ with open(tmp, "w") as f:
+ json.dump(state, f, indent=2)
+ os.replace(tmp, path)
+
+
+def render_manifest(
+ path: str, *, atespace: str, template_name: str, bucket_name: str, image: str
+) -> list[str]:
+ """Substitutes the template's ${ATESPACE}/${TEMPLATE_NAME}/${BUCKET_NAME} placeholders
+ and the __IMAGE__ marker, then splits the multi-document YAML on its own '---'
+ separators. Returns [namespace_and_workerpool_doc, actor_template_doc]."""
+ with open(path) as f:
+ raw = f.read()
+ rendered = (
+ string.Template(raw)
+ .safe_substitute(
+ ATESPACE=atespace, TEMPLATE_NAME=template_name, BUCKET_NAME=bucket_name
+ )
+ .replace("__IMAGE__", image)
+ )
+ docs = [d for d in rendered.split("\n---\n") if d.strip()]
+ if len(docs) != 3:
+ raise RuntimeError(
+ f"expected 3 YAML documents (Namespace, WorkerPool, ActorTemplate) in {path}, got {len(docs)}"
+ )
+ namespace_and_workerpool = f"{docs[0]}\n---\n{docs[1]}\n"
+ return [namespace_and_workerpool, docs[2] + "\n"]
+
+
+def verify_substrate_source(source: str, *, runner=subprocess.run) -> str:
+ """Require the exact source checkout that supplies the pinned ``ko`` module."""
+ root = Path(source).expanduser().resolve()
+ if not (root / ".git").exists():
+ raise RuntimeError(f"--substrate-src is not a git checkout: {root}")
+ for required in ("go.mod", ".ko.yaml"):
+ if not (root / required).is_file():
+ raise RuntimeError(f"--substrate-src is missing {required}: {root}")
+ result = runner(
+ ["git", "-C", str(root), "rev-parse", "HEAD"],
+ capture_output=True,
+ text=True,
+ check=True,
+ timeout=15,
+ )
+ commit = result.stdout.strip()
+ if commit != PINNED_SUBSTRATE_COMMIT:
+ raise RuntimeError(
+ f"--substrate-src must be pinned at {PINNED_SUBSTRATE_COMMIT}; found {commit!r}"
+ )
+ return str(root)
+
+
+def get_cluster_identity(
+ *, context: str, kubeconfig: str, runner=subprocess.run
+) -> dict[str, str]:
+ """Identify the selected cluster by its API URL and kube-system namespace UID."""
+ base = ["kubectl", "--context", context, "--kubeconfig", kubeconfig]
+ config = runner(
+ [*base, "config", "view", "--minify", "-o", "json"],
+ capture_output=True,
+ text=True,
+ check=True,
+ timeout=20,
+ )
+ config_doc = json.loads(config.stdout)
+ context_name = (config_doc.get("contexts") or [{}])[0].get("name")
+ api_server = ((config_doc.get("clusters") or [{}])[0].get("cluster") or {}).get(
+ "server"
+ )
+ if context_name != context or not api_server:
+ raise RuntimeError(
+ f"kubeconfig did not resolve the requested context {context!r} to an API server"
+ )
+ namespace = runner(
+ [*base, "get", "namespace", "kube-system", "-o", "json"],
+ capture_output=True,
+ text=True,
+ check=True,
+ timeout=20,
+ )
+ namespace_uid = (json.loads(namespace.stdout).get("metadata") or {}).get("uid")
+ if not namespace_uid:
+ raise RuntimeError("kube-system namespace response is missing metadata.uid")
+ return {"api_server_url": api_server, "kube_system_namespace_uid": namespace_uid}
+
+
+def prepare_run_state(args: argparse.Namespace, cluster: dict[str, str]) -> dict:
+ """Validate or persist the run intent before any cluster create/apply call."""
+ if not re.fullmatch(r".+@sha256:[0-9a-fA-F]{64}", args.image):
+ raise RuntimeError("--image must be pinned by a full @sha256 digest")
+ template_name = f"live-agent-gate-{args.template_version}"
+ identity = {
+ "context": args.context,
+ "cluster_identity": cluster,
+ "atespace": args.atespace,
+ "template_name": template_name,
+ "image_digest": args.image,
+ "actor_name": args.actor_name,
+ }
+ state = load_state(args.state_file)
+ if state:
+ missing = {"run_id", "template_uid", "actor_uid"} - state.keys()
+ mismatch = {
+ key: (state.get(key), value)
+ for key, value in identity.items()
+ if state.get(key) != value
+ }
+ if missing or mismatch:
+ details = []
+ if missing:
+ details.append(f"missing identity fields {sorted(missing)}")
+ if mismatch:
+ details.append(f"requested identity differs: {mismatch}")
+ raise RuntimeError(
+ "state file does not match this run (" + "; ".join(details) + ")"
+ )
+ return state
+
+ state = {
+ **identity,
+ "run_id": str(uuid.uuid4()),
+ "template_uid": None,
+ "actor_uid": None,
+ }
+ save_state(args.state_file, state)
+ return state
+
+
+def apply_worker_pool(
+ doc: str,
+ *,
+ kubeconfig: str,
+ context: str,
+ ko: str,
+ substrate_src: str,
+ runner=subprocess.run,
+) -> None:
+ """Resolve the WorkerPool's `ko://...` workerImage and apply it (and the Namespace doc
+ it's paired with) via `ko resolve | kubectl apply`. An unresolved ko:// reference
+ reaches the pod as an InvalidImageName, not a manifest-time error, so this step must not
+ be skipped even though `kubectl apply` alone would exit 0."""
+ substrate_src = verify_substrate_source(substrate_src, runner=runner)
+ ko_docker_repo = os.environ.get("KO_DOCKER_REPO")
+ if not ko_docker_repo:
+ raise RuntimeError(
+ "KO_DOCKER_REPO must be set (e.g. localhost:5001 for kind) to resolve ko:// images"
+ )
+ with tempfile.NamedTemporaryFile("w", suffix=".yaml", delete=False) as f:
+ f.write(doc)
+ doc_path = f.name
+ try:
+ resolved = (
+ runner( # nosec B603 - argv list, ko path is an operator-supplied flag
+ [ko, "resolve", "-f", doc_path],
+ env={**os.environ, "KO_DOCKER_REPO": ko_docker_repo},
+ capture_output=True,
+ text=True,
+ check=True,
+ timeout=180,
+ cwd=substrate_src,
+ )
+ )
+ runner( # nosec - argv list; kubectl is expected on PATH like git/uv
+ [
+ "kubectl",
+ "--context",
+ context,
+ "--kubeconfig",
+ kubeconfig,
+ "apply",
+ "-f",
+ "-",
+ ],
+ input=resolved.stdout,
+ capture_output=True,
+ text=True,
+ check=True,
+ timeout=60,
+ )
+ finally:
+ os.unlink(doc_path)
+
+
+def run_egress_tool(args: argparse.Namespace) -> None:
+ cmd = [
+ args.egress_tool,
+ "--kubeconfig",
+ args.kubeconfig,
+ "--context",
+ args.context,
+ "--atespace",
+ args.atespace,
+ "--actor",
+ args.actor_name,
+ ]
+ if args.egress_deny_all:
+ cmd.append("--deny-all")
+ elif args.egress_allow_all:
+ cmd.append("--allow-all")
+ else:
+ cmd += ["--cidr", args.egress_cidr]
+ subprocess.run(
+ cmd, check=True, timeout=60
+ ) # nosec B603 - argv list, path is an operator-supplied flag
+
+
+@contextlib.contextmanager
+def actor_router_tunnel(args: argparse.Namespace):
+ """Forward the Substrate router through the explicitly selected kube context."""
+ command = [
+ "kubectl",
+ "--context",
+ args.context,
+ "--kubeconfig",
+ args.kubeconfig,
+ "port-forward",
+ "--address",
+ "127.0.0.1",
+ "--namespace",
+ "ate-system",
+ "service/atenet-router",
+ f"{args.router_port}:80",
+ ]
+ process = (
+ subprocess.Popen( # nosec B603 - fixed kubectl argv, explicit kube context
+ command,
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ text=True,
+ )
+ )
+ try:
+ deadline = time.monotonic() + 20
+ while time.monotonic() < deadline:
+ if process.poll() is not None:
+ output = process.communicate()[0]
+ raise RuntimeError(f"router port-forward exited early: {output[-500:]}")
+ try:
+ with socket.create_connection(
+ ("127.0.0.1", args.router_port), timeout=0.2
+ ):
+ break
+ except OSError:
+ time.sleep(0.2)
+ else:
+ raise RuntimeError("router port-forward did not become ready within 20s")
+ yield args.router_port
+ finally:
+ if process.poll() is None:
+ process.terminate()
+ try:
+ process.wait(timeout=5)
+ except subprocess.TimeoutExpired:
+ process.kill()
+ process.wait(timeout=5)
+ if process.stdout:
+ process.stdout.close()
+
+
+def actor_health_check(
+ *, port: int, atespace: str, actor_name: str, timeout_s: float = 2
+) -> bool:
+ """Call the actor's non-default shim port through Substrate's HTTP CONNECT route."""
+ connection = http.client.HTTPConnection("127.0.0.1", port, timeout=timeout_s)
+ connection.set_tunnel(
+ f"actor-upstream:{ACTOR_SHIM_PORT}",
+ headers={"ate-target-actor": f"{atespace}/{actor_name}"},
+ )
+ try:
+ connection.request("GET", "/healthz")
+ response = connection.getresponse()
+ response.read()
+ return response.status == 200
+ except (OSError, http.client.HTTPException):
+ return False
+ finally:
+ connection.close()
+
+
+async def ensure_golden_template(
+ control: SubstrateControl,
+ args: argparse.Namespace,
+ template_name: str,
+ actor_template_doc: str,
+ state: dict,
+) -> str:
+ existing = await control.get_actor_template(args.atespace, template_name)
+ if existing is None:
+ print(f"-- creating actor-template {args.atespace}/{template_name}")
+ existing = await control.create_actor_template(
+ args.atespace, template_name, actor_template_doc
+ )
+ if existing.atespace != args.atespace or existing.name != template_name:
+ raise IdentityConflict(
+ "actor-template create/read returned a different identity"
+ )
+ if not existing.uid:
+ raise TransportError("actor-template response is missing metadata.uid")
+ if state.get("template_uid") and state["template_uid"] != existing.uid:
+ raise IdentityConflict(
+ f"actor-template {args.atespace}/{template_name} uid changed from "
+ f"{state['template_uid']} to {existing.uid}"
+ )
+ containers = existing.raw.get("containers") or []
+ template_images = [
+ item.get("image") for item in containers if isinstance(item, dict)
+ ]
+ if args.image not in template_images:
+ raise IdentityConflict(
+ f"actor-template {args.atespace}/{template_name} does not use requested "
+ "image digest"
+ )
+ state["template_uid"] = existing.uid
+ save_state(args.state_file, state)
+
+ if existing.golden_state is GoldenState.FAILED:
+ raise SystemExit(
+ f"actor-template {args.atespace}/{template_name} already failed its golden "
+ f"snapshot ({existing.error_message}); ActorTemplates are immutable -- pass a "
+ "new --template-version rather than reusing this one"
+ )
+ if existing.golden_state is not GoldenState.PENDING:
+ print(
+ f"-- actor-template {args.atespace}/{template_name} already exists (state={existing.golden_state.value}), awaiting its golden snapshot"
+ )
+
+ record = await wait_for_golden_snapshot(
+ control, args.atespace, template_name, timeout_s=args.golden_timeout
+ )
+ print(f"-- golden snapshot ready: {record.golden_tag}")
+ return record.uid or existing.uid
+
+
+async def ensure_actor(
+ control: SubstrateControl,
+ args: argparse.Namespace,
+ template_name: str,
+ template_uid: str,
+ state: dict,
+) -> str:
+ """Reconciles any persisted identity against the cluster's current state before
+ deciding whether to create or resume, then waits for RUNNING. Returns the actor uid.
+ """
+ persisted_uid = state.get("actor_uid")
+ live = await control.get_actor(args.atespace, args.actor_name)
+ outcome = reconcile_actor_identity(persisted_uid, live)
+
+ if outcome is IdentityOutcome.UNOWNED:
+ raise IdentityConflict(
+ f"actor {args.atespace}/{args.actor_name} already exists with uid {live.uid}, "
+ f"but {args.state_file} has no actor uid; refusing to adopt it. Use a new "
+ "--actor-name or reconcile the state file explicitly"
+ )
+ if outcome is IdentityOutcome.DIVERGED:
+ raise IdentityConflict(
+ f"actor {args.atespace}/{args.actor_name} exists with uid {live.uid}, but "
+ f"{args.state_file} recorded {persisted_uid} from a prior run -- refusing to "
+ "resume or recreate it; reconcile manually or use a different --actor-name"
+ )
+
+ if outcome is IdentityOutcome.ABSENT:
+ print(f"-- creating actor {args.atespace}/{args.actor_name}")
+ actor = await control.create_actor(
+ args.atespace, args.actor_name, template=template_name
+ )
+ if actor.current_actor_template_uid != template_uid:
+ raise IdentityConflict(
+ f"created actor {args.atespace}/{args.actor_name} references template uid "
+ f"{actor.current_actor_template_uid!r}, expected {template_uid!r}"
+ )
+ state["actor_uid"] = actor.uid
+ save_state(args.state_file, state)
+ else:
+ actor = live
+ if actor.current_actor_template_uid != template_uid:
+ raise IdentityConflict(
+ f"actor {args.atespace}/{args.actor_name} references template uid "
+ f"{actor.current_actor_template_uid!r}, requested {template_uid!r}; "
+ "refusing to resume it"
+ )
+ if actor.state in {ActorState.CRASHED, ActorState.DELETING}:
+ raise ActorFailedToStart(
+ f"actor {args.atespace}/{args.actor_name} is {actor.state.value}; "
+ "choose an explicit revert or a new --actor-name before retrying"
+ )
+ print(
+ f"-- actor {args.atespace}/{args.actor_name} already exists (uid matches persisted identity), state={actor.state.value}"
+ )
+
+ if actor.state in {ActorState.SUSPENDED, ActorState.PAUSED}:
+ print(f"-- resuming actor {args.atespace}/{args.actor_name}")
+ await control.resume_actor(args.atespace, args.actor_name)
+
+ actor = await wait_for_actor_running(
+ control, args.atespace, args.actor_name, timeout_s=args.actor_timeout
+ )
+ print(f"-- actor RUNNING: uid={actor.uid}")
+ return actor.uid
+
+
+async def async_main(args: argparse.Namespace) -> None:
+ substrate_src = verify_substrate_source(args.substrate_src)
+ cluster = get_cluster_identity(context=args.context, kubeconfig=args.kubeconfig)
+ state = prepare_run_state(args, cluster)
+ control = SubstrateControl(
+ kubeconfig=args.kubeconfig, context=args.context, cli=args.ate_cli
+ )
+ template_name = f"live-agent-gate-{args.template_version}"
+
+ print(f"-- registering atespace {args.atespace}")
+ await control.ensure_atespace(args.atespace)
+
+ namespace_and_workerpool_doc, actor_template_doc = render_manifest(
+ args.manifest,
+ atespace=args.atespace,
+ template_name=template_name,
+ bucket_name=args.bucket_name,
+ image=args.image,
+ )
+ print("-- resolving and applying the Namespace + WorkerPool")
+ apply_worker_pool(
+ namespace_and_workerpool_doc,
+ kubeconfig=args.kubeconfig,
+ context=args.context,
+ ko=args.ko,
+ substrate_src=substrate_src,
+ )
+
+ print(
+ f"-- waiting for an eligible worker in namespace={WORKER_NAMESPACE}, "
+ f"selector={WORKER_SELECTOR}, sandbox={WORKER_SANDBOX_CLASS}"
+ )
+ await wait_for_eligible_worker(
+ control,
+ WORKER_NAMESPACE,
+ WORKER_SELECTOR,
+ WORKER_SANDBOX_CLASS,
+ timeout_s=args.worker_timeout,
+ )
+
+ template_uid = await ensure_golden_template(
+ control, args, template_name, actor_template_doc, state
+ )
+ await ensure_actor(control, args, template_name, template_uid, state)
+
+ print("-- confirming current control-service health through the actor route")
+ with actor_router_tunnel(args) as route_port:
+ await wait_for_actor_health(
+ lambda: actor_health_check(
+ port=route_port,
+ atespace=args.atespace,
+ actor_name=args.actor_name,
+ ),
+ timeout_s=args.readiness_timeout,
+ )
+
+ print("-- applying the actor's EgressPolicy")
+ run_egress_tool(args)
+
+ print()
+ print(f"== actor {args.atespace}/{args.actor_name} is RUNNING, credential-free ==")
+ print(
+ "Credential injection and native-agent session launch are separate, still-gated steps"
+ )
+ print("(recovery plan steps 3-4) -- not performed by this script.")
+
+
+def main() -> None:
+ args = parse_args()
+ if args.context != "kind-substrate-preview":
+ raise SystemExit(
+ f"refusing to target context {args.context!r}: expected the dedicated "
+ "kind-substrate-preview context"
+ )
+ try:
+ asyncio.run(async_main(args))
+ except (subprocess.CalledProcessError, RuntimeError) as exc:
+ print(f"gate5_setup failed: {exc}", file=sys.stderr)
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/backend/src/mainloop/runtime/substrate.py b/backend/src/mainloop/runtime/substrate.py
index 5ba0dad..70158b2 100644
--- a/backend/src/mainloop/runtime/substrate.py
+++ b/backend/src/mainloop/runtime/substrate.py
@@ -14,8 +14,10 @@
import json
import logging
import shlex
+import time
from dataclasses import dataclass
from enum import StrEnum
+from typing import Callable
from mainloop.config import settings
@@ -99,6 +101,96 @@ def _actor_from_json(doc: dict) -> ActorRecord:
)
+class GoldenState(StrEnum):
+ """Mirrors the ``ateapipb.GoldenSnapshotStatus`` lifecycle for an ActorTemplate."""
+
+ PENDING = "pending"
+ READY = "ready"
+ FAILED = "failed"
+
+
+@dataclass(frozen=True, slots=True)
+class ActorTemplateRecord:
+ """Parsed subset of an ``ateapipb.ActorTemplate``, including golden-snapshot status."""
+
+ atespace: str
+ name: str
+ uid: str | None
+ golden_state: GoldenState
+ golden_tag: str | None
+ error_message: str
+ raw: dict
+
+
+def _actor_template_from_json(doc: dict) -> ActorTemplateRecord:
+ metadata = doc.get("metadata") or {}
+ status = doc.get("status") or {}
+ golden = status.get("goldenSnapshotStatus") or {}
+ error_message = golden.get("errorMessage", "")
+ golden_tag_ref = golden.get("goldenTag")
+ golden_tag = golden_tag_ref.get("name") if golden_tag_ref else None
+ if error_message:
+ golden_state = GoldenState.FAILED
+ elif golden_tag:
+ golden_state = GoldenState.READY
+ else:
+ golden_state = GoldenState.PENDING
+ return ActorTemplateRecord(
+ atespace=metadata.get("atespace", ""),
+ name=metadata.get("name", ""),
+ uid=metadata.get("uid"),
+ golden_state=golden_state,
+ golden_tag=golden_tag,
+ error_message=error_message,
+ raw=doc,
+ )
+
+
+class WaitTimeout(RuntimeError):
+ """A bounded poll reached its deadline without observing a terminal outcome. Callers
+ must treat this as failure, never as an implied success."""
+
+
+class GoldenSnapshotFailed(RuntimeError):
+ """The template controller reported an error while building the golden snapshot."""
+
+
+class GoldenSnapshotTimeout(WaitTimeout):
+ """The golden snapshot did not reach a terminal state within the bound. Not success."""
+
+
+class NoEligibleWorker(WaitTimeout):
+ """No worker registered for the atespace within the bound. Not success."""
+
+
+class IdentityOutcome(StrEnum):
+ """Result of reconciling a persisted actor UID against the cluster's current state,
+ before a retry decides whether to create, resume, or refuse to touch an actor."""
+
+ ABSENT = "absent" # no live actor with this name; safe to create fresh
+ UNOWNED = "unowned" # live actor exists but no actor uid was persisted
+ MATCHES = "matches" # live actor's uid matches the persisted identity
+ DIVERGED = "diverged" # live actor exists under this name with a different uid
+
+
+class IdentityConflict(RuntimeError):
+ """A live actor exists under the expected name but with a different uid than the
+ identity persisted from a prior run. Recreating or resuming it blindly could operate
+ on someone else's actor; this must be surfaced, not silently resolved."""
+
+
+def reconcile_actor_identity(
+ persisted_uid: str | None, live: ActorRecord | None
+) -> IdentityOutcome:
+ if live is None:
+ return IdentityOutcome.ABSENT
+ if persisted_uid is None:
+ return IdentityOutcome.UNOWNED
+ if persisted_uid == live.uid:
+ return IdentityOutcome.MATCHES
+ return IdentityOutcome.DIVERGED
+
+
class SubstrateControl:
"""Wraps ``kubectl ate`` for one (kubeconfig, context) pair. No retries, no caching."""
@@ -123,16 +215,22 @@ def _base_args(self) -> list[str]:
args += ["--context", self.context]
return args
- async def _exec(self, args: list[str], timeout: float = 45) -> ExecResult:
+ async def _exec(
+ self, args: list[str], timeout: float = 45, stdin: str | None = None
+ ) -> ExecResult:
command = self._base_args() + args
try:
proc = await asyncio.create_subprocess_exec(
*command,
+ stdin=asyncio.subprocess.PIPE if stdin is not None else None,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
try:
- out, err = await asyncio.wait_for(proc.communicate(), timeout=timeout)
+ out, err = await asyncio.wait_for(
+ proc.communicate(stdin.encode() if stdin is not None else None),
+ timeout=timeout,
+ )
except TimeoutError as exc:
proc.kill()
await proc.wait()
@@ -240,7 +338,326 @@ async def _require(self, atespace: str, name: str, res: ExecResult) -> ActorReco
raise TransportError(f"actor {atespace}/{name} not found after operation")
return actor
+ async def atespace_exists(self, name: str) -> bool:
+ res = await self._exec(["get", "atespaces", name, "-o", "json"])
+ if res.exit_code == 0:
+ return True
+ if "not found" in res.stderr.lower():
+ return False
+ raise TransportError(
+ f"get atespace failed (exit {res.exit_code}): {res.stderr.strip()[-300:]}"
+ )
+
+ async def ensure_atespace(self, name: str) -> None:
+ """Register the atespace via the control-plane API. Idempotent: a Kubernetes
+ Namespace of the same name is a separate, unrelated object and does not register
+ an atespace, so this call is required before an ActorTemplate or actor can be
+ created in it (``create actor-template``/``create actor`` require it to exist).
+ """
+ res = await self._exec(["create", "atespace", name, "-o", "json"])
+ if res.exit_code == 0:
+ return
+ if "already exists" in res.stderr.lower() or "AlreadyExists" in res.stderr:
+ return
+ raise TransportError(
+ f"create atespace failed (exit {res.exit_code}): {res.stderr.strip()[-300:]}"
+ )
+
+ async def get_actor_template(
+ self, atespace: str, name: str
+ ) -> ActorTemplateRecord | None:
+ res = await self._exec(
+ ["get", "actor-template", name, "-a", atespace, "-o", "json"]
+ )
+ if res.exit_code != 0:
+ if "not found" in res.stderr.lower() or "NotFound" in res.stderr:
+ return None
+ raise TransportError(
+ f"get actor-template failed (exit {res.exit_code}): "
+ f"{res.stderr.strip()[-300:]}"
+ )
+ text = res.stdout.strip()
+ if not text:
+ return None
+ return _actor_template_from_json(json.loads(text))
+
+ async def create_actor_template(
+ self, atespace: str, name: str, manifest: str
+ ) -> ActorTemplateRecord:
+ """``manifest`` is the protojson-shaped ActorTemplate document; its own
+ ``metadata.atespace``/``metadata.name`` select where it is created. The atespace
+ must already exist (``ensure_atespace``); templates are immutable, so a changed
+ manifest needs a new name, never a reused one. The pinned CLI prints a table by
+ default, so JSON is requested explicitly. If the create result is uncertain, read
+ the named template back before reporting failure; a later invocation also starts
+ with that read and cannot blindly repeat the create."""
+ try:
+ res = await self._exec(
+ ["create", "actor-template", "-f", "-", "-o", "json"],
+ stdin=manifest,
+ )
+ except TransportError as create_error:
+ return await self._read_uncertain_actor_template_create(
+ atespace, name, create_error
+ )
+
+ text = res.stdout.strip()
+ if res.exit_code != 0:
+ reason = (
+ f"create actor-template failed (exit {res.exit_code}): "
+ f"{res.stderr.strip()[-300:]}"
+ )
+ return await self._read_uncertain_actor_template_create(
+ atespace, name, TransportError(reason)
+ )
+ if not text:
+ return await self._read_uncertain_actor_template_create(
+ atespace,
+ name,
+ TransportError("create actor-template returned no output"),
+ )
+ try:
+ return _actor_template_from_json(json.loads(text))
+ except (json.JSONDecodeError, TypeError) as parse_error:
+ return await self._read_uncertain_actor_template_create(
+ atespace,
+ name,
+ TransportError(
+ f"create actor-template returned invalid JSON: {parse_error}"
+ ),
+ )
+
+ async def _read_uncertain_actor_template_create(
+ self, atespace: str, name: str, create_error: TransportError
+ ) -> ActorTemplateRecord:
+ try:
+ existing = await self.get_actor_template(atespace, name)
+ except TransportError as read_error:
+ raise TransportError(
+ f"actor-template create outcome is uncertain and read-back failed: "
+ f"{read_error}"
+ ) from create_error
+ if existing is not None:
+ return existing
+ raise create_error
+
+ async def get_eligible_workers(
+ self, namespace: str, selector: str, sandbox_class: str
+ ) -> int:
+ """Count active workers with free actor capacity matching this WorkerPool.
+
+ The pinned CLI's ``-a`` filter means "already hosting an actor in an atespace",
+ so it excludes the idle worker needed before the first actor exists. Namespace,
+ pool labels, and sandbox class identify the intended pool instead.
+ """
+ res = await self._exec(
+ [
+ "get",
+ "workers",
+ "-n",
+ namespace,
+ "-l",
+ selector,
+ "--sandbox-class",
+ sandbox_class,
+ "-o",
+ "json",
+ ]
+ )
+ if res.exit_code != 0:
+ raise TransportError(
+ f"get workers failed (exit {res.exit_code}): {res.stderr.strip()[-300:]}"
+ )
+ text = res.stdout.strip()
+ if not text:
+ return 0
+ try:
+ doc = json.loads(text)
+ except json.JSONDecodeError as exc:
+ raise TransportError(f"get workers returned invalid JSON: {exc}") from exc
+ if not isinstance(doc, dict) or "workers" not in doc:
+ raise TransportError("get workers JSON is missing its 'workers' list")
+ workers = doc["workers"]
+ if not isinstance(workers, list):
+ raise TransportError("get workers JSON field 'workers' is not a list")
+ return sum(
+ _worker_is_eligible(
+ worker,
+ namespace=namespace,
+ selector=selector,
+ sandbox_class=sandbox_class,
+ )
+ for worker in workers
+ )
+
def cli_quote(*parts: str) -> str:
"""Only for logging/evidence; commands are passed as argv, never through a shell."""
return " ".join(shlex.quote(p) for p in parts)
+
+
+def _worker_is_eligible(
+ worker: object, *, namespace: str, selector: str, sandbox_class: str
+) -> bool:
+ if not isinstance(worker, dict):
+ return False
+ if worker.get("workerNamespace") != namespace:
+ return False
+ if worker.get("sandboxClass") != sandbox_class:
+ return False
+ key, separator, value = selector.partition("=")
+ if not separator or not key or not value:
+ raise ValueError(f"unsupported worker label selector: {selector!r}")
+ labels = worker.get("labels") or {}
+ if not isinstance(labels, dict) or labels.get(key) != value:
+ return False
+
+ status = worker.get("status") or {}
+ if not isinstance(status, dict) or status.get("state") != "WORKER_STATE_ACTIVE":
+ return False
+ capacity = status.get("capacity") or {}
+ allocated = status.get("allocated") or {}
+ if isinstance(capacity, dict) and capacity.get("actors") is not None:
+ allocated_actors = (
+ allocated.get("actors", 0) if isinstance(allocated, dict) else 0
+ )
+ try:
+ if int(capacity["actors"]) - int(allocated_actors or 0) <= 0:
+ return False
+ except (TypeError, ValueError) as exc:
+ raise TransportError("worker actor capacity is not an integer") from exc
+ return True
+
+
+async def wait_for_golden_snapshot(
+ control: SubstrateControl,
+ atespace: str,
+ name: str,
+ *,
+ timeout_s: float,
+ poll_interval_s: float = 3,
+ sleep=asyncio.sleep,
+ clock=time.monotonic,
+) -> ActorTemplateRecord:
+ """Poll an ActorTemplate's golden-snapshot status to a terminal outcome. Raises
+ ``GoldenSnapshotFailed``/``GoldenSnapshotTimeout`` rather than returning normally on
+ anything but a completed golden tag -- a caller must never infer success from a log
+ line or from reaching this function without an exception being possible."""
+ deadline = clock() + timeout_s
+ while True:
+ record = await control.get_actor_template(atespace, name)
+ if record is None:
+ raise TransportError(
+ f"actor-template {atespace}/{name} disappeared while awaiting its "
+ "golden snapshot"
+ )
+ if record.golden_state is GoldenState.FAILED:
+ raise GoldenSnapshotFailed(
+ f"golden snapshot for {atespace}/{name} failed: {record.error_message}"
+ )
+ if record.golden_state is GoldenState.READY:
+ return record
+ if clock() >= deadline:
+ raise GoldenSnapshotTimeout(
+ f"golden snapshot for {atespace}/{name} did not complete within "
+ f"{timeout_s}s (last state: {record.golden_state.value})"
+ )
+ await sleep(poll_interval_s)
+
+
+async def wait_for_eligible_worker(
+ control: SubstrateControl,
+ namespace: str,
+ selector: str,
+ sandbox_class: str,
+ *,
+ timeout_s: float,
+ poll_interval_s: float = 3,
+ sleep=asyncio.sleep,
+ clock=time.monotonic,
+) -> int:
+ """Poll for at least one worker matching the intended WorkerPool. Raises on timeout rather
+ than proceeding to create an actor-template/actor against a pool with no capacity.
+ """
+ deadline = clock() + timeout_s
+ while True:
+ count = await control.get_eligible_workers(namespace, selector, sandbox_class)
+ if count > 0:
+ return count
+ if clock() >= deadline:
+ raise NoEligibleWorker(
+ f"no eligible worker for namespace={namespace}, selector={selector}, "
+ f"sandbox_class={sandbox_class} within {timeout_s}s"
+ )
+ await sleep(poll_interval_s)
+
+
+class ActorHealthTimeout(WaitTimeout):
+ """The actor route did not return a healthy response within the readiness bound."""
+
+
+async def wait_for_actor_health(
+ health_check: Callable[[], bool],
+ *,
+ timeout_s: float,
+ poll_interval_s: float = 2,
+ sleep=asyncio.sleep,
+ clock=time.monotonic,
+) -> None:
+ """Wait for a live actor-route ``/healthz`` check to succeed.
+
+ A restored actor need not replay startup logs, so readiness is based on current service
+ health rather than a boot marker.
+ """
+ deadline = clock() + timeout_s
+ while True:
+ if health_check():
+ return
+ if clock() >= deadline:
+ raise ActorHealthTimeout(
+ f"actor /healthz did not return 200 within {timeout_s}s"
+ )
+ await sleep(poll_interval_s)
+
+
+class ActorFailedToStart(RuntimeError):
+ """An actor reached a terminal, non-running state (e.g. CRASHED) while awaiting
+ readiness. Distinct from ``WaitTimeout``: this is a reported failure, not a bound
+ running out."""
+
+
+_ACTOR_TERMINAL_FAILURE_STATES = frozenset({ActorState.CRASHED, ActorState.DELETING})
+
+
+async def wait_for_actor_running(
+ control: SubstrateControl,
+ atespace: str,
+ name: str,
+ *,
+ timeout_s: float,
+ poll_interval_s: float = 2,
+ sleep=asyncio.sleep,
+ clock=time.monotonic,
+) -> ActorRecord:
+ """Poll an actor to RUNNING. Raises ``ActorFailedToStart`` on a terminal failure state
+ and ``WaitTimeout`` on exceeding the bound -- never returns normally except on RUNNING,
+ so a caller can never mistake "still starting" for success."""
+ deadline = clock() + timeout_s
+ while True:
+ actor = await control.get_actor(atespace, name)
+ if actor is None:
+ raise TransportError(
+ f"actor {atespace}/{name} disappeared while awaiting readiness"
+ )
+ if actor.state is ActorState.RUNNING:
+ return actor
+ if actor.state in _ACTOR_TERMINAL_FAILURE_STATES:
+ raise ActorFailedToStart(
+ f"actor {atespace}/{name} reached {actor.state.value} while awaiting readiness"
+ )
+ if clock() >= deadline:
+ raise WaitTimeout(
+ f"actor {atespace}/{name} did not reach RUNNING within {timeout_s}s "
+ f"(last state: {actor.state.value})"
+ )
+ await sleep(poll_interval_s)
diff --git a/backend/tests/runtime/fixtures/substrate/kubectl-ate-workers-pinned-cdac9ba.json b/backend/tests/runtime/fixtures/substrate/kubectl-ate-workers-pinned-cdac9ba.json
new file mode 100644
index 0000000..bce43a0
--- /dev/null
+++ b/backend/tests/runtime/fixtures/substrate/kubectl-ate-workers-pinned-cdac9ba.json
@@ -0,0 +1,25 @@
+{
+ "workers": [
+ {
+ "labels": {
+ "workload": "live-agent-gate"
+ },
+ "metadata": {
+ "name": "worker-1"
+ },
+ "sandboxClass": "gvisor",
+ "status": {
+ "allocated": {
+ "actors": 3
+ },
+ "capacity": {
+ "actors": 4
+ },
+ "state": "WORKER_STATE_ACTIVE"
+ },
+ "workerNamespace": "live-agent-gate",
+ "workerPod": "worker-1-pod",
+ "workerPool": "live-agent-gate-workers"
+ }
+ ]
+}
diff --git a/backend/tests/runtime/test_contracts.py b/backend/tests/runtime/test_contracts.py
index 9789d83..aaa0233 100644
--- a/backend/tests/runtime/test_contracts.py
+++ b/backend/tests/runtime/test_contracts.py
@@ -221,6 +221,31 @@ def test_takeover_can_retire_unsent_attempt_with_evidence(self):
self.store.create_attempt(attempt("retry", generation=2), 2)
self.assertEqual(len(self.store.attempts), 2)
+ def test_backend_restart_reconciles_persisted_recorded_attempt_before_retry(self):
+ # A restart leaves the durable attempt row intact. Ownership takeover is
+ # the in-memory contract's fake-backed model of loading that row under a
+ # new generation; no transport or database is involved.
+ recorded = self.store.create_attempt(attempt(), 1)
+ self.assertEqual(recorded.state, DeliveryState.RECORDED)
+ self.store.take_ownership(1)
+
+ historical = self.store.attempts[0]
+ self.assertEqual(historical.state, DeliveryState.RECORDED)
+ self.assertEqual(historical.ownership_generation, 1)
+ with self.assertRaises(ContractError):
+ self.store.create_attempt(attempt("retry", generation=2), 2)
+ with self.assertRaises(ContractError):
+ self.store.reconcile(self.evidence("delivered"), 2)
+
+ retired = self.store.reconcile(self.evidence("not_delivered"), 2)
+ self.assertEqual(retired.state, DeliveryState.FAILED)
+ self.assertEqual(retired.result, "not_delivered")
+ self.store.create_attempt(attempt("retry", generation=2), 2)
+ self.assertEqual(
+ [item.state for item in self.store.attempts],
+ [DeliveryState.FAILED, DeliveryState.RECORDED],
+ )
+
def test_takeover_reconnect_deduplicates_source_event(self):
original = self.store.ingest(attention(), 1)
self.store.take_ownership(1)
diff --git a/backend/tests/runtime/test_gate5_setup.py b/backend/tests/runtime/test_gate5_setup.py
new file mode 100644
index 0000000..fcfc2ac
--- /dev/null
+++ b/backend/tests/runtime/test_gate5_setup.py
@@ -0,0 +1,389 @@
+"""Credential-free regressions for the gate-5 setup script's build and rerun identity."""
+
+import json
+import tempfile
+import unittest
+from dataclasses import replace
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+from mainloop.runtime.substrate import (
+ ActorFailedToStart,
+ ActorRecord,
+ ActorState,
+ IdentityConflict,
+)
+
+from scripts import gate5_setup
+
+FIXTURE_KUBECONFIG = "/fixture/kubeconfig"
+FIXTURE_KO = "/fixture/substrate/bin/ko"
+
+
+def completed(argv, returncode=0, stdout="", stderr=""):
+ return SimpleNamespace(
+ args=argv, returncode=returncode, stdout=stdout, stderr=stderr
+ )
+
+
+class Gate5SourceAndBuildTests(unittest.TestCase):
+ def make_source(self, root: Path) -> Path:
+ (root / ".git").mkdir(parents=True)
+ (root / "go.mod").write_text("module fixture\n")
+ (root / ".ko.yaml").write_text("defaultBaseImage: scratch\n")
+ return root
+
+ def test_source_requires_pinned_checkout_and_required_files(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ source = self.make_source(Path(temp_dir) / "substrate")
+ calls = []
+
+ def runner(argv, **kwargs):
+ calls.append(argv)
+ return completed(
+ argv, stdout=gate5_setup.PINNED_SUBSTRATE_COMMIT + "\n"
+ )
+
+ self.assertEqual(
+ gate5_setup.verify_substrate_source(str(source), runner=runner),
+ str(source),
+ )
+ self.assertEqual(calls[0][:4], ["git", "-C", str(source), "rev-parse"])
+
+ def test_source_rejects_wrong_commit(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ source = self.make_source(Path(temp_dir) / "substrate")
+
+ def runner(argv, **kwargs):
+ return completed(argv, stdout="a" * 40)
+
+ with self.assertRaisesRegex(RuntimeError, "must be pinned"):
+ gate5_setup.verify_substrate_source(str(source), runner=runner)
+
+ def test_source_rejects_missing_go_module_files(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ source = Path(temp_dir) / "substrate"
+ source.mkdir()
+ (source / ".git").mkdir()
+ with self.assertRaisesRegex(RuntimeError, "missing go.mod"):
+ gate5_setup.verify_substrate_source(str(source))
+
+ def test_ko_resolve_uses_pinned_source_cwd_and_explicit_kube_target(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ source = self.make_source(Path(temp_dir) / "substrate")
+ calls = []
+
+ def runner(argv, **kwargs):
+ calls.append((argv, kwargs))
+ if argv[0] == "git":
+ return completed(
+ argv, stdout=gate5_setup.PINNED_SUBSTRATE_COMMIT + "\n"
+ )
+ if argv[1:3] == ["resolve", "-f"]:
+ return completed(argv, stdout="resolved-yaml")
+ return completed(argv)
+
+ with patch.dict("os.environ", {"KO_DOCKER_REPO": "localhost:5001"}):
+ gate5_setup.apply_worker_pool(
+ "apiVersion: v1\n",
+ kubeconfig=FIXTURE_KUBECONFIG,
+ context="kind-substrate-preview",
+ ko=FIXTURE_KO,
+ substrate_src=str(source),
+ runner=runner,
+ )
+
+ ko_argv, ko_kwargs = calls[1]
+ self.assertEqual(ko_argv[:3], [FIXTURE_KO, "resolve", "-f"])
+ self.assertEqual(ko_kwargs["cwd"], str(source))
+ apply_argv, apply_kwargs = calls[2]
+ self.assertEqual(
+ apply_argv[:5],
+ [
+ "kubectl",
+ "--context",
+ "kind-substrate-preview",
+ "--kubeconfig",
+ FIXTURE_KUBECONFIG,
+ ],
+ )
+ self.assertEqual(apply_kwargs["input"], "resolved-yaml")
+
+ def test_cluster_identity_uses_explicit_context_and_namespace_uid(self):
+ calls = []
+ results = [
+ json.dumps(
+ {
+ "contexts": [{"name": "kind-substrate-preview"}],
+ "clusters": [{"cluster": {"server": "https://127.0.0.1:45147"}}],
+ }
+ ),
+ json.dumps({"metadata": {"uid": "kube-system-uid"}}),
+ ]
+
+ def runner(argv, **kwargs):
+ calls.append(argv)
+ return completed(argv, stdout=results.pop(0))
+
+ identity = gate5_setup.get_cluster_identity(
+ context="kind-substrate-preview",
+ kubeconfig=FIXTURE_KUBECONFIG,
+ runner=runner,
+ )
+ self.assertEqual(
+ identity,
+ {
+ "api_server_url": "https://127.0.0.1:45147",
+ "kube_system_namespace_uid": "kube-system-uid",
+ },
+ )
+ self.assertTrue(
+ all("--context" in call and "--kubeconfig" in call for call in calls)
+ )
+
+ def test_actor_health_uses_actor_route_and_shim_port(self):
+ class FakeConnection:
+ def __init__(self, _host, _port, timeout):
+ self.timeout = timeout
+ self.tunnel = None
+ self.requested = None
+
+ def set_tunnel(self, target, *, headers):
+ self.tunnel = (target, headers)
+
+ def request(self, method, path):
+ self.requested = (method, path)
+
+ def getresponse(self):
+ return SimpleNamespace(status=200, read=lambda: b"ok")
+
+ def close(self):
+ pass
+
+ connections = []
+
+ def make_connection(*args, **kwargs):
+ connection = FakeConnection(*args, **kwargs)
+ connections.append(connection)
+ return connection
+
+ with patch.object(gate5_setup.http.client, "HTTPConnection", make_connection):
+ self.assertTrue(
+ gate5_setup.actor_health_check(
+ port=18091,
+ atespace="live-agent-gate",
+ actor_name="claude-gate5",
+ )
+ )
+ self.assertEqual(
+ connections[0].tunnel,
+ (
+ "actor-upstream:8090",
+ {"ate-target-actor": "live-agent-gate/claude-gate5"},
+ ),
+ )
+ self.assertEqual(connections[0].requested, ("GET", "/healthz"))
+
+
+class Gate5StateTests(unittest.TestCase):
+ def args(self, state_file: str, **overrides):
+ values = {
+ "context": "kind-substrate-preview",
+ "kubeconfig": FIXTURE_KUBECONFIG,
+ "atespace": "live-agent-gate",
+ "template_version": "v1",
+ "image": "localhost:5001/live-agent-gate@sha256:" + "a" * 64,
+ "actor_name": "claude-gate5",
+ "state_file": state_file,
+ }
+ values.update(overrides)
+ return SimpleNamespace(**values)
+
+ def cluster(self, *, uid="cluster-uid"):
+ return {
+ "api_server_url": "https://127.0.0.1:45147",
+ "kube_system_namespace_uid": uid,
+ }
+
+ def test_persists_run_intent_before_actor_or_template_uids_exist(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ args = self.args(path)
+ state = gate5_setup.prepare_run_state(args, self.cluster())
+ stored = json.loads(Path(path).read_text())
+ self.assertEqual(stored["run_id"], state["run_id"])
+ self.assertEqual(stored["template_name"], "live-agent-gate-v1")
+ self.assertEqual(stored["actor_name"], "claude-gate5")
+ self.assertIsNone(stored["template_uid"])
+ self.assertIsNone(stored["actor_uid"])
+ self.assertEqual(stored["cluster_identity"], self.cluster())
+
+ def test_rerun_refuses_changed_cluster_identity(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ args = self.args(path)
+ gate5_setup.prepare_run_state(args, self.cluster())
+ with self.assertRaisesRegex(RuntimeError, "requested identity differs"):
+ gate5_setup.prepare_run_state(args, self.cluster(uid="other-cluster"))
+
+ def test_rerun_refuses_changed_template_request(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ gate5_setup.prepare_run_state(self.args(path), self.cluster())
+ with self.assertRaisesRegex(RuntimeError, "template_name"):
+ gate5_setup.prepare_run_state(
+ self.args(path, template_version="v2"), self.cluster()
+ )
+
+
+class SetupControl:
+ def __init__(self, actor: ActorRecord | None):
+ self.actor = actor
+ self.created = []
+ self.resumed = 0
+
+ async def get_actor(self, _atespace, _name):
+ return self.actor
+
+ async def create_actor(self, atespace, name, *, template):
+ self.created.append((atespace, name, template))
+ self.actor = ActorRecord(
+ atespace=atespace,
+ name=name,
+ uid="actor-new",
+ state=ActorState.SUSPENDED,
+ external_snapshot_uri=None,
+ current_actor_template_uid="template-new",
+ raw={},
+ )
+ return self.actor
+
+ async def resume_actor(self, _atespace, _name):
+ self.resumed += 1
+ self.actor = replace(self.actor, state=ActorState.RUNNING)
+ return self.actor
+
+
+class Gate5ActorIdentityTests(unittest.TestCase):
+ def state(self, path, *, actor_uid="actor-1"):
+ state = {
+ "run_id": "run-1",
+ "actor_uid": actor_uid,
+ "actor_name": "claude-gate5",
+ "template_name": "live-agent-gate-v2",
+ "template_uid": "template-new",
+ }
+ gate5_setup.save_state(path, state)
+ return state
+
+ def actor(
+ self, *, uid="actor-1", template_uid="template-new", state=ActorState.RUNNING
+ ):
+ return ActorRecord(
+ atespace="live-agent-gate",
+ name="claude-gate5",
+ uid=uid,
+ state=state,
+ external_snapshot_uri=None,
+ current_actor_template_uid=template_uid,
+ raw={},
+ )
+
+ def args(self, path):
+ return SimpleNamespace(
+ state_file=path,
+ atespace="live-agent-gate",
+ actor_name="claude-gate5",
+ actor_timeout=5,
+ )
+
+ def test_exact_old_template_collision_with_new_template_is_refused(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ state = {
+ "run_id": "run-2",
+ "actor_uid": None,
+ "template_uid": "template-new",
+ }
+ control = SetupControl(self.actor(template_uid="template-old"))
+ with self.assertRaisesRegex(IdentityConflict, "no actor uid"):
+ asyncio_run(
+ gate5_setup.ensure_actor(
+ control,
+ self.args(path),
+ "live-agent-gate-v2",
+ "template-new",
+ state,
+ )
+ )
+
+ def test_owned_actor_with_template_mismatch_is_refused(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ state = self.state(path)
+ control = SetupControl(self.actor(template_uid="template-old"))
+ with self.assertRaisesRegex(IdentityConflict, "references template uid"):
+ asyncio_run(
+ gate5_setup.ensure_actor(
+ control,
+ self.args(path),
+ "live-agent-gate-v2",
+ "template-new",
+ state,
+ )
+ )
+
+ def test_crashed_and_deleting_actors_are_refused_before_resume(self):
+ for actor_state in (ActorState.CRASHED, ActorState.DELETING):
+ with self.subTest(
+ actor_state=actor_state
+ ), tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ state = self.state(path)
+ control = SetupControl(self.actor(state=actor_state))
+ with self.assertRaises(ActorFailedToStart):
+ asyncio_run(
+ gate5_setup.ensure_actor(
+ control,
+ self.args(path),
+ "live-agent-gate-v2",
+ "template-new",
+ state,
+ )
+ )
+ self.assertEqual(control.resumed, 0)
+
+ def test_new_actor_is_resumed_and_uid_is_saved(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ state = {
+ "run_id": "run-3",
+ "actor_uid": None,
+ "template_uid": "template-new",
+ }
+ control = SetupControl(None)
+ uid = asyncio_run(
+ gate5_setup.ensure_actor(
+ control,
+ self.args(path),
+ "live-agent-gate-v2",
+ "template-new",
+ state,
+ )
+ )
+ self.assertEqual(uid, "actor-new")
+ self.assertEqual(control.resumed, 1)
+ self.assertEqual(
+ json.loads(Path(path).read_text())["actor_uid"], "actor-new"
+ )
+
+
+def asyncio_run(coro):
+ import asyncio
+
+ return asyncio.run(coro)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/backend/tests/runtime/test_substrate.py b/backend/tests/runtime/test_substrate.py
index 51e3997..e4ff2d5 100644
--- a/backend/tests/runtime/test_substrate.py
+++ b/backend/tests/runtime/test_substrate.py
@@ -2,16 +2,43 @@
no credentials. Mirrors test_herdr.py's fake-transport pattern for the Herdr adapter."""
import asyncio
+import json
import unittest
+from pathlib import Path
from mainloop.runtime.substrate import (
+ ActorFailedToStart,
+ ActorHealthTimeout,
ActorState,
ExecResult,
+ GoldenSnapshotFailed,
+ GoldenSnapshotTimeout,
+ GoldenState,
+ IdentityOutcome,
+ NoEligibleWorker,
SubstrateControl,
TransportError,
+ WaitTimeout,
_actor_from_json,
+ _actor_template_from_json,
+ reconcile_actor_identity,
+ wait_for_actor_health,
+ wait_for_actor_running,
+ wait_for_eligible_worker,
+ wait_for_golden_snapshot,
)
+# Generated with kubectl-ate's PrintWorkersTo(..., "json") at pinned Substrate commit
+# cdac9baef81dd319b46086d695266e6161e9e592. It is CLI printer output with sanitized fixture
+# records, not a claim that a live worker was observed.
+PINNED_WORKERS_OUTPUT = (
+ Path(__file__).parent
+ / "fixtures"
+ / "substrate"
+ / "kubectl-ate-workers-pinned-cdac9ba.json"
+).read_text()
+PINNED_WORKERS_DOCUMENT = json.loads(PINNED_WORKERS_OUTPUT)
+
class FakeControl(SubstrateControl):
def __init__(self, results):
@@ -20,31 +47,92 @@ def __init__(self, results):
)
self.results = list(results)
self.calls: list[list[str]] = []
+ self.stdins: list[str | None] = []
- async def _exec(self, args, timeout=45):
+ async def _exec(self, args, timeout=45, stdin=None):
self.calls.append(args)
+ self.stdins.append(stdin)
+ if args and args[0] in {"get", "create", "resume", "suspend", "revert"}:
+ if "-o" not in args or args[args.index("-o") + 1] != "json":
+ return ExecResult(0, "NAME STATUS\nresource Pending\n", "")
result = self.results.pop(0)
if isinstance(result, Exception):
raise result
return result
+async def fake_sleep(_seconds: float) -> None:
+ return None
+
+
+class FakeClock:
+ """A monotonic clock that advances by a fixed step every time it's read, so a bounded
+ poll loop can be driven to its deadline deterministically without a real delay."""
+
+ def __init__(self, step: float = 1.0):
+ self.value = 0.0
+ self.step = step
+
+ def __call__(self) -> float:
+ current = self.value
+ self.value += self.step
+ return current
+
+
def run(coro):
return asyncio.run(coro)
-def actor_json(state: str, *, snapshot_uri: str | None = None) -> str:
+def actor_json(state: str, *, snapshot_uri: str | None = None, uid: str = "u-1") -> str:
doc = {
- "metadata": {"atespace": "mainloop-workspaces", "name": "ml-abc", "uid": "u-1"},
+ "metadata": {"atespace": "mainloop-workspaces", "name": "ml-abc", "uid": uid},
"status": {"state": state},
}
if snapshot_uri:
doc["status"]["externalSnapshot"] = {"snapshotUri": snapshot_uri}
- import json
+ return json.dumps(doc)
+
+def actor_template_json(
+ *, error_message: str = "", golden_tag: str | None = None, uid: str = "t-1"
+) -> str:
+ doc = {
+ "metadata": {
+ "atespace": "live-agent-gate",
+ "name": "live-agent-gate-v1",
+ "uid": uid,
+ },
+ "status": {"goldenSnapshotStatus": {}},
+ }
+ if error_message:
+ doc["status"]["goldenSnapshotStatus"]["errorMessage"] = error_message
+ if golden_tag:
+ doc["status"]["goldenSnapshotStatus"]["goldenTag"] = {
+ "atespace": "ate-golden",
+ "name": golden_tag,
+ }
return json.dumps(doc)
+def worker_record(
+ *,
+ state="WORKER_STATE_ACTIVE",
+ namespace="live-agent-gate",
+ pool_label="live-agent-gate",
+ sandbox_class="gvisor",
+ capacity=1,
+ allocated=0,
+):
+ record = json.loads(json.dumps(PINNED_WORKERS_DOCUMENT["workers"][0]))
+ record["workerNamespace"] = namespace
+ record["sandboxClass"] = sandbox_class
+ record["labels"]["workload"] = pool_label
+ record["status"]["state"] = state
+ record["status"]["capacity"]["actors"] = capacity
+ record["status"]["allocated"]["actors"] = allocated
+ return record
+
+
class ActorJsonParsingTests(unittest.TestCase):
def test_parses_running_actor_with_snapshot(self):
import json
@@ -150,5 +238,443 @@ def test_transport_error_on_transient_failure_is_not_retried(self):
self.assertEqual(len(ctl.calls), 1)
+class AtespaceTests(unittest.TestCase):
+ def test_ensure_atespace_creates(self):
+ ctl = FakeControl([ExecResult(0, "{}", "")])
+ run(ctl.ensure_atespace("live-agent-gate"))
+ self.assertEqual(
+ ctl.calls[0], ["create", "atespace", "live-agent-gate", "-o", "json"]
+ )
+
+ def test_ensure_atespace_tolerates_already_exists(self):
+ ctl = FakeControl(
+ [
+ ExecResult(
+ 1, "", 'Error: atespaces.ate.dev "live-agent-gate" already exists'
+ )
+ ]
+ )
+ run(ctl.ensure_atespace("live-agent-gate")) # does not raise
+
+ def test_ensure_atespace_other_failure_raises(self):
+ ctl = FakeControl([ExecResult(1, "", "connection refused")])
+ with self.assertRaises(TransportError):
+ run(ctl.ensure_atespace("live-agent-gate"))
+
+ def test_atespace_exists_false_on_not_found(self):
+ ctl = FakeControl([ExecResult(1, "", 'Error: atespaces.ate.dev "x" not found')])
+ self.assertFalse(run(ctl.atespace_exists("x")))
+
+
+class ActorTemplateJsonParsingTests(unittest.TestCase):
+ def test_pending_when_no_tag_and_no_error(self):
+ record = _actor_template_from_json(json.loads(actor_template_json()))
+ self.assertEqual(record.golden_state, GoldenState.PENDING)
+
+ def test_ready_when_golden_tag_present(self):
+ record = _actor_template_from_json(
+ json.loads(actor_template_json(golden_tag="golden-1"))
+ )
+ self.assertEqual(record.golden_state, GoldenState.READY)
+ self.assertEqual(record.golden_tag, "golden-1")
+
+ def test_failed_when_error_message_present(self):
+ record = _actor_template_from_json(
+ json.loads(actor_template_json(error_message="golden actor crashed"))
+ )
+ self.assertEqual(record.golden_state, GoldenState.FAILED)
+ self.assertEqual(record.error_message, "golden actor crashed")
+
+
+class ActorTemplateControlTests(unittest.TestCase):
+ def test_get_actor_template_uses_atespace_flag_not_a_composite_name(self):
+ # The prior harness bug: `get actor-template "/"` as a single
+ # positional argument, which the CLI requires as separate `-a `.
+ ctl = FakeControl([ExecResult(0, actor_template_json(golden_tag="g1"), "")])
+ run(ctl.get_actor_template("live-agent-gate", "live-agent-gate-v1"))
+ self.assertEqual(
+ ctl.calls[0],
+ [
+ "get",
+ "actor-template",
+ "live-agent-gate-v1",
+ "-a",
+ "live-agent-gate",
+ "-o",
+ "json",
+ ],
+ )
+
+ def test_get_actor_template_not_found_returns_none(self):
+ ctl = FakeControl(
+ [ExecResult(1, "", 'Error: actortemplates.ate.dev "x" not found')]
+ )
+ self.assertIsNone(run(ctl.get_actor_template("live-agent-gate", "x")))
+
+ def test_get_actor_template_other_failure_raises_transport_error(self):
+ ctl = FakeControl([ExecResult(1, "", "connection refused")])
+ with self.assertRaises(TransportError):
+ run(ctl.get_actor_template("live-agent-gate", "x"))
+
+ def test_create_actor_template_requests_json_and_passes_manifest_over_stdin(self):
+ ctl = FakeControl([ExecResult(0, actor_template_json(), "")])
+ run(
+ ctl.create_actor_template(
+ "live-agent-gate", "live-agent-gate-v1", "metadata:\n name: x\n"
+ )
+ )
+ self.assertEqual(
+ ctl.calls[0],
+ ["create", "actor-template", "-f", "-", "-o", "json"],
+ )
+ self.assertEqual(ctl.stdins[0], "metadata:\n name: x\n")
+
+ def test_uncertain_template_create_reads_before_returning_success(self):
+ ctl = FakeControl(
+ [
+ TransportError("request timed out after send"),
+ ExecResult(0, actor_template_json(golden_tag="g1"), ""),
+ ]
+ )
+ record = run(
+ ctl.create_actor_template(
+ "live-agent-gate", "live-agent-gate-v1", "manifest"
+ )
+ )
+ self.assertEqual(record.uid, "t-1")
+ self.assertEqual(len(ctl.calls), 2)
+ self.assertEqual(
+ ctl.calls[1][:3], ["get", "actor-template", "live-agent-gate-v1"]
+ )
+
+ def test_uncertain_template_create_reads_before_reporting_absent(self):
+ ctl = FakeControl(
+ [
+ TransportError("request timed out after send"),
+ ExecResult(1, "", 'Error: actortemplates.ate.dev "x" not found'),
+ ]
+ )
+ with self.assertRaises(TransportError):
+ run(ctl.create_actor_template("live-agent-gate", "x", "manifest"))
+ self.assertEqual(
+ len(ctl.calls), 2
+ ) # read-back happened before any caller retry
+
+
+class WaitForGoldenSnapshotTests(unittest.TestCase):
+ def test_returns_once_ready(self):
+ ctl = FakeControl(
+ [
+ ExecResult(0, actor_template_json(), ""), # pending
+ ExecResult(0, actor_template_json(golden_tag="g1"), ""), # ready
+ ]
+ )
+ record = run(
+ wait_for_golden_snapshot(
+ ctl,
+ "live-agent-gate",
+ "live-agent-gate-v1",
+ timeout_s=30,
+ poll_interval_s=0,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+ self.assertEqual(record.golden_state, GoldenState.READY)
+
+ def test_raises_on_reported_failure_not_just_returns(self):
+ ctl = FakeControl(
+ [ExecResult(0, actor_template_json(error_message="boom"), "")]
+ )
+ with self.assertRaises(GoldenSnapshotFailed):
+ run(
+ wait_for_golden_snapshot(
+ ctl,
+ "live-agent-gate",
+ "live-agent-gate-v1",
+ timeout_s=30,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+
+ def test_raises_timeout_rather_than_claiming_success(self):
+ # Every poll is still pending; the fake clock advances past the deadline. This is
+ # the "polling loop prints success after timeout" bug from the recovery review --
+ # here, timing out must raise, never return.
+ ctl = FakeControl([ExecResult(0, actor_template_json(), "") for _ in range(50)])
+ with self.assertRaises(GoldenSnapshotTimeout):
+ run(
+ wait_for_golden_snapshot(
+ ctl,
+ "live-agent-gate",
+ "live-agent-gate-v1",
+ timeout_s=5,
+ poll_interval_s=0,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+
+ def test_disappearing_template_is_a_transport_error_not_pending(self):
+ ctl = FakeControl(
+ [ExecResult(1, "", 'Error: actortemplates.ate.dev "x" not found')]
+ )
+ with self.assertRaises(TransportError):
+ run(
+ wait_for_golden_snapshot(
+ ctl,
+ "live-agent-gate",
+ "live-agent-gate-v1",
+ timeout_s=30,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+
+
+class GetEligibleWorkersTests(unittest.TestCase):
+ def get_workers(self, workers):
+ return FakeControl([ExecResult(0, json.dumps({"workers": workers}), "")])
+
+ def test_parses_pinned_cli_json_and_uses_pool_filters(self):
+ ctl = FakeControl([ExecResult(0, PINNED_WORKERS_OUTPUT, "")])
+ self.assertEqual(
+ run(
+ ctl.get_eligible_workers(
+ "live-agent-gate", "workload=live-agent-gate", "gvisor"
+ )
+ ),
+ 1,
+ )
+ self.assertEqual(
+ ctl.calls[0],
+ [
+ "get",
+ "workers",
+ "-n",
+ "live-agent-gate",
+ "-l",
+ "workload=live-agent-gate",
+ "--sandbox-class",
+ "gvisor",
+ "-o",
+ "json",
+ ],
+ )
+
+ def test_excludes_occupied_worker_using_capacity_minus_allocated(self):
+ ctl = self.get_workers([worker_record(capacity=1, allocated=1)])
+ self.assertEqual(
+ run(
+ ctl.get_eligible_workers(
+ "live-agent-gate", "workload=live-agent-gate", "gvisor"
+ )
+ ),
+ 0,
+ )
+
+ def test_excludes_draining_worker(self):
+ ctl = self.get_workers([worker_record(state="WORKER_STATE_DRAINING")])
+ self.assertEqual(
+ run(
+ ctl.get_eligible_workers(
+ "live-agent-gate", "workload=live-agent-gate", "gvisor"
+ )
+ ),
+ 0,
+ )
+
+ def test_excludes_wrong_pool_worker(self):
+ ctl = self.get_workers([worker_record(pool_label="other")])
+ self.assertEqual(
+ run(
+ ctl.get_eligible_workers(
+ "live-agent-gate", "workload=live-agent-gate", "gvisor"
+ )
+ ),
+ 0,
+ )
+
+ def test_empty_workers_list_returns_zero(self):
+ ctl = self.get_workers([])
+ self.assertEqual(
+ run(
+ ctl.get_eligible_workers(
+ "live-agent-gate", "workload=live-agent-gate", "gvisor"
+ )
+ ),
+ 0,
+ )
+
+ def test_missing_workers_field_is_a_contract_error(self):
+ ctl = FakeControl([ExecResult(0, json.dumps({"items": []}), "")])
+ with self.assertRaisesRegex(TransportError, "missing its 'workers' list"):
+ run(
+ ctl.get_eligible_workers(
+ "live-agent-gate", "workload=live-agent-gate", "gvisor"
+ )
+ )
+
+
+class WaitForEligibleWorkerTests(unittest.TestCase):
+ def test_returns_once_a_worker_is_eligible(self):
+ ctl = FakeControl(
+ [
+ ExecResult(0, json.dumps({"workers": []}), ""),
+ ExecResult(0, json.dumps({"workers": [worker_record()]}), ""),
+ ]
+ )
+ count = run(
+ wait_for_eligible_worker(
+ ctl,
+ "live-agent-gate",
+ "workload=live-agent-gate",
+ "gvisor",
+ timeout_s=30,
+ poll_interval_s=0,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+ self.assertEqual(count, 1)
+
+ def test_raises_no_eligible_worker_on_timeout(self):
+ ctl = FakeControl(
+ [ExecResult(0, json.dumps({"workers": []}), "") for _ in range(50)]
+ )
+ with self.assertRaises(NoEligibleWorker):
+ run(
+ wait_for_eligible_worker(
+ ctl,
+ "live-agent-gate",
+ "workload=live-agent-gate",
+ "gvisor",
+ timeout_s=5,
+ poll_interval_s=0,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+
+
+class WaitForActorRunningTests(unittest.TestCase):
+ def test_returns_once_running(self):
+ ctl = FakeControl(
+ [
+ ExecResult(0, actor_json("ACTOR_STATE_RESUMING"), ""),
+ ExecResult(0, actor_json("ACTOR_STATE_RUNNING"), ""),
+ ]
+ )
+ actor = run(
+ wait_for_actor_running(
+ ctl,
+ "mainloop-workspaces",
+ "ml-abc",
+ timeout_s=30,
+ poll_interval_s=0,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+ self.assertEqual(actor.state, ActorState.RUNNING)
+
+ def test_raises_actor_failed_to_start_on_crash_not_timeout(self):
+ ctl = FakeControl([ExecResult(0, actor_json("ACTOR_STATE_CRASHED"), "")])
+ with self.assertRaises(ActorFailedToStart):
+ run(
+ wait_for_actor_running(
+ ctl,
+ "mainloop-workspaces",
+ "ml-abc",
+ timeout_s=30,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+
+ def test_raises_wait_timeout_when_stuck_resuming(self):
+ ctl = FakeControl(
+ [ExecResult(0, actor_json("ACTOR_STATE_RESUMING"), "") for _ in range(50)]
+ )
+ with self.assertRaises(WaitTimeout):
+ run(
+ wait_for_actor_running(
+ ctl,
+ "mainloop-workspaces",
+ "ml-abc",
+ timeout_s=5,
+ poll_interval_s=0,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+
+
+class WaitForActorHealthTests(unittest.TestCase):
+ def test_slow_start_waits_until_health_route_returns_200(self):
+ checks = iter([False, False, True])
+ run(
+ wait_for_actor_health(
+ lambda: next(checks),
+ timeout_s=10,
+ poll_interval_s=0,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+
+ def test_failed_pane_or_shim_times_out_while_health_stays_non_200(self):
+ with self.assertRaises(ActorHealthTimeout):
+ run(
+ wait_for_actor_health(
+ lambda: False,
+ timeout_s=3,
+ poll_interval_s=0,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+
+ def test_restored_actor_passes_without_replaying_boot_marker(self):
+ # Restore resumes processes; current health, not a repeated startup log, is the
+ # readiness contract.
+ run(
+ wait_for_actor_health(
+ lambda: True,
+ timeout_s=3,
+ sleep=fake_sleep,
+ clock=FakeClock(step=1),
+ )
+ )
+
+
+class ReconcileActorIdentityTests(unittest.TestCase):
+ def test_absent_when_no_live_actor(self):
+ self.assertEqual(reconcile_actor_identity("u-1", None), IdentityOutcome.ABSENT)
+
+ def test_matches_when_uids_equal(self):
+ live = _actor_from_json(
+ json.loads(actor_json("ACTOR_STATE_RUNNING", uid="u-1"))
+ )
+ self.assertEqual(reconcile_actor_identity("u-1", live), IdentityOutcome.MATCHES)
+
+ def test_unowned_when_no_actor_uid_was_persisted(self):
+ live = _actor_from_json(
+ json.loads(actor_json("ACTOR_STATE_RUNNING", uid="u-1"))
+ )
+ self.assertEqual(reconcile_actor_identity(None, live), IdentityOutcome.UNOWNED)
+
+ def test_diverged_when_uids_differ(self):
+ # A rerun must not silently resume or recreate an actor that turned out to belong
+ # to a different run under the same name.
+ live = _actor_from_json(
+ json.loads(actor_json("ACTOR_STATE_RUNNING", uid="u-2"))
+ )
+ self.assertEqual(
+ reconcile_actor_identity("u-1", live), IdentityOutcome.DIVERGED
+ )
+
+
if __name__ == "__main__":
unittest.main()
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 56a9ee9..42206e9 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -24,7 +24,7 @@ drives its lifecycle through the real `kubectl ate` control-plane CLI.
| `dev-service-gate` WorkerPool + ActorTemplate: real `psql`, real external `postgres:16-alpine` StatefulSet, real `EgressPolicy` (CIDR rule, created via a small gRPC tool since `kubectl-ate` has no CLI verb for it) | Real |
| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
| File edits and shell commands run inside actors (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
-| `live-agent-gate` WorkerPool + ActorTemplate: real Claude/Codex CLIs, real cred-server | Built, not run -- blocked at cluster creation by Claude Code's own safety classifier ("Create Unsafe Agents"); see "Limits" |
+| `live-agent-gate` WorkerPool + ActorTemplate: real Claude/Codex CLIs, credential-free boot | Phase 1 harness repair complete; credential-free lifecycle rerun is pending. No native-agent session has been run. |
| Claude/Codex agent processes, credentials | Not run in this spike (see "Limits") |
| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
@@ -169,22 +169,59 @@ per `docs/network-egress.md`'s "CIDR/all policy: dial now" passthrough path) ins
## Limits / not attempted in this run
-- **Native-session gate, live**: infrastructure built and ready
- (`spikes/substrate-workspace-adapter/live-agent-image/`, `k8s/cred-server.yaml.tmpl`,
- `k8s/live-agent-gate-template.yaml.tmpl`) -- real Claude/Codex CLIs, credentials fetched at
- actor-runtime through the same egress-CIDR mechanism gate 4 proved enforces (never baked into
- a template), reusing `.tasknotes/plan.md`'s by-path Secret pattern. The trial was not run: the
- cluster-creation step was declined by Claude Code's own auto-mode safety classifier ("Create
- Unsafe Agents"), and the run stopped there rather than seeking a workaround, per the plan's own
- "missing permission" stopping criterion -- this involves the operator's real subscription
- credentials, so proceeding past a safety control without explicit human authorization was not
- appropriate. Only the fixture-level contract logic (`test_workspace_adapter.py`) and the
- generic restore-vs-reboot log evidence above are available for this gate.
+- **Native-session gate, live**: a prior owner-authorized attempt reached golden-snapshot
+ creation but failed when the boot-time credential fetch received HTTP 403. The earlier
+ description that the run was declined by a safety classifier was inaccurate: the run was
+ authorized, while tool policy rejected particular actions. Phase 1 removes the boot-time
+ fetch and repairs the harness; the credential-free lifecycle proof is pending. No Claude or
+ Codex session has been run. The old relay manifest and fetch helper have been removed. See
+ the finish plan for the bounded lifecycle proof and the separately gated Claude-only
+ credential-boundary attempt.
- **`workspace_bindings` orchestration functions** (`ensure_workspace`, `resume_workspace`, ...)
were not exercised against a live Postgres + running backend; only their extracted pure logic
(`plan_ensure`, `_binding_from_row`, `is_crashed`) is unit tested, and the transport layer
they call (`SubstrateControl`) is proved live as described above.
+### Native-session gate addendum: a later live attempt failed at golden creation, now repaired
+
+After the run above, a separate live attempt (outside this doc's own commits) did create the
+live-agent-gate infrastructure and hit a real failure during golden-snapshot creation, reviewed
+in `.tasknotes/gate5-review-and-recovery-plan-2026-09-22.md`: the golden actor's entrypoint
+fetched a credential from `cred-server` unconditionally at boot, that fetch was denied (`403`),
+and the golden actor exited before its snapshot was captured -- `runsc exit 128` on a later
+restore attempt is consistent with capturing a process that had already exited. The harness
+script driving that attempt (never committed; reviewed from a scratch copy) also applied an
+unresolved `ko://` WorkerPool image, never registered the atespace at the control-plane API
+(a Kubernetes Namespace of the same name is not an atespace), checked for an existing
+ActorTemplate with the atespace embedded in the name rather than the CLI's required `-a` flag,
+and printed success from a log line reached before the state it implied was actually confirmed.
+
+This Phase 1 change (recovery plan step 2) repairs those bugs and removes the root cause:
+
+- `entrypoint.sh` no longer fetches a credential or needs network access to reach a running
+ state. There is no credential-fetch helper or relay path in the image. Credential delivery
+ remains a separate, gated step and is never performed during golden-actor warmup.
+- `k8s/live-agent-gate-template.yaml.tmpl` no longer sets `CRED_SERVER` in the (shared,
+ immutable) container env, and its ActorTemplate name is now versioned
+ (`live-agent-gate-${TEMPLATE_VERSION}`) so a failed golden snapshot is never reused.
+- `backend/src/mainloop/runtime/substrate.py` gained `ensure_atespace`/`get_actor_template`/
+ `create_actor_template`/`get_eligible_workers`, plus bounded, exception-raising waits for
+ golden snapshots, eligible workers, actor state, and the live actor health route. Rerun
+ identity reconciliation distinguishes absent, unowned, matching, and diverged actors before
+ the harness creates or resumes one.
+- `backend/scripts/gate5_setup.py` registers the atespace, resolves the WorkerPool image with
+ `ko resolve` from the verified pinned checkout, waits for an eligible worker and a golden
+ snapshot, binds reruns to persisted cluster/template/actor identity, then confirms health
+ through the actor route before applying egress policy.
+- `egress-tool/main.go` fails closed: exactly one of `--deny-all`, `--cidr`, or `--allow-all`
+ must be explicit.
+
+**Scope of this repair**: code and fixture tests only. The focused Substrate, setup, and contract
+tests pass; the owner reports the full runtime suite passes 189/189 outside the restricted
+sandbox. No fresh lifecycle measurement is claimed here: `gate5_setup.py` has not yet been run
+against a fresh `kind-substrate-preview` cluster. Phase 2 must confirm the golden snapshot and
+restored readiness before this addendum can report a live result.
+
## Cleanup
Each of the three cluster lanes in this spike (adapter/CRASHED, preview-gate, dev-service-gate)
diff --git a/spikes/substrate-workspace-adapter/egress-tool/main.go b/spikes/substrate-workspace-adapter/egress-tool/main.go
index 1607afc..57d51ac 100644
--- a/spikes/substrate-workspace-adapter/egress-tool/main.go
+++ b/spikes/substrate-workspace-adapter/egress-tool/main.go
@@ -12,7 +12,12 @@
// cdac9baef81dd319b46086d695266e6161e9e592 when this was written).
//
// Usage: mainloop-egress-tool --kubeconfig --context --atespace --actor
-// [--cidr ] (omit --cidr to allow all destinations)
+// --deny-all | --cidr | --allow-all
+//
+// Fails closed: exactly one of --deny-all, --cidr, or --allow-all is required. An earlier version of this
+// tool silently allowed all destinations whenever --cidr was omitted (see
+// .tasknotes/gate5-review-and-recovery-plan-2026-09-22.md, "Make missing egress configuration
+// fail closed"); --allow-all must now be passed explicitly to get that behavior.
package main
import (
@@ -30,14 +35,39 @@ import (
"github.com/agent-substrate/substrate/pkg/proto/ateapipb"
)
+// validateEgressInput is the fail-closed check, isolated as a pure function so it can be
+// exercised without a cluster or a Substrate checkout.
+func validateEgressInput(cidr string, denyAll, allowAll bool) error {
+ selected := 0
+ if cidr != "" {
+ selected++
+ }
+ if denyAll {
+ selected++
+ }
+ if allowAll {
+ selected++
+ }
+ if selected != 1 {
+ return fmt.Errorf("exactly one of --deny-all, --cidr , or --allow-all is required")
+ }
+ return nil
+}
+
func main() {
kubeconfig := flag.String("kubeconfig", "", "")
context_ := flag.String("context", "", "")
atespace := flag.String("atespace", "", "")
actorName := flag.String("actor", "", "")
- cidr := flag.String("cidr", "", "CIDR to allow; empty means allow-all")
+ cidr := flag.String("cidr", "", "CIDR to allow")
+ denyAll := flag.Bool("deny-all", false, "explicitly deny all actor egress")
+ allowAll := flag.Bool("allow-all", false, "explicitly allow all egress destinations")
flag.Parse()
+ if err := validateEgressInput(*cidr, *denyAll, *allowAll); err != nil {
+ log.Fatalf("%v", err)
+ }
+
ctx := context.Background()
cli, err := ateclient.NewClient(ctx, *kubeconfig, *context_, "", "", false)
if err != nil {
@@ -47,15 +77,15 @@ func main() {
actorRef := resources.ActorRef{Atespace: *atespace, Name: *actorName}.ToObjectRef()
- var rule *ateapipb.EgressRule
- if *cidr == "" {
- rule = &ateapipb.EgressRule{All: &emptypb.Empty{}}
- } else {
- rule = &ateapipb.EgressRule{Cidrs: &ateapipb.CIDRRule{Cidrs: []string{*cidr}}}
+ var rules []*ateapipb.EgressRule
+ if *allowAll {
+ rules = []*ateapipb.EgressRule{{All: &emptypb.Empty{}}}
+ } else if *cidr != "" {
+ rules = []*ateapipb.EgressRule{{Cidrs: &ateapipb.CIDRRule{Cidrs: []string{*cidr}}}}
}
policy := &ateapipb.EgressPolicy{
Metadata: &ateapipb.ResourceMetadata{Atespace: *atespace, Name: "default"},
- Rules: []*ateapipb.EgressRule{rule},
+ Rules: rules,
}
_, err = cli.CreateActorEgressPolicy(ctx, &ateapipb.CreateActorEgressPolicyRequest{
diff --git a/spikes/substrate-workspace-adapter/egress-tool/main_test.go b/spikes/substrate-workspace-adapter/egress-tool/main_test.go
new file mode 100644
index 0000000..dcf8e0c
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/egress-tool/main_test.go
@@ -0,0 +1,28 @@
+package main
+
+import "testing"
+
+func TestValidateEgressInputRequiresExactlyOneMode(t *testing.T) {
+ tests := []struct {
+ name string
+ cidr string
+ denyAll bool
+ allowAll bool
+ wantErr bool
+ }{
+ {name: "deny all", denyAll: true},
+ {name: "cidr", cidr: "192.0.2.1/32"},
+ {name: "allow all", allowAll: true},
+ {name: "missing mode", wantErr: true},
+ {name: "conflicting modes", denyAll: true, allowAll: true, wantErr: true},
+ {name: "cidr and deny all", cidr: "192.0.2.1/32", denyAll: true, wantErr: true},
+ }
+ for _, tt := range tests {
+ t.Run(tt.name, func(t *testing.T) {
+ err := validateEgressInput(tt.cidr, tt.denyAll, tt.allowAll)
+ if (err != nil) != tt.wantErr {
+ t.Fatalf("validateEgressInput() error = %v, wantErr %v", err, tt.wantErr)
+ }
+ })
+ }
+}
diff --git a/spikes/substrate-workspace-adapter/k8s/cred-server.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/cred-server.yaml.tmpl
deleted file mode 100644
index f2b2459..0000000
--- a/spikes/substrate-workspace-adapter/k8s/cred-server.yaml.tmpl
+++ /dev/null
@@ -1,71 +0,0 @@
-# Serves the two credential files a real Claude/Codex CLI needs, from Kubernetes Secrets created
-# by path (never by value -- see build script). Reachable only by an actor whose EgressPolicy
-# explicitly allows this Service's ClusterIP: the same CIDR-scoped enforcement gate 4
-# (dev-service) proved denies everything else with a clean 403, reused here as the credential
-# boundary. Plain NGINX static-file serving; no application code. Deployed outside any atespace,
-# like the gate 4 Postgres target -- this represents a Mainloop-operated credential-relay
-# service, not part of the actor's own image or an atespace resource.
-apiVersion: v1
-kind: Namespace
-metadata:
- name: cred-server
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: cred-server-nginx-config
- namespace: cred-server
-data:
- nginx.conf: |
- events {}
- http {
- server {
- listen 80;
- location = /claude-token { alias /secrets/claude/token; }
- location = /codex-auth.json { alias /secrets/codex/auth.json; }
- location / { return 404; }
- }
- }
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: cred-server
- namespace: cred-server
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: cred-server
- template:
- metadata:
- labels:
- app: cred-server
- spec:
- containers:
- - name: nginx
- image: nginx:alpine
- ports:
- - containerPort: 80
- volumeMounts:
- - { name: config, mountPath: /etc/nginx/nginx.conf, subPath: nginx.conf }
- - { name: claude-token, mountPath: /secrets/claude, readOnly: true }
- - { name: codex-auth, mountPath: /secrets/codex, readOnly: true }
- volumes:
- - name: config
- configMap: { name: cred-server-nginx-config }
- - name: claude-token
- secret: { secretName: mainloop-claude-token }
- - name: codex-auth
- secret: { secretName: mainloop-codex-auth }
----
-apiVersion: v1
-kind: Service
-metadata:
- name: cred-server
- namespace: cred-server
-spec:
- selector:
- app: cred-server
- ports:
- - port: 80
diff --git a/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl
index a6e5350..dfb08a5 100644
--- a/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl
+++ b/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl
@@ -1,6 +1,10 @@
# WorkerPool + ActorTemplate for the bounded live native-agent proof (gate 5 in
# .tasknotes/plan.md): real Claude Code / Codex CLIs under Herdr, driven by the exec shim. See
# spikes/substrate-workspace-adapter/live-agent-image/.
+#
+# Deliberately no credential-relay env: the golden actor boots without credentials or external
+# network access. The pinned ActorTemplate API supports readyz; the probe waits for the control
+# server, shell pane, and exec shim before the golden snapshot or final actor is considered ready.
apiVersion: v1
kind: Namespace
metadata:
@@ -23,19 +27,22 @@ spec:
---
metadata:
atespace: ${ATESPACE}
- name: live-agent-gate
+ # Versioned, not "live-agent-gate": ActorTemplates are immutable, and a template whose
+ # golden snapshot failed must never be reused under the same name (recovery plan step 2).
+ name: ${TEMPLATE_NAME}
workerSelector:
matchLabels:
workload: live-agent-gate
containers:
- name: live-agent
image: __IMAGE__
- env:
- - { name: CRED_SERVER, value: "cred-server.cred-server.svc.cluster.local" }
resources:
limits:
- { name: cpu, quantity: "2" }
- { name: memory, quantity: 2Gi }
+ readyz:
+ httpGet: { path: /healthz, port: 8090 }
+ timeoutSeconds: 60
snapshotsConfig:
onPause: SNAPSHOT_CONTENT_SCOPE_FULL
onCommit: SNAPSHOT_CONTENT_SCOPE_FULL
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
index 824d030..bbd0b26 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
+++ b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
@@ -1,9 +1,10 @@
# Live-agent-gate spike image (gate 5, bounded live proof): real Herdr + real Claude Code /
# Codex CLIs + agentctl, driven by the same generic exec shim as the other gate images.
# herdr/claude/codex/codex-code-mode-host are copied from the host into the build context by the
-# build script (never committed). No credentials are baked into this image; they are fetched at
-# actor runtime from an in-cluster server reachable only via a narrow EgressPolicy (see
-# entrypoint.sh and docs/spikes/substrate-workspace-adapter.md).
+# build script (never committed). No credentials are baked into this image, and entrypoint.sh
+# never fetches one: the golden actor built from this image boots to a ready control service
+# with no credential and no external network access. Credential delivery remains deferred until
+# a reviewed boundary exists (see docs/spikes/substrate-workspace-adapter.md).
FROM node:22-bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends jq ca-certificates git curl procps ripgrep \
&& rm -rf /var/lib/apt/lists/* \
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
index 5f1547e..907aba7 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
+++ b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
@@ -3,23 +3,17 @@
# Live-agent-gate actor entrypoint (gate 5 in .tasknotes/plan.md, bounded live proof): a real
# Herdr server plus real Claude Code / Codex CLIs. Substrate has no Kubernetes-Secret-equivalent
# volume/env mechanism for an actor (see docs/spikes/substrate-workspace-adapter.md,
-# "Credential-injection gap"), so credentials are fetched over the network from a small in-cluster
-# server, reachable only because this actor's narrow EgressPolicy allows exactly that server's
-# ClusterIP -- the same CIDR-scoped access-control mechanism gate 4 (dev-service) proved actually
-# enforces (a non-allowed destination gets a clean 403), reused here as the auth boundary rather
-# than inventing a new one. Never echoed, never written to a template, never logged.
+# "Credential-injection gap").
+#
+# Credential-free by construction: this entrypoint never fetches a credential and never
+# requires network access to reach a running state. The template controller uses the
+# ActorTemplate's `/healthz` readiness check before accepting the golden actor. A boot path
+# that depends on a credential fetch succeeding can fail golden creation when its relay is
+# denied or unreachable, which is what happened. Credential delivery is deferred to a
+# reviewed boundary and is never performed during golden-actor warmup or from this entrypoint.
set -eu
mkdir -p "${HOME}" "${HOME}/.claude" "${CODEX_HOME}"
-if [[ -n ${CRED_SERVER-} ]]; then
- curl -fsS "http://${CRED_SERVER}/claude-token" -o "${HOME}/.claude-oauth-token"
- chmod 600 "${HOME}/.claude-oauth-token"
- CLAUDE_CODE_OAUTH_TOKEN="$(tr -d ' \r\n' <"${HOME}/.claude-oauth-token")"
- export CLAUDE_CODE_OAUTH_TOKEN
- curl -fsS "http://${CRED_SERVER}/codex-auth.json" -o "${CODEX_HOME}/auth.json"
- chmod 600 "${CODEX_HOME}/auth.json"
-fi
-
# Claude Code: onboarding done, workspace trusted, bypass-permissions warning accepted.
if [[ ! -s "${HOME}/.claude.json" ]]; then
jq -n --arg p "${WORKSPACE_PATH}" '{
@@ -39,19 +33,35 @@ grep -q '^\[notice\]' "${CODEX_HOME}/config.toml" || printf '\n[notice]\nhide_ra
mkdir -p "${WORKSPACE_PATH}"
[[ -d "${WORKSPACE_PATH}/.git" ]] || git -C "${WORKSPACE_PATH}" init -q
+# Credential-free identity/counter marker for the fake-payload proof (recovery plan step 2):
+# a plain file the exec shim can read/increment to verify golden restore and suspend/resume
+# without any real agent session or credential.
+[[ -f "${WORKSPACE_PATH}/gate5-counter" ]] || echo 0 >"${WORKSPACE_PATH}/gate5-counter"
-echo "herdr $(herdr --version) server starting (HOME=${HOME} session=${HERDR_SESSION})"
+echo "herdr $(herdr --version) starting (HOME=${HOME} session=${HERDR_SESSION})"
herdr --session "${HERDR_SESSION}" server &
HERDR_PID=$!
+# The persistent control service's readiness check: an explicit, confirmed status call,
+# not a fixed sleep or a pre-confirmation log line. The ActorTemplate's `/healthz` probe
+# checks the Herdr server and this shell pane before the controller captures its snapshot.
+ready=0
for _ in $(seq 1 60); do
- herdr --session "${HERDR_SESSION}" status server >/dev/null 2>&1 && break
+ if herdr --session "${HERDR_SESSION}" status server >/dev/null 2>&1; then
+ ready=1
+ break
+ fi
sleep 0.5
done
-
+if [[ ${ready} -ne 1 ]]; then
+ echo "CONTROL_SERVICE_READINESS_TIMEOUT: herdr server did not report ready within 30s" >&2
+ kill "${HERDR_PID}" 2>/dev/null || true
+ exit 1
+fi
shell_ws=$(herdr --session "${HERDR_SESSION}" workspace create --label shell --cwd "${WORKSPACE_PATH}")
shell_pane=$(echo "${shell_ws}" | jq -r '.result.root_pane.pane_id')
EXEC_SHIM_PANE_ID="${shell_pane}" HERDR_SESSION="${HERDR_SESSION}" node "${EXEC_SHIM}" &
+echo "CONTROL_SERVICE_READY session=${HERDR_SESSION} pane=${shell_pane}"
wait "${HERDR_PID}"
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
index 01250e7..8755b95 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
+++ b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
@@ -32,7 +32,29 @@ function herdr(args, res) {
});
}
+function healthz(res) {
+ execFile('herdr', ['--session', SESSION, 'status', 'server'], (statusErr, stdout) => {
+ if (statusErr || !/^status:\s+running\s*$/m.test(stdout)) {
+ res.writeHead(503, { 'content-type': 'text/plain' }).end('not ready');
+ return;
+ }
+ execFile('herdr', ['--session', SESSION, 'pane', 'read', PANE_ID], (paneErr) => {
+ if (paneErr) {
+ res.writeHead(503, { 'content-type': 'text/plain' }).end('not ready');
+ return;
+ }
+ // The request is served by this shim, Herdr reports a running server, and pane read
+ // confirms the shell pane still exists. Do not expose status output or pane contents.
+ res.writeHead(200, { 'content-type': 'text/plain' }).end('ok');
+ });
+ });
+}
+
const server = http.createServer((req, res) => {
+ if (req.method === 'GET' && req.url === '/healthz') {
+ healthz(res);
+ return;
+ }
if (req.method === 'GET' && req.url === '/read') {
herdr(['pane', 'read', PANE_ID], res);
return;
From 190b222f365892185e531c23c81e5cd98c391d75 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Wed, 23 Sep 2026 01:13:39 +0000
Subject: [PATCH 09/30] fix: correct Gate 5 rerun and router handling
Treat {} as an empty worker list, route the setup tunnel to CONNECT port 8081, and reconcile actor ownership before waiting for worker capacity. Add regressions and update the spike evidence.
---
backend/scripts/gate5_setup.py | 64 +++++++--
backend/src/mainloop/runtime/substrate.py | 13 +-
backend/tests/runtime/test_gate5_setup.py | 147 +++++++++++++++++++--
backend/tests/runtime/test_substrate.py | 11 ++
docs/spikes/substrate-workspace-adapter.md | 146 ++++++++++++++------
5 files changed, 311 insertions(+), 70 deletions(-)
diff --git a/backend/scripts/gate5_setup.py b/backend/scripts/gate5_setup.py
index 2271154..be894ea 100644
--- a/backend/scripts/gate5_setup.py
+++ b/backend/scripts/gate5_setup.py
@@ -351,7 +351,7 @@ def actor_router_tunnel(args: argparse.Namespace):
"--namespace",
"ate-system",
"service/atenet-router",
- f"{args.router_port}:80",
+ f"{args.router_port}:8081",
]
process = (
subprocess.Popen( # nosec B603 - fixed kubectl argv, explicit kube context
@@ -463,6 +463,55 @@ async def ensure_golden_template(
return record.uid or existing.uid
+async def wait_for_worker_if_actor_is_absent(
+ control: SubstrateControl, args: argparse.Namespace, state: dict
+) -> None:
+ """Reconcile actor ownership before waiting for capacity needed by a new actor."""
+ live = await control.get_actor(args.atespace, args.actor_name)
+ outcome = reconcile_actor_identity(state.get("actor_uid"), live)
+ if outcome is IdentityOutcome.UNOWNED:
+ raise IdentityConflict(
+ f"actor {args.atespace}/{args.actor_name} already exists with uid {live.uid}, "
+ f"but {args.state_file} has no actor uid; refusing to adopt it. Use a new "
+ "--actor-name or reconcile the state file explicitly"
+ )
+ if outcome is IdentityOutcome.DIVERGED:
+ raise IdentityConflict(
+ f"actor {args.atespace}/{args.actor_name} exists with uid {live.uid}, but "
+ f"{args.state_file} recorded {state.get('actor_uid')} from a prior run -- "
+ "refusing to resume or recreate it; reconcile manually or use a different "
+ "--actor-name"
+ )
+ if outcome is IdentityOutcome.MATCHES:
+ if (
+ state.get("template_uid")
+ and live.current_actor_template_uid != state["template_uid"]
+ ):
+ raise IdentityConflict(
+ f"actor {args.atespace}/{args.actor_name} references template uid "
+ f"{live.current_actor_template_uid!r}, but {args.state_file} recorded "
+ f"{state['template_uid']!r}"
+ )
+ if live.state in {ActorState.CRASHED, ActorState.DELETING}:
+ raise ActorFailedToStart(
+ f"actor {args.atespace}/{args.actor_name} is {live.state.value}; "
+ "choose an explicit revert or a new --actor-name before retrying"
+ )
+ return
+
+ print(
+ f"-- waiting for an eligible worker in namespace={WORKER_NAMESPACE}, "
+ f"selector={WORKER_SELECTOR}, sandbox={WORKER_SANDBOX_CLASS}"
+ )
+ await wait_for_eligible_worker(
+ control,
+ WORKER_NAMESPACE,
+ WORKER_SELECTOR,
+ WORKER_SANDBOX_CLASS,
+ timeout_s=args.worker_timeout,
+ )
+
+
async def ensure_actor(
control: SubstrateControl,
args: argparse.Namespace,
@@ -558,18 +607,7 @@ async def async_main(args: argparse.Namespace) -> None:
substrate_src=substrate_src,
)
- print(
- f"-- waiting for an eligible worker in namespace={WORKER_NAMESPACE}, "
- f"selector={WORKER_SELECTOR}, sandbox={WORKER_SANDBOX_CLASS}"
- )
- await wait_for_eligible_worker(
- control,
- WORKER_NAMESPACE,
- WORKER_SELECTOR,
- WORKER_SANDBOX_CLASS,
- timeout_s=args.worker_timeout,
- )
-
+ await wait_for_worker_if_actor_is_absent(control, args, state)
template_uid = await ensure_golden_template(
control, args, template_name, actor_template_doc, state
)
diff --git a/backend/src/mainloop/runtime/substrate.py b/backend/src/mainloop/runtime/substrate.py
index 70158b2..f50d063 100644
--- a/backend/src/mainloop/runtime/substrate.py
+++ b/backend/src/mainloop/runtime/substrate.py
@@ -475,9 +475,18 @@ async def get_eligible_workers(
doc = json.loads(text)
except json.JSONDecodeError as exc:
raise TransportError(f"get workers returned invalid JSON: {exc}") from exc
- if not isinstance(doc, dict) or "workers" not in doc:
+ if not isinstance(doc, dict):
raise TransportError("get workers JSON is missing its 'workers' list")
- workers = doc["workers"]
+ # The pinned CLI uses protojson's default omission behavior: a valid
+ # ListWorkers response with zero matches is `{}`, not `{"workers": []}`.
+ # Treat only that empty object as the empty list; a non-empty object
+ # without the field is still a contract error.
+ if "workers" not in doc:
+ if doc:
+ raise TransportError("get workers JSON is missing its 'workers' list")
+ workers = []
+ else:
+ workers = doc["workers"]
if not isinstance(workers, list):
raise TransportError("get workers JSON field 'workers' is not a list")
return sum(
diff --git a/backend/tests/runtime/test_gate5_setup.py b/backend/tests/runtime/test_gate5_setup.py
index fcfc2ac..0623bc8 100644
--- a/backend/tests/runtime/test_gate5_setup.py
+++ b/backend/tests/runtime/test_gate5_setup.py
@@ -185,6 +185,49 @@ def make_connection(*args, **kwargs):
)
self.assertEqual(connections[0].requested, ("GET", "/healthz"))
+ def test_actor_router_tunnel_forwards_to_connect_listener(self):
+ class FakeSocket:
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_exc):
+ return False
+
+ class FakeProcess:
+ def __init__(self):
+ self.terminated = False
+ self.stdout = None
+
+ def poll(self):
+ return None
+
+ def terminate(self):
+ self.terminated = True
+
+ def wait(self, timeout):
+ self.wait_timeout = timeout
+
+ args = SimpleNamespace(
+ context="kind-substrate-preview",
+ kubeconfig=FIXTURE_KUBECONFIG,
+ router_port=18081,
+ )
+ process = FakeProcess()
+ with (
+ patch.object(
+ gate5_setup.subprocess, "Popen", return_value=process
+ ) as popen,
+ patch.object(
+ gate5_setup.socket, "create_connection", return_value=FakeSocket()
+ ),
+ gate5_setup.actor_router_tunnel(args) as route_port,
+ ):
+ self.assertEqual(route_port, 18081)
+
+ command = popen.call_args.args[0]
+ self.assertIn("18081:8081", command)
+ self.assertTrue(process.terminated)
+
class Gate5StateTests(unittest.TestCase):
def args(self, state_file: str, **overrides):
@@ -242,11 +285,14 @@ def __init__(self, actor: ActorRecord | None):
self.actor = actor
self.created = []
self.resumed = 0
+ self.events = []
async def get_actor(self, _atespace, _name):
+ self.events.append("get_actor")
return self.actor
async def create_actor(self, atespace, name, *, template):
+ self.events.append("create_actor")
self.created.append((atespace, name, template))
self.actor = ActorRecord(
atespace=atespace,
@@ -260,6 +306,7 @@ async def create_actor(self, atespace, name, *, template):
return self.actor
async def resume_actor(self, _atespace, _name):
+ self.events.append("resume_actor")
self.resumed += 1
self.actor = replace(self.actor, state=ActorState.RUNNING)
return self.actor
@@ -296,6 +343,7 @@ def args(self, path):
atespace="live-agent-gate",
actor_name="claude-gate5",
actor_timeout=5,
+ worker_timeout=5,
)
def test_exact_old_template_collision_with_new_template_is_refused(self):
@@ -336,9 +384,10 @@ def test_owned_actor_with_template_mismatch_is_refused(self):
def test_crashed_and_deleting_actors_are_refused_before_resume(self):
for actor_state in (ActorState.CRASHED, ActorState.DELETING):
- with self.subTest(
- actor_state=actor_state
- ), tempfile.TemporaryDirectory() as temp_dir:
+ with (
+ self.subTest(actor_state=actor_state),
+ tempfile.TemporaryDirectory() as temp_dir,
+ ):
path = str(Path(temp_dir) / "state.json")
state = self.state(path)
control = SetupControl(self.actor(state=actor_state))
@@ -363,21 +412,97 @@ def test_new_actor_is_resumed_and_uid_is_saved(self):
"template_uid": "template-new",
}
control = SetupControl(None)
- uid = asyncio_run(
- gate5_setup.ensure_actor(
- control,
- self.args(path),
- "live-agent-gate-v2",
- "template-new",
- state,
+
+ async def wait_for_worker(*_args, **_kwargs):
+ control.events.append("wait_for_eligible_worker")
+
+ with patch.object(gate5_setup, "wait_for_eligible_worker", wait_for_worker):
+ asyncio_run(
+ gate5_setup.wait_for_worker_if_actor_is_absent(
+ control, self.args(path), state
+ )
+ )
+ uid = asyncio_run(
+ gate5_setup.ensure_actor(
+ control,
+ self.args(path),
+ "live-agent-gate-v2",
+ "template-new",
+ state,
+ )
)
- )
self.assertEqual(uid, "actor-new")
self.assertEqual(control.resumed, 1)
+ self.assertLess(
+ control.events.index("get_actor"),
+ control.events.index("wait_for_eligible_worker"),
+ )
+ self.assertLess(
+ control.events.index("wait_for_eligible_worker"),
+ control.events.index("create_actor"),
+ )
self.assertEqual(
json.loads(Path(path).read_text())["actor_uid"], "actor-new"
)
+ def test_owned_rerun_skips_worker_wait_when_its_actor_occupies_only_worker(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ state = self.state(path)
+ control = SetupControl(self.actor())
+
+ async def unexpected_worker_wait(*_args, **_kwargs):
+ self.fail("owned rerun must not wait for a spare worker")
+
+ with patch.object(
+ gate5_setup, "wait_for_eligible_worker", unexpected_worker_wait
+ ):
+ asyncio_run(
+ gate5_setup.wait_for_worker_if_actor_is_absent(
+ control, self.args(path), state
+ )
+ )
+ uid = asyncio_run(
+ gate5_setup.ensure_actor(
+ control,
+ self.args(path),
+ "live-agent-gate-v2",
+ "template-new",
+ state,
+ )
+ )
+
+ self.assertEqual(uid, "actor-1")
+ self.assertEqual(control.created, [])
+ self.assertEqual(control.resumed, 0)
+
+ def test_unowned_actor_is_refused_before_worker_wait(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ state = {
+ "run_id": "run-4",
+ "actor_uid": None,
+ "actor_name": "claude-gate5",
+ "template_name": "live-agent-gate-v2",
+ "template_uid": "template-new",
+ }
+ control = SetupControl(self.actor(template_uid="template-old"))
+
+ async def unexpected_worker_wait(*_args, **_kwargs):
+ self.fail("unowned actor must be refused before worker discovery")
+
+ with patch.object(
+ gate5_setup, "wait_for_eligible_worker", unexpected_worker_wait
+ ):
+ with self.assertRaisesRegex(IdentityConflict, "no actor uid"):
+ asyncio_run(
+ gate5_setup.wait_for_worker_if_actor_is_absent(
+ control, self.args(path), state
+ )
+ )
+
+ self.assertEqual(control.events, ["get_actor"])
+
def asyncio_run(coro):
import asyncio
diff --git a/backend/tests/runtime/test_substrate.py b/backend/tests/runtime/test_substrate.py
index e4ff2d5..e903daf 100644
--- a/backend/tests/runtime/test_substrate.py
+++ b/backend/tests/runtime/test_substrate.py
@@ -507,6 +507,17 @@ def test_empty_workers_list_returns_zero(self):
0,
)
+ def test_empty_object_matches_pinned_cli_empty_result(self):
+ ctl = FakeControl([ExecResult(0, "{}", "")])
+ self.assertEqual(
+ run(
+ ctl.get_eligible_workers(
+ "live-agent-gate", "workload=live-agent-gate", "gvisor"
+ )
+ ),
+ 0,
+ )
+
def test_missing_workers_field_is_a_contract_error(self):
ctl = FakeControl([ExecResult(0, json.dumps({"items": []}), "")])
with self.assertRaisesRegex(TransportError, "missing its 'workers' list"):
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 42206e9..427b3d3 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -16,17 +16,17 @@ drives its lifecycle through the real `kubectl ate` control-plane CLI.
## Real versus stand-in
-| Layer | Status |
-| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- |
-| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
-| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
-| `preview-gate` WorkerPool + ActorTemplate: real Herdr server, real Vite dev server, real NGINX header-proxy, real browser (`agent-browser`), real WebSocket HMR | Real |
-| `dev-service-gate` WorkerPool + ActorTemplate: real `psql`, real external `postgres:16-alpine` StatefulSet, real `EgressPolicy` (CIDR rule, created via a small gRPC tool since `kubectl-ate` has no CLI verb for it) | Real |
-| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
-| File edits and shell commands run inside actors (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
-| `live-agent-gate` WorkerPool + ActorTemplate: real Claude/Codex CLIs, credential-free boot | Phase 1 harness repair complete; credential-free lifecycle rerun is pending. No native-agent session has been run. |
-| Claude/Codex agent processes, credentials | Not run in this spike (see "Limits") |
-| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
+| Layer | Status |
+| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
+| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
+| `preview-gate` WorkerPool + ActorTemplate: real Herdr server, real Vite dev server, real NGINX header-proxy, real browser (`agent-browser`), real WebSocket HMR | Real |
+| `dev-service-gate` WorkerPool + ActorTemplate: real `psql`, real external `postgres:16-alpine` StatefulSet, real `EgressPolicy` (CIDR rule, created via a small gRPC tool since `kubectl-ate` has no CLI verb for it) | Real |
+| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
+| File edits and shell commands run inside actors (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
+| `live-agent-gate` WorkerPool + ActorTemplate: real Claude/Codex CLIs, credential-free boot | Phase 2: golden snapshot READY; actor RUNNING; `/healthz` and one suspend/resume proved live. Counter/marker restore and worker-loss revert remain unproved. No native-agent session has been run. |
+| Claude/Codex agent processes, credentials | Not run in this spike (see "Limits") |
+| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
## Why not a real agent for the preview-gate edit (credential-injection gap)
@@ -34,15 +34,19 @@ Substrate's pinned commit has no generic secret-injection mechanism equivalent t
Secret volume/env mount. `ActorTemplate` container env values are literal only (no
`envFrom`/`valueFrom`, and the template is immutable, so baking a token in would also mean
storing it permanently in a control-plane object -- unacceptable under this task's "credentials
-by path, never by value" rule). The only credential-shaped primitives are `SystemInfo` volumes
-(`actorMetadata`: the actor's own name/atespace/uid; `trustBundle`: a named, allowlisted CA
-bundle -- today only `egress-mitm.ate.dev`) and `pkg/proto/credproviderpb` (`CredentialProvider`,
-a plugin the _egress gateway_ calls to inject a credential into an actor's _outbound_ request,
-keyed by the actor's SPIFFE identity -- not a way to hand the actor's own process a local file or
-env var it can read directly, which is what the Claude Code / Codex CLIs need). A real
-native-agent proof (gate 5) therefore needs either an unsafe workaround or new plumbing (e.g. an
-authenticated credential-relay using the `MintActorJWT`/`MintActorCertificate` RPCs already in
-`ateapipb.Control`), out of scope for this spike. The preview-gate measurement below instead uses
+by path, never by value" rule). `SystemInfo` volumes are limited to actor metadata and an
+allowlisted CA bundle. `CredentialProvider` is an egress-gateway plugin, keyed by actor SPIFFE
+identity, that injects a credential into an outbound request; it does not hand a token to the
+CLI's local environment or filesystem.
+
+The pinned commit also has experimental static-header injection from a Kubernetes Secret URI
+into decrypted outbound requests. It requires Envoy with SDSMint and the experimental
+credential-injection flag. Envoy 1.39.1 crashed on this host, while agentgateway does not support
+the injection path. The revised Phase 3 uses a Mainloop-owned router NetworkPolicy and a per-actor
+shim token instead; credentials are delivered through that closed channel. This keeps the
+credential path separate from the unsupported Envoy feature, though actor snapshots will contain
+credentials after delivery. The earlier unauthenticated relay is not used. The preview-gate
+measurement below instead uses
a generic exec shim (`spikes/substrate-workspace-adapter/image/exec-shim.js`) that pastes text
into a real Herdr shell pane via `herdr pane run` -- a real shell executing a real command, just
not a credentialed agent's own tool call.
@@ -169,23 +173,25 @@ per `docs/network-egress.md`'s "CIDR/all policy: dial now" passthrough path) ins
## Limits / not attempted in this run
-- **Native-session gate, live**: a prior owner-authorized attempt reached golden-snapshot
- creation but failed when the boot-time credential fetch received HTTP 403. The earlier
- description that the run was declined by a safety classifier was inaccurate: the run was
- authorized, while tool policy rejected particular actions. Phase 1 removes the boot-time
- fetch and repairs the harness; the credential-free lifecycle proof is pending. No Claude or
- Codex session has been run. The old relay manifest and fetch helper have been removed. See
- the finish plan for the bounded lifecycle proof and the separately gated Claude-only
- credential-boundary attempt.
+- **Native-session gate, live**: no Claude or Codex session was run. The earlier attempt was
+ authorized; tool policy rejected particular actions after the boot-time credential fetch
+ received HTTP 403. Phase 1 removed the unauthenticated relay and repaired the harness. Phase 2
+ produced a READY golden snapshot, a RUNNING actor, a healthy `/healthz` through the documented
+ CONNECT port, and one successful suspend/resume. Counter/marker persistence and worker-loss
+ revert were not proved. Before any credential work, a tokenless caller reached `POST /run`
+ through the router and executed a harmless command. No existing ingress authorization primitive
+ closed this path under the earlier plan, so that credential attempt stopped and Gate 5 remains
+ partial/fixture pending the rewritten Phase 3 and Phase 4. The owner approved that plan on
+ 2026-09-23; no provider Secret was created or read in the earlier run.
- **`workspace_bindings` orchestration functions** (`ensure_workspace`, `resume_workspace`, ...)
were not exercised against a live Postgres + running backend; only their extracted pure logic
(`plan_ensure`, `_binding_from_row`, `is_crashed`) is unit tested, and the transport layer
they call (`SubstrateControl`) is proved live as described above.
-### Native-session gate addendum: a later live attempt failed at golden creation, now repaired
+### Native-session gate addendum: harness repair and Phase 2 partial live proof
-After the run above, a separate live attempt (outside this doc's own commits) did create the
-live-agent-gate infrastructure and hit a real failure during golden-snapshot creation, reviewed
+The original live attempt (outside this doc's own commits) created the live-agent-gate
+infrastructure and hit a real failure during golden-snapshot creation, reviewed
in `.tasknotes/gate5-review-and-recovery-plan-2026-09-22.md`: the golden actor's entrypoint
fetched a credential from `cred-server` unconditionally at boot, that fetch was denied (`403`),
and the golden actor exited before its snapshot was captured -- `runsc exit 128` on a later
@@ -196,7 +202,7 @@ unresolved `ko://` WorkerPool image, never registered the atespace at the contro
ActorTemplate with the atespace embedded in the name rather than the CLI's required `-a` flag,
and printed success from a log line reached before the state it implied was actually confirmed.
-This Phase 1 change (recovery plan step 2) repairs those bugs and removes the root cause:
+Phase 1 (recovery plan step 2) repairs the harness findings and removes the boot-time fetch:
- `entrypoint.sh` no longer fetches a credential or needs network access to reach a running
state. There is no credential-fetch helper or relay path in the image. Credential delivery
@@ -216,18 +222,70 @@ This Phase 1 change (recovery plan step 2) repairs those bugs and removes the ro
- `egress-tool/main.go` fails closed: exactly one of `--deny-all`, `--cidr`, or `--allow-all`
must be explicit.
-**Scope of this repair**: code and fixture tests only. The focused Substrate, setup, and contract
-tests pass; the owner reports the full runtime suite passes 189/189 outside the restricted
-sandbox. No fresh lifecycle measurement is claimed here: `gate5_setup.py` has not yet been run
-against a fresh `kind-substrate-preview` cluster. Phase 2 must confirm the golden snapshot and
-restored readiness before this addendum can report a live result.
+Phase 2 used pinned Substrate `cdac9baef81dd319b46086d695266e6161e9e592`, a fresh
+`kind-substrate-preview` cluster, agentgateway, and image
+`localhost:5001/live-agent-gate@sha256:a5ffadbede22382732067873c0239fa67a757f3bbde38b838942a5bfa20fbeda`.
+ActorTemplate `live-agent-gate-v1` (UID `b16cf365-0856-4623-87a9-479112767d46`) reached a READY
+golden snapshot; actor `claude-gate5` (UID `cbb4d70c-4ba9-4ce2-af08-20f866a11866`) reached
+RUNNING. The harness health probe timed out because it forwarded the router's HTTP port 80 while
+the documented non-default-port CONNECT listener is 8081. A direct `/healthz` through 8081
+returned 200. Suspend produced snapshot
+`gs://ate-snapshots/live-agent-gate/atespaces/live-agent-gate/actors/cbb4d70c-4ba9-4ce2-af08-20f866a11866/snapshots/fadab73f-5f8a-4346-8cfb-af67c85c893d`; resume returned the same actor UID to RUNNING, the health route returned 200, and logs showed gVisor's `restore -image-path` path. The marker/counter and process PID were not measured.
+
+The first setup pass exposed one additional pinned-CLI result shape: a valid zero-match worker
+query serializes as `{}`. The adapter now treats only an empty object as zero workers and still
+rejects non-empty objects missing `workers`. The 9 worker-discovery regression tests pass. The
+unchanged rerun did not create a duplicate actor, but waited for spare worker capacity before
+checking the persisted actor UID and failed because the single worker was already occupied by
+`claude-gate5`. The harness's own health check also used router Service port 80 instead of its
+CONNECT listener on 8081; direct `/healthz` through 8081 worked.
+
+Before any provider credential work, a separate tokenless Pod in `default` POSTed a harmless
+command through `atenet-router:8081` to `actor-upstream:8090/run` with the actor-routing header;
+the shim returned `200 OK`. The pinned router documentation says ingress treats request headers
+as unauthenticated input, while ingress authorization is future roadmap work. This is not
+closable with an existing Substrate actor-ingress primitive in this configuration. Under the
+earlier plan this meant fallback C; the 2026-09-23 owner decision supersedes that fallback with a
+Mainloop-owned NetworkPolicy, per-actor shim token, and closed-channel credential delivery. No
+provider credential Secret was created or read. Phase 4 was not attempted in that run.
+
+After deny-all egress was applied, operator `/run` calls did not complete; a pane read confirmed
+the baseline counter/marker command had not run. Therefore the suspend/resume result proves the
+actor and Herdr health path restored, but does not establish counter/marker persistence. The
+force-delete-worker/revert portion of the lane was not attempted without those markers.
+
+The reviewing session reports the full runtime suite passed 189/189 outside its restricted
+sandbox before the Phase 2 empty-result correction. After that correction, the focused
+Substrate, workspace-adapter, contract, and setup suites passed 91/91. The credential-free proof
+is partial live evidence; the native-session capability remains partial/fixture, and the full
+counter/marker and worker-loss checks are unproved.
+
+## CapabilityResult
+
+| Capability | State | Scope | Evidence and limit |
+| ---------------------------- | ------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `workspace_adapter_contract` | partial | fixture | `SubstrateControl` is exercised live; `workspace_bindings` orchestration is fixture-tested, not run against a live backend and Postgres. |
+| `substrate_actor_lifecycle` | partial | live | READY golden creation, actor RUNNING, health, suspend, and resume ran live; marker persistence was not proved, and the exact rerun stopped at the occupied single worker before identity reconciliation. |
+| `preview_hmr` | proved | live | Real Vite HMR socket and edits through the actor route; survives suspend/resume. |
+| `dev_service_postgres` | proved | live | Real Postgres query, narrow allow rule, denied destination, and reconnect after wake. |
+| `native_session_continuity` | partial | fixture | No provider session ran. A tokenless unrelated Pod reached `POST /run` through the router; no existing Substrate ingress authorization primitive closed this path. |
+| `failure_recovery` | partial | live | Actor CRASHED/revert mechanics were proved in the earlier live lane; backend restart with a durable `recorded` attempt is covered by a fake-backed contract test, not live Postgres delivery. |
+
+## Recommendation status at the Phase 2b checkpoint
+
+The earlier recommendation to defer native sessions is superseded by the owner's 2026-09-23
+decision. Current live evidence still does not prove native-session support. Phase 3 must close
+the router ingress boundary and deliver credentials through that channel; Phase 4 must then prove
+Claude and Codex continuity. The final adopt/defer recommendation remains open until those phases
+finish or stop on a named condition.
## Cleanup
-Each of the three cluster lanes in this spike (adapter/CRASHED, preview-gate, dev-service-gate)
-deleted its own test actors and target resources, then the `substrate-preview` cluster and its
-`kind-registry` (`hack/delete-kind-cluster.sh`), and pruned the locally built, unpushed-elsewhere
-Docker images. Final `kind get clusters` / `docker ps` showed only `mainloop-test` /
-`mainloop-test-control-plane` after every lane. Root disk stayed in the 19-27G-free range
-throughout (above the plan's 8G in-flight-trial abort threshold at all times); available RAM
-stayed above 8G. No Mainloop repository files outside this branch's own commits were changed.
+Phase 0 removed the approved `cred-server` relay resources and both named Secrets, then deleted
+the owner-confirmed failed-trial `kind` cluster and its `kind-registry`. The fresh Phase 2
+`kind-substrate-preview` cluster, its `kind-registry`, and the exact local image tag built by
+this run were deleted after evidence capture. Final inventory showed only `mainloop-test` and
+`mainloop-test-control-plane`; the run-specific image tag was absent and the unrelated `latest`
+tag was preserved. Root disk had 21 GiB available and RAM had 9.5 GiB available. `mainloop-test`
+was outside the cleanup scope. The owner handles rotation of the Claude and Codex credentials
+previously served by the relay.
From a1d26b5095f7285b277b606abf4b5b3603961e8c Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Wed, 23 Sep 2026 11:44:04 +0000
Subject: [PATCH 10/30] feat: add Phase 3 actor ingress boundary
Restrict router ingress to Mainloop control traffic and add bearer-token authentication to the exec shim. Add repeatable setup flags, router test tools, and sanitized shim tests. Correct the preview image readiness and proxy configuration, and update the spike documentation with the evidence and current stop state.
The preview and cross-actor checks passed. The hostname egress check still allowed an unlisted example.com request, so the actor policy was reset to deny-all and egress work is handed off. No provider credentials or live agent sessions were used.
---
backend/scripts/gate5_setup.py | 128 ++++++++++++-
backend/tests/runtime/test_gate5_setup.py | 104 ++++++++++
docs/spikes/substrate-workspace-adapter.md | 179 +++++++++++++-----
.../egress-tool/main.go | 20 +-
.../image/entrypoint.sh | 2 +-
.../k8s/preview-gate-template.yaml.tmpl | 12 +-
.../k8s/preview-proxy.yaml.tmpl | 6 +-
.../k8s/router-ingress-policy.yaml | 45 +++++
.../live-agent-image/entrypoint.sh | 8 +-
.../live-agent-image/exec-shim.js | 97 +++++++++-
.../tests/exec-shim.test.js | 130 +++++++++++++
.../tools/router-client.js | 110 +++++++++++
12 files changed, 767 insertions(+), 74 deletions(-)
create mode 100644 spikes/substrate-workspace-adapter/k8s/router-ingress-policy.yaml
create mode 100644 spikes/substrate-workspace-adapter/tests/exec-shim.test.js
create mode 100644 spikes/substrate-workspace-adapter/tools/router-client.js
diff --git a/backend/scripts/gate5_setup.py b/backend/scripts/gate5_setup.py
index be894ea..d1ac0d2 100644
--- a/backend/scripts/gate5_setup.py
+++ b/backend/scripts/gate5_setup.py
@@ -51,6 +51,7 @@
import json
import os
import re
+import secrets
import socket
import string
import subprocess # nosec B404 - drives trusted local kubectl/ko/egress-tool binaries, argv only
@@ -116,6 +117,13 @@ def parse_args() -> argparse.Namespace:
p.add_argument("--egress-tool", required=True)
egress = p.add_mutually_exclusive_group(required=True)
egress.add_argument("--egress-cidr", help="CIDR to allow")
+ egress.add_argument(
+ "--egress-hostname",
+ action="append",
+ dest="egress_hostnames",
+ metavar="HOSTNAME",
+ help="provider hostname to allow (repeatable)",
+ )
egress.add_argument("--egress-allow-all", action="store_true")
egress.add_argument("--egress-deny-all", action="store_true")
return p.parse_args()
@@ -130,9 +138,12 @@ def load_state(path: str) -> dict:
def save_state(path: str, state: dict) -> None:
tmp = f"{path}.tmp"
- with open(tmp, "w") as f:
+ fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_TRUNC, 0o600)
+ with os.fdopen(fd, "w") as f:
json.dump(state, f, indent=2)
+ os.chmod(tmp, 0o600)
os.replace(tmp, path)
+ os.chmod(path, 0o600)
def render_manifest(
@@ -329,6 +340,9 @@ def run_egress_tool(args: argparse.Namespace) -> None:
cmd.append("--deny-all")
elif args.egress_allow_all:
cmd.append("--allow-all")
+ elif args.egress_hostnames:
+ for hostname in args.egress_hostnames:
+ cmd.extend(["--hostname", hostname])
else:
cmd += ["--cidr", args.egress_cidr]
subprocess.run(
@@ -409,6 +423,114 @@ def actor_health_check(
connection.close()
+def actor_shim_request(
+ *,
+ port: int,
+ atespace: str,
+ actor_name: str,
+ method: str,
+ path: str,
+ token: str | None = None,
+ body: dict | None = None,
+ timeout_s: float = 5,
+) -> int | None:
+ """Send a request through the actor's CONNECT route without logging its body."""
+ connection = http.client.HTTPConnection("127.0.0.1", port, timeout=timeout_s)
+ connection.set_tunnel(
+ f"actor-upstream:{ACTOR_SHIM_PORT}",
+ headers={"ate-target-actor": f"{atespace}/{actor_name}"},
+ )
+ headers = {}
+ request_body = None
+ if body is not None:
+ request_body = json.dumps(body)
+ headers["Content-Type"] = "application/json"
+ if token is not None:
+ headers["Authorization"] = f"Bearer {token}"
+ try:
+ connection.request(method, path, body=request_body, headers=headers)
+ response = connection.getresponse()
+ response.read()
+ return response.status
+ except (OSError, http.client.HTTPException):
+ return None
+ finally:
+ connection.close()
+
+
+def ensure_shim_token(
+ *,
+ args: argparse.Namespace,
+ state: dict,
+ port: int,
+ requester=actor_shim_request,
+) -> None:
+ """Persist a per-actor token before installing it, then verify the auth boundary."""
+ token = state.get("shim_token")
+ if not token:
+ token = secrets.token_urlsafe(32)
+ state["shim_token"] = token
+ save_state(args.state_file, state)
+
+ status = requester(
+ port=port,
+ atespace=args.atespace,
+ actor_name=args.actor_name,
+ method="POST",
+ path="/token",
+ body={"token": token},
+ )
+ if status == 409:
+ already_installed = requester(
+ port=port,
+ atespace=args.atespace,
+ actor_name=args.actor_name,
+ method="GET",
+ path="/read",
+ token=token,
+ )
+ if already_installed != 200:
+ raise RuntimeError(
+ "the actor already has a shim token that does not match the private state; "
+ "manual reconciliation is required"
+ )
+ elif status != 201:
+ raise RuntimeError(
+ f"shim token installation failed (HTTP {status or 'no response'})"
+ )
+
+ checks = (
+ ("missing-token /read", "GET", "/read", None, None, 401),
+ ("wrong-token /read", "GET", "/read", f"{token}x", None, 401),
+ ("authenticated /read", "GET", "/read", token, None, 200),
+ (
+ "second /token",
+ "POST",
+ "/token",
+ None,
+ {"token": "one-time-install-probe"},
+ 409,
+ ),
+ ("open /healthz", "GET", "/healthz", None, None, 200),
+ )
+ for label, method, path, bearer, request_body, expected in checks:
+ observed = requester(
+ port=port,
+ atespace=args.atespace,
+ actor_name=args.actor_name,
+ method=method,
+ path=path,
+ token=bearer,
+ body=request_body,
+ )
+ if observed != expected:
+ raise RuntimeError(
+ f"shim token acceptance failed at {label} "
+ f"(HTTP {observed or 'no response'}, expected {expected})"
+ )
+ print("-- per-actor shim token installed; missing/wrong/correct and one-time checks passed")
+
+
async def ensure_golden_template(
control: SubstrateControl,
args: argparse.Namespace,
@@ -613,8 +735,10 @@ async def async_main(args: argparse.Namespace) -> None:
)
await ensure_actor(control, args, template_name, template_uid, state)
- print("-- confirming current control-service health through the actor route")
+ print("-- installing and checking the per-actor shim token through the actor route")
with actor_router_tunnel(args) as route_port:
+ ensure_shim_token(args=args, state=state, port=route_port)
+ print("-- confirming current control-service health through the actor route")
await wait_for_actor_health(
lambda: actor_health_check(
port=route_port,
diff --git a/backend/tests/runtime/test_gate5_setup.py b/backend/tests/runtime/test_gate5_setup.py
index 0623bc8..f02a663 100644
--- a/backend/tests/runtime/test_gate5_setup.py
+++ b/backend/tests/runtime/test_gate5_setup.py
@@ -1,6 +1,8 @@
"""Credential-free regressions for the gate-5 setup script's build and rerun identity."""
import json
+from contextlib import redirect_stdout
+from io import StringIO
import tempfile
import unittest
from dataclasses import replace
@@ -228,6 +230,38 @@ def wait(self, timeout):
self.assertIn("18081:8081", command)
self.assertTrue(process.terminated)
+ def test_egress_hostname_rules_are_passed_as_repeatable_flags(self):
+ args = SimpleNamespace(
+ egress_tool="/fixture/mainloop-egress-tool",
+ kubeconfig=FIXTURE_KUBECONFIG,
+ context="kind-substrate-preview",
+ atespace="live-agent-gate",
+ actor_name="claude-gate5",
+ egress_deny_all=False,
+ egress_allow_all=False,
+ egress_hostnames=["api.anthropic.com", "api.openai.com"],
+ )
+ with patch.object(gate5_setup.subprocess, "run") as run:
+ gate5_setup.run_egress_tool(args)
+ self.assertEqual(
+ run.call_args.args[0],
+ [
+ "/fixture/mainloop-egress-tool",
+ "--kubeconfig",
+ FIXTURE_KUBECONFIG,
+ "--context",
+ "kind-substrate-preview",
+ "--atespace",
+ "live-agent-gate",
+ "--actor",
+ "claude-gate5",
+ "--hostname",
+ "api.anthropic.com",
+ "--hostname",
+ "api.openai.com",
+ ],
+ )
+
class Gate5StateTests(unittest.TestCase):
def args(self, state_file: str, **overrides):
@@ -324,6 +358,76 @@ def state(self, path, *, actor_uid="actor-1"):
gate5_setup.save_state(path, state)
return state
+ def test_state_file_is_private_for_future_shim_token(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ gate5_setup.save_state(path, {"shim_token": "fixture-secret"})
+ self.assertEqual(Path(path).stat().st_mode & 0o777, 0o600)
+
+ def test_shim_token_is_persisted_before_body_only_install_and_verified(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ state = {"run_id": "run-token"}
+ args = SimpleNamespace(
+ state_file=path,
+ atespace="live-agent-gate",
+ actor_name="claude-gate5",
+ )
+ token = "fixture-shim-token-with-at-least-32-characters"
+ statuses = [201, 401, 401, 200, 409, 200]
+ calls = []
+
+ def requester(**kwargs):
+ calls.append(kwargs)
+ return statuses.pop(0)
+
+ output = StringIO()
+ with (
+ patch.object(gate5_setup.secrets, "token_urlsafe", return_value=token),
+ redirect_stdout(output),
+ ):
+ gate5_setup.ensure_shim_token(
+ args=args, state=state, port=18091, requester=requester
+ )
+
+ self.assertEqual(json.loads(Path(path).read_text())["shim_token"], token)
+ self.assertEqual(Path(path).stat().st_mode & 0o777, 0o600)
+ self.assertEqual(calls[0]["method"], "POST")
+ self.assertEqual(calls[0]["path"], "/token")
+ self.assertEqual(calls[0]["body"], {"token": token})
+ self.assertNotIn("token", calls[0])
+ self.assertEqual(calls[1]["token"], None)
+ self.assertEqual(calls[2]["token"], f"{token}x")
+ self.assertEqual(calls[3]["token"], token)
+ self.assertNotIn(token, output.getvalue())
+
+ def test_shim_token_rerun_verifies_existing_token_without_rotating_it(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ token = "existing-shim-token-with-at-least-32-characters"
+ state = {"run_id": "run-token", "shim_token": token}
+ gate5_setup.save_state(path, state)
+ args = SimpleNamespace(
+ state_file=path,
+ atespace="live-agent-gate",
+ actor_name="claude-gate5",
+ )
+ statuses = [409, 200, 401, 401, 200, 409, 200]
+ calls = []
+
+ def requester(**kwargs):
+ calls.append(kwargs)
+ return statuses.pop(0)
+
+ with patch.object(gate5_setup.secrets, "token_urlsafe") as generate:
+ gate5_setup.ensure_shim_token(
+ args=args, state=state, port=18091, requester=requester
+ )
+
+ generate.assert_not_called()
+ self.assertEqual(calls[1]["token"], token)
+ self.assertEqual(json.loads(Path(path).read_text())["shim_token"], token)
+
def actor(
self, *, uid="actor-1", template_uid="template-new", state=ActorState.RUNNING
):
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 427b3d3..7c2d255 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -24,8 +24,8 @@ drives its lifecycle through the real `kubectl ate` control-plane CLI.
| `dev-service-gate` WorkerPool + ActorTemplate: real `psql`, real external `postgres:16-alpine` StatefulSet, real `EgressPolicy` (CIDR rule, created via a small gRPC tool since `kubectl-ate` has no CLI verb for it) | Real |
| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
| File edits and shell commands run inside actors (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
-| `live-agent-gate` WorkerPool + ActorTemplate: real Claude/Codex CLIs, credential-free boot | Phase 2: golden snapshot READY; actor RUNNING; `/healthz` and one suspend/resume proved live. Counter/marker restore and worker-loss revert remain unproved. No native-agent session has been run. |
-| Claude/Codex agent processes, credentials | Not run in this spike (see "Limits") |
+| `live-agent-gate` WorkerPool + ActorTemplate: real Claude/Codex CLIs, credential-free boot | Golden snapshot READY and actor RUNNING proved live. A later credential-free preview actor hit the pinned gVisor restore error described below. No provider session ran. |
+| Claude/Codex agent processes, credentials | Not run; the 2026-09-23 live lane stopped before credential work (see "Limits") |
| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
## Why not a real agent for the preview-gate edit (credential-injection gap)
@@ -171,18 +171,23 @@ per `docs/network-egress.md`'s "CIDR/all policy: dial now" passthrough path) ins
**hostname rule** (HTTP/TLS-SNI-specific, and Postgres is neither), which the prior proof's HTTP
`fetch`-based trial did not have reason to distinguish.
-## Limits / not attempted in this run
-
-- **Native-session gate, live**: no Claude or Codex session was run. The earlier attempt was
- authorized; tool policy rejected particular actions after the boot-time credential fetch
- received HTTP 403. Phase 1 removed the unauthenticated relay and repaired the harness. Phase 2
- produced a READY golden snapshot, a RUNNING actor, a healthy `/healthz` through the documented
- CONNECT port, and one successful suspend/resume. Counter/marker persistence and worker-loss
- revert were not proved. Before any credential work, a tokenless caller reached `POST /run`
- through the router and executed a harmless command. No existing ingress authorization primitive
- closed this path under the earlier plan, so that credential attempt stopped and Gate 5 remains
- partial/fixture pending the rewritten Phase 3 and Phase 4. The owner approved that plan on
- 2026-09-23; no provider Secret was created or read in the earlier run.
+## Limits from the earlier Phase 2 checkpoint
+
+- **Native-session gate, live**: no Claude or Codex session ran in the earlier checkpoint. The
+ owner authorized the work; earlier tool-policy decisions rejected particular actions after
+ the old relay returned HTTP 403. The relay was removed. The first Phase 3 preview restore
+ error was later diagnosed as a dead app from the old image and missing readiness probe, then
+ corrected with a new image and template. The current Phase 3 run passed 3a–3c and stopped at
+ the hostname-egress bypass described below; no provider credential was delivered.
+- Preview/HMR under the router policy and actor-to-actor access were unverified at the earlier
+ checkpoint. In the current finish run, preview HMR passed, and an unrelated actor's CONNECT
+ to the shim was rejected at the actor egress gateway before reaching the router. The
+ per-actor shim token passed live checks, including suspend/resume persistence. Provider Secret
+ delivery and Phase 4 session continuity remain unproved; no provider Secret was created or
+ read.
+- Phase 2's counter/marker persistence and worker-loss revert were not proved in their original
+ run. The current token suspend/resume check did not cover worker loss or those markers; those
+ checks remain open.
- **`workspace_bindings` orchestration functions** (`ensure_workspace`, `resume_workspace`, ...)
were not exercised against a live Postgres + running backend; only their extracted pure logic
(`plan_ensure`, `_binding_from_row`, `is_crashed`) is unit tested, and the transport layer
@@ -194,9 +199,9 @@ The original live attempt (outside this doc's own commits) created the live-agen
infrastructure and hit a real failure during golden-snapshot creation, reviewed
in `.tasknotes/gate5-review-and-recovery-plan-2026-09-22.md`: the golden actor's entrypoint
fetched a credential from `cred-server` unconditionally at boot, that fetch was denied (`403`),
-and the golden actor exited before its snapshot was captured -- `runsc exit 128` on a later
-restore attempt is consistent with capturing a process that had already exited. The harness
-script driving that attempt (never committed; reviewed from a scratch copy) also applied an
+and the golden actor exited before its snapshot was captured. A later restore returned
+`runsc exit 128`; the original evidence did not establish that the denied fetch caused that
+restore error. The harness script driving that attempt (never committed; reviewed from a scratch copy) also applied an
unresolved `ko://` WorkerPool image, never registered the atespace at the control-plane API
(a Kubernetes Namespace of the same name is not an atespace), checked for an existing
ActorTemplate with the atespace embedded in the name rather than the CLI's required `-a` flag,
@@ -255,37 +260,115 @@ actor and Herdr health path restored, but does not establish counter/marker pers
force-delete-worker/revert portion of the lane was not attempted without those markers.
The reviewing session reports the full runtime suite passed 189/189 outside its restricted
-sandbox before the Phase 2 empty-result correction. After that correction, the focused
-Substrate, workspace-adapter, contract, and setup suites passed 91/91. The credential-free proof
-is partial live evidence; the native-session capability remains partial/fixture, and the full
-counter/marker and worker-loss checks are unproved.
+sandbox before the Phase 2 empty-result correction. The previous Phase 2 run's operator calls
+after deny-all egress did not complete. In the Phase 3 run, an in-cluster control namespace POST
+to `/run` returned 200 after deny-all egress was attached, so the ingress response path worked
+for that trusted request. The focused setup, Substrate, and contract suites now pass 85/85.
+The credential-free proof is partial live evidence; native-session continuity remains partial,
+and full counter/marker and worker-loss checks are unproved.
+
+### Phase 2b — rerun and router corrections
+
+The harness now forwards to the router CONNECT listener on Service port 8081, checks live actor
+identity before waiting for worker capacity, and handles the pinned CLI's `{}` zero-worker result.
+The focused 63-test setup/Substrate suite passed before the Phase 3 code; the latest setup,
+Substrate, and contract run passed 85/85. Phase 2b was committed locally as `3ca780a`.
+The in-sandbox `unittest discover` process IDs 2293648 and 2293673 were still alive but invisible
+to the sandbox check; the reviewing session killed them on 2026-09-23. The full suite result
+189/189 is attributed to that reviewing session.
+
+### Phase 3–4 finish run — router boundary, preview correction, and egress stop
+
+One fresh `kind-substrate-preview` cluster ran Kind v0.33.0, node image v1.37.0, enforcing
+kindnet `v20260820`, and pinned Substrate
+`cdac9baef81dd319b46086d695266e6161e9e592` with the agentgateway dataplane. Phase 3a passed:
+the client reached an HTTP probe before policy, was blocked by deny-all ingress, then reached it
+after a namespace-scoped allow.
+
+The tracked `k8s/router-ingress-policy.yaml` selects router Pods by `app=atenet-router`. It
+allows `mainloop-control` on router control ports, the preview proxy on HTTP only, and
+`otel-system` on stats port 15020. NetworkPolicy ports target Pod ports; Service port 80 maps
+to router container port 8080. A POST to `/run` through port 8081 returned 200 from
+`mainloop-control` and timed out from an unrelated Pod in `default`.
+
+The corrected `preview-gate-v2` template used the rebuilt image from
+`spikes/substrate-workspace-adapter/image/`, an explicit working directory, and `readyz`. The
+live Vite page passed HMR through the NGINX preview proxy: editing `main.js` changed the page
+heading in place, preserved the browser marker and time origin, and produced a successful HMR
+WebSocket upgrade. An unrelated preview actor's CONNECT attempt to
+`live-agent-gate/claude-gate5:8090` returned 405 at the actor egress gateway before reaching the
+router. The request was rejected, though that result is an egress-gateway denial rather than a
+router NetworkPolicy denial.
+
+The first immutable preview template remains abandoned. Its worker logged `npm error ENOENT`
+for `/package.json` before golden checkpoint. `savedMFOwners=[_pause:/]` records the surviving
+pause container after the application exited; the template lacked `readyz`, so the dead app was
+snapshotted. This is a harness/image/readiness error, not a gVisor restore blocker. Record it as
+a candidate upstream issue because Substrate checkpointed an exited container without a clear
+error. The corrected template name is `preview-gate-v2`.
+
+On `live-agent-gate/claude-gate5` (actor UID
+`6125c129-7312-453b-aea5-a2fae6745c62`), the live shim-token test passed: missing and wrong
+tokens returned 401, the correct token authorized `/run`, a second token install returned 409,
+and the token continued to authorize requests after suspend/resume with the same actor UID. A
+deny-all control check initially returned `/healthz` 200 and authenticated `/run` 200 with the
+command marker observed. After a later restore, `/healthz` intermittently timed out at the shim's
+serial Herdr status/pane-read check, while direct `herdr status server` and authenticated
+`/read` succeeded. The WorkerPool remained Ready; the failure was located at the shim health
+handler, not the router or NetworkPolicy.
+
+Provider discovery without credentials exited early (Claude rc 1, Codex rc 2) and emitted no
+hostnames. Since those were not CLI-measured destinations, the initial bounded check used
+provider/API/auth host candidates, informed by [Claude Code's network requirements](https://code.claude.com/docs/en/corporate-proxy),
+the [Codex ChatGPT sign-in flow](https://developers.openai.com/codex/auth), and the Codex file's
+`chatgpt` auth mode:
+`api.anthropic.com`, `platform.claude.com`, `api.openai.com`, `auth.openai.com`, and
+`chatgpt.com`. The live Substrate API confirmed exactly one hostname rule with those entries;
+its policy cache is 10 seconds. HTTPS to those hosts reached their public services, but an
+unlisted HTTPS request to `example.com` also returned 200. This fails Phase 3d's required
+unlisted-host 403 and shows hostname-only egress is not enforced on this agentgateway path.
+The actor was restored to a zero-rule deny-all EgressPolicy immediately afterward. No provider
+Secret was created or read, no credential was delivered, and no Claude or Codex session was
+started. Phase 4 and all snapshot-revert tests were not attempted.
+
+The restore diagnosis is clarified in the Phase 3 stop-condition section of the finish plan.
+The Codex sandbox could not see the stalled `unittest discover` processes 2293648 and 2293673;
+the reviewing session killed both on 2026-09-23. The full runtime result 189/189 is attributed
+to that reviewing session, not to a sandbox process killed by this agent.
## CapabilityResult
-| Capability | State | Scope | Evidence and limit |
-| ---------------------------- | ------- | ------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| `workspace_adapter_contract` | partial | fixture | `SubstrateControl` is exercised live; `workspace_bindings` orchestration is fixture-tested, not run against a live backend and Postgres. |
-| `substrate_actor_lifecycle` | partial | live | READY golden creation, actor RUNNING, health, suspend, and resume ran live; marker persistence was not proved, and the exact rerun stopped at the occupied single worker before identity reconciliation. |
-| `preview_hmr` | proved | live | Real Vite HMR socket and edits through the actor route; survives suspend/resume. |
-| `dev_service_postgres` | proved | live | Real Postgres query, narrow allow rule, denied destination, and reconnect after wake. |
-| `native_session_continuity` | partial | fixture | No provider session ran. A tokenless unrelated Pod reached `POST /run` through the router; no existing Substrate ingress authorization primitive closed this path. |
-| `failure_recovery` | partial | live | Actor CRASHED/revert mechanics were proved in the earlier live lane; backend restart with a durable `recorded` attempt is covered by a fake-backed contract test, not live Postgres delivery. |
-
-## Recommendation status at the Phase 2b checkpoint
-
-The earlier recommendation to defer native sessions is superseded by the owner's 2026-09-23
-decision. Current live evidence still does not prove native-session support. Phase 3 must close
-the router ingress boundary and deliver credentials through that channel; Phase 4 must then prove
-Claude and Codex continuity. The final adopt/defer recommendation remains open until those phases
-finish or stop on a named condition.
-
-## Cleanup
-
-Phase 0 removed the approved `cred-server` relay resources and both named Secrets, then deleted
-the owner-confirmed failed-trial `kind` cluster and its `kind-registry`. The fresh Phase 2
-`kind-substrate-preview` cluster, its `kind-registry`, and the exact local image tag built by
-this run were deleted after evidence capture. Final inventory showed only `mainloop-test` and
-`mainloop-test-control-plane`; the run-specific image tag was absent and the unrelated `latest`
-tag was preserved. Root disk had 21 GiB available and RAM had 9.5 GiB available. `mainloop-test`
-was outside the cleanup scope. The owner handles rotation of the Claude and Codex credentials
-previously served by the relay.
+| Capability | State | Scope | Evidence and limit |
+| ----------------------------- | ------- | ---------- | ------------------ |
+| `workspace_adapter_contract` | partial | fixture | `SubstrateControl` is exercised live; `workspace_bindings` orchestration is fixture-tested, not run against a live backend and Postgres. |
+| `substrate_actor_lifecycle` | partial | live | READY golden creation and actor RUNNING proved live; full marker and worker-loss recovery remain unproved. |
+| `preview_hmr` | proved | live | Real Vite HMR edit and WebSocket upgrade succeeded through the preview proxy. |
+| `dev_service_postgres` | proved | live | Real Postgres query, narrow allow rule, denied destination, and reconnect after wake. |
+| `phase3a_networkpolicy` | proved | live | Kindnet blocked and then allowed the in-cluster HTTP probe according to NetworkPolicy. |
+| `router_ingress_boundary` | proved | live | `default` was blocked; `mainloop-control` was admitted; unrelated actor CONNECT to the shim was rejected at actor egress; preview traffic passed. |
+| `shim_token_auth` | proved | live | 401/409 behavior, successful bearer use, and token persistence across suspend/resume passed on the final actor. |
+| `provider_egress` | failed | live | Exact hostname policy was confirmed, but HTTPS to unlisted `example.com` returned 200 through agentgateway. Actor returned to deny-all. |
+| `credential_delivery` | unknown | unverified | No provider Secret was created or read; credentials were not delivered. |
+| `native_session_continuity` | partial | unverified | Phase 3d failed before credential delivery; no native session was run. |
+| `failure_recovery` | partial | live | Earlier CRASHED/revert mechanics were proved; worker-loss recovery and the backend-restart `recorded` case remain unproved live. |
+
+## Recommendation
+
+The corrected preview image resolves the earlier `npm ENOENT`/dead-checkpoint harness error;
+that result does not justify deferring native-session adoption. The current agentgateway run
+found a separate security blocker: a hostname-only EgressPolicy allowed HTTPS to an unlisted
+host. Do not deliver provider credentials or promote native sessions to production until
+hostname enforcement is verified on a supported dataplane or another host-aware egress boundary
+is added. Production also needs a GitOps-managed router NetworkPolicy on an enforcing CNI,
+shim-token issuance by the real Mainloop backend, and snapshot-bucket access controls. The
+credential-bearing snapshot trade-off remains open because no credential entered an actor.
+
+## Cleanup and current cluster state
+
+Phase 0 removed the approved relay resources and both named Secrets, then deleted the
+owner-confirmed failed-trial cluster. Per the owner's 2026-09-23 instruction, cleanup of the
+fresh `kind-substrate-preview` cluster and its registry is stopped; both remain up for review.
+The actor's EgressPolicy is zero-rule deny-all. `mainloop-test` remains outside scope. No
+provider Secret or credential-bearing snapshot was created. Current Kind/Docker inventory and
+disk headroom are recorded in the task proof note. The owner handles rotation of credentials
+previously served by the removed relay.
diff --git a/spikes/substrate-workspace-adapter/egress-tool/main.go b/spikes/substrate-workspace-adapter/egress-tool/main.go
index 57d51ac..c3884b3 100644
--- a/spikes/substrate-workspace-adapter/egress-tool/main.go
+++ b/spikes/substrate-workspace-adapter/egress-tool/main.go
@@ -12,9 +12,9 @@
// cdac9baef81dd319b46086d695266e6161e9e592 when this was written).
//
// Usage: mainloop-egress-tool --kubeconfig --context --atespace --actor
-// --deny-all | --cidr | --allow-all
+// --deny-all | --cidr | --hostname ... | --allow-all
//
-// Fails closed: exactly one of --deny-all, --cidr, or --allow-all is required. An earlier version of this
+// Fails closed: exactly one of --deny-all, --cidr, --hostname, or --allow-all is required. An earlier version of this
// tool silently allowed all destinations whenever --cidr was omitted (see
// .tasknotes/gate5-review-and-recovery-plan-2026-09-22.md, "Make missing egress configuration
// fail closed"); --allow-all must now be passed explicitly to get that behavior.
@@ -37,11 +37,14 @@ import (
// validateEgressInput is the fail-closed check, isolated as a pure function so it can be
// exercised without a cluster or a Substrate checkout.
-func validateEgressInput(cidr string, denyAll, allowAll bool) error {
+func validateEgressInput(cidr string, hostnames []string, denyAll, allowAll bool) error {
selected := 0
if cidr != "" {
selected++
}
+ if len(hostnames) > 0 {
+ selected++
+ }
if denyAll {
selected++
}
@@ -49,7 +52,7 @@ func validateEgressInput(cidr string, denyAll, allowAll bool) error {
selected++
}
if selected != 1 {
- return fmt.Errorf("exactly one of --deny-all, --cidr , or --allow-all is required")
+ return fmt.Errorf("exactly one of --deny-all, --cidr , --hostname , or --allow-all is required")
}
return nil
}
@@ -60,11 +63,16 @@ func main() {
atespace := flag.String("atespace", "", "")
actorName := flag.String("actor", "", "")
cidr := flag.String("cidr", "", "CIDR to allow")
+ var hostnames []string
+ flag.Func("hostname", "exact hostname to allow (repeatable)", func(value string) error {
+ hostnames = append(hostnames, value)
+ return nil
+ })
denyAll := flag.Bool("deny-all", false, "explicitly deny all actor egress")
allowAll := flag.Bool("allow-all", false, "explicitly allow all egress destinations")
flag.Parse()
- if err := validateEgressInput(*cidr, *denyAll, *allowAll); err != nil {
+ if err := validateEgressInput(*cidr, hostnames, *denyAll, *allowAll); err != nil {
log.Fatalf("%v", err)
}
@@ -82,6 +90,8 @@ func main() {
rules = []*ateapipb.EgressRule{{All: &emptypb.Empty{}}}
} else if *cidr != "" {
rules = []*ateapipb.EgressRule{{Cidrs: &ateapipb.CIDRRule{Cidrs: []string{*cidr}}}}
+ } else if len(hostnames) > 0 {
+ rules = []*ateapipb.EgressRule{{Hostnames: &ateapipb.HostnameRule{Patterns: hostnames}}}
}
policy := &ateapipb.EgressPolicy{
Metadata: &ateapipb.ResourceMetadata{Atespace: *atespace, Name: "default"},
diff --git a/spikes/substrate-workspace-adapter/image/entrypoint.sh b/spikes/substrate-workspace-adapter/image/entrypoint.sh
index cf6385e..d3d6be9 100644
--- a/spikes/substrate-workspace-adapter/image/entrypoint.sh
+++ b/spikes/substrate-workspace-adapter/image/entrypoint.sh
@@ -25,7 +25,7 @@ shell_ws=$(herdr --session "${HERDR_SESSION}" workspace create --label shell --c
shell_pane=$(echo "${shell_ws}" | jq -r '.result.root_pane.pane_id')
echo "${shell_pane}" >"${STATE_DIR}/shell-pane-id"
-herdr --session "${HERDR_SESSION}" pane run "${dev_pane}" "npm run dev"
+herdr --session "${HERDR_SESSION}" pane run "${dev_pane}" "cd '${VITE_DIR}' && npm run dev -- --host 0.0.0.0"
EXEC_SHIM_PANE_ID="${shell_pane}" HERDR_SESSION="${HERDR_SESSION}" node "${EXEC_SHIM}" &
diff --git a/spikes/substrate-workspace-adapter/k8s/preview-gate-template.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/preview-gate-template.yaml.tmpl
index 0da7943..68c9ae2 100644
--- a/spikes/substrate-workspace-adapter/k8s/preview-gate-template.yaml.tmpl
+++ b/spikes/substrate-workspace-adapter/k8s/preview-gate-template.yaml.tmpl
@@ -11,10 +11,10 @@ metadata:
apiVersion: ate.dev/v1alpha1
kind: WorkerPool
metadata:
- name: preview-gate
+ name: ${TEMPLATE_NAME}
namespace: ${ATESPACE}
labels:
- workload: preview-gate
+ workload: ${TEMPLATE_NAME}
spec:
replicas: 1
workerImage: ko://github.com/agent-substrate/substrate/cmd/ateom-gvisor
@@ -29,10 +29,11 @@ spec:
---
metadata:
atespace: ${ATESPACE}
- name: preview-gate
+ # ActorTemplates are immutable; use a new name after the old image failed warmup.
+ name: ${TEMPLATE_NAME}
workerSelector:
matchLabels:
- workload: preview-gate
+ workload: ${TEMPLATE_NAME}
containers:
- name: preview
image: __IMAGE__
@@ -43,6 +44,9 @@ containers:
limits:
- { name: cpu, quantity: "1" }
- { name: memory, quantity: 1Gi }
+ readyz:
+ httpGet: { path: /, port: 80 }
+ timeoutSeconds: 60
snapshotsConfig:
onPause: SNAPSHOT_CONTENT_SCOPE_FULL
onCommit: SNAPSHOT_CONTENT_SCOPE_FULL
diff --git a/spikes/substrate-workspace-adapter/k8s/preview-proxy.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/preview-proxy.yaml.tmpl
index 24cc6d0..9fca364 100644
--- a/spikes/substrate-workspace-adapter/k8s/preview-proxy.yaml.tmpl
+++ b/spikes/substrate-workspace-adapter/k8s/preview-proxy.yaml.tmpl
@@ -19,6 +19,10 @@ data:
nginx.conf: |
events {}
http {
+ map $http_upgrade $connection_upgrade {
+ default upgrade;
+ '' close;
+ }
server {
listen 80;
location / {
@@ -29,7 +33,7 @@ data:
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Upgrade $http_upgrade;
- proxy_set_header Connection "upgrade";
+ proxy_set_header Connection $connection_upgrade;
}
}
}
diff --git a/spikes/substrate-workspace-adapter/k8s/router-ingress-policy.yaml b/spikes/substrate-workspace-adapter/k8s/router-ingress-policy.yaml
new file mode 100644
index 0000000..01901a3
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/k8s/router-ingress-policy.yaml
@@ -0,0 +1,45 @@
+# Mainloop-owned ingress boundary for the Substrate router. NetworkPolicy ports
+# are pod ports: Service port 80 maps to the router's HTTP listener on 8080.
+# The agentgateway router forwards requests to api.ate-system.svc:443 itself;
+# actor/provider egress goes through atenet-egress, so neither ate-system worker
+# pods nor the egress gateway are router ingress clients in this install.
+apiVersion: networking.k8s.io/v1
+kind: NetworkPolicy
+metadata:
+ name: atenet-router-mainloop-ingress
+ namespace: ate-system
+spec:
+ podSelector:
+ matchLabels:
+ app: atenet-router
+ policyTypes:
+ - Ingress
+ ingress:
+ # Trusted Mainloop backend. It uses HTTP plus the Substrate CONNECT listener
+ # for actor control and arbitrary-port workspace traffic.
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ mainloop.dev/role: control
+ ports:
+ - { protocol: TCP, port: 8080 }
+ - { protocol: TCP, port: 8081 }
+ - { protocol: TCP, port: 8443 }
+ - { protocol: TCP, port: 8444 }
+ # The fixed NGINX preview proxy injects ate-target-actor and only needs HTTP.
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ mainloop.dev/role: workspace
+ podSelector:
+ matchLabels:
+ app: preview-proxy
+ ports:
+ - { protocol: TCP, port: 8080 }
+ # The installer annotates the router for Prometheus scraping on 15020.
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ kubernetes.io/metadata.name: otel-system
+ ports:
+ - { protocol: TCP, port: 15020 }
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
index 907aba7..80ee2f0 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
+++ b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
@@ -6,11 +6,9 @@
# "Credential-injection gap").
#
# Credential-free by construction: this entrypoint never fetches a credential and never
-# requires network access to reach a running state. The template controller uses the
-# ActorTemplate's `/healthz` readiness check before accepting the golden actor. A boot path
-# that depends on a credential fetch succeeding can fail golden creation when its relay is
-# denied or unreachable, which is what happened. Credential delivery is deferred to a
-# reviewed boundary and is never performed during golden-actor warmup or from this entrypoint.
+# requires network access to reach a running state. Mainloop installs a per-actor shim token
+# and provider credentials only after the final actor is RUNNING. The golden actor stays clean.
+# The template controller checks `/healthz` before accepting the golden actor.
set -eu
mkdir -p "${HOME}" "${HOME}/.claude" "${CODEX_HOME}"
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
index 8755b95..7a308da 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
+++ b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
@@ -12,6 +12,9 @@
'use strict';
const http = require('node:http');
const { execFile } = require('node:child_process');
+const crypto = require('node:crypto');
+const fs = require('node:fs');
+const path = require('node:path');
const PANE_ID = process.env.EXEC_SHIM_PANE_ID;
const SESSION = process.env.HERDR_SESSION;
@@ -20,6 +23,56 @@ if (!PANE_ID || !SESSION) {
process.exit(1);
}
+const tokenPath = path.join(process.env.HOME || '/home/agent', '.mainloop', 'exec-shim-token');
+let bearerToken = null;
+try {
+ bearerToken = fs.readFileSync(tokenPath, 'utf8');
+} catch (err) {
+ if (err.code !== 'ENOENT') throw err;
+}
+
+function authorized(req) {
+ if (bearerToken === null) return true;
+ const header = req.headers.authorization;
+ if (typeof header !== 'string' || !header.startsWith('Bearer ')) return false;
+ const provided = Buffer.from(header.slice('Bearer '.length));
+ const expected = Buffer.from(bearerToken);
+ return provided.length === expected.length && crypto.timingSafeEqual(provided, expected);
+}
+
+function unauthorized(res) {
+ res.writeHead(401, { 'content-type': 'text/plain' }).end('unauthorized');
+}
+
+function requestBody(req, onBody) {
+ let body = '';
+ req.on('data', (chunk) => {
+ body += chunk;
+ if (body.length > 65536) req.destroy();
+ });
+ req.on('end', () => onBody(body));
+}
+
+function installToken(token) {
+ if (bearerToken !== null) return false;
+ if (typeof token !== 'string' || token.length < 32 || token.length > 4096) return null;
+ const directory = path.dirname(tokenPath);
+ fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+ fs.chmodSync(directory, 0o700);
+ const fd = fs.openSync(tokenPath, 'wx', 0o600);
+ try {
+ fs.writeFileSync(fd, token, 'utf8');
+ fs.fsyncSync(fd);
+ } catch (err) {
+ fs.closeSync(fd);
+ fs.rmSync(tokenPath, { force: true });
+ throw err;
+ }
+ fs.closeSync(fd);
+ bearerToken = token;
+ return true;
+}
+
function herdr(args, res) {
execFile('herdr', ['--session', SESSION, ...args], (err, stdout, stderr) => {
if (err) {
@@ -55,7 +108,38 @@ const server = http.createServer((req, res) => {
healthz(res);
return;
}
+ if (req.method === 'POST' && req.url === '/token') {
+ if (bearerToken !== null) {
+ res.writeHead(409).end('token already set');
+ return;
+ }
+ requestBody(req, (body) => {
+ let token;
+ try {
+ token = JSON.parse(body).token;
+ } catch {
+ res.writeHead(400).end('invalid json');
+ return;
+ }
+ try {
+ const installed = installToken(token);
+ if (installed === null) {
+ res.writeHead(400).end('invalid token');
+ return;
+ }
+ if (!installed) {
+ res.writeHead(409).end('token already set');
+ return;
+ }
+ res.writeHead(201).end('token set');
+ } catch {
+ res.writeHead(500).end('token could not be stored');
+ }
+ });
+ return;
+ }
if (req.method === 'GET' && req.url === '/read') {
+ if (!authorized(req)) return unauthorized(res);
herdr(['pane', 'read', PANE_ID], res);
return;
}
@@ -63,12 +147,8 @@ const server = http.createServer((req, res) => {
res.writeHead(404).end();
return;
}
- let body = '';
- req.on('data', (chunk) => {
- body += chunk;
- if (body.length > 65536) req.destroy();
- });
- req.on('end', () => {
+ if (!authorized(req)) return unauthorized(res);
+ requestBody(req, (body) => {
let command;
try {
command = JSON.parse(body).command;
@@ -84,6 +164,7 @@ const server = http.createServer((req, res) => {
});
});
-server.listen(8090, '0.0.0.0', () => {
- console.log('exec-shim listening on :8090, pane', PANE_ID);
+server.listen(Number(process.env.EXEC_SHIM_PORT || 8090), '0.0.0.0', () => {
+ const address = server.address();
+ console.log(`exec-shim listening on :${address.port}`);
});
diff --git a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
new file mode 100644
index 0000000..2428509
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
@@ -0,0 +1,130 @@
+'use strict';
+
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import http from 'node:http';
+import os from 'node:os';
+import path from 'node:path';
+import { spawn } from 'node:child_process';
+import { once } from 'node:events';
+import { test } from 'node:test';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const shim = path.resolve(__dirname, '../live-agent-image/exec-shim.js');
+
+async function startShim(home, fakeBin, shimPath) {
+ const child = spawn(process.execPath, [shimPath], {
+ env: {
+ ...process.env,
+ HOME: home,
+ PATH: `${fakeBin}:${process.env.PATH}`,
+ HERDR_SESSION: 'shim-test',
+ EXEC_SHIM_PANE_ID: 'pane-test',
+ EXEC_SHIM_PORT: '0',
+ },
+ stdio: ['ignore', 'pipe', 'pipe'],
+ });
+ let output = '';
+ const port = await new Promise((resolve, reject) => {
+ const timer = setTimeout(() => reject(new Error(`shim did not start: ${output}`)), 5000);
+ child.once('error', reject);
+ child.once('exit', (code) => reject(new Error(`shim exited ${code}: ${output}`)));
+ child.stdout.on('data', (chunk) => {
+ output += chunk;
+ const match = output.match(/exec-shim listening on :(\d+)/);
+ if (match) {
+ clearTimeout(timer);
+ resolve(Number(match[1]));
+ }
+ });
+ child.stderr.on('data', (chunk) => {
+ output += chunk;
+ });
+ });
+ return { child, port, output: () => output };
+}
+
+function request(port, method, route, { body, token } = {}) {
+ return new Promise((resolve, reject) => {
+ const headers = {};
+ if (body !== undefined) headers['content-type'] = 'application/json';
+ if (token !== undefined) headers.authorization = `Bearer ${token}`;
+ const req = http.request(
+ { host: '127.0.0.1', port, method, path: route, headers },
+ (res) => {
+ const chunks = [];
+ res.on('data', (chunk) => chunks.push(chunk));
+ res.on('end', () =>
+ resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString('utf8') }),
+ );
+ },
+ );
+ req.once('error', reject);
+ if (body !== undefined) req.end(JSON.stringify(body));
+ else req.end();
+ });
+}
+
+async function stop(child) {
+ if (child.exitCode !== null) return;
+ child.kill('SIGTERM');
+ await once(child, 'exit');
+}
+
+test('shim token gates run/read, is one-time, private, and survives process restart', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-test-'));
+ const home = path.join(root, 'home');
+ const fakeBin = path.join(root, 'bin');
+ const shimPath = path.join(root, 'exec-shim.js');
+ fs.mkdirSync(home);
+ fs.mkdirSync(fakeBin);
+ fs.copyFileSync(shim, shimPath);
+ const herdr = path.join(fakeBin, 'herdr');
+ fs.writeFileSync(
+ herdr,
+ '#!/bin/sh\nif [ "$3" = "status" ] && [ "$4" = "server" ]; then echo "status: running"; exit 0; fi\nif [ "$3" = "pane" ] && [ "$4" = "read" ]; then echo "fixture pane"; exit 0; fi\nexit 0\n',
+ { mode: 0o700 },
+ );
+
+ let running = await startShim(home, fakeBin, shimPath);
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+
+ assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
+ assert.equal((await request(running.port, 'GET', '/read')).status, 200);
+ assert.equal(
+ (await request(running.port, 'POST', '/run', { body: { command: 'echo fixture' } })).status,
+ 200,
+ );
+
+ const token = 'fixture-only-token-with-at-least-thirty-two-characters';
+ assert.equal(
+ (await request(running.port, 'POST', '/token', { body: { token } })).status,
+ 201,
+ );
+ const tokenPath = path.join(home, '.mainloop', 'exec-shim-token');
+ assert.equal(fs.statSync(tokenPath).mode & 0o777, 0o600);
+ assert.equal(fs.statSync(path.dirname(tokenPath)).mode & 0o777, 0o700);
+ assert.equal(fs.readFileSync(tokenPath, 'utf8'), token);
+ assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
+ assert.equal((await request(running.port, 'GET', '/read')).status, 401);
+ assert.equal((await request(running.port, 'GET', '/read', { token: `${token}-wrong` })).status, 401);
+ assert.equal((await request(running.port, 'GET', '/read', { token })).status, 200);
+ assert.equal((await request(running.port, 'POST', '/run', { body: { command: 'echo fixture' } })).status, 401);
+ assert.equal(
+ (await request(running.port, 'POST', '/run', { body: { command: 'echo fixture' }, token })).status,
+ 200,
+ );
+ assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 409);
+ assert.equal(running.output().includes(token), false);
+
+ await stop(running.child);
+ running = await startShim(home, fakeBin, shimPath);
+ assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
+ assert.equal((await request(running.port, 'GET', '/read')).status, 401);
+ assert.equal((await request(running.port, 'GET', '/read', { token })).status, 200);
+ assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 409);
+});
diff --git a/spikes/substrate-workspace-adapter/tools/router-client.js b/spikes/substrate-workspace-adapter/tools/router-client.js
new file mode 100644
index 0000000..4555f11
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tools/router-client.js
@@ -0,0 +1,110 @@
+// Small control-plane probe for the Substrate CONNECT router. Request data, including any
+// bearer token or credential body, is read from stdin so it never appears in argv or logs.
+'use strict';
+
+const http = require('node:http');
+
+let input = '';
+process.stdin.setEncoding('utf8');
+process.stdin.on('data', (chunk) => {
+ input += chunk;
+ if (input.length > 1_200_000) process.stdin.destroy();
+});
+process.stdin.on('end', () => {
+ let request;
+ try {
+ request = JSON.parse(input);
+ } catch {
+ process.stdout.write('{"error":"invalid request"}\n');
+ process.exitCode = 2;
+ return;
+ }
+
+ const targetPort = Number(request.targetPort || 8090);
+ if (
+ !/^[a-z0-9-]+$/.test(request.atespace || '') ||
+ !/^[a-z0-9-]+$/.test(request.actor || '') ||
+ !Number.isInteger(targetPort) || targetPort < 1 || targetPort > 65535 ||
+ !['GET', 'POST'].includes(request.method) ||
+ typeof request.path !== 'string' || !request.path.startsWith('/')
+ ) {
+ process.stdout.write('{"error":"invalid request"}\n');
+ process.exitCode = 2;
+ return;
+ }
+
+ let completed = false;
+ const finish = (result) => {
+ if (completed) return;
+ completed = true;
+ process.stdout.write(`${JSON.stringify(result)}\n`);
+ };
+
+ const tunnel = http.request({
+ hostname: 'atenet-router.ate-system.svc.cluster.local',
+ port: 8081,
+ method: 'CONNECT',
+ path: `actor-upstream:${targetPort}`,
+ headers: {
+ host: `actor-upstream:${targetPort}`,
+ 'ate-target-actor': `${request.atespace}/${request.actor}`,
+ },
+ timeout: 8000,
+ });
+ tunnel.on('connect', (response, socket) => {
+ if (response.statusCode !== 200) {
+ socket.destroy();
+ finish({ connectStatus: response.statusCode });
+ return;
+ }
+
+ const headers = { connection: 'close', host: `actor-upstream:${targetPort}` };
+ let body = Buffer.alloc(0);
+ if (request.body !== undefined) {
+ body = Buffer.from(JSON.stringify(request.body));
+ headers['content-type'] = 'application/json';
+ headers['content-length'] = String(body.length);
+ } else if (typeof request.rawBody === 'string') {
+ body = Buffer.from(request.rawBody);
+ headers['content-type'] = 'application/octet-stream';
+ headers['content-length'] = String(body.length);
+ }
+ if (typeof request.bearerToken === 'string') {
+ headers.authorization = `Bearer ${request.bearerToken}`;
+ }
+ const headerLines = Object.entries(headers).map(([name, value]) => `${name}: ${value}`);
+ const requestHeader = Buffer.from(
+ `${request.method} ${request.path} HTTP/1.1\r\n${headerLines.join('\r\n')}\r\n\r\n`,
+ );
+ const responseChunks = [];
+ if (response.head && response.head.length) responseChunks.push(response.head);
+ socket.setTimeout(15000, () => socket.destroy(new Error('response timeout')));
+ socket.on('data', (chunk) => responseChunks.push(chunk));
+ socket.on('end', () => {
+ const responseBytes = Buffer.concat(responseChunks);
+ const headerEnd = responseBytes.indexOf('\r\n\r\n');
+ if (headerEnd < 0) {
+ finish({ transportError: 'invalid-http-response', receivedBytes: responseBytes.length });
+ return;
+ }
+ const statusLine = responseBytes.subarray(0, headerEnd).toString('latin1').split('\r\n')[0];
+ const status = Number(statusLine.split(' ')[1]);
+ const responseBody = responseBytes.subarray(headerEnd + 4).toString('utf8');
+ finish(request.includeResponseBody
+ ? { status, body: responseBody }
+ : { status });
+ });
+ socket.on('error', (err) => finish({ transportError: err.code || 'request-failed' }));
+ socket.write(Buffer.concat([requestHeader, body]));
+ });
+ tunnel.on('response', (response) => {
+ response.resume();
+ finish({ connectStatus: response.statusCode });
+ });
+ tunnel.on('timeout', () => {
+ tunnel.destroy();
+ finish({ transportError: 'connect-timeout' });
+ });
+ tunnel.on('error', (err) => finish({ transportError: err.code || 'connect-failed' }));
+ tunnel.end();
+});
From 5536f6603bf3f5508c5995a3d1c6162fd16fe0da Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Wed, 23 Sep 2026 14:15:33 +0000
Subject: [PATCH 11/30] fix: preflight actor images and bound health probes
Verify the digest-addressed actor image through the active run registry before creating an immutable ActorTemplate. Add a fake-backed regression for the registry HEAD request and refusal cases.
Bound exec-shim health commands, share concurrent probes, and cache a recent healthy result so restored actors can satisfy readyz cheaply.
---
backend/scripts/gate5_setup.py | 51 ++++++++++++++
backend/tests/runtime/test_gate5_setup.py | 56 +++++++++++++++
.../live-agent-image/exec-shim.js | 69 +++++++++++++++----
.../tests/exec-shim.test.js | 46 ++++++++++++-
4 files changed, 207 insertions(+), 15 deletions(-)
diff --git a/backend/scripts/gate5_setup.py b/backend/scripts/gate5_setup.py
index d1ac0d2..0687384 100644
--- a/backend/scripts/gate5_setup.py
+++ b/backend/scripts/gate5_setup.py
@@ -58,6 +58,8 @@
import sys
import tempfile
import time
+import urllib.error
+import urllib.request
import uuid
from pathlib import Path
@@ -193,6 +195,54 @@ def verify_substrate_source(source: str, *, runner=subprocess.run) -> str:
return str(root)
+IMAGE_MANIFEST_ACCEPT = ", ".join(
+ (
+ "application/vnd.oci.image.index.v1+json",
+ "application/vnd.docker.distribution.manifest.v2+json",
+ "application/vnd.oci.image.manifest.v1+json",
+ )
+)
+
+
+def verify_image_manifest(image: str, *, opener=urllib.request.urlopen) -> None:
+ """Verify the digest-addressed image is present in the registry workers will use.
+
+ Local Docker RepoDigests can refer to a registry that has since been deleted. A HEAD
+ against the registry endpoint catches that setup error before creating an immutable
+ ActorTemplate and its golden actor.
+ """
+ reference, separator, digest = image.partition("@")
+ registry, slash, repository = reference.partition("/")
+ if (
+ not separator
+ or not slash
+ or not registry
+ or not repository
+ or not re.fullmatch(r"sha256:[0-9a-fA-F]{64}", digest)
+ ):
+ raise RuntimeError("--image must be a registry/repository pinned by a full sha256 digest")
+
+ request = urllib.request.Request(
+ f"http://{registry}/v2/{repository}/manifests/{digest}",
+ headers={"Accept": IMAGE_MANIFEST_ACCEPT},
+ method="HEAD",
+ )
+ try:
+ with opener(request, timeout=10) as response:
+ status = response.status
+ except urllib.error.HTTPError as exc:
+ raise RuntimeError(
+ f"image manifest preflight failed: registry returned HTTP {exc.code}"
+ ) from exc
+ except (urllib.error.URLError, OSError) as exc:
+ raise RuntimeError(f"image manifest preflight failed: {exc}") from exc
+ if not 200 <= status < 300:
+ raise RuntimeError(
+ f"image manifest preflight failed: registry returned HTTP {status}"
+ )
+ print(f"-- registry manifest confirmed for {repository}@{digest}")
+
+
def get_cluster_identity(
*, context: str, kubeconfig: str, runner=subprocess.run
) -> dict[str, str]:
@@ -702,6 +752,7 @@ async def ensure_actor(
async def async_main(args: argparse.Namespace) -> None:
+ verify_image_manifest(args.image)
substrate_src = verify_substrate_source(args.substrate_src)
cluster = get_cluster_identity(context=args.context, kubeconfig=args.kubeconfig)
state = prepare_run_state(args, cluster)
diff --git a/backend/tests/runtime/test_gate5_setup.py b/backend/tests/runtime/test_gate5_setup.py
index f02a663..6da8493 100644
--- a/backend/tests/runtime/test_gate5_setup.py
+++ b/backend/tests/runtime/test_gate5_setup.py
@@ -9,6 +9,7 @@
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
+from urllib.error import HTTPError
from mainloop.runtime.substrate import (
ActorFailedToStart,
@@ -30,6 +31,61 @@ def completed(argv, returncode=0, stdout="", stderr=""):
class Gate5SourceAndBuildTests(unittest.TestCase):
+ def test_image_manifest_preflight_checks_registry_endpoint_and_accept_types(self):
+ calls = []
+
+ class Response:
+ status = 200
+
+ def __enter__(self):
+ return self
+
+ def __exit__(self, *_args):
+ return False
+
+ def opener(request, *, timeout):
+ calls.append((request, timeout))
+ return Response()
+
+ image = "localhost:5001/live-agent-gate@sha256:" + "a" * 64
+ gate5_setup.verify_image_manifest(image, opener=opener)
+
+ request, timeout = calls[0]
+ self.assertEqual(
+ request.full_url,
+ f"http://localhost:5001/v2/live-agent-gate/manifests/sha256:{'a' * 64}",
+ )
+ self.assertEqual(request.get_method(), "HEAD")
+ self.assertEqual(request.get_header("Accept"), gate5_setup.IMAGE_MANIFEST_ACCEPT)
+ self.assertEqual(timeout, 10)
+
+ def test_image_manifest_preflight_fails_before_template_on_missing_manifest(self):
+ image = "localhost:5001/live-agent-gate@sha256:" + "b" * 64
+
+ def missing(request, *, timeout):
+ error = HTTPError(request.full_url, 404, "MANIFEST_UNKNOWN", {}, None)
+ error.close()
+ raise error
+
+ with self.assertRaisesRegex(RuntimeError, "registry returned HTTP 404"):
+ gate5_setup.verify_image_manifest(image, opener=missing)
+
+ def test_image_manifest_preflight_rejects_tag_or_malformed_digest(self):
+ calls = []
+
+ def opener(*_args, **_kwargs):
+ calls.append(True)
+
+ for image in (
+ "localhost:5001/live-agent-gate:latest",
+ "localhost:5001/live-agent-gate@sha256:bad",
+ ):
+ with self.subTest(image=image), self.assertRaisesRegex(
+ RuntimeError, "full sha256 digest"
+ ):
+ gate5_setup.verify_image_manifest(image, opener=opener)
+ self.assertEqual(calls, [])
+
def make_source(self, root: Path) -> Path:
(root / ".git").mkdir(parents=True)
(root / "go.mod").write_text("module fixture\n")
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
index 7a308da..34d6896 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
+++ b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
@@ -18,6 +18,8 @@ const path = require('node:path');
const PANE_ID = process.env.EXEC_SHIM_PANE_ID;
const SESSION = process.env.HERDR_SESSION;
+const HEALTH_COMMAND_TIMEOUT_MS = 1500;
+const HEALTH_CACHE_MS = 3000;
if (!PANE_ID || !SESSION) {
console.error('exec-shim: EXEC_SHIM_PANE_ID and HERDR_SESSION are required');
process.exit(1);
@@ -85,22 +87,63 @@ function herdr(args, res) {
});
}
+function runHealthCheck() {
+ return new Promise((resolve, reject) => {
+ execFile(
+ 'herdr',
+ ['--session', SESSION, 'status', 'server'],
+ { timeout: HEALTH_COMMAND_TIMEOUT_MS },
+ (statusErr, stdout) => {
+ if (statusErr || !/^status:\s+running\s*$/m.test(stdout)) {
+ reject(statusErr || new Error('Herdr server is not running'));
+ return;
+ }
+ execFile(
+ 'herdr',
+ ['--session', SESSION, 'pane', 'read', PANE_ID],
+ { timeout: HEALTH_COMMAND_TIMEOUT_MS },
+ (paneErr) => {
+ if (paneErr) {
+ reject(paneErr);
+ return;
+ }
+ resolve();
+ },
+ );
+ },
+ );
+ });
+}
+
+let lastGoodHealthAt = 0;
+let healthCheckInFlight = null;
+
function healthz(res) {
- execFile('herdr', ['--session', SESSION, 'status', 'server'], (statusErr, stdout) => {
- if (statusErr || !/^status:\s+running\s*$/m.test(stdout)) {
+ const respond = (ready) => {
+ if (res.destroyed) return;
+ if (ready) {
+ // The shim is responsive, Herdr reports a running server, and the shell pane exists.
+ // Never return status output or pane contents.
+ res.writeHead(200, { 'content-type': 'text/plain' }).end('ok');
+ } else {
res.writeHead(503, { 'content-type': 'text/plain' }).end('not ready');
- return;
}
- execFile('herdr', ['--session', SESSION, 'pane', 'read', PANE_ID], (paneErr) => {
- if (paneErr) {
- res.writeHead(503, { 'content-type': 'text/plain' }).end('not ready');
- return;
- }
- // The request is served by this shim, Herdr reports a running server, and pane read
- // confirms the shell pane still exists. Do not expose status output or pane contents.
- res.writeHead(200, { 'content-type': 'text/plain' }).end('ok');
- });
- });
+ };
+
+ if (Date.now() - lastGoodHealthAt < HEALTH_CACHE_MS) {
+ respond(true);
+ return;
+ }
+ if (!healthCheckInFlight) {
+ healthCheckInFlight = runHealthCheck()
+ .then(() => {
+ lastGoodHealthAt = Date.now();
+ })
+ .finally(() => {
+ healthCheckInFlight = null;
+ });
+ }
+ healthCheckInFlight.then(() => respond(true), () => respond(false));
}
const server = http.createServer((req, res) => {
diff --git a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
index 2428509..f3b991a 100644
--- a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
+++ b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
@@ -13,10 +13,11 @@ import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const shim = path.resolve(__dirname, '../live-agent-image/exec-shim.js');
-async function startShim(home, fakeBin, shimPath) {
+async function startShim(home, fakeBin, shimPath, extraEnv = {}) {
const child = spawn(process.execPath, [shimPath], {
env: {
...process.env,
+ ...extraEnv,
HOME: home,
PATH: `${fakeBin}:${process.env.PATH}`,
HERDR_SESSION: 'shim-test',
@@ -67,7 +68,7 @@ function request(port, method, route, { body, token } = {}) {
}
async function stop(child) {
- if (child.exitCode !== null) return;
+ if (child.exitCode !== null || child.signalCode !== null) return;
child.kill('SIGTERM');
await once(child, 'exit');
}
@@ -128,3 +129,44 @@ test('shim token gates run/read, is one-time, private, and survives process rest
assert.equal((await request(running.port, 'GET', '/read', { token })).status, 200);
assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 409);
});
+
+test('healthz bounds Herdr calls and reuses a recent successful check', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-health-'));
+ const home = path.join(root, 'home');
+ const fakeBin = path.join(root, 'bin');
+ const shimPath = path.join(root, 'exec-shim.js');
+ const logPath = path.join(root, 'herdr-calls.log');
+ fs.mkdirSync(home);
+ fs.mkdirSync(fakeBin);
+ fs.copyFileSync(shim, shimPath);
+ const herdr = path.join(fakeBin, 'herdr');
+ fs.writeFileSync(
+ herdr,
+ '#!/bin/sh\nif [ -n "${EXEC_SHIM_TEST_LOG:-}" ]; then printf "%s %s\\n" "$3" "$4" >> "$EXEC_SHIM_TEST_LOG"; fi\nif [ "$3" = "status" ] && [ "$4" = "server" ]; then if [ "${HERDR_TEST_SLOW_STATUS:-}" = "1" ]; then exec sleep 5; fi; echo "status: running"; exit 0; fi\nif [ "$3" = "pane" ] && [ "$4" = "read" ]; then echo "fixture pane"; exit 0; fi\nexit 0\n',
+ { mode: 0o700 },
+ );
+
+ const running = await startShim(home, fakeBin, shimPath, {
+ EXEC_SHIM_TEST_LOG: logPath,
+ });
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+
+ assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
+ assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
+ assert.deepEqual(fs.readFileSync(logPath, 'utf8').trim().split('\n'), [
+ 'status server',
+ 'pane read',
+ ]);
+
+ await stop(running.child);
+ const slow = await startShim(home, fakeBin, shimPath, {
+ HERDR_TEST_SLOW_STATUS: '1',
+ });
+ t.after(async () => stop(slow.child));
+ const startedAt = Date.now();
+ assert.equal((await request(slow.port, 'GET', '/healthz')).status, 503);
+ assert.ok(Date.now() - startedAt < 4000, 'hung Herdr status must be bounded by execFile timeout');
+});
From e362e95c82f4eb563a68036f59d863fc79e36b53 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Wed, 23 Sep 2026 20:50:10 +0000
Subject: [PATCH 12/30] feat: add Substrate egress tooling and native
credential delivery
Trust the run-specific Substrate MITM CA in the actor image and point Claude
and Codex at the system CA bundle. Require an installed shim token before
writing Codex auth, validate the payload as a JSON object, and cover the
one-time private write with fixtures.
Use the selected atespace when waiting for an eligible worker. Include the
Round 3 credential-provider and egress-injection sources and document the
measured hostname, injection, KPR, Claude, and Codex results.
Track A passed; Track B was skipped. KPR=true passed the Substrate core but
actor DNS to the kube-dns Service IP timed out; the exact component remains
unverified, so the preview uses KPR=false. Claude recalled its nonce after
suspend/resume. The Codex turn stopped at Envoy upstream SAN verification: the
IPv4 override fixed api.openai.com (upstream HTTP 401, expected without auth),
but chatgpt.com failed verification on a shared Cloudflare IP because Envoy
received api.openai.com SANs while expecting chatgpt.com. The cause remains
unverified. The live V4_PREFERRED override is an uncommitted cluster-only
change; production adoption remains deferred.
Checks: 24 fake-backed gate5 setup tests (uv run), 3 exec-shim tests, the
pinned Substrate source check, and git diff --check passed.
---
backend/scripts/gate5_setup.py | 5 +-
backend/tests/runtime/test_gate5_setup.py | 22 ++++
docs/spikes/substrate-workspace-adapter.md | 101 +++++++++++++++---
.../live-agent-image/Dockerfile | 13 +++
.../live-agent-image/exec-shim.js | 66 ++++++++++++
.../tests/exec-shim.test.js | 50 +++++++++
.../tools/README-round3-egress.md | 30 ++++++
.../tools/round3-claude-provider/main.go | 76 +++++++++++++
.../tools/round3-credprovider/main.go | 95 ++++++++++++++++
.../tools/round3-egress-injection/main.go | 44 ++++++++
10 files changed, 484 insertions(+), 18 deletions(-)
create mode 100644 spikes/substrate-workspace-adapter/tools/README-round3-egress.md
create mode 100644 spikes/substrate-workspace-adapter/tools/round3-claude-provider/main.go
create mode 100644 spikes/substrate-workspace-adapter/tools/round3-credprovider/main.go
create mode 100644 spikes/substrate-workspace-adapter/tools/round3-egress-injection/main.go
diff --git a/backend/scripts/gate5_setup.py b/backend/scripts/gate5_setup.py
index 0687384..715398a 100644
--- a/backend/scripts/gate5_setup.py
+++ b/backend/scripts/gate5_setup.py
@@ -81,7 +81,6 @@
)
PINNED_SUBSTRATE_COMMIT = "cdac9baef81dd319b46086d695266e6161e9e592"
-WORKER_NAMESPACE = "live-agent-gate"
WORKER_SELECTOR = "workload=live-agent-gate"
WORKER_SANDBOX_CLASS = "gvisor"
ACTOR_SHIM_PORT = 8090
@@ -672,12 +671,12 @@ async def wait_for_worker_if_actor_is_absent(
return
print(
- f"-- waiting for an eligible worker in namespace={WORKER_NAMESPACE}, "
+ f"-- waiting for an eligible worker in namespace={args.atespace}, "
f"selector={WORKER_SELECTOR}, sandbox={WORKER_SANDBOX_CLASS}"
)
await wait_for_eligible_worker(
control,
- WORKER_NAMESPACE,
+ args.atespace,
WORKER_SELECTOR,
WORKER_SANDBOX_CLASS,
timeout_s=args.worker_timeout,
diff --git a/backend/tests/runtime/test_gate5_setup.py b/backend/tests/runtime/test_gate5_setup.py
index 6da8493..724aa0d 100644
--- a/backend/tests/runtime/test_gate5_setup.py
+++ b/backend/tests/runtime/test_gate5_setup.py
@@ -605,6 +605,28 @@ async def wait_for_worker(*_args, **_kwargs):
json.loads(Path(path).read_text())["actor_uid"], "actor-new"
)
+ def test_worker_wait_uses_the_selected_atespace_namespace(self):
+ with tempfile.TemporaryDirectory() as temp_dir:
+ path = str(Path(temp_dir) / "state.json")
+ args = self.args(path)
+ args.atespace = "native-codex"
+ control = SetupControl(None)
+ observed = {}
+
+ async def wait_for_worker(_control, namespace, *_args, **_kwargs):
+ observed["namespace"] = namespace
+
+ with patch.object(gate5_setup, "wait_for_eligible_worker", wait_for_worker):
+ asyncio_run(
+ gate5_setup.wait_for_worker_if_actor_is_absent(
+ control,
+ args,
+ {"run_id": "run-codex", "actor_uid": None, "template_uid": "template-codex"},
+ )
+ )
+
+ self.assertEqual(observed["namespace"], "native-codex")
+
def test_owned_rerun_skips_worker_wait_when_its_actor_occupies_only_worker(self):
with tempfile.TemporaryDirectory() as temp_dir:
path = str(Path(temp_dir) / "state.json")
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 7c2d255..e6ba0da 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -5,6 +5,15 @@ Status: local spike, not a product feature. Adapter code lives in
lives in `spikes/substrate-workspace-adapter/`. See `docs/spikes/k8s-herdr-agents.md` for the
native-session/Herdr spike this one builds on and does not replace.
+## Current status — Round 3 and Phase 4 (2026-09-23)
+
+Substrate actors, the Cilium-backed preview cluster, router ingress controls, per-actor shim
+tokens, and Envoy hostname enforcement were measured live. Claude completed a native turn and
+recalled a nonce after suspend/resume. Codex's auth file was installed safely, but its native
+turn failed at Envoy's upstream connection to the OpenAI API (HTTP 503, reset before response
+headers). Gate 5 is partial/live, so defer production adoption of the native-session path until
+Codex egress and session continuity are proved.
+
## What it shows
[Substrate](https://github.com/agent-substrate/substrate) can provide the per-session isolated
@@ -14,7 +23,7 @@ owner of the session<->actor mapping, delivery, and audit state. A Mainloop-auth
Substrate actor instead of a fixed StatefulSet pod, and `backend/src/mainloop/runtime/substrate.py`
drives its lifecycle through the real `kubectl ate` control-plane CLI.
-## Real versus stand-in
+## Earlier real-versus-stand-in inventory (before Round 3)
| Layer | Status |
| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
@@ -28,7 +37,7 @@ drives its lifecycle through the real `kubectl ate` control-plane CLI.
| Claude/Codex agent processes, credentials | Not run; the 2026-09-23 live lane stopped before credential work (see "Limits") |
| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
-## Why not a real agent for the preview-gate edit (credential-injection gap)
+## Preview-gate edit path and Round 3 credential boundary
Substrate's pinned commit has no generic secret-injection mechanism equivalent to a Kubernetes
Secret volume/env mount. `ActorTemplate` container env values are literal only (no
@@ -40,16 +49,19 @@ identity, that injects a credential into an outbound request; it does not hand a
CLI's local environment or filesystem.
The pinned commit also has experimental static-header injection from a Kubernetes Secret URI
-into decrypted outbound requests. It requires Envoy with SDSMint and the experimental
-credential-injection flag. Envoy 1.39.1 crashed on this host, while agentgateway does not support
-the injection path. The revised Phase 3 uses a Mainloop-owned router NetworkPolicy and a per-actor
-shim token instead; credentials are delivered through that closed channel. This keeps the
-credential path separate from the unsupported Envoy feature, though actor snapshots will contain
-credentials after delivery. The earlier unauthenticated relay is not used. The preview-gate
-measurement below instead uses
-a generic exec shim (`spikes/substrate-workspace-adapter/image/exec-shim.js`) that pastes text
-into a real Herdr shell pane via `herdr pane run` -- a real shell executing a real command, just
-not a credentialed agent's own tool call.
+into decrypted outbound requests. HTTPS hostname rules require the Envoy sdsmint overlay and
+`--experimental-egress-credential-injection`; the plain Envoy overlay has no MITM egress, and
+agentgateway does not implement this injection path. Envoy 1.39.1 crashed during an earlier
+install attempt, but the rebuilt Round 3 cluster ran Envoy with sdsmint and the credential
+provider. Claude's credential was injected on the upstream leg and stayed out of actor snapshots.
+Codex instead received `auth.json` through the authenticated shim because Codex refreshes that
+file locally; its actor snapshots therefore contain that credential. Neither credential value
+was logged or copied into a golden snapshot. The old unauthenticated relay was not used.
+
+The preview/HMR edit itself still uses a generic exec shim
+(`spikes/substrate-workspace-adapter/image/exec-shim.js`) that pastes text into a real Herdr shell
+pane via `herdr pane run` -- a real shell executing a real command, but not a native agent's own
+Bash tool.
## Run it
@@ -336,7 +348,7 @@ The Codex sandbox could not see the stalled `unittest discover` processes 229364
the reviewing session killed both on 2026-09-23. The full runtime result 189/189 is attributed
to that reviewing session, not to a sandbox process killed by this agent.
-## CapabilityResult
+## Historical CapabilityResult (before Round 3)
| Capability | State | Scope | Evidence and limit |
| ----------------------------- | ------- | ---------- | ------------------ |
@@ -352,7 +364,7 @@ to that reviewing session, not to a sandbox process killed by this agent.
| `native_session_continuity` | partial | unverified | Phase 3d failed before credential delivery; no native session was run. |
| `failure_recovery` | partial | live | Earlier CRASHED/revert mechanics were proved; worker-loss recovery and the backend-restart `recorded` case remain unproved live. |
-## Recommendation
+## Historical recommendation (superseded by the Round 3/Phase 4 result)
The corrected preview image resolves the earlier `npm ENOENT`/dead-checkpoint harness error;
that result does not justify deferring native-session adoption. The current agentgateway run
@@ -363,7 +375,7 @@ is added. Production also needs a GitOps-managed router NetworkPolicy on an enfo
shim-token issuance by the real Mainloop backend, and snapshot-bucket access controls. The
credential-bearing snapshot trade-off remains open because no credential entered an actor.
-## Cleanup and current cluster state
+## Historical cleanup note (superseded by the Phase 4 owner-review hold)
Phase 0 removed the approved relay resources and both named Secrets, then deleted the
owner-confirmed failed-trial cluster. Per the owner's 2026-09-23 instruction, cleanup of the
@@ -372,3 +384,62 @@ The actor's EgressPolicy is zero-rule deny-all. `mainloop-test` remains outside
provider Secret or credential-bearing snapshot was created. Current Kind/Docker inventory and
disk headroom are recorded in the task proof note. The owner handles rotation of credentials
previously served by the removed relay.
+
+## Current CapabilityResult — Round 3 and Phase 4
+
+| Capability | State | Scope | Evidence and limit |
+| --- | --- | --- | --- |
+| `workspace_adapter_contract` | partial | fixture | Contract and identity reconciliation are covered by fakes; the `workspace_bindings` orchestration was not run against live Postgres and a running backend. |
+| `substrate_actor_lifecycle` | proved | live | Golden, actor readiness, router ingress, and suspend/resume passed on the Cilium preview cluster. |
+| `preview_hmr` | proved | live | The earlier real Vite/WebSocket HMR route remains measured and passed. |
+| `dev_service_postgres` | proved | live | The earlier real PostgreSQL query, narrow CIDR rule, denied destination, and wake reconnection remain measured and passed. |
+| `networkpolicy_enforcement` | proved | live | Cilium blocked the denied probe and allowed the control-namespace probe. |
+| `router_ingress_boundary` | proved | live | `default` was denied; `mainloop-control` was admitted; the preview route remained functional. |
+| `shim_token_auth` | proved | live | Missing/wrong/correct token, one-time install, and suspend/resume persistence passed. |
+| `image_manifest_preflight` | proved | live | The exact pushed digest returned registry HTTP 200 before template creation; fake-backed tests cover the manifest check and rejection cases. |
+| `shim_healthz` | proved | live | Readiness checks pass through the actor route; bounded Herdr calls and cached health have fixture coverage. |
+| `cilium_kube_proxy_replacement` | partial | live | KPR=true core checks passed. DNS to the CoreDNS Pod IP worked, while DNS to the kube-dns Service IP timed out. The exact failing component was not isolated; kube-proxy replacement ClusterIP translation from the nested actor network is only a hypothesis. The cluster fell back to KPR=false as directed. |
+| `provider_hostname_egress` | proved | live | Actor A's listed Claude host returned 404 while unlisted `example.com` and raw IP returned 403. Actor B's differing rule allowed `example.com` to reach an upstream 503, denied `api.anthropic.com`, and denied raw IP. Track B was skipped because Track A passed. |
+| `dummy_header_injection` | proved | live | The compare-only provider returned a fixed match boolean; the final actor had no injected value in its env/files and received no echoed value. Earlier dummy-only diagnostic history is in the task proof note. |
+| `credential_delivery` | proved | live | Claude's credential was retrieved by the actor-bound provider and injected on the Envoy upstream leg. Codex `auth.json` was installed through the authenticated shim, once, mode 0600. These delivery proofs do not imply successful authentication for both CLIs. |
+| `claude_native_session` | proved | live | Claude Code 2.1.280 completed a turn, preserved its session ID through suspend/resume, and recalled a prior nonce. |
+| `codex_native_session` | partial | live | Codex CLI 0.156.1 created a thread, but Envoy returned an upstream-connect 503 before the turn completed; the model was not reported, and there was no marker or recall. No Codex revert was attempted. |
+| `native_session_continuity` | partial | live | Claude suspend/resume recall passed. Claude post-worker-loss recall and transcript rollback after snapshot revert remain unverified; Codex continuity did not pass. |
+| `snapshot_revert` | partial | live | Claude state-A restore and actor lifecycle passed, but transcript/history rollback was not conclusively measured. Codex revert was not attempted. |
+| `worker_loss_recovery` | partial | live | The Claude actor recovered on a replacement worker from its completed snapshot. Post-loss native recall did not complete. |
+| `backend_restart_delivery_reconciliation` | proved | fixture | The `ContractStore` fake test models a persisted `recorded` row across ownership restart, requires `not_delivered` evidence before retry, and rejects a duplicate attempt. |
+| `snapshot_bucket_access_control` | unknown | unverified | Read access to snapshot storage was not established in this install. |
+
+## Current recommendation
+
+**Defer production adoption of the Substrate native-session path.** The live run proves that
+Substrate can host the isolated workspace lifecycle, enforce router ingress and actor hostname
+egress, inject Claude credentials outside the actor snapshot, and preserve a Claude session
+through suspend/resume. It does not prove the required two-agent path: Codex could not complete a
+turn because the egress Envoy reset its OpenAI upstream connection before response headers
+(HTTP 503). The same credential-free API request returned `server: envoy` and an
+`upstream connect error`, while TLS verification succeeded and the unlisted-host rule still
+returned 403. The bounded review found no image, CA, install-flag, actor identity, or hostname
+policy mismatch.
+
+Before production, resolve and retest that Envoy-to-OpenAI upstream hop, then prove Codex
+authentication and nonce recall across suspend/resume. Production also needs a GitOps-managed
+router NetworkPolicy, a CNI that enforces it, shim-token issuance from the real Mainloop backend,
+and snapshot-bucket access control. The snapshot containing Codex `auth.json` must be removed
+when the actor is deleted; the owner handles credential rotation.
+
+## Current cleanup and review hold
+
+The Phase 4 owner-review brief requires both actors to remain SUSPENDED for review and their
+credential Secrets to be deleted. The dedicated preview cluster and run-built images remain
+available for that review; `mainloop-test` remains untouched. Temporary delivery Jobs,
+ConfigMaps, shim-token Secrets, the Claude provider, and the Claude/Codex credential Secrets are
+removed. The actor snapshots are intentionally retained for owner inspection, so the Codex
+snapshot still contains `auth.json`. The proof note records the final inventory and identifies
+this review hold as the reason the older finish-plan teardown was not applied.
+
+The Round 3 helper sources are under `spikes/substrate-workspace-adapter/tools/`; the captured
+sources match the pinned Substrate checkout. The Codex file-write route now requires an installed
+shim token even on a tokenless golden and validates a JSON object. That source hardening is
+fixture-tested, but the retained actor image digest predates the change; rebuild before using
+that route in another run.
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
index bbd0b26..fae870c 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
+++ b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
@@ -1,3 +1,4 @@
+# syntax=docker/dockerfile:1.7
# Live-agent-gate spike image (gate 5, bounded live proof): real Herdr + real Claude Code /
# Codex CLIs + agentctl, driven by the same generic exec shim as the other gate images.
# herdr/claude/codex/codex-code-mode-host are copied from the host into the build context by the
@@ -9,6 +10,16 @@ FROM node:22-bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends jq ca-certificates git curl procps ripgrep \
&& rm -rf /var/lib/apt/lists/* \
&& useradd -m -u 10001 agent
+# The run-specific Substrate MITM CA is a public trust anchor. BuildKit mounts
+# it without retaining the input file or putting the PEM in the build command.
+ARG EGRESS_MITM_CA_SHA256=""
+RUN --mount=type=secret,id=egress-mitm-ca,target=/run/secrets/egress-mitm-ca,required=false \
+ if [ -s /run/secrets/egress-mitm-ca ]; then \
+ test -n "${EGRESS_MITM_CA_SHA256}" && \
+ test "$(sha256sum /run/secrets/egress-mitm-ca | cut -d' ' -f1)" = "${EGRESS_MITM_CA_SHA256}" && \
+ install -m 0644 /run/secrets/egress-mitm-ca /usr/local/share/ca-certificates/substrate-egress-mitm.crt && \
+ update-ca-certificates; \
+ fi
COPY herdr /usr/local/bin/herdr
COPY claude /usr/local/bin/claude
COPY codex /usr/local/bin/codex
@@ -25,5 +36,7 @@ ENV WORKSPACE_PATH=/work/repo
ENV CODEX_HOME=/home/agent/.codex
ENV AGENT_CONFIG_DIR=/etc/agent-config
ENV HERDR_SESSION=mainloop-live-agent
+ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt
+ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
USER 10001:10001
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
index 34d6896..11c19aa 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
+++ b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
@@ -75,6 +75,45 @@ function installToken(token) {
return true;
}
+function installCodexAuth(contentBase64) {
+ if (typeof contentBase64 !== 'string') return false;
+ const contents = Buffer.from(contentBase64, 'base64');
+ if (
+ contents.length === 0 ||
+ contents.length > 65536 ||
+ contents.toString('base64') !== contentBase64
+ ) return false;
+ let auth;
+ try {
+ auth = JSON.parse(contents.toString('utf8'));
+ } catch {
+ return false;
+ }
+ if (!auth || typeof auth !== 'object' || Array.isArray(auth)) return false;
+ const codexHome = process.env.CODEX_HOME || path.join(process.env.HOME || '/home/agent', '.codex');
+ fs.mkdirSync(codexHome, { recursive: true, mode: 0o700 });
+ fs.chmodSync(codexHome, 0o700);
+ const destination = path.join(codexHome, 'auth.json');
+ let fd;
+ try {
+ fd = fs.openSync(destination, 'wx', 0o600);
+ } catch (err) {
+ if (err.code === 'EEXIST') return null;
+ throw err;
+ }
+ try {
+ fs.writeFileSync(fd, contents);
+ fs.fsyncSync(fd);
+ fs.closeSync(fd);
+ fs.chmodSync(destination, 0o600);
+ } catch (err) {
+ try { fs.closeSync(fd); } catch {}
+ fs.rmSync(destination, { force: true });
+ throw err;
+ }
+ return true;
+}
+
function herdr(args, res) {
execFile('herdr', ['--session', SESSION, ...args], (err, stdout, stderr) => {
if (err) {
@@ -186,6 +225,33 @@ const server = http.createServer((req, res) => {
herdr(['pane', 'read', PANE_ID], res);
return;
}
+ if (req.method === 'POST' && req.url === '/write-codex-auth') {
+ if (bearerToken === null || !authorized(req)) return unauthorized(res);
+ requestBody(req, (body) => {
+ let contentBase64;
+ try {
+ contentBase64 = JSON.parse(body).contentBase64;
+ } catch {
+ res.writeHead(400).end('invalid json');
+ return;
+ }
+ try {
+ const installed = installCodexAuth(contentBase64);
+ if (installed === null) {
+ res.writeHead(409).end('auth file already exists');
+ return;
+ }
+ if (!installed) {
+ res.writeHead(400).end('invalid auth payload');
+ return;
+ }
+ res.writeHead(201).end('Codex auth installed');
+ } catch {
+ res.writeHead(500).end('Codex auth could not be stored');
+ }
+ });
+ return;
+ }
if (req.method !== 'POST' || req.url !== '/run') {
res.writeHead(404).end();
return;
diff --git a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
index f3b991a..783e6c8 100644
--- a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
+++ b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
@@ -130,6 +130,56 @@ test('shim token gates run/read, is one-time, private, and survives process rest
assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 409);
});
+test('Codex auth write requires an installed shim token and creates a private one-time file', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-codex-auth-'));
+ const home = path.join(root, 'home');
+ const fakeBin = path.join(root, 'bin');
+ const shimPath = path.join(root, 'exec-shim.js');
+ const codexHome = path.join(home, '.codex');
+ fs.mkdirSync(home);
+ fs.mkdirSync(fakeBin);
+ fs.copyFileSync(shim, shimPath);
+ const herdr = path.join(fakeBin, 'herdr');
+ fs.writeFileSync(
+ herdr,
+ '#!/bin/sh\nif [ "$3" = "status" ] && [ "$4" = "server" ]; then echo "status: running"; exit 0; fi\nif [ "$3" = "pane" ] && [ "$4" = "read" ]; then echo "fixture pane"; exit 0; fi\nexit 0\n',
+ { mode: 0o700 },
+ );
+
+ const running = await startShim(home, fakeBin, shimPath, { CODEX_HOME: codexHome });
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+
+ const authContents = JSON.stringify({ fixture: 'codex-auth-never-logged' });
+ const contentBase64 = Buffer.from(authContents).toString('base64');
+ const write = (encoded, token) =>
+ request(running.port, 'POST', '/write-codex-auth', {
+ body: { contentBase64: encoded },
+ ...(token === undefined ? {} : { token }),
+ });
+
+ assert.equal((await write(contentBase64)).status, 401, 'golden actor must reject writes before token installation');
+ assert.equal(fs.existsSync(codexHome), false);
+
+ const token = 'fixture-only-codex-shim-token-at-least-thirty-two-chars';
+ assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 201);
+ assert.equal((await write(contentBase64)).status, 401);
+ assert.equal((await write(contentBase64, `${token}-wrong`)).status, 401);
+ assert.equal((await write(Buffer.from('[]').toString('base64'), token)).status, 400);
+ assert.equal((await write('!!!!', token)).status, 400);
+
+ assert.equal((await write(contentBase64, token)).status, 201);
+ const authPath = path.join(codexHome, 'auth.json');
+ assert.equal(fs.readFileSync(authPath, 'utf8'), authContents);
+ assert.equal(fs.statSync(codexHome).mode & 0o777, 0o700);
+ assert.equal(fs.statSync(authPath).mode & 0o777, 0o600);
+ assert.equal((await write(contentBase64, token)).status, 409, 'auth file must not be overwritten');
+ assert.equal(running.output().includes(authContents), false);
+ assert.equal(running.output().includes(contentBase64), false);
+});
+
test('healthz bounds Herdr calls and reuses a recent successful check', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-health-'));
const home = path.join(root, 'home');
diff --git a/spikes/substrate-workspace-adapter/tools/README-round3-egress.md b/spikes/substrate-workspace-adapter/tools/README-round3-egress.md
new file mode 100644
index 0000000..1cdb39b
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tools/README-round3-egress.md
@@ -0,0 +1,30 @@
+# Round 3 egress helpers
+
+These source files are a live-only, dummy-credential harness for the 2026-09-23
+Substrate preview run. They are not production credential-provider code.
+
+- `round3-credprovider/main.go` implements a temporary mTLS gRPC credential
+ provider and HTTPS echo endpoint. It reads only a Kubernetes Secret created
+ specifically for this test. The echo endpoint compares the injected header
+ to that dummy value and returns a boolean result; it never returns the value.
+- `round3-egress-injection/main.go` updates one actor's egress policy to inject
+ a credential for one hostname. `--prefix` supports bearer-token headers;
+ omit it for the dummy echo check.
+- `round3-claude-provider/main.go` is the separate Phase 3e preview provider.
+ It is pinned to the Claude Secret URI and one actor SPIFFE ID, reads the
+ credential from a read-only Secret mount, and logs only a success marker.
+ It is test-run scaffolding, not a general-purpose or production provider.
+
+Build from the pinned Substrate checkout (`cdac9baef81dd319b46086d695266e6161e9e592`),
+where the imported internal packages and protobuf modules are available:
+
+```sh
+CGO_ENABLED=0 GOFLAGS=-mod=vendor go build -o /tmp/round3-credprovider ./cmd/round3-credprovider
+CGO_ENABLED=0 GOFLAGS=-mod=vendor go build -o /tmp/round3-egress-injection ./cmd/round3-egress-injection
+CGO_ENABLED=0 GOFLAGS=-mod=vendor go build -o /tmp/round3-claude-provider ./cmd/round3-claude-provider
+```
+
+Use only a throwaway dummy Secret with the first provider in a disposable
+preview cluster. The Phase 3e provider is separately actor-bound; pass its
+credential only as a read-only Secret mount. Never put a credential in source,
+logs, actor commands, or echo responses.
diff --git a/spikes/substrate-workspace-adapter/tools/round3-claude-provider/main.go b/spikes/substrate-workspace-adapter/tools/round3-claude-provider/main.go
new file mode 100644
index 0000000..cc7371c
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tools/round3-claude-provider/main.go
@@ -0,0 +1,76 @@
+package main
+
+import (
+ "context"
+ "crypto/tls"
+ "crypto/x509"
+ "log"
+ "net"
+ "os"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/status"
+
+ "github.com/agent-substrate/substrate/pkg/proto/credproviderpb"
+)
+
+const (
+ expectedURI = "ate-secret://kubernetes.io/mainloop-control/claude-oauth/oauth-token"
+ expectedActorID = "spiffe://substrate-actor.local/atespace/live-agent-gate/actor/egress-actor-a"
+ credentialPath = "/run/claude/oauth-token"
+ servingBundlePath = "/run/servicedns/credential-bundle.pem"
+ clientCAPath = "/run/podidentity-ca/trust-bundle.pem"
+)
+
+type provider struct {
+ credproviderpb.UnimplementedCredentialProviderServer
+}
+
+func (provider) FetchSecret(_ context.Context, req *credproviderpb.FetchSecretRequest) (*credproviderpb.FetchSecretResponse, error) {
+ if req.GetUri() != expectedURI || req.GetActorSpiffeId() != expectedActorID {
+ return nil, status.Error(codes.PermissionDenied, "credential request rejected")
+ }
+ value, err := os.ReadFile(credentialPath)
+ if err != nil {
+ log.Printf("credential_fetch=unavailable actor_identity_match=true")
+ return nil, status.Error(codes.Unavailable, "credential unavailable")
+ }
+ if len(value) == 0 {
+ return nil, status.Error(codes.NotFound, "credential unavailable")
+ }
+ log.Printf("credential_fetch=ok actor_identity_match=true credential_kind=claude-oauth")
+ return &credproviderpb.FetchSecretResponse{OpaqueBytes: value}, nil
+}
+
+func main() {
+ servingCert, err := tls.LoadX509KeyPair(servingBundlePath, servingBundlePath)
+ if err != nil {
+ log.Fatal("serving certificate unavailable")
+ }
+ caBytes, err := os.ReadFile(clientCAPath)
+ if err != nil {
+ log.Fatal("client CA unavailable")
+ }
+ clientCAs := x509.NewCertPool()
+ if !clientCAs.AppendCertsFromPEM(caBytes) {
+ log.Fatal("client CA bundle invalid")
+ }
+ tlsConfig := &tls.Config{
+ MinVersion: tls.VersionTLS12,
+ Certificates: []tls.Certificate{servingCert},
+ ClientAuth: tls.RequireAndVerifyClientCert,
+ ClientCAs: clientCAs,
+ }
+ grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig)))
+ credproviderpb.RegisterCredentialProviderServer(grpcServer, provider{})
+ listener, err := net.Listen("tcp", ":50051")
+ if err != nil {
+ log.Fatal("gRPC listener unavailable")
+ }
+ log.Printf("credential_provider_ready=true")
+ if err := grpcServer.Serve(listener); err != nil {
+ log.Fatal("gRPC server failed")
+ }
+}
diff --git a/spikes/substrate-workspace-adapter/tools/round3-credprovider/main.go b/spikes/substrate-workspace-adapter/tools/round3-credprovider/main.go
new file mode 100644
index 0000000..a08dbb7
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tools/round3-credprovider/main.go
@@ -0,0 +1,95 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/tls"
+ "crypto/x509"
+ "io"
+ "log"
+ "net"
+ "net/http"
+ "os"
+ "strings"
+
+ "google.golang.org/grpc"
+ "google.golang.org/grpc/codes"
+ "google.golang.org/grpc/credentials"
+ "google.golang.org/grpc/status"
+
+ "github.com/agent-substrate/substrate/pkg/proto/credproviderpb"
+)
+
+const expectedURI = "ate-secret://kubernetes.io/ate-system/round3-dummy/token"
+
+type provider struct {
+ credproviderpb.UnimplementedCredentialProviderServer
+ secretPath string
+}
+
+func (p provider) FetchSecret(_ context.Context, req *credproviderpb.FetchSecretRequest) (*credproviderpb.FetchSecretResponse, error) {
+ if req.GetUri() != expectedURI || !strings.HasPrefix(req.GetActorSpiffeId(), "spiffe://") {
+ return nil, status.Error(codes.PermissionDenied, "dummy provider request rejected")
+ }
+ value, err := os.ReadFile(p.secretPath)
+ if err != nil {
+ return nil, status.Error(codes.Unavailable, "dummy value unavailable")
+ }
+ log.Printf("credential_fetch=ok actor_identity_present=true")
+ return &credproviderpb.FetchSecretResponse{OpaqueBytes: bytes.TrimSpace(value)}, nil
+}
+
+func main() {
+ bundle := "/run/servicedns/credential-bundle.pem"
+ servingCert, err := tls.LoadX509KeyPair(bundle, bundle)
+ if err != nil {
+ log.Fatal("serving certificate unavailable")
+ }
+ caBytes, err := os.ReadFile("/run/podidentity-ca/trust-bundle.pem")
+ if err != nil {
+ log.Fatal("client CA unavailable")
+ }
+ clientCAs := x509.NewCertPool()
+ if !clientCAs.AppendCertsFromPEM(caBytes) {
+ log.Fatal("client CA bundle invalid")
+ }
+ tlsConfig := &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{servingCert}, ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: clientCAs}
+ grpcServer := grpc.NewServer(grpc.Creds(credentials.NewTLS(tlsConfig)))
+ credproviderpb.RegisterCredentialProviderServer(grpcServer, provider{secretPath: "/run/dummy/token"})
+ listener, err := net.Listen("tcp", ":50051")
+ if err != nil {
+ log.Fatal("gRPC listener unavailable")
+ }
+ go func() {
+ log.Printf("credential_provider_ready=true")
+ if err := grpcServer.Serve(listener); err != nil {
+ log.Fatal("gRPC server failed")
+ }
+ }()
+ http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+ w.Header().Set("content-type", "text/plain")
+ if r.URL.Path == "/readyz" {
+ w.WriteHeader(http.StatusNoContent)
+ return
+ }
+ want, err := os.ReadFile("/run/dummy/token")
+ if err != nil {
+ http.Error(w, "dummy value unavailable", http.StatusServiceUnavailable)
+ return
+ }
+ if bytes.Equal(bytes.TrimSpace(want), []byte(r.Header.Get("X-Mainloop-Dummy"))) {
+ _, _ = io.WriteString(w, "injected-header-matched")
+ return
+ }
+ w.WriteHeader(http.StatusForbidden)
+ _, _ = io.WriteString(w, "injected-header-mismatch")
+ })
+ server := &http.Server{
+ Addr: ":8443",
+ Handler: nil,
+ TLSConfig: &tls.Config{MinVersion: tls.VersionTLS12, Certificates: []tls.Certificate{servingCert}},
+ }
+ if err := server.ListenAndServeTLS("", ""); err != nil {
+ log.Fatal("TLS echo listener unavailable")
+ }
+}
diff --git a/spikes/substrate-workspace-adapter/tools/round3-egress-injection/main.go b/spikes/substrate-workspace-adapter/tools/round3-egress-injection/main.go
new file mode 100644
index 0000000..9b4b6a6
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tools/round3-egress-injection/main.go
@@ -0,0 +1,44 @@
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "log"
+
+ "github.com/agent-substrate/substrate/internal/ateclient"
+ "github.com/agent-substrate/substrate/internal/resources"
+ "github.com/agent-substrate/substrate/pkg/proto/ateapipb"
+)
+
+func main() {
+ kubeconfig := flag.String("kubeconfig", "", "")
+ contextName := flag.String("context", "", "")
+ atespace := flag.String("atespace", "", "")
+ actor := flag.String("actor", "", "")
+ hostname := flag.String("hostname", "", "")
+ header := flag.String("header", "", "")
+ prefix := flag.String("prefix", "", "")
+ uri := flag.String("credential-uri", "", "")
+ flag.Parse()
+ ctx := context.Background()
+ cli, err := ateclient.NewClient(ctx, *kubeconfig, *contextName, "", "", false)
+ if err != nil {
+ log.Fatal("ateapi client unavailable")
+ }
+ defer cli.Close()
+ ref := resources.ActorRef{Atespace: *atespace, Name: *actor}.ToObjectRef()
+ existing, err := cli.GetActorEgressPolicy(ctx, &ateapipb.GetActorEgressPolicyRequest{Actor: ref})
+ if err != nil {
+ log.Fatal("actor egress policy unavailable")
+ }
+ policy := &ateapipb.EgressPolicy{Metadata: existing.GetMetadata(), Rules: []*ateapipb.EgressRule{{
+ Hostnames: &ateapipb.HostnameRule{Patterns: []string{*hostname}, Effects: &ateapipb.EgressRuleEffects{
+ InjectStaticHeaders: []*ateapipb.CredentialHeaderInjection{{Header: *header, Prefix: *prefix, CredentialUri: *uri}},
+ }},
+ }}}
+ if _, err := cli.UpdateActorEgressPolicy(ctx, &ateapipb.UpdateActorEgressPolicyRequest{Actor: ref, EgressPolicy: policy}); err != nil {
+ log.Fatal("injection policy update failed")
+ }
+ fmt.Println("egress_header_injection_policy=updated")
+}
From 7db742b077c4c91761e4b7372e0b7fbeb5c66fec Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Wed, 23 Sep 2026 21:22:43 +0000
Subject: [PATCH 13/30] feat: add native-agent credential delivery and startup
Add one authenticated /credential endpoint for allowlisted Claude and Codex
credentials, preserving Codex JSON validation and exclusive one-time writes.
Add control-side path-based delivery through temporary control Secrets and an
actor-scoped router tunnel. Seed native CLI first-run configuration and add
explicit Claude and Codex launchers; Claude loads its token into the CLI
environment with telemetry and update traffic disabled.
Checks: 9 fake-backed Node tests, 4 fake-backed Python tests, shell and
JavaScript syntax checks, and git diff --check passed. The live Phase 4 proof
did not use this code; this draft contains local code only.
---
backend/scripts/gate5_deliver_credentials.py | 479 ++++++++++++++++++
.../tests/runtime/test_gate5_credentials.py | 138 +++++
.../live-agent-image/Dockerfile | 4 +-
.../bin/prepare-native-agent-config.cjs | 51 ++
.../live-agent-image/bin/start-native-agent | 47 ++
.../live-agent-image/entrypoint.sh | 21 +-
.../live-agent-image/exec-shim.js | 113 +++--
.../tests/deliver-credentials.test.js | 214 ++++++++
.../tests/exec-shim.test.js | 40 +-
.../tools/phase4/deliver-credentials.cjs | 155 ++++++
10 files changed, 1185 insertions(+), 77 deletions(-)
create mode 100644 backend/scripts/gate5_deliver_credentials.py
create mode 100644 backend/tests/runtime/test_gate5_credentials.py
create mode 100644 spikes/substrate-workspace-adapter/live-agent-image/bin/prepare-native-agent-config.cjs
create mode 100755 spikes/substrate-workspace-adapter/live-agent-image/bin/start-native-agent
create mode 100644 spikes/substrate-workspace-adapter/tests/deliver-credentials.test.js
create mode 100644 spikes/substrate-workspace-adapter/tools/phase4/deliver-credentials.cjs
diff --git a/backend/scripts/gate5_deliver_credentials.py b/backend/scripts/gate5_deliver_credentials.py
new file mode 100644
index 0000000..a420cb9
--- /dev/null
+++ b/backend/scripts/gate5_deliver_credentials.py
@@ -0,0 +1,479 @@
+#!/usr/bin/env python3
+"""Deliver one provider credential from a control-namespace Secret to its final actor.
+
+The source file is passed only as a path. kubectl reads it through an inherited file
+descriptor into a Secret in mainloop-control; a short-lived control Job mounts that Secret
+and sends its contents in the authenticated exec-shim request body. Nothing reads a Secret
+back through the Kubernetes API, and neither secret value is placed in argv, env, or logs.
+
+This is a separate command from gate5_setup.py, so golden creation and credential-free actor
+setup never invoke delivery.
+"""
+
+import argparse
+import json
+import os
+import re
+import secrets
+import stat
+import subprocess # nosec B404 - fixed kubectl commands, secret data via file descriptors
+import sys
+import tempfile
+from pathlib import Path
+from typing import Callable
+
+CONTROL_NAMESPACE = "mainloop-control"
+DEFAULT_CONTEXT = "kind-substrate-preview"
+DEFAULT_KUBECONFIG = "/tmp/substrate-preview-kubeconfig"
+DEFAULT_IMAGE = (
+ "localhost:5001/live-agent-gate@sha256:"
+ "8ec007c56b070a2357f20203807197e42ebdb8d0c0e155d57a3bd48fb8d10f57"
+)
+MAX_SECRET_BYTES = 1024 * 1024
+ACTOR_NAMESPACES = {"claude": "native-claude", "codex": "native-codex"}
+SHARED_CLUSTER_NOTE = (
+ Path(__file__).resolve().parents[2]
+ / ".tasknotes"
+ / "shared-cluster-2026-09-23.md"
+)
+DELIVERY_SCRIPT = (
+ Path(__file__).resolve().parents[2]
+ / "spikes"
+ / "substrate-workspace-adapter"
+ / "tools"
+ / "phase4"
+ / "deliver-credentials.cjs"
+)
+
+
+def parse_args() -> argparse.Namespace:
+ parser = argparse.ArgumentParser(description=__doc__)
+ parser.add_argument("--context", default=DEFAULT_CONTEXT)
+ parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG)
+ parser.add_argument("--actor-namespace", required=True)
+ parser.add_argument("--actor-name", required=True)
+ parser.add_argument("--state-file", required=True)
+ parser.add_argument("--credential", choices=("claude", "codex"), required=True)
+ parser.add_argument(
+ "--claude-token-file", default=str(Path.home() / ".claude-token")
+ )
+ parser.add_argument(
+ "--codex-auth-file", default=str(Path.home() / ".codex" / "auth.json")
+ )
+ parser.add_argument("--image", default=DEFAULT_IMAGE)
+ return parser.parse_args()
+
+
+def read_private_state(path: str, *, actor_namespace: str, actor_name: str) -> dict:
+ try:
+ with open(path, encoding="utf-8") as stream:
+ state = json.load(stream)
+ except (OSError, ValueError):
+ raise RuntimeError("private actor state is unavailable") from None
+
+ if not isinstance(state, dict) or (
+ state.get("context") != DEFAULT_CONTEXT
+ or state.get("atespace") != actor_namespace
+ or state.get("actor_name") != actor_name
+ or not state.get("actor_uid")
+ ):
+ raise RuntimeError("private actor state does not own the requested final actor")
+ token = state.get("shim_token")
+ if (
+ not isinstance(token, str)
+ or not 32 <= len(token) <= 4096
+ or re.search(r"\s", token)
+ ):
+ raise RuntimeError("private actor state has no valid shim token")
+ return state
+
+
+def require_handover(path: Path = SHARED_CLUSTER_NOTE) -> None:
+ try:
+ note = path.read_text(encoding="utf-8")
+ except OSError:
+ raise RuntimeError("shared-cluster handover note is unavailable") from None
+ if not re.search(r"(?m)^\*\*Handover:\*\*\s+done(?:\s|$)", note):
+ raise RuntimeError("shared-cluster handover is not done; no resources were created")
+
+
+def kubectl_prefix(context: str, kubeconfig: str) -> list[str]:
+ if context != DEFAULT_CONTEXT:
+ raise RuntimeError(f"refusing context {context!r}; expected {DEFAULT_CONTEXT!r}")
+ return ["kubectl", "--context", context, "--kubeconfig", kubeconfig]
+
+
+def run_kubectl(
+ argv: list[str],
+ *,
+ runner: Callable = subprocess.run,
+ input_bytes: bytes | None = None,
+ pass_fds: tuple[int, ...] = (),
+ timeout: int = 60,
+ action: str,
+):
+ try:
+ return runner(
+ argv,
+ input=input_bytes,
+ pass_fds=pass_fds,
+ capture_output=True,
+ check=True,
+ timeout=timeout,
+ )
+ except (subprocess.CalledProcessError, OSError, subprocess.TimeoutExpired):
+ raise RuntimeError(f"{action} failed") from None
+
+
+def create_secret_from_path(
+ *,
+ context: str,
+ kubeconfig: str,
+ namespace: str,
+ name: str,
+ key: str,
+ source_path: str,
+ runner: Callable = subprocess.run,
+) -> None:
+ """Create a Secret without placing the source path or bytes in child argv/env/logs."""
+ try:
+ source = open(source_path, "rb")
+ except OSError:
+ raise RuntimeError("credential source file is unavailable") from None
+ with source:
+ details = os.fstat(source.fileno())
+ if (
+ not stat.S_ISREG(details.st_mode)
+ or details.st_size <= 0
+ or details.st_size > MAX_SECRET_BYTES
+ ):
+ raise RuntimeError("credential source file has an invalid size or type")
+ source_fd = source.fileno()
+ descriptor_path = f"/proc/self/fd/{source_fd}"
+ create = [
+ *kubectl_prefix(context, kubeconfig),
+ "create",
+ "secret",
+ "generic",
+ name,
+ "--namespace",
+ namespace,
+ f"--from-file={key}={descriptor_path}",
+ "--dry-run=client",
+ "-o",
+ "json",
+ ]
+ result = run_kubectl(
+ create,
+ runner=runner,
+ pass_fds=(source_fd,),
+ action="credential Secret rendering",
+ )
+ apply = [*kubectl_prefix(context, kubeconfig), "apply", "-f", "-"]
+ run_kubectl(
+ apply,
+ runner=runner,
+ input_bytes=result.stdout,
+ action="credential Secret creation",
+ )
+
+
+def apply_json(
+ *,
+ context: str,
+ kubeconfig: str,
+ manifest: dict,
+ runner: Callable = subprocess.run,
+ action: str,
+) -> None:
+ encoded = json.dumps(manifest, separators=(",", ":")).encode("utf-8")
+ run_kubectl(
+ [*kubectl_prefix(context, kubeconfig), "apply", "-f", "-"],
+ runner=runner,
+ input_bytes=encoded,
+ action=action,
+ )
+
+
+def delete_delivery_resources(
+ *, context: str, kubeconfig: str, names: list[str], runner: Callable
+) -> None:
+ run_kubectl(
+ [
+ *kubectl_prefix(context, kubeconfig),
+ "delete",
+ "--namespace",
+ CONTROL_NAMESPACE,
+ "--ignore-not-found=true",
+ "--wait=true",
+ "--timeout=15s",
+ *names,
+ ],
+ runner=runner,
+ timeout=20,
+ action="delivery resource cleanup",
+ )
+
+
+def build_job(
+ *,
+ name: str,
+ image: str,
+ credential: str,
+ credential_secret: str,
+ shim_secret: str,
+ actor_namespace: str,
+ actor_name: str,
+) -> dict:
+ return {
+ "apiVersion": "batch/v1",
+ "kind": "Job",
+ "metadata": {"name": name, "namespace": CONTROL_NAMESPACE},
+ "spec": {
+ "activeDeadlineSeconds": 60,
+ "backoffLimit": 0,
+ "ttlSecondsAfterFinished": 120,
+ "template": {
+ "metadata": {"labels": {"mainloop.dev/role": "control"}},
+ "spec": {
+ "automountServiceAccountToken": False,
+ "restartPolicy": "Never",
+ "securityContext": {
+ "runAsNonRoot": True,
+ "runAsUser": 10001,
+ "runAsGroup": 10001,
+ "fsGroup": 10001,
+ },
+ "containers": [
+ {
+ "name": "deliver-credential",
+ "image": image,
+ "imagePullPolicy": "IfNotPresent",
+ "command": [
+ "node",
+ "/var/run/phase4-delivery/deliver-credentials.cjs",
+ ],
+ "env": [
+ {"name": "CREDENTIAL_KIND", "value": credential},
+ {"name": "ACTOR_NAMESPACE", "value": actor_namespace},
+ {"name": "ACTOR_NAME", "value": actor_name},
+ {
+ "name": "ROUTER_HOST",
+ "value": "atenet-router.ate-system.svc.cluster.local",
+ },
+ {"name": "ROUTER_PORT", "value": "8081"},
+ ],
+ "volumeMounts": [
+ {
+ "name": "delivery-script",
+ "mountPath": "/var/run/phase4-delivery",
+ "readOnly": True,
+ },
+ {
+ "name": "credential",
+ "mountPath": "/var/run/phase4-credentials",
+ "readOnly": True,
+ },
+ {
+ "name": "shim-auth",
+ "mountPath": "/var/run/phase4-shim-auth",
+ "readOnly": True,
+ },
+ ],
+ "securityContext": {
+ "allowPrivilegeEscalation": False,
+ "readOnlyRootFilesystem": True,
+ "capabilities": {"drop": ["ALL"]},
+ },
+ }
+ ],
+ "volumes": [
+ {
+ "name": "delivery-script",
+ "configMap": {
+ "name": name,
+ "items": [
+ {
+ "key": "deliver-credentials.cjs",
+ "path": "deliver-credentials.cjs",
+ }
+ ],
+ },
+ },
+ {
+ "name": "credential",
+ "secret": {
+ "secretName": credential_secret,
+ "defaultMode": 0o440,
+ "items": [
+ {"key": "credential", "path": "credential"}
+ ],
+ },
+ },
+ {
+ "name": "shim-auth",
+ "secret": {
+ "secretName": shim_secret,
+ "defaultMode": 0o440,
+ "items": [{"key": "token", "path": "token"}],
+ },
+ },
+ ],
+ },
+ },
+ },
+ }
+
+
+def deliver_credentials(
+ args: argparse.Namespace,
+ *,
+ runner: Callable = subprocess.run,
+ handover_note: Path = SHARED_CLUSTER_NOTE,
+) -> None:
+ require_handover(handover_note)
+ expected_namespace = ACTOR_NAMESPACES[args.credential]
+ if args.actor_namespace != expected_namespace:
+ raise RuntimeError("credential kind and actor namespace do not match")
+ if not re.fullmatch(
+ r"localhost:5001/live-agent-gate@sha256:[0-9a-f]{64}", args.image
+ ):
+ raise RuntimeError("delivery image must be the digest-pinned live-agent-gate image")
+
+ state = read_private_state(
+ args.state_file,
+ actor_namespace=args.actor_namespace,
+ actor_name=args.actor_name,
+ )
+ credential_path = (
+ args.claude_token_file if args.credential == "claude" else args.codex_auth_file
+ )
+ credential_secret = f"phase4-{args.credential}-{secrets.token_hex(4)}"
+ shim_secret = f"phase4-shim-{secrets.token_hex(4)}"
+ job_name = f"phase4-delivery-{args.credential}-{secrets.token_hex(4)}"
+ resource_names = [
+ f"job/{job_name}",
+ f"configmap/{job_name}",
+ f"secret/{credential_secret}",
+ f"secret/{shim_secret}",
+ ]
+ failed = False
+ try:
+ create_secret_from_path(
+ context=args.context,
+ kubeconfig=args.kubeconfig,
+ namespace=CONTROL_NAMESPACE,
+ name=credential_secret,
+ key="credential",
+ source_path=credential_path,
+ runner=runner,
+ )
+ with tempfile.TemporaryFile(mode="w+b") as shim_token_file:
+ shim_token_file.write(state["shim_token"].encode("utf-8"))
+ shim_token_file.flush()
+ shim_token_file.seek(0)
+ shim_fd = shim_token_file.fileno()
+ shim_key_path = f"/proc/self/fd/{shim_fd}"
+ create = [
+ *kubectl_prefix(args.context, args.kubeconfig),
+ "create",
+ "secret",
+ "generic",
+ shim_secret,
+ "--namespace",
+ CONTROL_NAMESPACE,
+ f"--from-file=token={shim_key_path}",
+ "--dry-run=client",
+ "-o",
+ "json",
+ ]
+ created = run_kubectl(
+ create,
+ runner=runner,
+ pass_fds=(shim_fd,),
+ action="shim Secret rendering",
+ )
+ run_kubectl(
+ [*kubectl_prefix(args.context, args.kubeconfig), "apply", "-f", "-"],
+ runner=runner,
+ input_bytes=created.stdout,
+ action="shim Secret creation",
+ )
+
+ configmap = {
+ "apiVersion": "v1",
+ "kind": "ConfigMap",
+ "metadata": {"name": job_name, "namespace": CONTROL_NAMESPACE},
+ "data": {
+ "deliver-credentials.cjs": DELIVERY_SCRIPT.read_text(encoding="utf-8")
+ },
+ }
+ apply_json(
+ context=args.context,
+ kubeconfig=args.kubeconfig,
+ manifest=configmap,
+ runner=runner,
+ action="delivery ConfigMap creation",
+ )
+ job = build_job(
+ name=job_name,
+ image=args.image,
+ credential=args.credential,
+ credential_secret=credential_secret,
+ shim_secret=shim_secret,
+ actor_namespace=args.actor_namespace,
+ actor_name=args.actor_name,
+ )
+ apply_json(
+ context=args.context,
+ kubeconfig=args.kubeconfig,
+ manifest=job,
+ runner=runner,
+ action="delivery Job creation",
+ )
+ run_kubectl(
+ [
+ *kubectl_prefix(args.context, args.kubeconfig),
+ "wait",
+ "--namespace",
+ CONTROL_NAMESPACE,
+ f"job/{job_name}",
+ "--for=condition=complete",
+ "--timeout=75s",
+ ],
+ runner=runner,
+ timeout=80,
+ action="credential delivery Job",
+ )
+ print(
+ f"credential delivered for {args.credential} to "
+ f"{args.actor_namespace}/{args.actor_name}"
+ )
+ except Exception:
+ failed = True
+ raise
+ finally:
+ try:
+ delete_delivery_resources(
+ context=args.context,
+ kubeconfig=args.kubeconfig,
+ names=resource_names,
+ runner=runner,
+ )
+ except RuntimeError:
+ if not failed:
+ raise
+ print("credential delivery resource cleanup failed", file=sys.stderr)
+
+
+def main() -> None:
+ args = parse_args()
+ try:
+ deliver_credentials(args)
+ except RuntimeError as exc:
+ print(f"gate5_deliver_credentials failed: {exc}", file=sys.stderr)
+ sys.exit(1)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/backend/tests/runtime/test_gate5_credentials.py b/backend/tests/runtime/test_gate5_credentials.py
new file mode 100644
index 0000000..ca0ff6c
--- /dev/null
+++ b/backend/tests/runtime/test_gate5_credentials.py
@@ -0,0 +1,138 @@
+"""Fake-backed checks for the private, actor-targeted credential delivery path."""
+
+import json
+import os
+from contextlib import redirect_stdout
+from io import StringIO
+from pathlib import Path
+import tempfile
+import unittest
+from types import SimpleNamespace
+
+from scripts import gate5_deliver_credentials
+
+
+class Gate5CredentialDeliveryTests(unittest.TestCase):
+ def setUp(self):
+ self.tempdir = tempfile.TemporaryDirectory()
+ self.addCleanup(self.tempdir.cleanup)
+ self.root = Path(self.tempdir.name)
+ self.credential_path = self.root / "private-provider-file"
+ self.credential_value = b"fixture-provider-credential-never-for-argv-or-logs"
+ self.credential_path.write_bytes(self.credential_value)
+ self.state_path = self.root / "private-state.json"
+ self.state_path.write_text(
+ json.dumps(
+ {
+ "context": "kind-substrate-preview",
+ "atespace": "native-claude",
+ "actor_name": "claude-final",
+ "actor_uid": "actor-uid-fixture",
+ "shim_token": "fixture-shim-token-with-at-least-32-characters",
+ }
+ ),
+ encoding="utf-8",
+ )
+ self.handover_path = self.root / "shared-cluster.md"
+ self.handover_path.write_text("**Handover:** done — 2026-09-23 17:00 UTC\n", encoding="utf-8")
+ self.args = SimpleNamespace(
+ context="kind-substrate-preview",
+ kubeconfig="/fixture/kubeconfig",
+ actor_namespace="native-claude",
+ actor_name="claude-final",
+ state_file=str(self.state_path),
+ credential="claude",
+ claude_token_file=str(self.credential_path),
+ codex_auth_file="/fixture/not-used",
+ image=gate5_deliver_credentials.DEFAULT_IMAGE,
+ )
+
+ def fake_runner(self, calls, data_seen):
+ def runner(argv, **kwargs):
+ calls.append((argv, kwargs))
+ if "create" in argv and "secret" in argv:
+ fds = kwargs.get("pass_fds", ())
+ self.assertEqual(len(fds), 1)
+ data_seen.append(os.pread(fds[0], 1024 * 1024, 0))
+ return SimpleNamespace(stdout=b'{"apiVersion":"v1","kind":"Secret"}')
+ return SimpleNamespace(stdout=b"completed")
+
+ return runner
+
+ def test_credentials_flow_through_control_secret_and_authenticated_job_without_path_or_value_in_argv(self):
+ calls = []
+ data_seen = []
+ output = StringIO()
+ with redirect_stdout(output):
+ gate5_deliver_credentials.deliver_credentials(
+ self.args,
+ runner=self.fake_runner(calls, data_seen),
+ handover_note=self.handover_path,
+ )
+
+ self.assertEqual(data_seen[0], self.credential_value)
+ self.assertNotIn(self.args.state_file.encode(), b"".join(data_seen))
+ self.assertEqual(len(calls), 8)
+ for argv, _kwargs in calls:
+ self.assertEqual(argv[0], "kubectl")
+ self.assertIn("--context", argv)
+ self.assertIn("kind-substrate-preview", argv)
+ self.assertIn("--kubeconfig", argv)
+ self.assertIn("/fixture/kubeconfig", argv)
+ joined = " ".join(map(str, argv))
+ self.assertNotIn(str(self.credential_path), joined)
+ self.assertNotIn(self.credential_value.decode(), joined)
+ self.assertNotIn(self.args.state_file, joined)
+
+ secret_argv = calls[0][0]
+ from_file = next(part for part in secret_argv if part.startswith("--from-file="))
+ self.assertRegex(from_file, r"^--from-file=credential=/proc/self/fd/\d+$")
+ job_manifest = json.loads(calls[5][1]["input"])
+ self.assertEqual(job_manifest["kind"], "Job")
+ self.assertEqual(job_manifest["metadata"]["namespace"], "mainloop-control")
+ pod_spec = job_manifest["spec"]["template"]["spec"]
+ self.assertEqual(pod_spec["securityContext"]["fsGroup"], 10001)
+ self.assertEqual(pod_spec["volumes"][1]["secret"]["defaultMode"], 0o440)
+ container = pod_spec["containers"][0]
+ self.assertEqual(container["env"][0], {"name": "CREDENTIAL_KIND", "value": "claude"})
+ self.assertEqual(container["env"][1], {"name": "ACTOR_NAMESPACE", "value": "native-claude"})
+ self.assertNotIn(self.credential_value.decode(), str(job_manifest))
+ self.assertNotIn(str(self.credential_path), str(job_manifest))
+ self.assertEqual(
+ output.getvalue().strip(),
+ "credential delivered for claude to native-claude/claude-final",
+ )
+
+ def test_pending_handover_refuses_before_any_kubectl_call(self):
+ calls = []
+ self.handover_path.write_text("**Handover:** pending\n", encoding="utf-8")
+ with self.assertRaisesRegex(RuntimeError, "handover is not done"):
+ gate5_deliver_credentials.deliver_credentials(
+ self.args,
+ runner=self.fake_runner(calls, []),
+ handover_note=self.handover_path,
+ )
+ self.assertEqual(calls, [])
+
+ def test_actor_ownership_mismatch_refuses_before_any_kubectl_call(self):
+ calls = []
+ state = json.loads(self.state_path.read_text(encoding="utf-8"))
+ state["atespace"] = "native-codex"
+ self.state_path.write_text(json.dumps(state), encoding="utf-8")
+ with self.assertRaisesRegex(RuntimeError, "does not own"):
+ gate5_deliver_credentials.deliver_credentials(
+ self.args,
+ runner=self.fake_runner(calls, []),
+ handover_note=self.handover_path,
+ )
+ self.assertEqual(calls, [])
+
+ def test_credentials_are_not_wired_into_the_golden_setup_flow(self):
+ setup_source = Path(gate5_deliver_credentials.__file__).with_name("gate5_setup.py")
+ source = setup_source.read_text(encoding="utf-8")
+ self.assertNotIn("gate5_deliver_credentials", source)
+ self.assertNotIn("deliver_credentials(", source)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
index fae870c..f7724a4 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
+++ b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
@@ -25,10 +25,12 @@ COPY claude /usr/local/bin/claude
COPY codex /usr/local/bin/codex
COPY codex-code-mode-host /usr/local/bin/codex-code-mode-host
COPY bin/agentctl bin/mainloop /usr/local/bin/
+COPY bin/start-native-agent /usr/local/bin/start-native-agent
+COPY bin/prepare-native-agent-config.cjs /usr/local/bin/prepare-native-agent-config.cjs
COPY agent-config /etc/agent-config
COPY exec-shim.js /usr/local/bin/exec-shim.js
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
-RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/agentctl /usr/local/bin/mainloop \
+RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/agentctl /usr/local/bin/mainloop /usr/local/bin/start-native-agent \
&& mkdir -p /work && chown -R agent:agent /work
ENV EXEC_SHIM=/usr/local/bin/exec-shim.js
ENV HOME=/home/agent
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/bin/prepare-native-agent-config.cjs b/spikes/substrate-workspace-adapter/live-agent-image/bin/prepare-native-agent-config.cjs
new file mode 100644
index 0000000..8217864
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/bin/prepare-native-agent-config.cjs
@@ -0,0 +1,51 @@
+'use strict';
+
+const fs = require('node:fs');
+const path = require('node:path');
+
+const home = process.env.HOME || '/home/agent';
+const workspace = process.env.WORKSPACE_PATH || '/work/repo';
+const codexHome = process.env.CODEX_HOME || path.join(home, '.codex');
+const claudeConfig = path.join(home, '.claude.json');
+const claudeSettingsDir = path.join(home, '.claude');
+const claudeSettings = path.join(claudeSettingsDir, 'settings.json');
+const codexConfig = path.join(codexHome, 'config.toml');
+
+fs.mkdirSync(claudeSettingsDir, { recursive: true, mode: 0o700 });
+fs.mkdirSync(codexHome, { recursive: true, mode: 0o700 });
+
+if (!fs.existsSync(claudeConfig) || fs.statSync(claudeConfig).size === 0) {
+ fs.writeFileSync(claudeConfig, `${JSON.stringify({
+ hasCompletedOnboarding: true,
+ numStartups: 1,
+ theme: 'dark',
+ projects: {
+ [workspace]: {
+ hasTrustDialogAccepted: true,
+ hasCompletedProjectOnboarding: true,
+ allowedTools: [],
+ },
+ },
+ }, null, 2)}\n`, { mode: 0o600 });
+}
+if (!fs.existsSync(claudeSettings) || fs.statSync(claudeSettings).size === 0) {
+ fs.writeFileSync(claudeSettings, '{"skipDangerousModePermissionPrompt":true}\n', { mode: 0o600 });
+}
+
+if (!fs.existsSync(codexConfig) || fs.statSync(codexConfig).size === 0) {
+ const quotedWorkspace = JSON.stringify(workspace);
+ const defaults = [
+ 'check_for_update_on_startup = false',
+ '',
+ '[tui]',
+ 'theme = "dark"',
+ '',
+ `[projects.${quotedWorkspace}]`,
+ 'trust_level = "trusted"',
+ '',
+ '[notice]',
+ 'hide_rate_limit_model_nudge = true',
+ '',
+ ].join('\n');
+ fs.writeFileSync(codexConfig, defaults, { mode: 0o600 });
+}
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/bin/start-native-agent b/spikes/substrate-workspace-adapter/live-agent-image/bin/start-native-agent
new file mode 100755
index 0000000..0a61910
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/live-agent-image/bin/start-native-agent
@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+# Start one native CLI in the actor's persistent Herdr shell pane after credential delivery.
+set -euo pipefail
+
+kind="${1:?usage: start-native-agent claude|codex}"
+if [[ $# -ne 1 ]]; then
+ echo 'usage: start-native-agent claude|codex' >&2
+ exit 2
+fi
+
+workspace="${WORKSPACE_PATH:-/work/repo}"
+cd "${workspace}"
+
+case "${kind}" in
+ claude)
+ token_file="${HOME}/.mainloop/claude-token"
+ if [[ ! -s "${token_file}" ]]; then
+ echo 'Claude credential is not installed' >&2
+ exit 1
+ fi
+ CLAUDE_CODE_OAUTH_TOKEN="$(tr -d ' \r\n' <"${token_file}")"
+ if [[ -z "${CLAUDE_CODE_OAUTH_TOKEN}" ]]; then
+ echo 'Claude credential is empty' >&2
+ exit 1
+ fi
+ export CLAUDE_CODE_OAUTH_TOKEN
+ export DISABLE_TELEMETRY=1
+ export DISABLE_ERROR_REPORTING=1
+ export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1
+ export DISABLE_AUTOUPDATER=1
+ exec claude \
+ --dangerously-skip-permissions \
+ --append-system-prompt-file "${AGENT_SYSTEM_PROMPT_FILE:-/etc/agent-config/mainloop-system.txt}"
+ ;;
+ codex)
+ auth_file="${CODEX_HOME:-${HOME}/.codex}/auth.json"
+ if [[ ! -s "${auth_file}" ]]; then
+ echo 'Codex credential is not installed' >&2
+ exit 1
+ fi
+ exec codex --dangerously-bypass-approvals-and-sandbox
+ ;;
+ *)
+ echo 'unsupported native CLI' >&2
+ exit 2
+ ;;
+esac
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
index 80ee2f0..82dc5e1 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
+++ b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
@@ -10,24 +10,9 @@
# and provider credentials only after the final actor is RUNNING. The golden actor stays clean.
# The template controller checks `/healthz` before accepting the golden actor.
set -eu
-mkdir -p "${HOME}" "${HOME}/.claude" "${CODEX_HOME}"
-
-# Claude Code: onboarding done, workspace trusted, bypass-permissions warning accepted.
-if [[ ! -s "${HOME}/.claude.json" ]]; then
- jq -n --arg p "${WORKSPACE_PATH}" '{
- hasCompletedOnboarding: true,
- numStartups: 1,
- theme: "dark",
- projects: {($p): {hasTrustDialogAccepted: true, hasCompletedProjectOnboarding: true, allowedTools: []}}
- }' >"${HOME}/.claude.json"
-fi
-[[ -s "${HOME}/.claude/settings.json" ]] || echo '{"skipDangerousModePermissionPrompt": true}' >"${HOME}/.claude/settings.json"
-
-# Codex: trust the workspace.
-if [[ ! -s "${CODEX_HOME}/config.toml" ]]; then
- printf '[projects."%s"]\ntrust_level = "trusted"\n' "${WORKSPACE_PATH}" >"${CODEX_HOME}/config.toml"
-fi
-grep -q '^\[notice\]' "${CODEX_HOME}/config.toml" || printf '\n[notice]\nhide_rate_limit_model_nudge = true\n' >>"${CODEX_HOME}/config.toml"
+# Seed supported first-run defaults before either CLI is started. Native sessions start only
+# through start-native-agent, after the final actor receives its credential.
+node /usr/local/bin/prepare-native-agent-config.cjs
mkdir -p "${WORKSPACE_PATH}"
[[ -d "${WORKSPACE_PATH}/.git" ]] || git -C "${WORKSPACE_PATH}" init -q
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
index 11c19aa..be75d1c 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
+++ b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
@@ -1,10 +1,7 @@
-// Minimal generic command executor for the preview-gate spike: POST /run { command } pastes
-// `command` as literal text into a real Herdr shell pane via `herdr pane run` (fire-and-forget;
-// the pane is a real bash shell, so this is a real shell write, not a purpose-built edit
-// endpoint). GET /read returns the pane's current terminal buffer (`herdr pane read`), since
-// `pane run` itself never captures output. Stands in for a credentialed native agent's own Bash
-// tool -- see entrypoint.sh and docs/spikes/substrate-workspace-adapter.md for why a real agent
-// could not be used here.
+// Minimal preview-gate command executor: POST /run { command } pastes `command` as literal
+// text into a real Herdr shell pane, and GET /read returns its current terminal buffer.
+// Authenticated POST /credential { name, contents } writes only one of the fixed credential
+// files used by the native-agent launchers. Request bodies are never logged.
// Listens on port 8090, separate from the Vite dev server's port 80. Reached only through
// atenet-router's arbitrary-port CONNECT tunnel with the ate-target-actor header (see
// docs/api-guide.md "Workload Connectivity"), from test orchestration on the host -- never
@@ -20,12 +17,21 @@ const PANE_ID = process.env.EXEC_SHIM_PANE_ID;
const SESSION = process.env.HERDR_SESSION;
const HEALTH_COMMAND_TIMEOUT_MS = 1500;
const HEALTH_CACHE_MS = 3000;
+const MAX_REQUEST_BODY_BYTES = 64 * 1024;
+const MAX_CREDENTIAL_REQUEST_BODY_BYTES = 512 * 1024;
+const MAX_CREDENTIAL_BYTES = 64 * 1024;
if (!PANE_ID || !SESSION) {
console.error('exec-shim: EXEC_SHIM_PANE_ID and HERDR_SESSION are required');
process.exit(1);
}
const tokenPath = path.join(process.env.HOME || '/home/agent', '.mainloop', 'exec-shim-token');
+const homePath = path.resolve(process.env.HOME || '/home/agent');
+const codexHomePath = path.resolve(process.env.CODEX_HOME || path.join(homePath, '.codex'));
+const credentialPaths = new Map([
+ ['claude-token', path.join(homePath, '.mainloop', 'claude-token')],
+ ['codex-auth', path.join(codexHomePath, 'auth.json')],
+]);
let bearerToken = null;
try {
bearerToken = fs.readFileSync(tokenPath, 'utf8');
@@ -46,13 +52,22 @@ function unauthorized(res) {
res.writeHead(401, { 'content-type': 'text/plain' }).end('unauthorized');
}
-function requestBody(req, onBody) {
- let body = '';
+function requestBody(req, res, onBody, maxBytes = MAX_REQUEST_BODY_BYTES) {
+ const chunks = [];
+ let size = 0;
+ let tooLarge = false;
req.on('data', (chunk) => {
- body += chunk;
- if (body.length > 65536) req.destroy();
+ size += chunk.length;
+ if (size > maxBytes) {
+ if (!tooLarge) res.writeHead(413).end('request too large');
+ tooLarge = true;
+ return;
+ }
+ if (!tooLarge) chunks.push(chunk);
+ });
+ req.on('end', () => {
+ if (!tooLarge) onBody(Buffer.concat(chunks).toString('utf8'));
});
- req.on('end', () => onBody(body));
}
function installToken(token) {
@@ -75,34 +90,38 @@ function installToken(token) {
return true;
}
-function installCodexAuth(contentBase64) {
- if (typeof contentBase64 !== 'string') return false;
- const contents = Buffer.from(contentBase64, 'base64');
+function installCredential(name, contents) {
+ const destination = credentialPaths.get(name);
+ if (!destination) return false;
if (
- contents.length === 0 ||
- contents.length > 65536 ||
- contents.toString('base64') !== contentBase64
- ) return false;
- let auth;
- try {
- auth = JSON.parse(contents.toString('utf8'));
- } catch {
- return false;
+ typeof contents !== 'string' ||
+ !contents ||
+ Buffer.byteLength(contents, 'utf8') > MAX_CREDENTIAL_BYTES
+ ) return null;
+ if (name === 'codex-auth') {
+ let auth;
+ try {
+ auth = JSON.parse(contents);
+ } catch {
+ return null;
+ }
+ if (!auth || typeof auth !== 'object' || Array.isArray(auth)) return null;
}
- if (!auth || typeof auth !== 'object' || Array.isArray(auth)) return false;
- const codexHome = process.env.CODEX_HOME || path.join(process.env.HOME || '/home/agent', '.codex');
- fs.mkdirSync(codexHome, { recursive: true, mode: 0o700 });
- fs.chmodSync(codexHome, 0o700);
- const destination = path.join(codexHome, 'auth.json');
+
+ const directory = path.dirname(destination);
+ fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+ const directoryStat = fs.lstatSync(directory);
+ if (!directoryStat.isDirectory() || directoryStat.isSymbolicLink()) return null;
+ fs.chmodSync(directory, 0o700);
let fd;
try {
fd = fs.openSync(destination, 'wx', 0o600);
} catch (err) {
- if (err.code === 'EEXIST') return null;
+ if (err.code === 'EEXIST') return 'exists';
throw err;
}
try {
- fs.writeFileSync(fd, contents);
+ fs.writeFileSync(fd, contents, 'utf8');
fs.fsyncSync(fd);
fs.closeSync(fd);
fs.chmodSync(destination, 0o600);
@@ -195,7 +214,7 @@ const server = http.createServer((req, res) => {
res.writeHead(409).end('token already set');
return;
}
- requestBody(req, (body) => {
+ requestBody(req, res, (body) => {
let token;
try {
token = JSON.parse(body).token;
@@ -225,31 +244,39 @@ const server = http.createServer((req, res) => {
herdr(['pane', 'read', PANE_ID], res);
return;
}
- if (req.method === 'POST' && req.url === '/write-codex-auth') {
+ if (req.method === 'POST' && req.url === '/credential') {
if (bearerToken === null || !authorized(req)) return unauthorized(res);
- requestBody(req, (body) => {
- let contentBase64;
+ requestBody(req, res, (body) => {
+ let document;
try {
- contentBase64 = JSON.parse(body).contentBase64;
+ document = JSON.parse(body);
} catch {
res.writeHead(400).end('invalid json');
return;
}
+ if (!document || typeof document !== 'object' || Array.isArray(document)) {
+ res.writeHead(400).end('invalid credential');
+ return;
+ }
try {
- const installed = installCodexAuth(contentBase64);
+ const installed = installCredential(document.name, document.contents);
if (installed === null) {
- res.writeHead(409).end('auth file already exists');
+ res.writeHead(400).end('invalid credential');
return;
}
if (!installed) {
- res.writeHead(400).end('invalid auth payload');
+ res.writeHead(403).end('credential name is not allowlisted');
return;
}
- res.writeHead(201).end('Codex auth installed');
+ if (installed === 'exists') {
+ res.writeHead(409).end('credential file already exists');
+ return;
+ }
+ res.writeHead(201).end('credential stored');
} catch {
- res.writeHead(500).end('Codex auth could not be stored');
+ res.writeHead(500).end('credential could not be stored');
}
- });
+ }, MAX_CREDENTIAL_REQUEST_BODY_BYTES);
return;
}
if (req.method !== 'POST' || req.url !== '/run') {
@@ -257,7 +284,7 @@ const server = http.createServer((req, res) => {
return;
}
if (!authorized(req)) return unauthorized(res);
- requestBody(req, (body) => {
+ requestBody(req, res, (body) => {
let command;
try {
command = JSON.parse(body).command;
diff --git a/spikes/substrate-workspace-adapter/tests/deliver-credentials.test.js b/spikes/substrate-workspace-adapter/tests/deliver-credentials.test.js
new file mode 100644
index 0000000..47c2a01
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tests/deliver-credentials.test.js
@@ -0,0 +1,214 @@
+'use strict';
+
+import assert from 'node:assert/strict';
+import fs from 'node:fs';
+import http from 'node:http';
+import { createRequire } from 'node:module';
+import os from 'node:os';
+import path from 'node:path';
+import { spawnSync } from 'node:child_process';
+import { once } from 'node:events';
+import { test } from 'node:test';
+import { fileURLToPath } from 'node:url';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const require = createRequire(import.meta.url);
+const { buildCredentialPayload, deliverFromMountedFiles, postViaRouter } = require(
+ path.resolve(__dirname, '../tools/phase4/deliver-credentials.cjs'),
+);
+const liveAgentImage = path.resolve(__dirname, '../live-agent-image');
+
+test('credential payload uses the fixed shim allowlist and validates Codex auth JSON', () => {
+ assert.deepEqual(buildCredentialPayload('claude', 'fixture token\r\n'), {
+ name: 'claude-token',
+ contents: 'fixturetoken',
+ });
+ assert.deepEqual(buildCredentialPayload('codex', '{"access_token":"fixture"}'), {
+ name: 'codex-auth',
+ contents: '{"access_token":"fixture"}',
+ });
+ assert.throws(() => buildCredentialPayload('../../etc/passwd', 'fixture'), /unsupported/);
+ assert.throws(() => buildCredentialPayload('codex', 'not-json'), SyntaxError);
+});
+
+test('control delivery reads mounted paths and sends credential contents only in the request payload', async () => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'phase4-delivery-'));
+ const credentialFile = path.join(root, 'credential');
+ const shimTokenFile = path.join(root, 'shim-token');
+ fs.writeFileSync(credentialFile, 'fixture-claude-token\n', { mode: 0o600 });
+ fs.writeFileSync(shimTokenFile, 'fixture-shim-token-with-at-least-32-characters', { mode: 0o600 });
+ const calls = [];
+ try {
+ const result = await deliverFromMountedFiles({
+ kind: 'claude',
+ credentialFile,
+ shimTokenFile,
+ namespace: 'native-claude',
+ actor: 'claude-final',
+ request: async (request) => {
+ calls.push(request);
+ return 201;
+ },
+ });
+ assert.deepEqual(result, {
+ kind: 'claude',
+ namespace: 'native-claude',
+ actor: 'claude-final',
+ });
+ assert.equal(calls.length, 1);
+ assert.equal(calls[0].payload.name, 'claude-token');
+ assert.equal(calls[0].payload.contents, 'fixture-claude-token');
+ assert.equal(calls[0].token, 'fixture-shim-token-with-at-least-32-characters');
+ assert.equal(calls[0].namespace, 'native-claude');
+ assert.equal(calls[0].actor, 'claude-final');
+ } finally {
+ fs.rmSync(root, { recursive: true, force: true });
+ }
+});
+
+test('router delivery authenticates the CONNECT target and posts the body without logging it', async (t) => {
+ const received = {};
+ const proxy = http.createServer();
+ proxy.on('connect', (request, socket, head) => {
+ received.target = request.url;
+ received.actorHeader = request.headers['ate-target-actor'];
+ if (head.length) socket.unshift(head);
+ socket.write('HTTP/1.1 200 Connection Established\r\n\r\n');
+ let bytes = Buffer.alloc(0);
+ socket.on('data', (chunk) => {
+ bytes = Buffer.concat([bytes, chunk]);
+ const boundary = bytes.indexOf('\r\n\r\n');
+ if (boundary === -1) return;
+ const headers = bytes.subarray(0, boundary).toString('latin1');
+ const length = Number(/^content-length:\s*(\d+)\s*$/im.exec(headers)?.[1]);
+ if (!Number.isFinite(length) || bytes.length < boundary + 4 + length) return;
+ received.requestHeaders = headers;
+ received.payload = JSON.parse(bytes.subarray(boundary + 4, boundary + 4 + length).toString('utf8'));
+ socket.end('HTTP/1.1 201 Created\r\nContent-Length: 0\r\nConnection: close\r\n\r\n');
+ });
+ });
+ proxy.listen(0, '127.0.0.1');
+ await once(proxy, 'listening');
+ t.after(() => proxy.close());
+
+ const { port } = proxy.address();
+ const status = await postViaRouter({
+ host: '127.0.0.1',
+ port,
+ namespace: 'native-codex',
+ actor: 'codex-final',
+ token: 'fixture-shim-token-with-at-least-32-characters',
+ payload: { name: 'codex-auth', contents: '{"fixture":"provider"}' },
+ });
+ assert.equal(status, 201);
+ assert.equal(received.target, 'actor-upstream:8090');
+ assert.equal(received.actorHeader, 'native-codex/codex-final');
+ assert.match(received.requestHeaders, /Authorization: Bearer fixture-shim-token-with-at-least-32-characters/i);
+ assert.deepEqual(received.payload, { name: 'codex-auth', contents: '{"fixture":"provider"}' });
+});
+
+test('native-agent launcher reads Claude auth from its file into the process environment only', (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'phase4-launch-claude-'));
+ const home = path.join(root, 'home');
+ const bin = path.join(root, 'bin');
+ const workspace = path.join(root, 'repo');
+ const tokenFile = path.join(home, '.mainloop', 'claude-token');
+ const tokenCapture = path.join(root, 'token.capture');
+ const argsCapture = path.join(root, 'args.capture');
+ fs.mkdirSync(path.dirname(tokenFile), { recursive: true });
+ fs.mkdirSync(bin);
+ fs.mkdirSync(workspace);
+ fs.writeFileSync(tokenFile, 'fixture-claude-oauth-value\r\n', { mode: 0o600 });
+ fs.writeFileSync(
+ path.join(bin, 'claude'),
+ '#!/bin/sh\nprintf "%s" "$CLAUDE_CODE_OAUTH_TOKEN" >"$TOKEN_CAPTURE"\nprintf "%s\\n" "$DISABLE_TELEMETRY" "$DISABLE_ERROR_REPORTING" "$CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC" "$DISABLE_AUTOUPDATER" >"$FLAGS_CAPTURE"\nprintf "%s\\n" "$@" >"$ARGS_CAPTURE"\n',
+ { mode: 0o700 },
+ );
+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+
+ const result = spawnSync('/bin/bash', [path.join(liveAgentImage, 'bin/start-native-agent'), 'claude'], {
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ HOME: home,
+ PATH: `${bin}:${process.env.PATH}`,
+ WORKSPACE_PATH: workspace,
+ TOKEN_CAPTURE: tokenCapture,
+ FLAGS_CAPTURE: path.join(root, 'flags.capture'),
+ ARGS_CAPTURE: argsCapture,
+ AGENT_SYSTEM_PROMPT_FILE: path.join(root, 'system-prompt.fixture'),
+ },
+ });
+ assert.equal(result.status, 0, result.stderr);
+ assert.equal(fs.readFileSync(tokenCapture, 'utf8'), 'fixture-claude-oauth-value');
+ assert.deepEqual(fs.readFileSync(path.join(root, 'flags.capture'), 'utf8').trim().split('\n'), [
+ '1', '1', '1', '1',
+ ]);
+ const args = fs.readFileSync(argsCapture, 'utf8');
+ assert.match(args, /--dangerously-skip-permissions/);
+ assert.match(args, /--append-system-prompt-file/);
+ assert.equal(args.includes('fixture-claude-oauth-value'), false);
+ assert.equal(result.stdout.includes('fixture-claude-oauth-value'), false);
+});
+
+test('native-agent launcher starts Codex only when its installed auth file exists', (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'phase4-launch-codex-'));
+ const home = path.join(root, 'home');
+ const codexHome = path.join(home, '.codex');
+ const bin = path.join(root, 'bin');
+ const workspace = path.join(root, 'repo');
+ const argsCapture = path.join(root, 'args.capture');
+ fs.mkdirSync(codexHome, { recursive: true });
+ fs.mkdirSync(bin);
+ fs.mkdirSync(workspace);
+ fs.writeFileSync(path.join(codexHome, 'auth.json'), '{"fixture":"codex-auth"}', { mode: 0o600 });
+ fs.writeFileSync(path.join(bin, 'codex'), '#!/bin/sh\nprintf "%s\\n" "$@" >"$ARGS_CAPTURE"\n', { mode: 0o700 });
+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+
+ const env = {
+ ...process.env,
+ HOME: home,
+ CODEX_HOME: codexHome,
+ PATH: `${bin}:${process.env.PATH}`,
+ WORKSPACE_PATH: workspace,
+ ARGS_CAPTURE: argsCapture,
+ };
+ const result = spawnSync('/bin/bash', [path.join(liveAgentImage, 'bin/start-native-agent'), 'codex'], {
+ encoding: 'utf8',
+ env,
+ });
+ assert.equal(result.status, 0, result.stderr);
+ assert.equal(fs.readFileSync(argsCapture, 'utf8').trim(), '--dangerously-bypass-approvals-and-sandbox');
+ assert.equal(result.stdout.includes('fixture'), false);
+
+ fs.rmSync(path.join(codexHome, 'auth.json'));
+ const missing = spawnSync('/bin/bash', [path.join(liveAgentImage, 'bin/start-native-agent'), 'codex'], {
+ encoding: 'utf8',
+ env,
+ });
+ assert.equal(missing.status, 1);
+ assert.equal(missing.stderr.includes('auth.json'), false);
+});
+
+test('golden boot seeds trusted workspaces, themes, and disabled update checks without starting a CLI', (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'phase4-config-'));
+ const home = path.join(root, 'home');
+ const codexHome = path.join(home, '.codex');
+ const workspace = path.join(root, 'repo');
+ const result = spawnSync(process.execPath, [path.join(liveAgentImage, 'bin/prepare-native-agent-config.cjs')], {
+ encoding: 'utf8',
+ env: { ...process.env, HOME: home, CODEX_HOME: codexHome, WORKSPACE_PATH: workspace },
+ });
+ t.after(() => fs.rmSync(root, { recursive: true, force: true }));
+ assert.equal(result.status, 0, result.stderr);
+
+ const claudeConfig = JSON.parse(fs.readFileSync(path.join(home, '.claude.json'), 'utf8'));
+ assert.equal(claudeConfig.hasCompletedOnboarding, true);
+ assert.equal(claudeConfig.theme, 'dark');
+ assert.equal(claudeConfig.projects[workspace].hasTrustDialogAccepted, true);
+ const codexConfig = fs.readFileSync(path.join(codexHome, 'config.toml'), 'utf8');
+ assert.match(codexConfig, /^check_for_update_on_startup = false$/m);
+ assert.match(codexConfig, /^theme = "dark"$/m);
+ assert.ok(codexConfig.includes(`[projects.${JSON.stringify(workspace)}]`));
+ assert.match(codexConfig, /^trust_level = "trusted"$/m);
+});
diff --git a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
index 783e6c8..5006a60 100644
--- a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
+++ b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
@@ -130,8 +130,8 @@ test('shim token gates run/read, is one-time, private, and survives process rest
assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 409);
});
-test('Codex auth write requires an installed shim token and creates a private one-time file', async (t) => {
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-codex-auth-'));
+test('credential delivery requires a shim token and writes only allowlisted private files once', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-credentials-'));
const home = path.join(root, 'home');
const fakeBin = path.join(root, 'bin');
const shimPath = path.join(root, 'exec-shim.js');
@@ -152,32 +152,42 @@ test('Codex auth write requires an installed shim token and creates a private on
fs.rmSync(root, { recursive: true, force: true });
});
+ const claudeContents = 'fixture-claude-token-never-logged';
const authContents = JSON.stringify({ fixture: 'codex-auth-never-logged' });
- const contentBase64 = Buffer.from(authContents).toString('base64');
- const write = (encoded, token) =>
- request(running.port, 'POST', '/write-codex-auth', {
- body: { contentBase64: encoded },
+ const write = (name, contents, token) =>
+ request(running.port, 'POST', '/credential', {
+ body: { name, contents },
...(token === undefined ? {} : { token }),
});
- assert.equal((await write(contentBase64)).status, 401, 'golden actor must reject writes before token installation');
+ assert.equal((await write('codex-auth', authContents)).status, 401, 'golden actor must reject writes before token installation');
assert.equal(fs.existsSync(codexHome), false);
const token = 'fixture-only-codex-shim-token-at-least-thirty-two-chars';
assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 201);
- assert.equal((await write(contentBase64)).status, 401);
- assert.equal((await write(contentBase64, `${token}-wrong`)).status, 401);
- assert.equal((await write(Buffer.from('[]').toString('base64'), token)).status, 400);
- assert.equal((await write('!!!!', token)).status, 400);
-
- assert.equal((await write(contentBase64, token)).status, 201);
+ assert.equal((await write('claude-token', claudeContents)).status, 401);
+ assert.equal((await write('claude-token', claudeContents, `${token}-wrong`)).status, 401);
+ assert.equal((await write('../outside', claudeContents, token)).status, 403);
+ assert.equal(fs.existsSync(path.join(root, 'outside')), false);
+ assert.equal((await write('codex-auth', '[]', token)).status, 400);
+ assert.equal((await write('codex-auth', 'not-json', token)).status, 400);
+ assert.equal((await request(running.port, 'POST', '/write-codex-auth', { token, body: {} })).status, 404);
+
+ assert.equal((await write('claude-token', claudeContents, token)).status, 201);
+ const claudePath = path.join(home, '.mainloop', 'claude-token');
+ assert.equal(fs.readFileSync(claudePath, 'utf8'), claudeContents);
+ assert.equal(fs.statSync(claudePath).mode & 0o777, 0o600);
+ assert.equal(fs.statSync(path.dirname(claudePath)).mode & 0o777, 0o700);
+ assert.equal((await write('claude-token', 'replacement', token)).status, 409);
+
+ assert.equal((await write('codex-auth', authContents, token)).status, 201);
const authPath = path.join(codexHome, 'auth.json');
assert.equal(fs.readFileSync(authPath, 'utf8'), authContents);
assert.equal(fs.statSync(codexHome).mode & 0o777, 0o700);
assert.equal(fs.statSync(authPath).mode & 0o777, 0o600);
- assert.equal((await write(contentBase64, token)).status, 409, 'auth file must not be overwritten');
+ assert.equal((await write('codex-auth', authContents, token)).status, 409, 'auth file must not be overwritten');
+ assert.equal(running.output().includes(claudeContents), false);
assert.equal(running.output().includes(authContents), false);
- assert.equal(running.output().includes(contentBase64), false);
});
test('healthz bounds Herdr calls and reuses a recent successful check', async (t) => {
diff --git a/spikes/substrate-workspace-adapter/tools/phase4/deliver-credentials.cjs b/spikes/substrate-workspace-adapter/tools/phase4/deliver-credentials.cjs
new file mode 100644
index 0000000..5ebb55e
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tools/phase4/deliver-credentials.cjs
@@ -0,0 +1,155 @@
+'use strict';
+
+const fs = require('node:fs');
+const http = require('node:http');
+
+const CREDENTIALS = Object.freeze({
+ claude: Object.freeze({ name: 'claude-token' }),
+ codex: Object.freeze({ name: 'codex-auth' }),
+});
+
+function validateActorIdentity(namespace, actor) {
+ const dnsLabel = /^[a-z0-9](?:[-a-z0-9]*[a-z0-9])?$/;
+ if (!dnsLabel.test(namespace || '') || !dnsLabel.test(actor || '')) {
+ throw new Error('invalid actor identity');
+ }
+}
+
+function buildCredentialPayload(kind, rawContents) {
+ const credential = CREDENTIALS[kind];
+ if (!credential) throw new Error('unsupported credential kind');
+ if (typeof rawContents !== 'string' || !rawContents) {
+ throw new Error('empty credential');
+ }
+ const contents = kind === 'claude' ? rawContents.replace(/[ \r\n]/g, '') : rawContents;
+ if (!contents || Buffer.byteLength(contents, 'utf8') > 1024 * 1024) {
+ throw new Error('invalid credential size');
+ }
+ if (kind === 'codex') {
+ const parsed = JSON.parse(contents);
+ if (!parsed || typeof parsed !== 'object' || Array.isArray(parsed)) {
+ throw new Error('invalid Codex auth document');
+ }
+ }
+ return { name: credential.name, contents };
+}
+
+function postViaRouter({ host, port, namespace, actor, token, payload, timeoutMs = 10000 }) {
+ validateActorIdentity(namespace, actor);
+ if (typeof token !== 'string' || !token || /[\r\n]/.test(token)) {
+ return Promise.reject(new Error('invalid shim token'));
+ }
+ const body = Buffer.from(JSON.stringify(payload), 'utf8');
+
+ return new Promise((resolve, reject) => {
+ let settled = false;
+ const finish = (error, status) => {
+ if (settled) return;
+ settled = true;
+ if (error) reject(error);
+ else resolve(status);
+ };
+
+ const tunnel = http.request({
+ host,
+ port,
+ method: 'CONNECT',
+ path: 'actor-upstream:8090',
+ headers: { 'ate-target-actor': `${namespace}/${actor}` },
+ });
+ tunnel.setTimeout(timeoutMs, () => tunnel.destroy(new Error('router timed out')));
+ tunnel.once('error', (error) => finish(error));
+ tunnel.once('connect', (response, socket, head) => {
+ if (response.statusCode !== 200) {
+ socket.destroy();
+ finish(new Error('router rejected actor tunnel'));
+ return;
+ }
+ if (head.length) socket.unshift(head);
+
+ let responseHeaders = Buffer.alloc(0);
+ socket.setTimeout(timeoutMs, () => socket.destroy(new Error('shim timed out')));
+ socket.once('error', (error) => finish(error));
+ socket.on('data', (chunk) => {
+ responseHeaders = Buffer.concat([responseHeaders, chunk]);
+ const boundary = responseHeaders.indexOf('\r\n\r\n');
+ if (boundary === -1) return;
+ const statusLine = responseHeaders
+ .subarray(0, boundary)
+ .toString('latin1')
+ .split('\r\n', 1)[0];
+ const match = /^HTTP\/1\.[01] (\d{3})(?: |$)/.exec(statusLine);
+ if (!match) {
+ finish(new Error('invalid shim response'));
+ socket.destroy();
+ return;
+ }
+ finish(null, Number(match[1]));
+ socket.end();
+ });
+
+ const headers = Buffer.from([
+ 'POST /credential HTTP/1.1',
+ 'Host: actor-upstream:8090',
+ `Authorization: Bearer ${token}`,
+ 'Content-Type: application/json',
+ `Content-Length: ${body.length}`,
+ 'Connection: close',
+ '',
+ '',
+ ].join('\r\n'), 'ascii');
+ socket.write(Buffer.concat([headers, body]));
+ });
+ tunnel.end();
+ });
+}
+
+async function deliverFromMountedFiles({
+ kind,
+ credentialFile,
+ shimTokenFile,
+ namespace,
+ actor,
+ host = 'atenet-router.ate-system.svc.cluster.local',
+ port = 8081,
+ request = postViaRouter,
+}) {
+ validateActorIdentity(namespace, actor);
+ const payload = buildCredentialPayload(kind, fs.readFileSync(credentialFile, 'utf8'));
+ const token = fs.readFileSync(shimTokenFile, 'utf8').trim();
+ if (!token || token.length > 4096 || /\s/.test(token)) {
+ throw new Error('invalid shim token');
+ }
+ const status = await request({
+ host,
+ port,
+ namespace,
+ actor,
+ token,
+ payload,
+ });
+ if (status !== 201) throw new Error('shim rejected credential delivery');
+ return { kind, namespace, actor };
+}
+
+async function main() {
+ await deliverFromMountedFiles({
+ kind: process.env.CREDENTIAL_KIND,
+ credentialFile: '/var/run/phase4-credentials/credential',
+ shimTokenFile: '/var/run/phase4-shim-auth/token',
+ namespace: process.env.ACTOR_NAMESPACE,
+ actor: process.env.ACTOR_NAME,
+ host: process.env.ROUTER_HOST || 'atenet-router.ate-system.svc.cluster.local',
+ port: Number(process.env.ROUTER_PORT || '8081'),
+ });
+ process.stdout.write(`credential delivered for ${process.env.CREDENTIAL_KIND}\n`);
+}
+
+if (require.main === module) {
+ main().catch(() => {
+ process.stderr.write('credential delivery failed\n');
+ process.exitCode = 1;
+ });
+}
+
+module.exports = { buildCredentialPayload, deliverFromMountedFiles, postViaRouter };
From 147e2a009414f178dd18d467d78772bed4523e88 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Wed, 23 Sep 2026 23:40:41 +0000
Subject: [PATCH 14/30] spike: run native turns headlessly and close out Phase
5 evidence
Replace the live-agent actor's Herdr pane shim with authenticated,
turn-shaped /turn and /run endpoints. Prompts go to the native CLIs on
stdin; turns return parsed events, native ids and final messages, and a
second concurrent turn for one agent returns 409.
Start only the actor-local shim at boot and make readiness depend on the
shim and workspace. Remove Herdr, agentctl and HERDR_SESSION from the actor
image and templates.
Update setup checks and the spike doc with the headless per-turn design,
the no-attachable-TUI trade-off, gate states and a deferred production
recommendation. The rebuilt headless image is not yet live-proved.
Add sanitized Claude stream-json and Codex JSONL fixtures, fake-backed shim
coverage, and a ContractStore restart case for a persisted recorded delivery
that blocks retry until reconciliation. Record the worker-pool isolation
finding: worker selection is not scoped by atespace or namespace, so
per-tenant pool names and selectors must be unique. Require an explicit
kubeconfig in the credential-delivery CLI.
---
backend/scripts/gate5_deliver_credentials.py | 23 +-
backend/scripts/gate5_setup.py | 48 +-
backend/src/mainloop/runtime/substrate.py | 11 +-
backend/tests/runtime/test_contracts.py | 55 ++
.../tests/runtime/test_gate5_credentials.py | 57 +-
backend/tests/runtime/test_gate5_setup.py | 30 +-
docs/spikes/substrate-workspace-adapter.md | 248 ++++---
.../k8s/actor-template.yaml.tmpl | 11 +-
.../k8s/live-agent-gate-template.yaml.tmpl | 9 +-
.../k8s/router-ingress-policy.yaml | 58 +-
.../live-agent-image/.gitignore | 4 +-
.../live-agent-image/Dockerfile | 22 +-
.../bin/prepare-native-agent-config.cjs | 33 +-
.../live-agent-image/bin/start-native-agent | 88 +--
.../live-agent-image/entrypoint.sh | 49 +-
.../live-agent-image/exec-shim.js | 629 +++++++++++++-----
.../tests/deliver-credentials.test.js | 111 ++--
.../tests/exec-shim.test.js | 434 ++++++++----
.../fixtures/native/claude-stream-json.jsonl | 3 +
.../tests/fixtures/native/codex-jsonl.jsonl | 4 +
.../tools/phase4/deliver-credentials.cjs | 33 +-
.../tools/router-client.js | 17 +-
22 files changed, 1354 insertions(+), 623 deletions(-)
create mode 100644 spikes/substrate-workspace-adapter/tests/fixtures/native/claude-stream-json.jsonl
create mode 100644 spikes/substrate-workspace-adapter/tests/fixtures/native/codex-jsonl.jsonl
diff --git a/backend/scripts/gate5_deliver_credentials.py b/backend/scripts/gate5_deliver_credentials.py
index a420cb9..f088340 100644
--- a/backend/scripts/gate5_deliver_credentials.py
+++ b/backend/scripts/gate5_deliver_credentials.py
@@ -24,7 +24,6 @@
CONTROL_NAMESPACE = "mainloop-control"
DEFAULT_CONTEXT = "kind-substrate-preview"
-DEFAULT_KUBECONFIG = "/tmp/substrate-preview-kubeconfig"
DEFAULT_IMAGE = (
"localhost:5001/live-agent-gate@sha256:"
"8ec007c56b070a2357f20203807197e42ebdb8d0c0e155d57a3bd48fb8d10f57"
@@ -32,9 +31,7 @@
MAX_SECRET_BYTES = 1024 * 1024
ACTOR_NAMESPACES = {"claude": "native-claude", "codex": "native-codex"}
SHARED_CLUSTER_NOTE = (
- Path(__file__).resolve().parents[2]
- / ".tasknotes"
- / "shared-cluster-2026-09-23.md"
+ Path(__file__).resolve().parents[2] / ".tasknotes" / "shared-cluster-2026-09-23.md"
)
DELIVERY_SCRIPT = (
Path(__file__).resolve().parents[2]
@@ -49,7 +46,7 @@
def parse_args() -> argparse.Namespace:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--context", default=DEFAULT_CONTEXT)
- parser.add_argument("--kubeconfig", default=DEFAULT_KUBECONFIG)
+ parser.add_argument("--kubeconfig", required=True)
parser.add_argument("--actor-namespace", required=True)
parser.add_argument("--actor-name", required=True)
parser.add_argument("--state-file", required=True)
@@ -94,12 +91,16 @@ def require_handover(path: Path = SHARED_CLUSTER_NOTE) -> None:
except OSError:
raise RuntimeError("shared-cluster handover note is unavailable") from None
if not re.search(r"(?m)^\*\*Handover:\*\*\s+done(?:\s|$)", note):
- raise RuntimeError("shared-cluster handover is not done; no resources were created")
+ raise RuntimeError(
+ "shared-cluster handover is not done; no resources were created"
+ )
def kubectl_prefix(context: str, kubeconfig: str) -> list[str]:
if context != DEFAULT_CONTEXT:
- raise RuntimeError(f"refusing context {context!r}; expected {DEFAULT_CONTEXT!r}")
+ raise RuntimeError(
+ f"refusing context {context!r}; expected {DEFAULT_CONTEXT!r}"
+ )
return ["kubectl", "--context", context, "--kubeconfig", kubeconfig]
@@ -305,9 +306,7 @@ def build_job(
"secret": {
"secretName": credential_secret,
"defaultMode": 0o440,
- "items": [
- {"key": "credential", "path": "credential"}
- ],
+ "items": [{"key": "credential", "path": "credential"}],
},
},
{
@@ -338,7 +337,9 @@ def deliver_credentials(
if not re.fullmatch(
r"localhost:5001/live-agent-gate@sha256:[0-9a-f]{64}", args.image
):
- raise RuntimeError("delivery image must be the digest-pinned live-agent-gate image")
+ raise RuntimeError(
+ "delivery image must be the digest-pinned live-agent-gate image"
+ )
state = read_private_state(
args.state_file,
diff --git a/backend/scripts/gate5_setup.py b/backend/scripts/gate5_setup.py
index 715398a..fc965dc 100644
--- a/backend/scripts/gate5_setup.py
+++ b/backend/scripts/gate5_setup.py
@@ -3,7 +3,7 @@
Gate 5 (native-session continuity, .tasknotes/plan.md) credential-free harness: registers
the atespace, resolves and applies the WorkerPool, creates a *versioned* ActorTemplate and
waits for its golden snapshot, then creates/resumes one actor and waits for it to become
-RUNNING with its persistent control service (Herdr) confirmed ready -- all without a
+RUNNING with its actor-local headless shim confirmed ready -- all without a
provider credential, a credential server, or a native Claude/Codex session.
Implements recovery step 2 of .tasknotes/gate5-review-and-recovery-plan-2026-09-22.md. That
@@ -15,8 +15,9 @@
credential fetch built into the shared, immutable template, which this script's manifest no
longer has (see spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh).
-Deliberately out of scope here (recovery plan steps 3-4, a separate task): fetching a real
-credential, attaching it to a running actor, and starting a native Claude/Codex session.
+This setup harness remains credential-free: it installs and checks the shim token but does not
+fetch provider credentials or submit a native turn. The image's authenticated /turn endpoint
+runs one headless native CLI process per request; Mainloop owns delivery and retry decisions.
Prerequisites:
kubectl, kubectl-ate, and ko built from the pinned Substrate checkout.
@@ -39,7 +40,7 @@
--egress-tool /tmp/substrate-preview-src/bin/mainloop-egress-tool \\
--egress-deny-all
-Re-running with the same --state-file reconciles the persisted actor uid against the
+ Re-running with the same --state-file reconciles the persisted actor uid against the
cluster's current state rather than blindly creating or resuming; a name collision with a
*different* uid is refused, not silently overwritten.
"""
@@ -219,7 +220,9 @@ def verify_image_manifest(image: str, *, opener=urllib.request.urlopen) -> None:
or not repository
or not re.fullmatch(r"sha256:[0-9a-fA-F]{64}", digest)
):
- raise RuntimeError("--image must be a registry/repository pinned by a full sha256 digest")
+ raise RuntimeError(
+ "--image must be a registry/repository pinned by a full sha256 digest"
+ )
request = urllib.request.Request(
f"http://{registry}/v2/{repository}/manifests/{digest}",
@@ -535,10 +538,10 @@ def ensure_shim_token(
atespace=args.atespace,
actor_name=args.actor_name,
method="GET",
- path="/read",
+ path="/turn/00000000-0000-4000-8000-000000000000",
token=token,
)
- if already_installed != 200:
+ if already_installed != 404:
raise RuntimeError(
"the actor already has a shim token that does not match the private state; "
"manual reconciliation is required"
@@ -549,9 +552,30 @@ def ensure_shim_token(
)
checks = (
- ("missing-token /read", "GET", "/read", None, None, 401),
- ("wrong-token /read", "GET", "/read", f"{token}x", None, 401),
- ("authenticated /read", "GET", "/read", token, None, 200),
+ (
+ "missing-token /turn/",
+ "GET",
+ "/turn/00000000-0000-4000-8000-000000000000",
+ None,
+ None,
+ 401,
+ ),
+ (
+ "wrong-token /turn/",
+ "GET",
+ "/turn/00000000-0000-4000-8000-000000000000",
+ f"{token}x",
+ None,
+ 401,
+ ),
+ (
+ "authenticated unknown /turn/",
+ "GET",
+ "/turn/00000000-0000-4000-8000-000000000000",
+ token,
+ None,
+ 404,
+ ),
(
"second /token",
"POST",
@@ -577,7 +601,9 @@ def ensure_shim_token(
f"shim token acceptance failed at {label} "
f"(HTTP {observed or 'no response'}, expected {expected})"
)
- print("-- per-actor shim token installed; missing/wrong/correct and one-time checks passed")
+ print(
+ "-- per-actor shim token installed; missing/wrong/correct and one-time checks passed"
+ )
async def ensure_golden_template(
diff --git a/backend/src/mainloop/runtime/substrate.py b/backend/src/mainloop/runtime/substrate.py
index f50d063..b063a2c 100644
--- a/backend/src/mainloop/runtime/substrate.py
+++ b/backend/src/mainloop/runtime/substrate.py
@@ -1,11 +1,12 @@
"""Thin Substrate adapter: drives ``kubectl ate`` (control-plane CLI over gRPC to
``ate-api-server``) to manage per-session actors as Mainloop workspaces.
-Unlike ``herdr.py`` (pod-exec into an already-running workspace), this adapter talks to
-Substrate's cluster-level control plane: actors are created, suspended, resumed, reverted and
-deleted through ``ateapipb.Control`` (see the pinned checkout's ``pkg/proto/ateapipb/ateapi.proto``
-and ``cmd/kubectl-ate/internal/cmd/actor.go``). ``TransportError`` means the outcome of the call
-is unknown; callers must inspect the actor before retrying rather than blindly re-creating it.
+The actor image is designed for headless, per-turn native CLI invocations; no Herdr server or
+terminal manager runs inside it. This adapter talks to Substrate's cluster-level control plane:
+actors are created, suspended, resumed, reverted and deleted through ``ateapipb.Control`` (see
+the pinned checkout's ``pkg/proto/ateapipb/ateapi.proto`` and
+``cmd/kubectl-ate/internal/cmd/actor.go``). ``TransportError`` means the outcome of the call is
+unknown; callers must inspect the actor before retrying rather than blindly re-creating it.
"""
from __future__ import annotations
diff --git a/backend/tests/runtime/test_contracts.py b/backend/tests/runtime/test_contracts.py
index aaa0233..78f5f17 100644
--- a/backend/tests/runtime/test_contracts.py
+++ b/backend/tests/runtime/test_contracts.py
@@ -246,6 +246,61 @@ def test_backend_restart_reconciles_persisted_recorded_attempt_before_retry(self
[DeliveryState.FAILED, DeliveryState.RECORDED],
)
+ def test_restart_rehydrates_recorded_prompt_without_replay_or_loss(self):
+ # Model a backend restart with fake durable rows, then load those rows into
+ # a fresh ContractStore before taking ownership. ContractStore has no I/O;
+ # this proves the contract semantics, not a database or transport restart.
+ before_restart = ContractStore(binding())
+ prompt = {**message(), "payload_ref": "fixture://prompts/restart-window"}
+ before_restart.record_message(prompt, 1)
+ before_restart.create_attempt(attempt(), 1)
+ fake_database = {
+ "binding": before_restart.binding.model_dump(mode="json"),
+ "messages": [
+ item.model_dump(mode="json") for item in before_restart.messages
+ ],
+ "attempts": [
+ item.model_dump(mode="json") for item in before_restart.attempts
+ ],
+ }
+
+ after_restart = ContractStore(fake_database["binding"])
+ for persisted_message in fake_database["messages"]:
+ after_restart.record_message(persisted_message, 1)
+ for persisted_attempt in fake_database["attempts"]:
+ after_restart.create_attempt(persisted_attempt, 1)
+
+ binding_after_takeover = after_restart.take_ownership(1)
+ self.assertEqual(binding_after_takeover.ownership_generation, 2)
+ self.assertEqual(after_restart.messages[0].payload_ref, prompt["payload_ref"])
+ pending = after_restart.checkpoint(2).pending_delivery
+ self.assertEqual(len(pending), 1)
+ self.assertEqual(pending[0].state, DeliveryState.RECORDED)
+ with self.assertRaises(ContractError):
+ after_restart.create_attempt(attempt("replay", generation=2), 2)
+
+ retired = after_restart.reconcile(
+ {
+ "attempt_id": "attempt",
+ "binding_id": "binding",
+ "evidence_ref": "fixture://journal/no-prompt-receipt",
+ "observed_at": NOW,
+ "outcome": "not_delivered",
+ },
+ 2,
+ )
+ self.assertEqual(retired.state, DeliveryState.FAILED)
+ self.assertEqual(after_restart.messages[0].payload_ref, prompt["payload_ref"])
+ retry = after_restart.create_attempt(attempt("retry", generation=2), 2)
+ self.assertEqual(retry.logical_message_id, "message")
+ self.assertEqual(
+ [
+ (item.attempt_id, item.state)
+ for item in after_restart.checkpoint(2).pending_delivery
+ ],
+ [("retry", DeliveryState.RECORDED)],
+ )
+
def test_takeover_reconnect_deduplicates_source_event(self):
original = self.store.ingest(attention(), 1)
self.store.take_ownership(1)
diff --git a/backend/tests/runtime/test_gate5_credentials.py b/backend/tests/runtime/test_gate5_credentials.py
index ca0ff6c..7884298 100644
--- a/backend/tests/runtime/test_gate5_credentials.py
+++ b/backend/tests/runtime/test_gate5_credentials.py
@@ -2,12 +2,13 @@
import json
import os
+import tempfile
+import unittest
from contextlib import redirect_stdout
from io import StringIO
from pathlib import Path
-import tempfile
-import unittest
from types import SimpleNamespace
+from unittest.mock import patch
from scripts import gate5_deliver_credentials
@@ -34,7 +35,9 @@ def setUp(self):
encoding="utf-8",
)
self.handover_path = self.root / "shared-cluster.md"
- self.handover_path.write_text("**Handover:** done — 2026-09-23 17:00 UTC\n", encoding="utf-8")
+ self.handover_path.write_text(
+ "**Handover:** done — 2026-09-23 17:00 UTC\n", encoding="utf-8"
+ )
self.args = SimpleNamespace(
context="kind-substrate-preview",
kubeconfig="/fixture/kubeconfig",
@@ -47,6 +50,34 @@ def setUp(self):
image=gate5_deliver_credentials.DEFAULT_IMAGE,
)
+ def test_kubeconfig_is_required_by_cli(self):
+ required = [
+ "--actor-namespace",
+ "native-claude",
+ "--actor-name",
+ "claude-final",
+ "--state-file",
+ "/fixture/state.json",
+ "--credential",
+ "claude",
+ ]
+ with patch("sys.argv", ["gate5_deliver_credentials.py", *required]):
+ with self.assertRaises(SystemExit) as raised:
+ gate5_deliver_credentials.parse_args()
+ self.assertEqual(raised.exception.code, 2)
+
+ with patch(
+ "sys.argv",
+ [
+ "gate5_deliver_credentials.py",
+ *required,
+ "--kubeconfig",
+ "/fixture/kubeconfig",
+ ],
+ ):
+ parsed = gate5_deliver_credentials.parse_args()
+ self.assertEqual(parsed.kubeconfig, "/fixture/kubeconfig")
+
def fake_runner(self, calls, data_seen):
def runner(argv, **kwargs):
calls.append((argv, kwargs))
@@ -59,7 +90,9 @@ def runner(argv, **kwargs):
return runner
- def test_credentials_flow_through_control_secret_and_authenticated_job_without_path_or_value_in_argv(self):
+ def test_credentials_flow_through_control_secret_and_authenticated_job_without_path_or_value_in_argv(
+ self,
+ ):
calls = []
data_seen = []
output = StringIO()
@@ -85,7 +118,9 @@ def test_credentials_flow_through_control_secret_and_authenticated_job_without_p
self.assertNotIn(self.args.state_file, joined)
secret_argv = calls[0][0]
- from_file = next(part for part in secret_argv if part.startswith("--from-file="))
+ from_file = next(
+ part for part in secret_argv if part.startswith("--from-file=")
+ )
self.assertRegex(from_file, r"^--from-file=credential=/proc/self/fd/\d+$")
job_manifest = json.loads(calls[5][1]["input"])
self.assertEqual(job_manifest["kind"], "Job")
@@ -94,8 +129,12 @@ def test_credentials_flow_through_control_secret_and_authenticated_job_without_p
self.assertEqual(pod_spec["securityContext"]["fsGroup"], 10001)
self.assertEqual(pod_spec["volumes"][1]["secret"]["defaultMode"], 0o440)
container = pod_spec["containers"][0]
- self.assertEqual(container["env"][0], {"name": "CREDENTIAL_KIND", "value": "claude"})
- self.assertEqual(container["env"][1], {"name": "ACTOR_NAMESPACE", "value": "native-claude"})
+ self.assertEqual(
+ container["env"][0], {"name": "CREDENTIAL_KIND", "value": "claude"}
+ )
+ self.assertEqual(
+ container["env"][1], {"name": "ACTOR_NAMESPACE", "value": "native-claude"}
+ )
self.assertNotIn(self.credential_value.decode(), str(job_manifest))
self.assertNotIn(str(self.credential_path), str(job_manifest))
self.assertEqual(
@@ -128,7 +167,9 @@ def test_actor_ownership_mismatch_refuses_before_any_kubectl_call(self):
self.assertEqual(calls, [])
def test_credentials_are_not_wired_into_the_golden_setup_flow(self):
- setup_source = Path(gate5_deliver_credentials.__file__).with_name("gate5_setup.py")
+ setup_source = Path(gate5_deliver_credentials.__file__).with_name(
+ "gate5_setup.py"
+ )
source = setup_source.read_text(encoding="utf-8")
self.assertNotIn("gate5_deliver_credentials", source)
self.assertNotIn("deliver_credentials(", source)
diff --git a/backend/tests/runtime/test_gate5_setup.py b/backend/tests/runtime/test_gate5_setup.py
index 724aa0d..5cbfcd3 100644
--- a/backend/tests/runtime/test_gate5_setup.py
+++ b/backend/tests/runtime/test_gate5_setup.py
@@ -1,11 +1,11 @@
"""Credential-free regressions for the gate-5 setup script's build and rerun identity."""
import json
-from contextlib import redirect_stdout
-from io import StringIO
import tempfile
import unittest
+from contextlib import redirect_stdout
from dataclasses import replace
+from io import StringIO
from pathlib import Path
from types import SimpleNamespace
from unittest.mock import patch
@@ -56,7 +56,9 @@ def opener(request, *, timeout):
f"http://localhost:5001/v2/live-agent-gate/manifests/sha256:{'a' * 64}",
)
self.assertEqual(request.get_method(), "HEAD")
- self.assertEqual(request.get_header("Accept"), gate5_setup.IMAGE_MANIFEST_ACCEPT)
+ self.assertEqual(
+ request.get_header("Accept"), gate5_setup.IMAGE_MANIFEST_ACCEPT
+ )
self.assertEqual(timeout, 10)
def test_image_manifest_preflight_fails_before_template_on_missing_manifest(self):
@@ -80,8 +82,9 @@ def opener(*_args, **_kwargs):
"localhost:5001/live-agent-gate:latest",
"localhost:5001/live-agent-gate@sha256:bad",
):
- with self.subTest(image=image), self.assertRaisesRegex(
- RuntimeError, "full sha256 digest"
+ with (
+ self.subTest(image=image),
+ self.assertRaisesRegex(RuntimeError, "full sha256 digest"),
):
gate5_setup.verify_image_manifest(image, opener=opener)
self.assertEqual(calls, [])
@@ -429,8 +432,8 @@ def test_shim_token_is_persisted_before_body_only_install_and_verified(self):
atespace="live-agent-gate",
actor_name="claude-gate5",
)
- token = "fixture-shim-token-with-at-least-32-characters"
- statuses = [201, 401, 401, 200, 409, 200]
+ token = "fixture-shim-" + "v" * 40
+ statuses = [201, 401, 401, 404, 409, 200]
calls = []
def requester(**kwargs):
@@ -452,6 +455,9 @@ def requester(**kwargs):
self.assertEqual(calls[0]["path"], "/token")
self.assertEqual(calls[0]["body"], {"token": token})
self.assertNotIn("token", calls[0])
+ self.assertEqual(
+ calls[1]["path"], "/turn/00000000-0000-4000-8000-000000000000"
+ )
self.assertEqual(calls[1]["token"], None)
self.assertEqual(calls[2]["token"], f"{token}x")
self.assertEqual(calls[3]["token"], token)
@@ -460,7 +466,7 @@ def requester(**kwargs):
def test_shim_token_rerun_verifies_existing_token_without_rotating_it(self):
with tempfile.TemporaryDirectory() as temp_dir:
path = str(Path(temp_dir) / "state.json")
- token = "existing-shim-token-with-at-least-32-characters"
+ token = "existing-shim-" + "v" * 40
state = {"run_id": "run-token", "shim_token": token}
gate5_setup.save_state(path, state)
args = SimpleNamespace(
@@ -468,7 +474,7 @@ def test_shim_token_rerun_verifies_existing_token_without_rotating_it(self):
atespace="live-agent-gate",
actor_name="claude-gate5",
)
- statuses = [409, 200, 401, 401, 200, 409, 200]
+ statuses = [409, 404, 401, 401, 404, 409, 200]
calls = []
def requester(**kwargs):
@@ -621,7 +627,11 @@ async def wait_for_worker(_control, namespace, *_args, **_kwargs):
gate5_setup.wait_for_worker_if_actor_is_absent(
control,
args,
- {"run_id": "run-codex", "actor_uid": None, "template_uid": "template-codex"},
+ {
+ "run_id": "run-codex",
+ "actor_uid": None,
+ "template_uid": "template-codex",
+ },
)
)
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index e6ba0da..5293a1f 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -2,40 +2,63 @@
Status: local spike, not a product feature. Adapter code lives in
`backend/src/mainloop/runtime/substrate.py` and `workspace_adapter.py`; the actor manifest
-lives in `spikes/substrate-workspace-adapter/`. See `docs/spikes/k8s-herdr-agents.md` for the
-native-session/Herdr spike this one builds on and does not replace.
-
-## Current status — Round 3 and Phase 4 (2026-09-23)
-
-Substrate actors, the Cilium-backed preview cluster, router ingress controls, per-actor shim
-tokens, and Envoy hostname enforcement were measured live. Claude completed a native turn and
-recalled a nonce after suspend/resume. Codex's auth file was installed safely, but its native
-turn failed at Envoy's upstream connection to the OpenAI API (HTTP 503, reset before response
-headers). Gate 5 is partial/live, so defer production adoption of the native-session path until
-Codex egress and session continuity are proved.
+lives in `spikes/substrate-workspace-adapter/`. The earlier live proof used a Herdr-backed
+actor; the target described by this closeout has no Herdr server or terminal manager in the
+actor and invokes native CLIs headlessly once per turn. See
+`docs/spikes/k8s-herdr-agents.md` for the historical native-session/Herdr spike.
+
+## Current status — Phase 5 closeout (2026-09-23)
+
+The Round 3 and Phase 4 runs measured Substrate actors, Cilium-enforced router ingress,
+per-actor shim tokens, Envoy hostname egress, and native sessions for both Claude Code and
+Codex in the earlier Herdr-backed actor. Both agents completed a native turn and recalled a
+nonce in the same session after suspend/resume. Measured suspend/resume times were 393/408 ms
+for Claude and 573/789 ms for Codex. Those results do not prove the headless per-turn actor
+model. Its rebuilt image is **not yet live-proved**. The Codex file marker was not confirmed in
+the recorded run; its follow-up result, Claude recall after worker loss, and Claude history
+after revert remain pending in the cluster lane.
+
+The earlier Codex API HTTP 503 came from Envoy selecting unreachable IPv6 upstream addresses
+when its DNS caches used `ALL`; `V4_PREFERRED` corrected that path. A separate ChatGPT SAN
+failure was traced by reviewer-provided evidence to cross-SNI upstream TLS session reuse in
+Envoy 1.39.1. After applying both settings, a credential-free probe returned the expected API
+401 and ChatGPT 200 with no TLS verification failures, then the native Codex turn and recall
+completed. Native-session support behind the Phase 3 boundary is proved for both agents in the
+earlier actor design. The requested headless actor design remains unproved, the broader
+continuity gate is partial, and production adoption is deferred pending live proof of its
+rebuilt image and the requirements below.
## What it shows
[Substrate](https://github.com/agent-substrate/substrate) can provide the per-session isolated
compute Mainloop's roadmap calls for ("Workspace platform"), while Mainloop stays the durable
-owner of the session<->actor mapping, delivery, and audit state. A Mainloop-authored
-`ActorTemplate` (Herdr + `agentctl`, the same image contents as the Herdr spike) runs as a
-Substrate actor instead of a fixed StatefulSet pod, and `backend/src/mainloop/runtime/substrate.py`
-drives its lifecycle through the real `kubectl ate` control-plane CLI.
+owner of session-to-actor mapping, delivery, and audit state. The target actor has no Herdr
+server or terminal manager; Mainloop delivers each turn to a headless native CLI process. The
+actor holds files between turns, with no attachable TUI; callers watch progress through streamed
+events. The rebuilt image for that design is not yet live-proved. Existing live lifecycle and
+native-session results below came from the earlier Herdr-backed actor and are historical evidence
+for Substrate and the Phase 3 boundary, not proof of the target image.
+
+The headless shim exposes authenticated `POST /turn` and `GET /turn/:id` endpoints. Prompts are
+sent to the native CLI on stdin; the shim stores bounded JSONL events per turn and reports the
+native session/thread id and final message. It permits one in-flight turn per agent and returns
+409 instead of queueing. `POST /run` starts a shell command with a bounded timeout and actor-local
+output file; `GET /run/:id` reports its status and bounded output. `/healthz` and `/readyz` check
+only the shim and workspace. The actor image and these routes still need live proof.
## Earlier real-versus-stand-in inventory (before Round 3)
-| Layer | Status |
-| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
-| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
-| `preview-gate` WorkerPool + ActorTemplate: real Herdr server, real Vite dev server, real NGINX header-proxy, real browser (`agent-browser`), real WebSocket HMR | Real |
-| `dev-service-gate` WorkerPool + ActorTemplate: real `psql`, real external `postgres:16-alpine` StatefulSet, real `EgressPolicy` (CIDR rule, created via a small gRPC tool since `kubectl-ate` has no CLI verb for it) | Real |
-| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
-| File edits and shell commands run inside actors (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
-| `live-agent-gate` WorkerPool + ActorTemplate: real Claude/Codex CLIs, credential-free boot | Golden snapshot READY and actor RUNNING proved live. A later credential-free preview actor hit the pinned gVisor restore error described below. No provider session ran. |
-| Claude/Codex agent processes, credentials | Not run; the 2026-09-23 live lane stopped before credential work (see "Limits") |
-| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
+| Layer | Status |
+| --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------ |
+| kind cluster `substrate-preview`, pinned Substrate `cdac9baef8...` (ate-system + agentgateway dataplane) | Real |
+| `mainloop-workspace` WorkerPool + ActorTemplate, actor create/get/resume/suspend/revert/delete | Real, driven through `backend/src/mainloop/runtime/substrate.py`'s actual code (not a separate probe script's own CLI calls) |
+| `preview-gate` WorkerPool + ActorTemplate: real Herdr server, real Vite dev server, real NGINX header-proxy, real browser (`agent-browser`), real WebSocket HMR | Real |
+| `dev-service-gate` WorkerPool + ActorTemplate: real `psql`, real external `postgres:16-alpine` StatefulSet, real `EgressPolicy` (CIDR rule, created via a small gRPC tool since `kubectl-ate` has no CLI verb for it) | Real |
+| Herdr + `agentctl` inside the actor images | Real (same image contents as `spikes/k8s-herdr-agents`), without the real Claude/Codex CLIs |
+| File edits and shell commands run inside actors (a generic `herdr pane run` shim, not a native agent's own Bash tool) | Stand-in -- see "Why not a real agent" below |
+| `live-agent-gate` WorkerPool + ActorTemplate: real Claude/Codex CLIs, credential-free boot | Golden snapshot READY and actor RUNNING proved live. A later credential-free preview actor hit the pinned gVisor restore error described below. No provider session ran. |
+| Claude/Codex agent processes, credentials | Not run; the 2026-09-23 live lane stopped before credential work (see "Limits") |
+| `workspace_bindings` durable mapping (Postgres) | Fixture/unit-tested only; not exercised against a live backend + database in this run |
## Preview-gate edit path and Round 3 credential boundary
@@ -51,12 +74,21 @@ CLI's local environment or filesystem.
The pinned commit also has experimental static-header injection from a Kubernetes Secret URI
into decrypted outbound requests. HTTPS hostname rules require the Envoy sdsmint overlay and
`--experimental-egress-credential-injection`; the plain Envoy overlay has no MITM egress, and
-agentgateway does not implement this injection path. Envoy 1.39.1 crashed during an earlier
-install attempt, but the rebuilt Round 3 cluster ran Envoy with sdsmint and the credential
-provider. Claude's credential was injected on the upstream leg and stayed out of actor snapshots.
-Codex instead received `auth.json` through the authenticated shim because Codex refreshes that
-file locally; its actor snapshots therefore contain that credential. Neither credential value
-was logged or copied into a golden snapshot. The old unauthenticated relay was not used.
+agentgateway does not implement this injection path. An earlier Envoy 1.39.1 router install
+crashed, while the later Round 3 Cilium setup ran Envoy egress with sdsmint and the
+actor-bound credential provider. Claude's credential was injected on the upstream leg and stayed
+out of actor snapshots. Codex instead received `auth.json` through the authenticated shim because
+Codex refreshes that file locally; its actor snapshots therefore contain that credential. Moving
+Codex credentials out of snapshots remains a production requirement. Neither credential value
+was logged or copied into a golden snapshot, and the old unauthenticated relay was not used.
+
+The Round 3 egress run also found that `dns_lookup_family: ALL` selected unreachable IPv6
+addresses on the IPv4-only Kind node. Setting the five egress DNS caches to `V4_PREFERRED`
+allowed the credential-free API and ChatGPT probes to reach upstreams. A separate
+reviewer-provided diagnosis tied the earlier ChatGPT SAN failures to cross-SNI TLS session
+resumption; setting `max_session_keys: 0` on the four approved upstream TLS clusters preserved
+SAN verification and the later probe passed. These are Envoy-specific settings; they do not make
+agentgateway hostname enforcement safe.
The preview/HMR edit itself still uses a generic exec shim
(`spikes/substrate-workspace-adapter/image/exec-shim.js`) that pastes text into a real Herdr shell
@@ -183,23 +215,21 @@ per `docs/network-egress.md`'s "CIDR/all policy: dial now" passthrough path) ins
**hostname rule** (HTTP/TLS-SNI-specific, and Postgres is neither), which the prior proof's HTTP
`fetch`-based trial did not have reason to distinguish.
-## Limits from the earlier Phase 2 checkpoint
+## Historical limits from the earlier Phase 2 checkpoint
-- **Native-session gate, live**: no Claude or Codex session ran in the earlier checkpoint. The
- owner authorized the work; earlier tool-policy decisions rejected particular actions after
- the old relay returned HTTP 403. The relay was removed. The first Phase 3 preview restore
- error was later diagnosed as a dead app from the old image and missing readiness probe, then
- corrected with a new image and template. The current Phase 3 run passed 3a–3c and stopped at
- the hostname-egress bypass described below; no provider credential was delivered.
+- **Native-session gate, live at that checkpoint**: no Claude or Codex session ran then. The
+ owner authorized the work; a tool policy rejected particular actions after the old relay
+ returned HTTP 403. The relay was removed. The first Phase 3 preview restore error was later
+ diagnosed as a dead app from the old image and missing readiness probe, then corrected with a
+ new image and template. Those earlier limits were superseded by the Round 3 and Phase 4 results
+ above.
- Preview/HMR under the router policy and actor-to-actor access were unverified at the earlier
checkpoint. In the current finish run, preview HMR passed, and an unrelated actor's CONNECT
to the shim was rejected at the actor egress gateway before reaching the router. The
- per-actor shim token passed live checks, including suspend/resume persistence. Provider Secret
- delivery and Phase 4 session continuity remain unproved; no provider Secret was created or
- read.
-- Phase 2's counter/marker persistence and worker-loss revert were not proved in their original
- run. The current token suspend/resume check did not cover worker loss or those markers; those
- checks remain open.
+ per-actor shim token passed live checks, including suspend/resume persistence.
+- Phase 2's counter/marker persistence and worker-loss revert were not proved in that original
+ run. Later Phase 4 checks proved Claude's file marker across suspend/resume; the three pending
+ Phase 4 checks are listed in the current results below.
- **`workspace_bindings` orchestration functions** (`ensure_workspace`, `resume_workspace`, ...)
were not exercised against a live Postgres + running backend; only their extracted pure logic
(`plan_ensure`, `_binding_from_row`, `is_crashed`) is unit tested, and the transport layer
@@ -251,11 +281,13 @@ returned 200. Suspend produced snapshot
The first setup pass exposed one additional pinned-CLI result shape: a valid zero-match worker
query serializes as `{}`. The adapter now treats only an empty object as zero workers and still
-rejects non-empty objects missing `workers`. The 9 worker-discovery regression tests pass. The
-unchanged rerun did not create a duplicate actor, but waited for spare worker capacity before
-checking the persisted actor UID and failed because the single worker was already occupied by
-`claude-gate5`. The harness's own health check also used router Service port 80 instead of its
-CONNECT listener on 8081; direct `/healthz` through 8081 worked.
+rejects non-empty objects missing `workers`. The affected correction run passed 30 tests: 9
+worker-discovery tests and 21 contract tests. The live Phase 2 run reached a READY golden and a
+RUNNING actor, returned `/healthz` 200 through the CONNECT listener on port 8081, and suspended
+then resumed the same actor to RUNNING with `/healthz` 200. Its unchanged rerun did not create a
+duplicate actor, but waited for spare worker capacity before checking the persisted actor UID and
+failed because the single worker was already occupied. The marker/counter and process PID were
+not measured in that run.
Before any provider credential work, a separate tokenless Pod in `default` POSTed a harmless
command through `atenet-router:8081` to `actor-upstream:8090/run` with the actor-routing header;
@@ -350,19 +382,19 @@ to that reviewing session, not to a sandbox process killed by this agent.
## Historical CapabilityResult (before Round 3)
-| Capability | State | Scope | Evidence and limit |
-| ----------------------------- | ------- | ---------- | ------------------ |
-| `workspace_adapter_contract` | partial | fixture | `SubstrateControl` is exercised live; `workspace_bindings` orchestration is fixture-tested, not run against a live backend and Postgres. |
-| `substrate_actor_lifecycle` | partial | live | READY golden creation and actor RUNNING proved live; full marker and worker-loss recovery remain unproved. |
-| `preview_hmr` | proved | live | Real Vite HMR edit and WebSocket upgrade succeeded through the preview proxy. |
-| `dev_service_postgres` | proved | live | Real Postgres query, narrow allow rule, denied destination, and reconnect after wake. |
-| `phase3a_networkpolicy` | proved | live | Kindnet blocked and then allowed the in-cluster HTTP probe according to NetworkPolicy. |
-| `router_ingress_boundary` | proved | live | `default` was blocked; `mainloop-control` was admitted; unrelated actor CONNECT to the shim was rejected at actor egress; preview traffic passed. |
-| `shim_token_auth` | proved | live | 401/409 behavior, successful bearer use, and token persistence across suspend/resume passed on the final actor. |
-| `provider_egress` | failed | live | Exact hostname policy was confirmed, but HTTPS to unlisted `example.com` returned 200 through agentgateway. Actor returned to deny-all. |
-| `credential_delivery` | unknown | unverified | No provider Secret was created or read; credentials were not delivered. |
-| `native_session_continuity` | partial | unverified | Phase 3d failed before credential delivery; no native session was run. |
-| `failure_recovery` | partial | live | Earlier CRASHED/revert mechanics were proved; worker-loss recovery and the backend-restart `recorded` case remain unproved live. |
+| Capability | State | Scope | Evidence and limit |
+| ---------------------------- | ------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `workspace_adapter_contract` | partial | fixture | `SubstrateControl` is exercised live; `workspace_bindings` orchestration is fixture-tested, not run against a live backend and Postgres. |
+| `substrate_actor_lifecycle` | partial | live | READY golden creation and actor RUNNING proved live; full marker and worker-loss recovery remain unproved. |
+| `preview_hmr` | proved | live | Real Vite HMR edit and WebSocket upgrade succeeded through the preview proxy. |
+| `dev_service_postgres` | proved | live | Real Postgres query, narrow allow rule, denied destination, and reconnect after wake. |
+| `phase3a_networkpolicy` | proved | live | Kindnet blocked and then allowed the in-cluster HTTP probe according to NetworkPolicy. |
+| `router_ingress_boundary` | proved | live | `default` was blocked; `mainloop-control` was admitted; unrelated actor CONNECT to the shim was rejected at actor egress; preview traffic passed. |
+| `shim_token_auth` | proved | live | 401/409 behavior, successful bearer use, and token persistence across suspend/resume passed on the final actor. |
+| `provider_egress` | failed | live | Exact hostname policy was confirmed, but HTTPS to unlisted `example.com` returned 200 through agentgateway. Actor returned to deny-all. |
+| `credential_delivery` | unknown | unverified | No provider Secret was created or read; credentials were not delivered. |
+| `native_session_continuity` | partial | unverified | Phase 3d failed before credential delivery; no native session was run. |
+| `failure_recovery` | partial | live | Earlier CRASHED/revert mechanics were proved; worker-loss recovery and the backend-restart `recorded` case remain unproved live. |
## Historical recommendation (superseded by the Round 3/Phase 4 result)
@@ -385,48 +417,54 @@ provider Secret or credential-bearing snapshot was created. Current Kind/Docker
disk headroom are recorded in the task proof note. The owner handles rotation of credentials
previously served by the removed relay.
-## Current CapabilityResult — Round 3 and Phase 4
-
-| Capability | State | Scope | Evidence and limit |
-| --- | --- | --- | --- |
-| `workspace_adapter_contract` | partial | fixture | Contract and identity reconciliation are covered by fakes; the `workspace_bindings` orchestration was not run against live Postgres and a running backend. |
-| `substrate_actor_lifecycle` | proved | live | Golden, actor readiness, router ingress, and suspend/resume passed on the Cilium preview cluster. |
-| `preview_hmr` | proved | live | The earlier real Vite/WebSocket HMR route remains measured and passed. |
-| `dev_service_postgres` | proved | live | The earlier real PostgreSQL query, narrow CIDR rule, denied destination, and wake reconnection remain measured and passed. |
-| `networkpolicy_enforcement` | proved | live | Cilium blocked the denied probe and allowed the control-namespace probe. |
-| `router_ingress_boundary` | proved | live | `default` was denied; `mainloop-control` was admitted; the preview route remained functional. |
-| `shim_token_auth` | proved | live | Missing/wrong/correct token, one-time install, and suspend/resume persistence passed. |
-| `image_manifest_preflight` | proved | live | The exact pushed digest returned registry HTTP 200 before template creation; fake-backed tests cover the manifest check and rejection cases. |
-| `shim_healthz` | proved | live | Readiness checks pass through the actor route; bounded Herdr calls and cached health have fixture coverage. |
-| `cilium_kube_proxy_replacement` | partial | live | KPR=true core checks passed. DNS to the CoreDNS Pod IP worked, while DNS to the kube-dns Service IP timed out. The exact failing component was not isolated; kube-proxy replacement ClusterIP translation from the nested actor network is only a hypothesis. The cluster fell back to KPR=false as directed. |
-| `provider_hostname_egress` | proved | live | Actor A's listed Claude host returned 404 while unlisted `example.com` and raw IP returned 403. Actor B's differing rule allowed `example.com` to reach an upstream 503, denied `api.anthropic.com`, and denied raw IP. Track B was skipped because Track A passed. |
-| `dummy_header_injection` | proved | live | The compare-only provider returned a fixed match boolean; the final actor had no injected value in its env/files and received no echoed value. Earlier dummy-only diagnostic history is in the task proof note. |
-| `credential_delivery` | proved | live | Claude's credential was retrieved by the actor-bound provider and injected on the Envoy upstream leg. Codex `auth.json` was installed through the authenticated shim, once, mode 0600. These delivery proofs do not imply successful authentication for both CLIs. |
-| `claude_native_session` | proved | live | Claude Code 2.1.280 completed a turn, preserved its session ID through suspend/resume, and recalled a prior nonce. |
-| `codex_native_session` | partial | live | Codex CLI 0.156.1 created a thread, but Envoy returned an upstream-connect 503 before the turn completed; the model was not reported, and there was no marker or recall. No Codex revert was attempted. |
-| `native_session_continuity` | partial | live | Claude suspend/resume recall passed. Claude post-worker-loss recall and transcript rollback after snapshot revert remain unverified; Codex continuity did not pass. |
-| `snapshot_revert` | partial | live | Claude state-A restore and actor lifecycle passed, but transcript/history rollback was not conclusively measured. Codex revert was not attempted. |
-| `worker_loss_recovery` | partial | live | The Claude actor recovered on a replacement worker from its completed snapshot. Post-loss native recall did not complete. |
-| `backend_restart_delivery_reconciliation` | proved | fixture | The `ContractStore` fake test models a persisted `recorded` row across ownership restart, requires `not_delivered` evidence before retry, and rejects a duplicate attempt. |
-| `snapshot_bucket_access_control` | unknown | unverified | Read access to snapshot storage was not established in this install. |
-
-## Current recommendation
-
-**Defer production adoption of the Substrate native-session path.** The live run proves that
-Substrate can host the isolated workspace lifecycle, enforce router ingress and actor hostname
-egress, inject Claude credentials outside the actor snapshot, and preserve a Claude session
-through suspend/resume. It does not prove the required two-agent path: Codex could not complete a
-turn because the egress Envoy reset its OpenAI upstream connection before response headers
-(HTTP 503). The same credential-free API request returned `server: envoy` and an
-`upstream connect error`, while TLS verification succeeded and the unlisted-host rule still
-returned 403. The bounded review found no image, CA, install-flag, actor identity, or hostname
-policy mismatch.
-
-Before production, resolve and retest that Envoy-to-OpenAI upstream hop, then prove Codex
-authentication and nonce recall across suspend/resume. Production also needs a GitOps-managed
-router NetworkPolicy, a CNI that enforces it, shim-token issuance from the real Mainloop backend,
-and snapshot-bucket access control. The snapshot containing Codex `auth.json` must be removed
-when the actor is deleted; the owner handles credential rotation.
+## Current CapabilityResult — gates 1–6 and Phase 4
+
+| Gate / capability | State | Scope | Evidence and limit |
+| ----------------------------------------------- | ------- | ---------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| Gate 1 — native contract | proved | fixture | Record transitions, ownership fencing, and reconciliation are exercised with `ContractStore` fakes; this is not a database-backed restart proof. |
+| Gate 2 — workspace binding adapter | partial | fixture | Planning and row-mapping logic use fakes; `workspace_bindings` orchestration was not run against live Postgres and a running backend. |
+| Gate 2 — Substrate control adapter | proved | live | Actor lifecycle operations used the real `kubectl ate` adapter and the Cilium preview cluster. |
+| Gate 3 — preview/HMR | proved | live | Real Vite edits updated the open browser session over WebSocket, including across suspend/resume. |
+| Gate 4 — dev-service access | proved | live | Real PostgreSQL query, narrow CIDR policy, denied destination, and reconnect after wake passed. |
+| Gate 5 — native sessions in prior actor design | proved | live | Claude and Codex each completed a native turn and recalled a nonce after suspend/resume in the same Herdr-backed actor. Suspend/resume measured 393/408 ms for Claude and 573/789 ms for Codex. |
+| Gate 5 — headless per-turn CLI actor | unknown | unverified | Target design has no Herdr server or terminal manager in the actor. Its rebuilt image is not yet live-proved; the earlier Phase 4 measurements do not establish this gate. |
+| Gate 5 — complete continuity | partial | live | In the earlier actor design, the Codex file marker was not confirmed. Claude recall after worker loss, Claude history after revert, and the Codex marker follow-up remain pending. |
+| Gate 6 — actor failure recovery | partial | live | CRASHED-to-revert-to-resume and replacement-worker restore were measured; native recall after worker loss and history after revert are not established. |
+| Phase 4 — Claude native session (prior actor) | proved | live | In the earlier Herdr-backed actor, Claude Code completed a turn and same-session nonce recall after suspend/resume; the post-worker-loss and post-revert history checks remain pending. |
+| Phase 4 — Codex native session (prior actor) | partial | live | In the earlier Herdr-backed actor, Codex CLI completed a turn and same-session nonce recall after suspend/resume; file-marker continuity remains pending. |
+| `backend_restart_delivery_reconciliation` | proved | fixture | A new fake-backed test reloads a persisted message and `recorded` attempt into a fresh `ContractStore`; retry is blocked until `not_delivered` evidence, and the same payload reference remains pending. It does not exercise Postgres, a transport, or the production delivery loop. |
+| `router_ingress_boundary` and `shim_token_auth` | proved | live | The unrelated namespace was denied, control-namespace access succeeded, and missing/wrong/correct token plus one-time install and suspend/resume checks passed. |
+| `provider_hostname_egress` | proved | live | Envoy/Cilium actor policies denied unlisted hosts; the earlier agentgateway HTTPS path allowed an unlisted host and is not a supported hostname boundary. |
+| `credential_delivery` | proved | live | Claude was injected on the Envoy upstream leg. Codex `auth.json` was installed through the shim and remains in the actor snapshot; production must move Codex credentials to egress injection. |
+| `cilium_kube_proxy_replacement` | partial | live | KPR=true CoreDNS Pod-IP queries worked, but kube-dns Service-IP queries timed out. The cause was not isolated; the run continued with KPR=false. |
+| `snapshot_bucket_access_control` | unknown | unverified | Snapshot-bucket read access was not established. |
+| `stuck_state_timeouts` | unknown | unverified | Production timeouts and surfaced recovery for stuck states remain to be implemented and measured. |
+
+## Current recommendation and production requirements
+
+**Defer production adoption of the headless per-turn actor design.** The earlier Herdr-backed
+actor proved that Substrate and the Phase 3 boundary can host native Claude and Codex sessions;
+both completed a turn and same-session nonce recall across suspend/resume. This does not prove
+the target actor, which has no Herdr server or terminal manager and invokes native CLIs headlessly
+per turn. Its rebuilt image is not yet live-proved. Production readiness also remains partial
+because the Codex marker and two Claude post-worker-loss/revert-history checks are pending, and
+durable backend integration and snapshot access controls are not proved.
+
+Production requirements:
+
+- Isolate worker selection by tenant. Substrate does not scope selection by atespace or
+ namespace; require unique per-tenant WorkerPool names/selectors and enforce uniqueness with
+ admission policy.
+- Manage the router `NetworkPolicy` through GitOps and use a CNI that enforces it.
+- Issue and rotate per-actor shim tokens from the real Mainloop backend.
+- Enforce snapshot-bucket access controls. The Codex snapshot currently contains `auth.json`.
+- Move Codex credentials from the actor snapshot to a supported egress-injection flow.
+- Use Envoy sdsmint for experimental credential injection; agentgateway does not implement it.
+- Set `dns_lookup_family: V4_PREFERRED` on the egress DNS caches and `max_session_keys: 0` on
+ the relevant upstream TLS clusters; retain hostname and SAN verification.
+- Resolve KPR=true actor DNS: CoreDNS Pod-IP lookup worked, but kube-dns Service-IP lookup
+ timed out and the exact cause remains unknown.
+- Add bounded timeouts and visible recovery for actors and deliveries that remain stuck.
## Current cleanup and review hold
diff --git a/spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl
index f4542e2..fb24de2 100644
--- a/spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl
+++ b/spikes/substrate-workspace-adapter/k8s/actor-template.yaml.tmpl
@@ -31,11 +31,9 @@ spec:
cpu: 250m
memory: 2Gi
---
-# ActorTemplate: one Herdr + agentctl container per actor, same image and entrypoint as
-# spikes/k8s-herdr-agents (see that spike's Dockerfile/bin/entrypoint.sh), running under
-# Substrate instead of a StatefulSet. snapshotsConfig captures full process memory on
-# suspend/commit so a resumed actor's native agent session (Claude/Codex under Herdr) continues
-# rather than cold-booting -- this is the behavior gate 5 (native-session continuity) measures.
+# ActorTemplate: one headless per-turn native-agent container per actor, running under Substrate.
+# Substrate owns lifecycle and snapshots; the actor retains workspace files between turns and
+# has no Herdr server or terminal manager. Gate 5 must live-prove the rebuilt image.
metadata:
atespace: ${ATESPACE}
name: ${TEMPLATE_NAME}
@@ -50,10 +48,9 @@ containers:
env:
- { name: HOME, value: /workspace/.home }
- { name: WORKSPACE_PATH, value: /workspace/repo }
- - { name: STANDIN_STATE_DIR, value: /workspace/.standin }
- - { name: HERDR_SESSION, value: mainloop-substrate }
- { name: AGENT_CONFIG_DIR, value: /etc/agent-config }
- { name: CODEX_HOME, value: /workspace/.codex }
+ - { name: EXEC_SHIM_STATE_DIR, value: /workspace/.mainloop/exec-shim }
volumeMounts:
- { name: workspace, mountPath: /workspace }
# ateapipb.SecurityContext only models Linux capability adjustments (no
diff --git a/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl b/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl
index dfb08a5..f231ecc 100644
--- a/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl
+++ b/spikes/substrate-workspace-adapter/k8s/live-agent-gate-template.yaml.tmpl
@@ -1,10 +1,11 @@
# WorkerPool + ActorTemplate for the bounded live native-agent proof (gate 5 in
-# .tasknotes/plan.md): real Claude Code / Codex CLIs under Herdr, driven by the exec shim. See
-# spikes/substrate-workspace-adapter/live-agent-image/.
+# .tasknotes/plan.md): headless per-turn Claude Code / Codex CLIs, driven by the actor-local
+# shim. Substrate owns suspend/resume/revert; no terminal manager runs in the actor.
+# See spikes/substrate-workspace-adapter/live-agent-image/.
#
# Deliberately no credential-relay env: the golden actor boots without credentials or external
-# network access. The pinned ActorTemplate API supports readyz; the probe waits for the control
-# server, shell pane, and exec shim before the golden snapshot or final actor is considered ready.
+# network access. The pinned ActorTemplate API supports readyz; the probe waits for the shim and
+# workspace to become healthy before the golden snapshot or final actor is considered ready.
apiVersion: v1
kind: Namespace
metadata:
diff --git a/spikes/substrate-workspace-adapter/k8s/router-ingress-policy.yaml b/spikes/substrate-workspace-adapter/k8s/router-ingress-policy.yaml
index 01901a3..0a18299 100644
--- a/spikes/substrate-workspace-adapter/k8s/router-ingress-policy.yaml
+++ b/spikes/substrate-workspace-adapter/k8s/router-ingress-policy.yaml
@@ -13,33 +13,33 @@ spec:
matchLabels:
app: atenet-router
policyTypes:
- - Ingress
+ - Ingress
ingress:
- # Trusted Mainloop backend. It uses HTTP plus the Substrate CONNECT listener
- # for actor control and arbitrary-port workspace traffic.
- - from:
- - namespaceSelector:
- matchLabels:
- mainloop.dev/role: control
- ports:
- - { protocol: TCP, port: 8080 }
- - { protocol: TCP, port: 8081 }
- - { protocol: TCP, port: 8443 }
- - { protocol: TCP, port: 8444 }
- # The fixed NGINX preview proxy injects ate-target-actor and only needs HTTP.
- - from:
- - namespaceSelector:
- matchLabels:
- mainloop.dev/role: workspace
- podSelector:
- matchLabels:
- app: preview-proxy
- ports:
- - { protocol: TCP, port: 8080 }
- # The installer annotates the router for Prometheus scraping on 15020.
- - from:
- - namespaceSelector:
- matchLabels:
- kubernetes.io/metadata.name: otel-system
- ports:
- - { protocol: TCP, port: 15020 }
+ # Trusted Mainloop backend. It uses HTTP plus the Substrate CONNECT listener
+ # for actor control and arbitrary-port workspace traffic.
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ mainloop.dev/role: control
+ ports:
+ - { protocol: TCP, port: 8080 }
+ - { protocol: TCP, port: 8081 }
+ - { protocol: TCP, port: 8443 }
+ - { protocol: TCP, port: 8444 }
+ # The fixed NGINX preview proxy injects ate-target-actor and only needs HTTP.
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ mainloop.dev/role: workspace
+ podSelector:
+ matchLabels:
+ app: preview-proxy
+ ports:
+ - { protocol: TCP, port: 8080 }
+ # The installer annotates the router for Prometheus scraping on 15020.
+ - from:
+ - namespaceSelector:
+ matchLabels:
+ kubernetes.io/metadata.name: otel-system
+ ports:
+ - { protocol: TCP, port: 15020 }
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/.gitignore b/spikes/substrate-workspace-adapter/live-agent-image/.gitignore
index 8ab7127..64443b6 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/.gitignore
+++ b/spikes/substrate-workspace-adapter/live-agent-image/.gitignore
@@ -1,4 +1,6 @@
-herdr
+/claude
+/codex
+/codex-code-mode-host
claude
codex
codex-code-mode-host
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
index f7724a4..d71d199 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
+++ b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
@@ -1,11 +1,8 @@
# syntax=docker/dockerfile:1.7
-# Live-agent-gate spike image (gate 5, bounded live proof): real Herdr + real Claude Code /
-# Codex CLIs + agentctl, driven by the same generic exec shim as the other gate images.
-# herdr/claude/codex/codex-code-mode-host are copied from the host into the build context by the
-# build script (never committed). No credentials are baked into this image, and entrypoint.sh
-# never fetches one: the golden actor built from this image boots to a ready control service
-# with no credential and no external network access. Credential delivery remains deferred until
-# a reviewed boundary exists (see docs/spikes/substrate-workspace-adapter.md).
+# Headless Substrate actor image: Substrate owns lifecycle; the shim starts one native CLI
+# process per delivered turn. Claude/Codex binaries are supplied in the local build context.
+# No credentials are baked in or fetched during startup; the golden actor reaches readiness
+# without credentials or external network access.
FROM node:22-bookworm-slim
RUN apt-get update && apt-get install -y --no-install-recommends jq ca-certificates git curl procps ripgrep \
&& rm -rf /var/lib/apt/lists/* \
@@ -20,24 +17,23 @@ RUN --mount=type=secret,id=egress-mitm-ca,target=/run/secrets/egress-mitm-ca,req
install -m 0644 /run/secrets/egress-mitm-ca /usr/local/share/ca-certificates/substrate-egress-mitm.crt && \
update-ca-certificates; \
fi
-COPY herdr /usr/local/bin/herdr
COPY claude /usr/local/bin/claude
COPY codex /usr/local/bin/codex
COPY codex-code-mode-host /usr/local/bin/codex-code-mode-host
-COPY bin/agentctl bin/mainloop /usr/local/bin/
COPY bin/start-native-agent /usr/local/bin/start-native-agent
COPY bin/prepare-native-agent-config.cjs /usr/local/bin/prepare-native-agent-config.cjs
-COPY agent-config /etc/agent-config
+COPY agent-config/mainloop-system.txt /etc/agent-config/mainloop-system.txt
COPY exec-shim.js /usr/local/bin/exec-shim.js
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
-RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/agentctl /usr/local/bin/mainloop /usr/local/bin/start-native-agent \
+RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/start-native-agent \
&& mkdir -p /work && chown -R agent:agent /work
ENV EXEC_SHIM=/usr/local/bin/exec-shim.js
+ENV NATIVE_AGENT_LAUNCHER=/usr/local/bin/start-native-agent
ENV HOME=/home/agent
ENV WORKSPACE_PATH=/work/repo
ENV CODEX_HOME=/home/agent/.codex
-ENV AGENT_CONFIG_DIR=/etc/agent-config
-ENV HERDR_SESSION=mainloop-live-agent
+ENV EXEC_SHIM_STATE_DIR=/work/repo/.mainloop/exec-shim
+ENV AGENT_SYSTEM_PROMPT_FILE=/etc/agent-config/mainloop-system.txt
ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt
ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
USER 10001:10001
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/bin/prepare-native-agent-config.cjs b/spikes/substrate-workspace-adapter/live-agent-image/bin/prepare-native-agent-config.cjs
index 8217864..e2de1bc 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/bin/prepare-native-agent-config.cjs
+++ b/spikes/substrate-workspace-adapter/live-agent-image/bin/prepare-native-agent-config.cjs
@@ -1,3 +1,4 @@
+// Seed stable first-run defaults for the headless per-turn Claude and Codex CLIs.
'use strict';
const fs = require('node:fs');
@@ -15,18 +16,26 @@ fs.mkdirSync(claudeSettingsDir, { recursive: true, mode: 0o700 });
fs.mkdirSync(codexHome, { recursive: true, mode: 0o700 });
if (!fs.existsSync(claudeConfig) || fs.statSync(claudeConfig).size === 0) {
- fs.writeFileSync(claudeConfig, `${JSON.stringify({
- hasCompletedOnboarding: true,
- numStartups: 1,
- theme: 'dark',
- projects: {
- [workspace]: {
- hasTrustDialogAccepted: true,
- hasCompletedProjectOnboarding: true,
- allowedTools: [],
+ fs.writeFileSync(
+ claudeConfig,
+ `${JSON.stringify(
+ {
+ hasCompletedOnboarding: true,
+ numStartups: 1,
+ theme: 'dark',
+ projects: {
+ [workspace]: {
+ hasTrustDialogAccepted: true,
+ hasCompletedProjectOnboarding: true,
+ allowedTools: []
+ }
+ }
},
- },
- }, null, 2)}\n`, { mode: 0o600 });
+ null,
+ 2
+ )}\n`,
+ { mode: 0o600 }
+ );
}
if (!fs.existsSync(claudeSettings) || fs.statSync(claudeSettings).size === 0) {
fs.writeFileSync(claudeSettings, '{"skipDangerousModePermissionPrompt":true}\n', { mode: 0o600 });
@@ -45,7 +54,7 @@ if (!fs.existsSync(codexConfig) || fs.statSync(codexConfig).size === 0) {
'',
'[notice]',
'hide_rate_limit_model_nudge = true',
- '',
+ ''
].join('\n');
fs.writeFileSync(codexConfig, defaults, { mode: 0o600 });
}
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/bin/start-native-agent b/spikes/substrate-workspace-adapter/live-agent-image/bin/start-native-agent
index 0a61910..726816d 100755
--- a/spikes/substrate-workspace-adapter/live-agent-image/bin/start-native-agent
+++ b/spikes/substrate-workspace-adapter/live-agent-image/bin/start-native-agent
@@ -1,47 +1,61 @@
#!/usr/bin/env bash
-# Start one native CLI in the actor's persistent Herdr shell pane after credential delivery.
+# Run one headless native CLI turn. The prompt is inherited on stdin, never passed in argv.
set -euo pipefail
-kind="${1:?usage: start-native-agent claude|codex}"
-if [[ $# -ne 1 ]]; then
- echo 'usage: start-native-agent claude|codex' >&2
- exit 2
+kind="${1:?usage: start-native-agent claude|codex [native-session-id]}"
+if [[ $# -gt 2 ]]; then
+ echo 'usage: start-native-agent claude|codex [native-session-id]' >&2
+ exit 2
+fi
+session_id="${2-}"
+if [[ -n ${session_id} && ! ${session_id} =~ ^[A-Za-z0-9._:-]{1,256}$ ]]; then
+ echo 'invalid native session id' >&2
+ exit 2
fi
workspace="${WORKSPACE_PATH:-/work/repo}"
cd "${workspace}"
+# Keep telemetry and automatic network chatter disabled for every native invocation.
+export DISABLE_TELEMETRY=1
+export DISABLE_ERROR_REPORTING=1
+export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1
+export DISABLE_AUTOUPDATER=1
+
case "${kind}" in
- claude)
- token_file="${HOME}/.mainloop/claude-token"
- if [[ ! -s "${token_file}" ]]; then
- echo 'Claude credential is not installed' >&2
- exit 1
- fi
- CLAUDE_CODE_OAUTH_TOKEN="$(tr -d ' \r\n' <"${token_file}")"
- if [[ -z "${CLAUDE_CODE_OAUTH_TOKEN}" ]]; then
- echo 'Claude credential is empty' >&2
- exit 1
- fi
- export CLAUDE_CODE_OAUTH_TOKEN
- export DISABLE_TELEMETRY=1
- export DISABLE_ERROR_REPORTING=1
- export CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC=1
- export DISABLE_AUTOUPDATER=1
- exec claude \
- --dangerously-skip-permissions \
- --append-system-prompt-file "${AGENT_SYSTEM_PROMPT_FILE:-/etc/agent-config/mainloop-system.txt}"
- ;;
- codex)
- auth_file="${CODEX_HOME:-${HOME}/.codex}/auth.json"
- if [[ ! -s "${auth_file}" ]]; then
- echo 'Codex credential is not installed' >&2
- exit 1
- fi
- exec codex --dangerously-bypass-approvals-and-sandbox
- ;;
- *)
- echo 'unsupported native CLI' >&2
- exit 2
- ;;
+claude)
+ token_file="${HOME}/.mainloop/claude-token"
+ if [[ ! -s ${token_file} ]]; then
+ echo 'Claude credential is not installed' >&2
+ exit 1
+ fi
+ CLAUDE_CODE_OAUTH_TOKEN="$(tr -d ' \r\n' <"${token_file}")"
+ if [[ -z ${CLAUDE_CODE_OAUTH_TOKEN} ]]; then
+ echo 'Claude credential is empty' >&2
+ exit 1
+ fi
+ export CLAUDE_CODE_OAUTH_TOKEN
+ args=(-p)
+ if [[ -n ${session_id} ]]; then
+ args+=(--resume "${session_id}")
+ fi
+ args+=(--output-format stream-json --dangerously-skip-permissions
+ --append-system-prompt-file "${AGENT_SYSTEM_PROMPT_FILE:-/etc/agent-config/mainloop-system.txt}")
+ exec claude "${args[@]}"
+ ;;
+codex)
+ auth_file="${CODEX_HOME:-${HOME}/.codex}/auth.json"
+ if [[ ! -s ${auth_file} ]]; then
+ echo 'Codex credential is not installed' >&2
+ exit 1
+ fi
+ if [[ -n ${session_id} ]]; then
+ exec codex exec resume --json "${session_id}" --dangerously-bypass-approvals-and-sandbox
+ fi
+ exec codex exec --json --dangerously-bypass-approvals-and-sandbox
+ ;;
+*)
+ echo 'unsupported native CLI' >&2
+ exit 2
+ ;;
esac
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
index 82dc5e1..3ecbb62 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
+++ b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
@@ -1,50 +1,37 @@
#!/usr/bin/env bash
-# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
-# Live-agent-gate actor entrypoint (gate 5 in .tasknotes/plan.md, bounded live proof): a real
-# Herdr server plus real Claude Code / Codex CLIs. Substrate has no Kubernetes-Secret-equivalent
-# volume/env mechanism for an actor (see docs/spikes/substrate-workspace-adapter.md,
-# "Credential-injection gap").
-#
-# Credential-free by construction: this entrypoint never fetches a credential and never
-# requires network access to reach a running state. Mainloop installs a per-actor shim token
-# and provider credentials only after the final actor is RUNNING. The golden actor stays clean.
-# The template controller checks `/healthz` before accepting the golden actor.
+# Start only the actor-local shim. Native CLIs run headlessly once per delivered turn.
set -eu
-# Seed supported first-run defaults before either CLI is started. Native sessions start only
-# through start-native-agent, after the final actor receives its credential.
-node /usr/local/bin/prepare-native-agent-config.cjs
+node /usr/local/bin/prepare-native-agent-config.cjs
mkdir -p "${WORKSPACE_PATH}"
[[ -d "${WORKSPACE_PATH}/.git" ]] || git -C "${WORKSPACE_PATH}" init -q
-# Credential-free identity/counter marker for the fake-payload proof (recovery plan step 2):
-# a plain file the exec shim can read/increment to verify golden restore and suspend/resume
-# without any real agent session or credential.
+
+# Credential-free marker retained for the bounded Gate 5 restore/suspend-resume probe.
[[ -f "${WORKSPACE_PATH}/gate5-counter" ]] || echo 0 >"${WORKSPACE_PATH}/gate5-counter"
-echo "herdr $(herdr --version) starting (HOME=${HOME} session=${HERDR_SESSION})"
-herdr --session "${HERDR_SESSION}" server &
-HERDR_PID=$!
+node "${EXEC_SHIM}" &
+SHIM_PID=$!
+trap 'kill "${SHIM_PID}" 2>/dev/null || true' EXIT INT TERM
-# The persistent control service's readiness check: an explicit, confirmed status call,
-# not a fixed sleep or a pre-confirmation log line. The ActorTemplate's `/healthz` probe
-# checks the Herdr server and this shell pane before the controller captures its snapshot.
ready=0
for _ in $(seq 1 60); do
- if herdr --session "${HERDR_SESSION}" status server >/dev/null 2>&1; then
+ if curl --fail --silent http://127.0.0.1:"${EXEC_SHIM_PORT:-8090}"/healthz >/dev/null; then
ready=1
break
fi
+ if ! kill -0 "${SHIM_PID}" 2>/dev/null; then
+ wait "${SHIM_PID}" || true
+ echo 'CONTROL_SERVICE_READINESS_FAILED: shim exited before healthz passed' >&2
+ exit 1
+ fi
sleep 0.5
done
if [[ ${ready} -ne 1 ]]; then
- echo "CONTROL_SERVICE_READINESS_TIMEOUT: herdr server did not report ready within 30s" >&2
- kill "${HERDR_PID}" 2>/dev/null || true
+ echo 'CONTROL_SERVICE_READINESS_TIMEOUT: shim/workspace not ready within 30s' >&2
+ kill "${SHIM_PID}" 2>/dev/null || true
+ wait "${SHIM_PID}" || true
exit 1
fi
-shell_ws=$(herdr --session "${HERDR_SESSION}" workspace create --label shell --cwd "${WORKSPACE_PATH}")
-shell_pane=$(echo "${shell_ws}" | jq -r '.result.root_pane.pane_id')
-
-EXEC_SHIM_PANE_ID="${shell_pane}" HERDR_SESSION="${HERDR_SESSION}" node "${EXEC_SHIM}" &
-echo "CONTROL_SERVICE_READY session=${HERDR_SESSION} pane=${shell_pane}"
-wait "${HERDR_PID}"
+echo "CONTROL_SERVICE_READY workspace=${WORKSPACE_PATH} port=${EXEC_SHIM_PORT:-8090}"
+wait "${SHIM_PID}"
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
index be75d1c..c92b73b 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
+++ b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
@@ -1,36 +1,40 @@
-// Minimal preview-gate command executor: POST /run { command } pastes `command` as literal
-// text into a real Herdr shell pane, and GET /read returns its current terminal buffer.
-// Authenticated POST /credential { name, contents } writes only one of the fixed credential
-// files used by the native-agent launchers. Request bodies are never logged.
-// Listens on port 8090, separate from the Vite dev server's port 80. Reached only through
-// atenet-router's arbitrary-port CONNECT tunnel with the ate-target-actor header (see
-// docs/api-guide.md "Workload Connectivity"), from test orchestration on the host -- never
-// through the previewed route a browser uses (port 80 via the NGINX header-proxy).
+// Authenticated, headless turn and command service for the Substrate actor.
+// Prompts are piped to the native CLI on stdin and are never logged or persisted.
'use strict';
const http = require('node:http');
-const { execFile } = require('node:child_process');
+const { spawn } = require('node:child_process');
const crypto = require('node:crypto');
const fs = require('node:fs');
const path = require('node:path');
-const PANE_ID = process.env.EXEC_SHIM_PANE_ID;
-const SESSION = process.env.HERDR_SESSION;
-const HEALTH_COMMAND_TIMEOUT_MS = 1500;
-const HEALTH_CACHE_MS = 3000;
const MAX_REQUEST_BODY_BYTES = 64 * 1024;
const MAX_CREDENTIAL_REQUEST_BODY_BYTES = 512 * 1024;
const MAX_CREDENTIAL_BYTES = 64 * 1024;
-if (!PANE_ID || !SESSION) {
- console.error('exec-shim: EXEC_SHIM_PANE_ID and HERDR_SESSION are required');
- process.exit(1);
+const MAX_JOB_OUTPUT_BYTES = 2 * 1024 * 1024;
+const MAX_RESPONSE_OUTPUT_BYTES = 32 * 1024;
+const MAX_RETURN_EVENTS = 256;
+const DEFAULT_RUN_TIMEOUT_MS = 60_000;
+const DEFAULT_TURN_TIMEOUT_MS = 10 * 60_000;
+const MAX_TIMEOUT_MS = 10 * 60_000;
+const WORKSPACE_PATH = path.resolve(process.env.WORKSPACE_PATH || '/work/repo');
+const HOME_PATH = path.resolve(process.env.HOME || '/home/agent');
+const CODEX_HOME_PATH = path.resolve(process.env.CODEX_HOME || path.join(HOME_PATH, '.codex'));
+const STATE_DIR = path.resolve(
+ process.env.EXEC_SHIM_STATE_DIR || path.join(WORKSPACE_PATH, '.mainloop')
+);
+const RUN_DIR = path.join(STATE_DIR, 'runs');
+const TURN_DIR = path.join(STATE_DIR, 'turns');
+const LAUNCHER = process.env.NATIVE_AGENT_LAUNCHER || '/usr/local/bin/start-native-agent';
+
+for (const directory of [STATE_DIR, RUN_DIR, TURN_DIR]) {
+ fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
+ fs.chmodSync(directory, 0o700);
}
-const tokenPath = path.join(process.env.HOME || '/home/agent', '.mainloop', 'exec-shim-token');
-const homePath = path.resolve(process.env.HOME || '/home/agent');
-const codexHomePath = path.resolve(process.env.CODEX_HOME || path.join(homePath, '.codex'));
+const tokenPath = path.join(HOME_PATH, '.mainloop', 'exec-shim-token');
const credentialPaths = new Map([
- ['claude-token', path.join(homePath, '.mainloop', 'claude-token')],
- ['codex-auth', path.join(codexHomePath, 'auth.json')],
+ ['claude-token', path.join(HOME_PATH, '.mainloop', 'claude-token')],
+ ['codex-auth', path.join(CODEX_HOME_PATH, 'auth.json')]
]);
let bearerToken = null;
try {
@@ -39,8 +43,15 @@ try {
if (err.code !== 'ENOENT') throw err;
}
+const jobs = new Map();
+const activeTurns = new Map();
+
+function json(res, status, document) {
+ res.writeHead(status, { 'content-type': 'application/json' }).end(JSON.stringify(document));
+}
+
function authorized(req) {
- if (bearerToken === null) return true;
+ if (bearerToken === null) return false;
const header = req.headers.authorization;
if (typeof header !== 'string' || !header.startsWith('Bearer ')) return false;
const provided = Buffer.from(header.slice('Bearer '.length));
@@ -66,7 +77,7 @@ function requestBody(req, res, onBody, maxBytes = MAX_REQUEST_BODY_BYTES) {
if (!tooLarge) chunks.push(chunk);
});
req.on('end', () => {
- if (!tooLarge) onBody(Buffer.concat(chunks).toString('utf8'));
+ if (!tooLarge && !res.destroyed) onBody(Buffer.concat(chunks).toString('utf8'));
});
}
@@ -97,7 +108,8 @@ function installCredential(name, contents) {
typeof contents !== 'string' ||
!contents ||
Buffer.byteLength(contents, 'utf8') > MAX_CREDENTIAL_BYTES
- ) return null;
+ )
+ return null;
if (name === 'codex-auth') {
let auth;
try {
@@ -126,87 +138,422 @@ function installCredential(name, contents) {
fs.closeSync(fd);
fs.chmodSync(destination, 0o600);
} catch (err) {
- try { fs.closeSync(fd); } catch {}
+ try {
+ fs.closeSync(fd);
+ } catch {}
fs.rmSync(destination, { force: true });
throw err;
}
return true;
}
-function herdr(args, res) {
- execFile('herdr', ['--session', SESSION, ...args], (err, stdout, stderr) => {
- if (err) {
- res.writeHead(502).end(String(err));
- return;
+function atomicJsonWrite(file, document) {
+ const temporary = file + '.' + process.pid + '.tmp';
+ fs.writeFileSync(temporary, JSON.stringify(document), { mode: 0o600 });
+ fs.renameSync(temporary, file);
+}
+
+function metadataPath(directory, id) {
+ return path.join(directory, id + '.json');
+}
+
+function loadJobs(directory, kind) {
+ for (const name of fs.readdirSync(directory)) {
+ if (!name.endsWith('.json')) continue;
+ try {
+ const record = JSON.parse(fs.readFileSync(path.join(directory, name), 'utf8'));
+ if (!record || typeof record.id !== 'string') continue;
+ if (record.status === 'running') {
+ record.status = 'interrupted';
+ record.exit_code = null;
+ record.blocking = kind === 'turn';
+ record.updated_at = new Date().toISOString();
+ atomicJsonWrite(path.join(directory, name), record);
+ }
+ jobs.set(record.id, { ...record, kind });
+ if (kind === 'turn' && record.blocking && record.agent) {
+ activeTurns.set(record.agent, record.id);
+ }
+ } catch {
+ // Ignore an incomplete or corrupt record; it cannot safely be resumed.
}
- res
- .writeHead(200, { 'content-type': 'application/json' })
- .end(JSON.stringify({ ok: true, stdout, stderr }));
- });
+ }
}
-function runHealthCheck() {
- return new Promise((resolve, reject) => {
- execFile(
- 'herdr',
- ['--session', SESSION, 'status', 'server'],
- { timeout: HEALTH_COMMAND_TIMEOUT_MS },
- (statusErr, stdout) => {
- if (statusErr || !/^status:\s+running\s*$/m.test(stdout)) {
- reject(statusErr || new Error('Herdr server is not running'));
- return;
- }
- execFile(
- 'herdr',
- ['--session', SESSION, 'pane', 'read', PANE_ID],
- { timeout: HEALTH_COMMAND_TIMEOUT_MS },
- (paneErr) => {
- if (paneErr) {
- reject(paneErr);
- return;
- }
- resolve();
- },
- );
- },
- );
+loadJobs(RUN_DIR, 'run');
+loadJobs(TURN_DIR, 'turn');
+
+function createJob(kind, fields) {
+ const id = crypto.randomUUID();
+ const directory = kind === 'run' ? RUN_DIR : TURN_DIR;
+ const job = {
+ id,
+ kind,
+ status: 'running',
+ exit_code: null,
+ created_at: new Date().toISOString(),
+ updated_at: new Date().toISOString(),
+ truncated: false,
+ stored_bytes: 0,
+ ...fields
+ };
+ jobs.set(id, job);
+ atomicJsonWrite(metadataPath(directory, id), publicMetadata(job));
+ return job;
+}
+
+function publicMetadata(job) {
+ const {
+ id,
+ kind,
+ status,
+ exit_code,
+ created_at,
+ updated_at,
+ truncated,
+ agent,
+ native_session_id,
+ blocking
+ } = job;
+ return {
+ id,
+ kind,
+ status,
+ exit_code,
+ created_at,
+ updated_at,
+ truncated,
+ agent,
+ native_session_id,
+ blocking
+ };
+}
+
+function saveJob(job) {
+ job.updated_at = new Date().toISOString();
+ const directory = job.kind === 'run' ? RUN_DIR : TURN_DIR;
+ atomicJsonWrite(metadataPath(directory, job.id), publicMetadata(job));
+}
+
+function appendBounded(job, filename, chunk) {
+ const remaining = MAX_JOB_OUTPUT_BYTES - job.stored_bytes;
+ if (remaining <= 0) {
+ job.truncated = true;
+ return;
+ }
+ const bytes = Buffer.isBuffer(chunk) ? chunk : Buffer.from(chunk);
+ const stored = bytes.length > remaining ? bytes.subarray(0, remaining) : bytes;
+ fs.appendFileSync(filename, stored, { mode: 0o600 });
+ job.stored_bytes += stored.length;
+ if (stored.length < bytes.length) job.truncated = true;
+}
+
+function readBounded(filename, limit = MAX_RESPONSE_OUTPUT_BYTES) {
+ try {
+ const fd = fs.openSync(filename, 'r');
+ try {
+ const size = fs.fstatSync(fd).size;
+ const length = Math.min(size, limit);
+ const buffer = Buffer.alloc(length);
+ fs.readSync(fd, buffer, 0, length, Math.max(0, size - length));
+ return { text: buffer.toString('utf8'), truncated: size > limit };
+ } finally {
+ fs.closeSync(fd);
+ }
+ } catch (err) {
+ if (err.code === 'ENOENT') return { text: '', truncated: false };
+ throw err;
+ }
+}
+
+function runOutputPath(job) {
+ return path.join(RUN_DIR, job.id + '.output');
+}
+
+function turnEventsPath(job) {
+ return path.join(TURN_DIR, job.id + '.events.jsonl');
+}
+
+function turnStderrPath(job) {
+ return path.join(TURN_DIR, job.id + '.stderr');
+}
+
+function killProcessGroup(child, signal) {
+ if (!child.pid) return;
+ try {
+ process.kill(-child.pid, signal);
+ } catch (err) {
+ if (err.code !== 'ESRCH') child.kill(signal);
+ }
+}
+
+function runChild(job, child, timeoutMs, onStart) {
+ let timedOut = false;
+ let spawnError = null;
+ let finalized = false;
+ let exitCode = null;
+ const finish = (code) => {
+ if (finalized) return;
+ finalized = true;
+ if (timedOut) job.status = 'timed_out';
+ else if (spawnError) job.status = 'failed';
+ else job.status = code === 0 ? 'completed' : 'failed';
+ job.exit_code = code;
+ job.blocking = false;
+ if (spawnError) job.error = 'process could not start';
+ saveJob(job);
+ if (job.kind === 'turn' && activeTurns.get(job.agent) === job.id) {
+ activeTurns.delete(job.agent);
+ }
+ };
+ const timer = setTimeout(() => {
+ timedOut = true;
+ job.timed_out = true;
+ killProcessGroup(child, 'SIGTERM');
+ const killTimer = setTimeout(() => {
+ killProcessGroup(child, 'SIGKILL');
+ finish(exitCode);
+ }, 500);
+ killTimer.unref();
+ }, timeoutMs);
+ timer.unref();
+ child.once('error', (err) => {
+ spawnError = err;
+ });
+ onStart(child);
+ child.once('close', (code) => {
+ clearTimeout(timer);
+ exitCode = code;
+ if (!timedOut) finish(code);
});
}
-let lastGoodHealthAt = 0;
-let healthCheckInFlight = null;
-
-function healthz(res) {
- const respond = (ready) => {
- if (res.destroyed) return;
- if (ready) {
- // The shim is responsive, Herdr reports a running server, and the shell pane exists.
- // Never return status output or pane contents.
- res.writeHead(200, { 'content-type': 'text/plain' }).end('ok');
- } else {
- res.writeHead(503, { 'content-type': 'text/plain' }).end('not ready');
+function parseTurn(job) {
+ const file = turnEventsPath(job);
+ const content = readBounded(file, MAX_JOB_OUTPUT_BYTES).text;
+ const lines = content.split('\n');
+ const parsed = [];
+ let nativeSessionId = job.native_session_id || null;
+ let finalMessage = null;
+ for (const line of lines) {
+ if (!line.trim()) continue;
+ let event;
+ try {
+ event = JSON.parse(line);
+ } catch {
+ continue;
}
+ if (!event || typeof event !== 'object' || Array.isArray(event)) continue;
+ if (event.session_id && job.agent === 'claude') nativeSessionId = event.session_id;
+ if (event.thread_id && job.agent === 'codex') nativeSessionId = event.thread_id;
+ if (event.type === 'thread.started' && event.thread_id) nativeSessionId = event.thread_id;
+ if (event.type === 'result' && typeof event.result === 'string') finalMessage = event.result;
+ const item = event.item || (event.params && event.params.item);
+ if (
+ job.agent === 'codex' &&
+ item &&
+ (item.type === 'agent_message' || item.type === 'agentMessage') &&
+ typeof item.text === 'string'
+ )
+ finalMessage = item.text;
+ parsed.push(event);
+ }
+ return {
+ native_session_id: nativeSessionId,
+ final_message: finalMessage,
+ events: parsed.slice(-MAX_RETURN_EVENTS)
+ };
+}
+
+function turnResponse(job) {
+ const parsed = parseTurn(job);
+ if (parsed.native_session_id !== job.native_session_id) {
+ job.native_session_id = parsed.native_session_id;
+ saveJob(job);
+ }
+ const stderr = readBounded(turnStderrPath(job));
+ return {
+ id: job.id,
+ agent: job.agent,
+ status: job.status,
+ exit_code: job.exit_code,
+ native_session_id: parsed.native_session_id,
+ final_message: parsed.final_message,
+ events: parsed.events,
+ stderr: stderr.text,
+ truncated: job.truncated || stderr.truncated
};
+}
+
+function workspaceReady() {
+ try {
+ const stat = fs.statSync(WORKSPACE_PATH);
+ if (!stat.isDirectory()) return false;
+ fs.accessSync(WORKSPACE_PATH, fs.constants.R_OK | fs.constants.W_OK);
+ return true;
+ } catch {
+ return false;
+ }
+}
- if (Date.now() - lastGoodHealthAt < HEALTH_CACHE_MS) {
- respond(true);
+function validateTimeout(value, fallback) {
+ if (value === undefined) return fallback;
+ if (!Number.isInteger(value) || value < 1 || value > MAX_TIMEOUT_MS) return null;
+ return value;
+}
+
+function handleBody(req, res, callback, maxBytes = MAX_REQUEST_BODY_BYTES) {
+ requestBody(
+ req,
+ res,
+ (body) => {
+ let document;
+ try {
+ document = JSON.parse(body);
+ } catch {
+ res.writeHead(400).end('invalid json');
+ return;
+ }
+ if (!document || typeof document !== 'object' || Array.isArray(document)) {
+ res.writeHead(400).end('invalid request');
+ return;
+ }
+ callback(document);
+ },
+ maxBytes
+ );
+}
+
+function startRun(document, res) {
+ const command = document.command;
+ const timeoutMs = validateTimeout(document.timeout_ms, DEFAULT_RUN_TIMEOUT_MS);
+ if (typeof command !== 'string' || command.length === 0) {
+ res.writeHead(400).end('missing command');
return;
}
- if (!healthCheckInFlight) {
- healthCheckInFlight = runHealthCheck()
- .then(() => {
- lastGoodHealthAt = Date.now();
- })
- .finally(() => {
- healthCheckInFlight = null;
- });
+ if (timeoutMs === null) {
+ res.writeHead(400).end('invalid timeout_ms');
+ return;
+ }
+ if (!workspaceReady()) {
+ res.writeHead(503).end('workspace is not ready');
+ return;
+ }
+ const job = createJob('run', { blocking: false });
+ try {
+ const child = spawn('/bin/sh', ['-lc', command], {
+ cwd: WORKSPACE_PATH,
+ env: process.env,
+ stdio: ['ignore', 'pipe', 'pipe'],
+ detached: true
+ });
+ runChild(job, child, timeoutMs, (processChild) => {
+ processChild.stdout.on('data', (chunk) => appendBounded(job, runOutputPath(job), chunk));
+ processChild.stderr.on('data', (chunk) => appendBounded(job, runOutputPath(job), chunk));
+ });
+ json(res, 202, { id: job.id, status: job.status });
+ } catch {
+ job.status = 'failed';
+ job.error = 'process could not start';
+ saveJob(job);
+ json(res, 500, { id: job.id, status: job.status });
+ }
+}
+
+function startTurn(document, res) {
+ const agent = document.agent;
+ const prompt = document.prompt;
+ const sessionId = document.session_id;
+ const timeoutMs = validateTimeout(document.timeout_ms, DEFAULT_TURN_TIMEOUT_MS);
+ if (agent !== 'claude' && agent !== 'codex') {
+ res.writeHead(400).end('unsupported agent');
+ return;
+ }
+ if (typeof prompt !== 'string' || prompt.length === 0) {
+ res.writeHead(400).end('missing prompt');
+ return;
+ }
+ if (
+ sessionId !== undefined &&
+ (typeof sessionId !== 'string' || !/^[A-Za-z0-9._:-]{1,256}$/.test(sessionId))
+ ) {
+ res.writeHead(400).end('invalid session_id');
+ return;
+ }
+ if (timeoutMs === null) {
+ res.writeHead(400).end('invalid timeout_ms');
+ return;
+ }
+ if (!workspaceReady()) {
+ res.writeHead(503).end('workspace is not ready');
+ return;
+ }
+ if (activeTurns.has(agent)) {
+ res.writeHead(409).end('turn already in flight for agent');
+ return;
+ }
+
+ const job = createJob('turn', {
+ agent,
+ native_session_id: sessionId || null,
+ blocking: true
+ });
+ activeTurns.set(agent, job.id);
+ try {
+ const child = spawn(LAUNCHER, [agent, ...(sessionId ? [sessionId] : [])], {
+ cwd: WORKSPACE_PATH,
+ env: process.env,
+ stdio: ['pipe', 'pipe', 'pipe'],
+ detached: true
+ });
+ runChild(job, child, timeoutMs, (processChild) => {
+ processChild.stdout.on('data', (chunk) => appendBounded(job, turnEventsPath(job), chunk));
+ processChild.stderr.on('data', (chunk) => appendBounded(job, turnStderrPath(job), chunk));
+ processChild.stdin.on('error', () => {});
+ processChild.stdin.end(prompt, 'utf8');
+ });
+ json(res, 202, { id: job.id, status: job.status });
+ } catch {
+ activeTurns.delete(agent);
+ job.status = 'failed';
+ job.blocking = false;
+ job.error = 'process could not start';
+ saveJob(job);
+ json(res, 500, { id: job.id, status: job.status });
}
- healthCheckInFlight.then(() => respond(true), () => respond(false));
+}
+
+function getJob(kind, id, res) {
+ if (!/^[0-9a-f-]{36}$/i.test(id)) {
+ res.writeHead(404).end('not found');
+ return;
+ }
+ const job = jobs.get(id);
+ if (!job || job.kind !== kind) {
+ res.writeHead(404).end('not found');
+ return;
+ }
+ if (kind === 'turn') {
+ json(res, 200, turnResponse(job));
+ return;
+ }
+ const output = readBounded(runOutputPath(job));
+ json(res, 200, {
+ id: job.id,
+ status: job.status,
+ exit_code: job.exit_code,
+ output: output.text,
+ truncated: job.truncated || output.truncated
+ });
}
const server = http.createServer((req, res) => {
- if (req.method === 'GET' && req.url === '/healthz') {
- healthz(res);
+ if (req.method === 'GET' && (req.url === '/healthz' || req.url === '/readyz')) {
+ if (!workspaceReady()) {
+ res.writeHead(503, { 'content-type': 'text/plain' }).end('workspace not ready');
+ return;
+ }
+ res.writeHead(200, { 'content-type': 'text/plain' }).end('ok');
return;
}
if (req.method === 'POST' && req.url === '/token') {
@@ -214,16 +561,9 @@ const server = http.createServer((req, res) => {
res.writeHead(409).end('token already set');
return;
}
- requestBody(req, res, (body) => {
- let token;
+ handleBody(req, res, (document) => {
try {
- token = JSON.parse(body).token;
- } catch {
- res.writeHead(400).end('invalid json');
- return;
- }
- try {
- const installed = installToken(token);
+ const installed = installToken(document.token);
if (installed === null) {
res.writeHead(400).end('invalid token');
return;
@@ -239,68 +579,59 @@ const server = http.createServer((req, res) => {
});
return;
}
- if (req.method === 'GET' && req.url === '/read') {
+ if (req.method === 'POST' && req.url === '/credential') {
if (!authorized(req)) return unauthorized(res);
- herdr(['pane', 'read', PANE_ID], res);
+ handleBody(
+ req,
+ res,
+ (document) => {
+ try {
+ const installed = installCredential(document.name, document.contents);
+ if (installed === null) {
+ res.writeHead(400).end('invalid credential');
+ return;
+ }
+ if (!installed) {
+ res.writeHead(403).end('credential name is not allowlisted');
+ return;
+ }
+ if (installed === 'exists') {
+ res.writeHead(409).end('credential file already exists');
+ return;
+ }
+ res.writeHead(201).end('credential stored');
+ } catch {
+ res.writeHead(500).end('credential could not be stored');
+ }
+ },
+ MAX_CREDENTIAL_REQUEST_BODY_BYTES
+ );
return;
}
- if (req.method === 'POST' && req.url === '/credential') {
- if (bearerToken === null || !authorized(req)) return unauthorized(res);
- requestBody(req, res, (body) => {
- let document;
- try {
- document = JSON.parse(body);
- } catch {
- res.writeHead(400).end('invalid json');
- return;
- }
- if (!document || typeof document !== 'object' || Array.isArray(document)) {
- res.writeHead(400).end('invalid credential');
- return;
- }
- try {
- const installed = installCredential(document.name, document.contents);
- if (installed === null) {
- res.writeHead(400).end('invalid credential');
- return;
- }
- if (!installed) {
- res.writeHead(403).end('credential name is not allowlisted');
- return;
- }
- if (installed === 'exists') {
- res.writeHead(409).end('credential file already exists');
- return;
- }
- res.writeHead(201).end('credential stored');
- } catch {
- res.writeHead(500).end('credential could not be stored');
- }
- }, MAX_CREDENTIAL_REQUEST_BODY_BYTES);
+ if (req.method === 'POST' && req.url === '/run') {
+ if (!authorized(req)) return unauthorized(res);
+ handleBody(req, res, (document) => startRun(document, res));
return;
}
- if (req.method !== 'POST' || req.url !== '/run') {
- res.writeHead(404).end();
+ if (req.method === 'POST' && req.url === '/turn') {
+ if (!authorized(req)) return unauthorized(res);
+ handleBody(req, res, (document) => startTurn(document, res));
return;
}
- if (!authorized(req)) return unauthorized(res);
- requestBody(req, res, (body) => {
- let command;
- try {
- command = JSON.parse(body).command;
- } catch {
- res.writeHead(400).end('invalid json');
- return;
- }
- if (typeof command !== 'string' || !command) {
- res.writeHead(400).end('missing command');
- return;
- }
- herdr(['pane', 'run', PANE_ID, command], res);
- });
+ const match = req.method === 'GET' && req.url.match(/^\/(run|turn)\/([^/?]+)$/);
+ if (match) {
+ if (!authorized(req)) return unauthorized(res);
+ getJob(match[1] === 'run' ? 'run' : 'turn', match[2], res);
+ return;
+ }
+ res.writeHead(404).end();
});
-server.listen(Number(process.env.EXEC_SHIM_PORT || 8090), '0.0.0.0', () => {
- const address = server.address();
- console.log(`exec-shim listening on :${address.port}`);
-});
+server.listen(
+ Number(process.env.EXEC_SHIM_PORT || 8090),
+ process.env.EXEC_SHIM_HOST || '0.0.0.0',
+ () => {
+ const address = server.address();
+ console.log('exec-shim listening on :' + address.port);
+ }
+);
diff --git a/spikes/substrate-workspace-adapter/tests/deliver-credentials.test.js b/spikes/substrate-workspace-adapter/tests/deliver-credentials.test.js
index 47c2a01..0d7c7c7 100644
--- a/spikes/substrate-workspace-adapter/tests/deliver-credentials.test.js
+++ b/spikes/substrate-workspace-adapter/tests/deliver-credentials.test.js
@@ -14,18 +14,18 @@ import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const require = createRequire(import.meta.url);
const { buildCredentialPayload, deliverFromMountedFiles, postViaRouter } = require(
- path.resolve(__dirname, '../tools/phase4/deliver-credentials.cjs'),
+ path.resolve(__dirname, '../tools/phase4/deliver-credentials.cjs')
);
const liveAgentImage = path.resolve(__dirname, '../live-agent-image');
test('credential payload uses the fixed shim allowlist and validates Codex auth JSON', () => {
assert.deepEqual(buildCredentialPayload('claude', 'fixture token\r\n'), {
name: 'claude-token',
- contents: 'fixturetoken',
+ contents: 'fixturetoken'
});
assert.deepEqual(buildCredentialPayload('codex', '{"access_token":"fixture"}'), {
name: 'codex-auth',
- contents: '{"access_token":"fixture"}',
+ contents: '{"access_token":"fixture"}'
});
assert.throws(() => buildCredentialPayload('../../etc/passwd', 'fixture'), /unsupported/);
assert.throws(() => buildCredentialPayload('codex', 'not-json'), SyntaxError);
@@ -36,7 +36,9 @@ test('control delivery reads mounted paths and sends credential contents only in
const credentialFile = path.join(root, 'credential');
const shimTokenFile = path.join(root, 'shim-token');
fs.writeFileSync(credentialFile, 'fixture-claude-token\n', { mode: 0o600 });
- fs.writeFileSync(shimTokenFile, 'fixture-shim-token-with-at-least-32-characters', { mode: 0o600 });
+ fs.writeFileSync(shimTokenFile, 'fixture-shim-token-with-at-least-32-characters', {
+ mode: 0o600
+ });
const calls = [];
try {
const result = await deliverFromMountedFiles({
@@ -48,12 +50,12 @@ test('control delivery reads mounted paths and sends credential contents only in
request: async (request) => {
calls.push(request);
return 201;
- },
+ }
});
assert.deepEqual(result, {
kind: 'claude',
namespace: 'native-claude',
- actor: 'claude-final',
+ actor: 'claude-final'
});
assert.equal(calls.length, 1);
assert.equal(calls[0].payload.name, 'claude-token');
@@ -83,7 +85,9 @@ test('router delivery authenticates the CONNECT target and posts the body withou
const length = Number(/^content-length:\s*(\d+)\s*$/im.exec(headers)?.[1]);
if (!Number.isFinite(length) || bytes.length < boundary + 4 + length) return;
received.requestHeaders = headers;
- received.payload = JSON.parse(bytes.subarray(boundary + 4, boundary + 4 + length).toString('utf8'));
+ received.payload = JSON.parse(
+ bytes.subarray(boundary + 4, boundary + 4 + length).toString('utf8')
+ );
socket.end('HTTP/1.1 201 Created\r\nContent-Length: 0\r\nConnection: close\r\n\r\n');
});
});
@@ -98,12 +102,15 @@ test('router delivery authenticates the CONNECT target and posts the body withou
namespace: 'native-codex',
actor: 'codex-final',
token: 'fixture-shim-token-with-at-least-32-characters',
- payload: { name: 'codex-auth', contents: '{"fixture":"provider"}' },
+ payload: { name: 'codex-auth', contents: '{"fixture":"provider"}' }
});
assert.equal(status, 201);
assert.equal(received.target, 'actor-upstream:8090');
assert.equal(received.actorHeader, 'native-codex/codex-final');
- assert.match(received.requestHeaders, /Authorization: Bearer fixture-shim-token-with-at-least-32-characters/i);
+ assert.match(
+ received.requestHeaders,
+ /Authorization: Bearer fixture-shim-token-with-at-least-32-characters/i
+ );
assert.deepEqual(received.payload, { name: 'codex-auth', contents: '{"fixture":"provider"}' });
});
@@ -122,29 +129,38 @@ test('native-agent launcher reads Claude auth from its file into the process env
fs.writeFileSync(
path.join(bin, 'claude'),
'#!/bin/sh\nprintf "%s" "$CLAUDE_CODE_OAUTH_TOKEN" >"$TOKEN_CAPTURE"\nprintf "%s\\n" "$DISABLE_TELEMETRY" "$DISABLE_ERROR_REPORTING" "$CLAUDE_CODE_DISABLE_NONESSENTIAL_TRAFFIC" "$DISABLE_AUTOUPDATER" >"$FLAGS_CAPTURE"\nprintf "%s\\n" "$@" >"$ARGS_CAPTURE"\n',
- { mode: 0o700 },
+ { mode: 0o700 }
);
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
- const result = spawnSync('/bin/bash', [path.join(liveAgentImage, 'bin/start-native-agent'), 'claude'], {
- encoding: 'utf8',
- env: {
- ...process.env,
- HOME: home,
- PATH: `${bin}:${process.env.PATH}`,
- WORKSPACE_PATH: workspace,
- TOKEN_CAPTURE: tokenCapture,
- FLAGS_CAPTURE: path.join(root, 'flags.capture'),
- ARGS_CAPTURE: argsCapture,
- AGENT_SYSTEM_PROMPT_FILE: path.join(root, 'system-prompt.fixture'),
- },
- });
+ const result = spawnSync(
+ '/bin/bash',
+ [path.join(liveAgentImage, 'bin/start-native-agent'), 'claude'],
+ {
+ encoding: 'utf8',
+ env: {
+ ...process.env,
+ HOME: home,
+ PATH: `${bin}:${process.env.PATH}`,
+ WORKSPACE_PATH: workspace,
+ TOKEN_CAPTURE: tokenCapture,
+ FLAGS_CAPTURE: path.join(root, 'flags.capture'),
+ ARGS_CAPTURE: argsCapture,
+ AGENT_SYSTEM_PROMPT_FILE: path.join(root, 'system-prompt.fixture')
+ }
+ }
+ );
assert.equal(result.status, 0, result.stderr);
assert.equal(fs.readFileSync(tokenCapture, 'utf8'), 'fixture-claude-oauth-value');
assert.deepEqual(fs.readFileSync(path.join(root, 'flags.capture'), 'utf8').trim().split('\n'), [
- '1', '1', '1', '1',
+ '1',
+ '1',
+ '1',
+ '1'
]);
const args = fs.readFileSync(argsCapture, 'utf8');
+ assert.match(args, /-p/);
+ assert.match(args, /--output-format/);
assert.match(args, /--dangerously-skip-permissions/);
assert.match(args, /--append-system-prompt-file/);
assert.equal(args.includes('fixture-claude-oauth-value'), false);
@@ -162,7 +178,9 @@ test('native-agent launcher starts Codex only when its installed auth file exist
fs.mkdirSync(bin);
fs.mkdirSync(workspace);
fs.writeFileSync(path.join(codexHome, 'auth.json'), '{"fixture":"codex-auth"}', { mode: 0o600 });
- fs.writeFileSync(path.join(bin, 'codex'), '#!/bin/sh\nprintf "%s\\n" "$@" >"$ARGS_CAPTURE"\n', { mode: 0o700 });
+ fs.writeFileSync(path.join(bin, 'codex'), '#!/bin/sh\nprintf "%s\\n" "$@" >"$ARGS_CAPTURE"\n', {
+ mode: 0o700
+ });
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
const env = {
@@ -171,21 +189,32 @@ test('native-agent launcher starts Codex only when its installed auth file exist
CODEX_HOME: codexHome,
PATH: `${bin}:${process.env.PATH}`,
WORKSPACE_PATH: workspace,
- ARGS_CAPTURE: argsCapture,
+ ARGS_CAPTURE: argsCapture
};
- const result = spawnSync('/bin/bash', [path.join(liveAgentImage, 'bin/start-native-agent'), 'codex'], {
- encoding: 'utf8',
- env,
- });
+ const result = spawnSync(
+ '/bin/bash',
+ [path.join(liveAgentImage, 'bin/start-native-agent'), 'codex'],
+ {
+ encoding: 'utf8',
+ env
+ }
+ );
assert.equal(result.status, 0, result.stderr);
- assert.equal(fs.readFileSync(argsCapture, 'utf8').trim(), '--dangerously-bypass-approvals-and-sandbox');
+ assert.equal(
+ fs.readFileSync(argsCapture, 'utf8').trim(),
+ 'exec\n--json\n--dangerously-bypass-approvals-and-sandbox'
+ );
assert.equal(result.stdout.includes('fixture'), false);
fs.rmSync(path.join(codexHome, 'auth.json'));
- const missing = spawnSync('/bin/bash', [path.join(liveAgentImage, 'bin/start-native-agent'), 'codex'], {
- encoding: 'utf8',
- env,
- });
+ const missing = spawnSync(
+ '/bin/bash',
+ [path.join(liveAgentImage, 'bin/start-native-agent'), 'codex'],
+ {
+ encoding: 'utf8',
+ env
+ }
+ );
assert.equal(missing.status, 1);
assert.equal(missing.stderr.includes('auth.json'), false);
});
@@ -195,10 +224,14 @@ test('golden boot seeds trusted workspaces, themes, and disabled update checks w
const home = path.join(root, 'home');
const codexHome = path.join(home, '.codex');
const workspace = path.join(root, 'repo');
- const result = spawnSync(process.execPath, [path.join(liveAgentImage, 'bin/prepare-native-agent-config.cjs')], {
- encoding: 'utf8',
- env: { ...process.env, HOME: home, CODEX_HOME: codexHome, WORKSPACE_PATH: workspace },
- });
+ const result = spawnSync(
+ process.execPath,
+ [path.join(liveAgentImage, 'bin/prepare-native-agent-config.cjs')],
+ {
+ encoding: 'utf8',
+ env: { ...process.env, HOME: home, CODEX_HOME: codexHome, WORKSPACE_PATH: workspace }
+ }
+ );
t.after(() => fs.rmSync(root, { recursive: true, force: true }));
assert.equal(result.status, 0, result.stderr);
diff --git a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
index 5006a60..63c0f46 100644
--- a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
+++ b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
@@ -11,20 +11,36 @@ import { test } from 'node:test';
import { fileURLToPath } from 'node:url';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
-const shim = path.resolve(__dirname, '../live-agent-image/exec-shim.js');
+const image = path.resolve(__dirname, '../live-agent-image');
+const shim = path.join(image, 'exec-shim.js');
+const launcher = path.join(image, 'bin/start-native-agent');
+const fixtures = path.join(__dirname, 'fixtures/native');
+const token = 'fixture-only-shim-token-long-enough-for-testing';
-async function startShim(home, fakeBin, shimPath, extraEnv = {}) {
- const child = spawn(process.execPath, [shimPath], {
+async function startShim(root, extraEnv = {}) {
+ const home = path.join(root, 'home');
+ const workspace = path.join(root, 'repo');
+ const state = path.join(root, 'state');
+ const fakeBin = path.join(root, 'bin');
+ const shimCopy = path.join(root, 'exec-shim.js');
+ fs.mkdirSync(home, { recursive: true });
+ fs.mkdirSync(workspace, { recursive: true });
+ fs.mkdirSync(fakeBin, { recursive: true });
+ fs.copyFileSync(shim, shimCopy);
+ const child = spawn(process.execPath, [shimCopy], {
env: {
...process.env,
...extraEnv,
HOME: home,
- PATH: `${fakeBin}:${process.env.PATH}`,
- HERDR_SESSION: 'shim-test',
- EXEC_SHIM_PANE_ID: 'pane-test',
+ CODEX_HOME: path.join(home, '.codex'),
+ WORKSPACE_PATH: workspace,
+ EXEC_SHIM_STATE_DIR: state,
+ EXEC_SHIM_HOST: '127.0.0.1',
EXEC_SHIM_PORT: '0',
+ NATIVE_AGENT_LAUNCHER: launcher,
+ PATH: `${fakeBin}:${process.env.PATH}`
},
- stdio: ['ignore', 'pipe', 'pipe'],
+ stdio: ['ignore', 'pipe', 'pipe']
});
let output = '';
const port = await new Promise((resolve, reject) => {
@@ -43,110 +59,132 @@ async function startShim(home, fakeBin, shimPath, extraEnv = {}) {
output += chunk;
});
});
- return { child, port, output: () => output };
+ return { child, port, output: () => output, home, workspace, fakeBin, state };
}
-function request(port, method, route, { body, token } = {}) {
+function request(port, method, route, { body, bearer } = {}) {
return new Promise((resolve, reject) => {
const headers = {};
if (body !== undefined) headers['content-type'] = 'application/json';
- if (token !== undefined) headers.authorization = `Bearer ${token}`;
- const req = http.request(
- { host: '127.0.0.1', port, method, path: route, headers },
- (res) => {
- const chunks = [];
- res.on('data', (chunk) => chunks.push(chunk));
- res.on('end', () =>
- resolve({ status: res.statusCode, body: Buffer.concat(chunks).toString('utf8') }),
- );
- },
- );
+ if (bearer !== undefined) headers.authorization = `Bearer ${bearer}`;
+ const req = http.request({ host: '127.0.0.1', port, method, path: route, headers }, (res) => {
+ const chunks = [];
+ res.on('data', (chunk) => chunks.push(chunk));
+ res.on('end', () =>
+ resolve({
+ status: res.statusCode,
+ body: Buffer.concat(chunks).toString('utf8')
+ })
+ );
+ });
req.once('error', reject);
if (body !== undefined) req.end(JSON.stringify(body));
else req.end();
});
}
+async function installToken(running) {
+ const result = await request(running.port, 'POST', '/token', { body: { token } });
+ assert.equal(result.status, 201, result.body);
+}
+
+async function installCredential(running, name, contents) {
+ const result = await request(running.port, 'POST', '/credential', {
+ bearer: token,
+ body: { name, contents }
+ });
+ assert.equal(result.status, 201, result.body);
+}
+
+async function waitForJob(running, kind, id, bearer = token) {
+ for (let attempt = 0; attempt < 100; attempt += 1) {
+ const result = await request(running.port, 'GET', `/${kind}/${id}`, { bearer });
+ assert.equal(result.status, 200, result.body);
+ const job = JSON.parse(result.body);
+ if (job.status !== 'running') return job;
+ await new Promise((resolve) => setTimeout(resolve, 20));
+ }
+ throw new Error(`${kind} job ${id} did not finish`);
+}
+
async function stop(child) {
if (child.exitCode !== null || child.signalCode !== null) return;
child.kill('SIGTERM');
await once(child, 'exit');
}
-test('shim token gates run/read, is one-time, private, and survives process restart', async (t) => {
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-test-'));
- const home = path.join(root, 'home');
- const fakeBin = path.join(root, 'bin');
- const shimPath = path.join(root, 'exec-shim.js');
- fs.mkdirSync(home);
- fs.mkdirSync(fakeBin);
- fs.copyFileSync(shim, shimPath);
- const herdr = path.join(fakeBin, 'herdr');
- fs.writeFileSync(
- herdr,
- '#!/bin/sh\nif [ "$3" = "status" ] && [ "$4" = "server" ]; then echo "status: running"; exit 0; fi\nif [ "$3" = "pane" ] && [ "$4" = "read" ]; then echo "fixture pane"; exit 0; fi\nexit 0\n',
- { mode: 0o700 },
- );
-
- let running = await startShim(home, fakeBin, shimPath);
+function fakeCli(file, script) {
+ fs.writeFileSync(file, script, { mode: 0o700 });
+}
+
+test('token is private and one-time; readiness is workspace-only and work routes require auth', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-token-'));
+ let running = await startShim(root);
t.after(async () => {
await stop(running.child);
fs.rmSync(root, { recursive: true, force: true });
});
assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
- assert.equal((await request(running.port, 'GET', '/read')).status, 200);
+ assert.equal((await request(running.port, 'GET', '/readyz')).status, 200);
assert.equal(
- (await request(running.port, 'POST', '/run', { body: { command: 'echo fixture' } })).status,
- 200,
+ (await request(running.port, 'POST', '/run', { body: { command: 'true' } })).status,
+ 401
);
-
- const token = 'fixture-only-token-with-at-least-thirty-two-characters';
assert.equal(
- (await request(running.port, 'POST', '/token', { body: { token } })).status,
- 201,
+ (await request(running.port, 'POST', '/turn', { body: { agent: 'claude', prompt: 'fixture' } }))
+ .status,
+ 401
);
- const tokenPath = path.join(home, '.mainloop', 'exec-shim-token');
+ await installToken(running);
+
+ const tokenPath = path.join(running.home, '.mainloop', 'exec-shim-token');
assert.equal(fs.statSync(tokenPath).mode & 0o777, 0o600);
assert.equal(fs.statSync(path.dirname(tokenPath)).mode & 0o777, 0o700);
assert.equal(fs.readFileSync(tokenPath, 'utf8'), token);
- assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
- assert.equal((await request(running.port, 'GET', '/read')).status, 401);
- assert.equal((await request(running.port, 'GET', '/read', { token: `${token}-wrong` })).status, 401);
- assert.equal((await request(running.port, 'GET', '/read', { token })).status, 200);
- assert.equal((await request(running.port, 'POST', '/run', { body: { command: 'echo fixture' } })).status, 401);
assert.equal(
- (await request(running.port, 'POST', '/run', { body: { command: 'echo fixture' }, token })).status,
- 200,
+ (await request(running.port, 'GET', '/turn/00000000-0000-4000-8000-000000000000')).status,
+ 401
+ );
+ assert.equal(
+ (
+ await request(running.port, 'GET', '/turn/00000000-0000-4000-8000-000000000000', {
+ bearer: `${token}-wrong`
+ })
+ ).status,
+ 401
+ );
+ assert.equal(
+ (
+ await request(running.port, 'GET', '/turn/00000000-0000-4000-8000-000000000000', {
+ bearer: token
+ })
+ ).status,
+ 404
);
assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 409);
assert.equal(running.output().includes(token), false);
await stop(running.child);
- running = await startShim(home, fakeBin, shimPath);
+ running = await startShim(root);
assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
- assert.equal((await request(running.port, 'GET', '/read')).status, 401);
- assert.equal((await request(running.port, 'GET', '/read', { token })).status, 200);
- assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 409);
+ assert.equal(
+ (await request(running.port, 'GET', '/turn/00000000-0000-4000-8000-000000000000')).status,
+ 401
+ );
+ assert.equal(
+ (
+ await request(running.port, 'GET', '/turn/00000000-0000-4000-8000-000000000000', {
+ bearer: token
+ })
+ ).status,
+ 404
+ );
});
-test('credential delivery requires a shim token and writes only allowlisted private files once', async (t) => {
+test('credential delivery remains token-gated, allowlisted, private, and one-time', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-credentials-'));
- const home = path.join(root, 'home');
- const fakeBin = path.join(root, 'bin');
- const shimPath = path.join(root, 'exec-shim.js');
- const codexHome = path.join(home, '.codex');
- fs.mkdirSync(home);
- fs.mkdirSync(fakeBin);
- fs.copyFileSync(shim, shimPath);
- const herdr = path.join(fakeBin, 'herdr');
- fs.writeFileSync(
- herdr,
- '#!/bin/sh\nif [ "$3" = "status" ] && [ "$4" = "server" ]; then echo "status: running"; exit 0; fi\nif [ "$3" = "pane" ] && [ "$4" = "read" ]; then echo "fixture pane"; exit 0; fi\nexit 0\n',
- { mode: 0o700 },
- );
-
- const running = await startShim(home, fakeBin, shimPath, { CODEX_HOME: codexHome });
+ const running = await startShim(root);
t.after(async () => {
await stop(running.child);
fs.rmSync(root, { recursive: true, force: true });
@@ -154,79 +192,219 @@ test('credential delivery requires a shim token and writes only allowlisted priv
const claudeContents = 'fixture-claude-token-never-logged';
const authContents = JSON.stringify({ fixture: 'codex-auth-never-logged' });
- const write = (name, contents, token) =>
- request(running.port, 'POST', '/credential', {
- body: { name, contents },
- ...(token === undefined ? {} : { token }),
- });
-
- assert.equal((await write('codex-auth', authContents)).status, 401, 'golden actor must reject writes before token installation');
- assert.equal(fs.existsSync(codexHome), false);
-
- const token = 'fixture-only-codex-shim-token-at-least-thirty-two-chars';
- assert.equal((await request(running.port, 'POST', '/token', { body: { token } })).status, 201);
- assert.equal((await write('claude-token', claudeContents)).status, 401);
- assert.equal((await write('claude-token', claudeContents, `${token}-wrong`)).status, 401);
- assert.equal((await write('../outside', claudeContents, token)).status, 403);
- assert.equal(fs.existsSync(path.join(root, 'outside')), false);
- assert.equal((await write('codex-auth', '[]', token)).status, 400);
- assert.equal((await write('codex-auth', 'not-json', token)).status, 400);
- assert.equal((await request(running.port, 'POST', '/write-codex-auth', { token, body: {} })).status, 404);
-
- assert.equal((await write('claude-token', claudeContents, token)).status, 201);
- const claudePath = path.join(home, '.mainloop', 'claude-token');
+ assert.equal(
+ (
+ await request(running.port, 'POST', '/credential', {
+ body: { name: 'codex-auth', contents: authContents }
+ })
+ ).status,
+ 401
+ );
+ await installToken(running);
+ assert.equal(
+ (
+ await request(running.port, 'POST', '/credential', {
+ bearer: `${token}-wrong`,
+ body: { name: 'claude-token', contents: claudeContents }
+ })
+ ).status,
+ 401
+ );
+ assert.equal(
+ (
+ await request(running.port, 'POST', '/credential', {
+ bearer: token,
+ body: { name: '../outside', contents: claudeContents }
+ })
+ ).status,
+ 403
+ );
+ assert.equal(
+ (
+ await request(running.port, 'POST', '/credential', {
+ bearer: token,
+ body: { name: 'codex-auth', contents: '[]' }
+ })
+ ).status,
+ 400
+ );
+ assert.equal(
+ (
+ await request(running.port, 'POST', '/credential', {
+ bearer: token,
+ body: { name: 'claude-token', contents: claudeContents }
+ })
+ ).status,
+ 201
+ );
+ const claudePath = path.join(running.home, '.mainloop', 'claude-token');
assert.equal(fs.readFileSync(claudePath, 'utf8'), claudeContents);
assert.equal(fs.statSync(claudePath).mode & 0o777, 0o600);
- assert.equal(fs.statSync(path.dirname(claudePath)).mode & 0o777, 0o700);
- assert.equal((await write('claude-token', 'replacement', token)).status, 409);
-
- assert.equal((await write('codex-auth', authContents, token)).status, 201);
- const authPath = path.join(codexHome, 'auth.json');
+ assert.equal(
+ (
+ await request(running.port, 'POST', '/credential', {
+ bearer: token,
+ body: { name: 'claude-token', contents: 'replacement' }
+ })
+ ).status,
+ 409
+ );
+ assert.equal(
+ (
+ await request(running.port, 'POST', '/credential', {
+ bearer: token,
+ body: { name: 'codex-auth', contents: authContents }
+ })
+ ).status,
+ 201
+ );
+ const authPath = path.join(running.home, '.codex', 'auth.json');
assert.equal(fs.readFileSync(authPath, 'utf8'), authContents);
- assert.equal(fs.statSync(codexHome).mode & 0o777, 0o700);
assert.equal(fs.statSync(authPath).mode & 0o777, 0o600);
- assert.equal((await write('codex-auth', authContents, token)).status, 409, 'auth file must not be overwritten');
assert.equal(running.output().includes(claudeContents), false);
assert.equal(running.output().includes(authContents), false);
});
-test('healthz bounds Herdr calls and reuses a recent successful check', async (t) => {
- const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-health-'));
- const home = path.join(root, 'home');
- const fakeBin = path.join(root, 'bin');
- const shimPath = path.join(root, 'exec-shim.js');
- const logPath = path.join(root, 'herdr-calls.log');
- fs.mkdirSync(home);
- fs.mkdirSync(fakeBin);
- fs.copyFileSync(shim, shimPath);
- const herdr = path.join(fakeBin, 'herdr');
- fs.writeFileSync(
- herdr,
- '#!/bin/sh\nif [ -n "${EXEC_SHIM_TEST_LOG:-}" ]; then printf "%s %s\\n" "$3" "$4" >> "$EXEC_SHIM_TEST_LOG"; fi\nif [ "$3" = "status" ] && [ "$4" = "server" ]; then if [ "${HERDR_TEST_SLOW_STATUS:-}" = "1" ]; then exec sleep 5; fi; echo "status: running"; exit 0; fi\nif [ "$3" = "pane" ] && [ "$4" = "read" ]; then echo "fixture pane"; exit 0; fi\nexit 0\n',
- { mode: 0o700 },
+test('turn prompt is piped on stdin and never appears in argv or shim logs', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-stdin-'));
+ const running = await startShim(root, {
+ ARGS_CAPTURE: path.join(root, 'claude-args'),
+ PROMPT_CAPTURE: path.join(root, 'claude-prompt')
+ });
+ fakeCli(
+ path.join(running.fakeBin, 'claude'),
+ '#!/bin/sh\nprintf "%s\\n" "$@" >"$ARGS_CAPTURE"\ncat >"$PROMPT_CAPTURE"\ncat <<\'EVENTS\'\n{"type":"system","subtype":"init","session_id":"fixture-session"}\n{"type":"result","result":"fixture answer","session_id":"fixture-session"}\nEVENTS\n'
+ );
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+ await installToken(running);
+ await installCredential(running, 'claude-token', 'fixture-claude-token');
+ const prompt = 'private fixture prompt must travel only on stdin';
+ const started = await request(running.port, 'POST', '/turn', {
+ bearer: token,
+ body: { agent: 'claude', prompt }
+ });
+ assert.equal(started.status, 202, started.body);
+ const { id } = JSON.parse(started.body);
+ const result = await waitForJob(running, 'turn', id);
+ assert.equal(result.status, 'completed');
+ assert.equal(result.final_message, 'fixture answer');
+ assert.equal(result.native_session_id, 'fixture-session');
+ assert.equal(fs.readFileSync(path.join(root, 'claude-prompt'), 'utf8'), prompt);
+ const argv = fs.readFileSync(path.join(root, 'claude-args'), 'utf8');
+ assert.match(argv, /-p/);
+ assert.match(argv, /--output-format/);
+ assert.equal(argv.includes(prompt), false);
+ assert.equal(running.output().includes(prompt), false);
+ assert.equal(
+ fs
+ .readFileSync(path.join(running.state, 'turns', id + '.events.jsonl'), 'utf8')
+ .includes(prompt),
+ false
+ );
+});
+
+test('a second concurrent turn for the same agent receives 409 and is not queued', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-concurrency-'));
+ const running = await startShim(root, { TURN_DELAY: '0.4' });
+ fakeCli(
+ path.join(running.fakeBin, 'claude'),
+ '#!/bin/sh\ncat >/dev/null\nsleep "$TURN_DELAY"\nprintf \'%s\\n\' \'{"type":"result","session_id":"fixture-session","result":"done"}\'\n'
);
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+ await installToken(running);
+ await installCredential(running, 'claude-token', 'fixture-claude-token');
+ const body = { agent: 'claude', prompt: 'turn prompt' };
+ const first = await request(running.port, 'POST', '/turn', { bearer: token, body });
+ assert.equal(first.status, 202, first.body);
+ const second = await request(running.port, 'POST', '/turn', { bearer: token, body });
+ assert.equal(second.status, 409);
+ const result = await waitForJob(running, 'turn', JSON.parse(first.body).id);
+ assert.equal(result.status, 'completed');
+});
+
+test('/run executes a command, stores bounded output, and reports timeout', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-run-'));
+ const running = await startShim(root);
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+ await installToken(running);
+ const quick = await request(running.port, 'POST', '/run', {
+ bearer: token,
+ body: { command: 'printf fixture-output', timeout_ms: 1000 }
+ });
+ assert.equal(quick.status, 202, quick.body);
+ const completed = await waitForJob(running, 'run', JSON.parse(quick.body).id);
+ assert.equal(completed.status, 'completed');
+ assert.equal(completed.exit_code, 0);
+ assert.equal(completed.output, 'fixture-output');
+
+ const slow = await request(running.port, 'POST', '/run', {
+ bearer: token,
+ body: { command: 'sleep 5; printf should-not-finish', timeout_ms: 50 }
+ });
+ assert.equal(slow.status, 202, slow.body);
+ const timeout = await waitForJob(running, 'run', JSON.parse(slow.body).id);
+ assert.equal(timeout.status, 'timed_out');
+ assert.equal(timeout.output.includes('should-not-finish'), false);
+});
- const running = await startShim(home, fakeBin, shimPath, {
- EXEC_SHIM_TEST_LOG: logPath,
+test('Claude stream-json and Codex JSONL produce native ids and final messages', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-events-'));
+ const running = await startShim(root, {
+ CLAUDE_EVENTS: path.join(fixtures, 'claude-stream-json.jsonl'),
+ CODEX_EVENTS: path.join(fixtures, 'codex-jsonl.jsonl'),
+ CLAUDE_ARGS_CAPTURE: path.join(root, 'claude-args'),
+ CODEX_ARGS_CAPTURE: path.join(root, 'codex-args')
});
+ fakeCli(
+ path.join(running.fakeBin, 'claude'),
+ '#!/bin/sh\nprintf "%s\\n" "$@" >"$CLAUDE_ARGS_CAPTURE"\ncat >/dev/null\ncat "$CLAUDE_EVENTS"\n'
+ );
+ fakeCli(
+ path.join(running.fakeBin, 'codex'),
+ '#!/bin/sh\nprintf "%s\\n" "$@" >"$CODEX_ARGS_CAPTURE"\ncat >/dev/null\ncat "$CODEX_EVENTS"\n'
+ );
t.after(async () => {
await stop(running.child);
fs.rmSync(root, { recursive: true, force: true });
});
+ await installToken(running);
+ await installCredential(running, 'claude-token', 'fixture-claude-token');
+ await installCredential(running, 'codex-auth', '{"fixture":"auth"}');
- assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
- assert.equal((await request(running.port, 'GET', '/healthz')).status, 200);
- assert.deepEqual(fs.readFileSync(logPath, 'utf8').trim().split('\n'), [
- 'status server',
- 'pane read',
- ]);
+ const claudeStart = await request(running.port, 'POST', '/turn', {
+ bearer: token,
+ body: { agent: 'claude', session_id: 'resume-session-001', prompt: 'claude fixture prompt' }
+ });
+ assert.equal(claudeStart.status, 202, claudeStart.body);
+ const claude = await waitForJob(running, 'turn', JSON.parse(claudeStart.body).id);
+ assert.equal(claude.native_session_id, 'fixture-claude-session');
+ assert.equal(claude.final_message, 'fixture Claude final');
+ assert.equal(claude.events.length, 3);
+ assert.match(
+ fs.readFileSync(path.join(root, 'claude-args'), 'utf8'),
+ /--resume\s+resume-session-001/
+ );
- await stop(running.child);
- const slow = await startShim(home, fakeBin, shimPath, {
- HERDR_TEST_SLOW_STATUS: '1',
+ const codexStart = await request(running.port, 'POST', '/turn', {
+ bearer: token,
+ body: { agent: 'codex', session_id: 'resume-thread-001', prompt: 'codex fixture prompt' }
});
- t.after(async () => stop(slow.child));
- const startedAt = Date.now();
- assert.equal((await request(slow.port, 'GET', '/healthz')).status, 503);
- assert.ok(Date.now() - startedAt < 4000, 'hung Herdr status must be bounded by execFile timeout');
+ assert.equal(codexStart.status, 202, codexStart.body);
+ const codex = await waitForJob(running, 'turn', JSON.parse(codexStart.body).id);
+ assert.equal(codex.native_session_id, 'fixture-codex-thread');
+ assert.equal(codex.final_message, 'fixture Codex final');
+ assert.equal(codex.events.length, 4);
+ assert.equal(
+ fs.readFileSync(path.join(root, 'codex-args'), 'utf8').trim(),
+ 'exec\nresume\n--json\nresume-thread-001\n--dangerously-bypass-approvals-and-sandbox'
+ );
});
diff --git a/spikes/substrate-workspace-adapter/tests/fixtures/native/claude-stream-json.jsonl b/spikes/substrate-workspace-adapter/tests/fixtures/native/claude-stream-json.jsonl
new file mode 100644
index 0000000..5d05fa4
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tests/fixtures/native/claude-stream-json.jsonl
@@ -0,0 +1,3 @@
+{"type":"system","subtype":"init","session_id":"fixture-claude-session","model":"claude-fixture"}
+{"type":"assistant","message":{"content":[{"type":"text","text":"fixture Claude response"}]}}
+{"type":"result","subtype":"success","session_id":"fixture-claude-session","result":"fixture Claude final"}
diff --git a/spikes/substrate-workspace-adapter/tests/fixtures/native/codex-jsonl.jsonl b/spikes/substrate-workspace-adapter/tests/fixtures/native/codex-jsonl.jsonl
new file mode 100644
index 0000000..c2da5e8
--- /dev/null
+++ b/spikes/substrate-workspace-adapter/tests/fixtures/native/codex-jsonl.jsonl
@@ -0,0 +1,4 @@
+{"type":"thread.started","thread_id":"fixture-codex-thread"}
+{"type":"turn.started","thread_id":"fixture-codex-thread"}
+{"type":"item.completed","item":{"id":"fixture-message-id","type":"agent_message","text":"fixture Codex final"}}
+{"type":"turn.completed","thread_id":"fixture-codex-thread"}
diff --git a/spikes/substrate-workspace-adapter/tools/phase4/deliver-credentials.cjs b/spikes/substrate-workspace-adapter/tools/phase4/deliver-credentials.cjs
index 5ebb55e..7cf3167 100644
--- a/spikes/substrate-workspace-adapter/tools/phase4/deliver-credentials.cjs
+++ b/spikes/substrate-workspace-adapter/tools/phase4/deliver-credentials.cjs
@@ -5,7 +5,7 @@ const http = require('node:http');
const CREDENTIALS = Object.freeze({
claude: Object.freeze({ name: 'claude-token' }),
- codex: Object.freeze({ name: 'codex-auth' }),
+ codex: Object.freeze({ name: 'codex-auth' })
});
function validateActorIdentity(namespace, actor) {
@@ -55,7 +55,7 @@ function postViaRouter({ host, port, namespace, actor, token, payload, timeoutMs
port,
method: 'CONNECT',
path: 'actor-upstream:8090',
- headers: { 'ate-target-actor': `${namespace}/${actor}` },
+ headers: { 'ate-target-actor': `${namespace}/${actor}` }
});
tunnel.setTimeout(timeoutMs, () => tunnel.destroy(new Error('router timed out')));
tunnel.once('error', (error) => finish(error));
@@ -88,16 +88,19 @@ function postViaRouter({ host, port, namespace, actor, token, payload, timeoutMs
socket.end();
});
- const headers = Buffer.from([
- 'POST /credential HTTP/1.1',
- 'Host: actor-upstream:8090',
- `Authorization: Bearer ${token}`,
- 'Content-Type: application/json',
- `Content-Length: ${body.length}`,
- 'Connection: close',
- '',
- '',
- ].join('\r\n'), 'ascii');
+ const headers = Buffer.from(
+ [
+ 'POST /credential HTTP/1.1',
+ 'Host: actor-upstream:8090',
+ `Authorization: Bearer ${token}`,
+ 'Content-Type: application/json',
+ `Content-Length: ${body.length}`,
+ 'Connection: close',
+ '',
+ ''
+ ].join('\r\n'),
+ 'ascii'
+ );
socket.write(Buffer.concat([headers, body]));
});
tunnel.end();
@@ -112,7 +115,7 @@ async function deliverFromMountedFiles({
actor,
host = 'atenet-router.ate-system.svc.cluster.local',
port = 8081,
- request = postViaRouter,
+ request = postViaRouter
}) {
validateActorIdentity(namespace, actor);
const payload = buildCredentialPayload(kind, fs.readFileSync(credentialFile, 'utf8'));
@@ -126,7 +129,7 @@ async function deliverFromMountedFiles({
namespace,
actor,
token,
- payload,
+ payload
});
if (status !== 201) throw new Error('shim rejected credential delivery');
return { kind, namespace, actor };
@@ -140,7 +143,7 @@ async function main() {
namespace: process.env.ACTOR_NAMESPACE,
actor: process.env.ACTOR_NAME,
host: process.env.ROUTER_HOST || 'atenet-router.ate-system.svc.cluster.local',
- port: Number(process.env.ROUTER_PORT || '8081'),
+ port: Number(process.env.ROUTER_PORT || '8081')
});
process.stdout.write(`credential delivered for ${process.env.CREDENTIAL_KIND}\n`);
}
diff --git a/spikes/substrate-workspace-adapter/tools/router-client.js b/spikes/substrate-workspace-adapter/tools/router-client.js
index 4555f11..bb29c97 100644
--- a/spikes/substrate-workspace-adapter/tools/router-client.js
+++ b/spikes/substrate-workspace-adapter/tools/router-client.js
@@ -24,9 +24,12 @@ process.stdin.on('end', () => {
if (
!/^[a-z0-9-]+$/.test(request.atespace || '') ||
!/^[a-z0-9-]+$/.test(request.actor || '') ||
- !Number.isInteger(targetPort) || targetPort < 1 || targetPort > 65535 ||
+ !Number.isInteger(targetPort) ||
+ targetPort < 1 ||
+ targetPort > 65535 ||
!['GET', 'POST'].includes(request.method) ||
- typeof request.path !== 'string' || !request.path.startsWith('/')
+ typeof request.path !== 'string' ||
+ !request.path.startsWith('/')
) {
process.stdout.write('{"error":"invalid request"}\n');
process.exitCode = 2;
@@ -47,9 +50,9 @@ process.stdin.on('end', () => {
path: `actor-upstream:${targetPort}`,
headers: {
host: `actor-upstream:${targetPort}`,
- 'ate-target-actor': `${request.atespace}/${request.actor}`,
+ 'ate-target-actor': `${request.atespace}/${request.actor}`
},
- timeout: 8000,
+ timeout: 8000
});
tunnel.on('connect', (response, socket) => {
if (response.statusCode !== 200) {
@@ -74,7 +77,7 @@ process.stdin.on('end', () => {
}
const headerLines = Object.entries(headers).map(([name, value]) => `${name}: ${value}`);
const requestHeader = Buffer.from(
- `${request.method} ${request.path} HTTP/1.1\r\n${headerLines.join('\r\n')}\r\n\r\n`,
+ `${request.method} ${request.path} HTTP/1.1\r\n${headerLines.join('\r\n')}\r\n\r\n`
);
const responseChunks = [];
if (response.head && response.head.length) responseChunks.push(response.head);
@@ -90,9 +93,7 @@ process.stdin.on('end', () => {
const statusLine = responseBytes.subarray(0, headerEnd).toString('latin1').split('\r\n')[0];
const status = Number(statusLine.split(' ')[1]);
const responseBody = responseBytes.subarray(headerEnd + 4).toString('utf8');
- finish(request.includeResponseBody
- ? { status, body: responseBody }
- : { status });
+ finish(request.includeResponseBody ? { status, body: responseBody } : { status });
});
socket.on('error', (err) => finish({ transportError: err.code || 'request-failed' }));
socket.write(Buffer.concat([requestHeader, body]));
From d9fc9e2dc4e635dcab628fe63dbaaa9c45204e4f Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Thu, 24 Sep 2026 01:58:38 +0000
Subject: [PATCH 15/30] feat: add Substrate transport for native sessions
Add a config-selected Substrate workspace transport that attaches to pre-created actors through
the CONNECT router and reads per-actor shim tokens from Kubernetes Secrets. Keep Herdr as the
default runtime. Extend the headless shim with authenticated credential readiness, turn status,
bounded native journal pages, and stop handling; cover delivery through native_sessions with fake
router and shim tests. Document the runtime configuration and its fixture-backed evidence.
Start the actor entrypoint as root only for ownership preparation, then exec it as UID/GID
10001 with initialized groups, an empty capability bounding set, and no-new-privileges. Install
setpriv in the image and refuse root in the shim outside its explicit test-only override. Cover
the root guard and image privilege-drop wiring with local tests.
---
backend/src/mainloop/config.py | 25 +-
.../src/mainloop/runtime/native_sessions.py | 53 ++-
.../mainloop/runtime/substrate_workspace.py | 418 ++++++++++++++++++
.../tests/runtime/test_substrate_workspace.py | 375 ++++++++++++++++
docs/spikes/substrate-workspace-adapter.md | 17 +
.../live-agent-image/Dockerfile | 9 +-
.../live-agent-image/entrypoint.sh | 36 ++
.../live-agent-image/exec-shim.js | 229 +++++++++-
.../tests/exec-shim.test.js | 184 +++++++-
9 files changed, 1332 insertions(+), 14 deletions(-)
create mode 100644 backend/src/mainloop/runtime/substrate_workspace.py
create mode 100644 backend/tests/runtime/test_substrate_workspace.py
diff --git a/backend/src/mainloop/config.py b/backend/src/mainloop/config.py
index 94814a7..efa7244 100644
--- a/backend/src/mainloop/config.py
+++ b/backend/src/mainloop/config.py
@@ -1,11 +1,22 @@
"""Configuration management."""
+from typing import Literal
from urllib.parse import quote_plus
-from pydantic import computed_field
+from pydantic import BaseModel, ConfigDict, Field, computed_field
from pydantic_settings import BaseSettings, SettingsConfigDict
+class SubstrateActorBinding(BaseModel):
+ """Deployment-provided route and token Secret for one pre-created actor."""
+
+ atespace: str
+ actor: str
+ shim_token_secret_name: str
+
+ model_config = ConfigDict(extra="forbid", frozen=True)
+
+
class Settings(BaseSettings):
"""Application settings."""
@@ -37,6 +48,18 @@ def database_url(self) -> str:
"main-0" # pod that runs the native main thread (scratch cwd, no repo)
)
+ # Native-session workspace transport. Herdr remains the default; Substrate attaches to
+ # pre-created actors through the CONNECT router and never creates or resumes actors itself.
+ workspace_runtime: Literal["herdr", "substrate"] = "herdr"
+ substrate_router_address: str = (
+ "http://atenet-router.ate-system.svc.cluster.local:8081"
+ )
+ substrate_shim_secret_namespace: str = "mainloop-control"
+ substrate_actor_bindings: dict[
+ Literal["claude", "codex"], SubstrateActorBinding
+ ] = Field(default_factory=dict)
+ substrate_resume_timeout_seconds: float = 120.0
+
# Substrate workspace-runtime adapter (bounded integration spike; see
# docs/architecture/native-agent-inventory.md and .tasknotes/plan.md). Empty
# kubeconfig/context falls back to the ambient kubeconfig. One actor per session
diff --git a/backend/src/mainloop/runtime/native_sessions.py b/backend/src/mainloop/runtime/native_sessions.py
index 21667ea..3c2cfcb 100644
--- a/backend/src/mainloop/runtime/native_sessions.py
+++ b/backend/src/mainloop/runtime/native_sessions.py
@@ -31,6 +31,7 @@
from mainloop.runtime.herdr import HerdrWorkspace, TransportError, WorkspaceUnavailable
from mainloop.runtime.journal import completed_turns, parse_journal
from mainloop.runtime.standing import content_hash
+from mainloop.runtime.substrate_workspace import SubstrateWorkspace
from models import NativeDeliveryInfo, NativeSessionInfo, SessionStatus
@@ -44,7 +45,7 @@
DELIVERED_MAX_AGE = timedelta(minutes=30)
_NS = uuid.UUID("6f0f7f0e-3f1e-4a3c-9d3b-0e4b6f5c2a11")
_locks: dict[str, asyncio.Lock] = {}
-_workspaces: dict[str, HerdrWorkspace] = {}
+_workspaces: dict[tuple[str, ...], HerdrWorkspace | SubstrateWorkspace] = {}
_rotating: set[str] = set()
OPEN_STATES = ("recorded", "sending", "delivered")
# Ended by the user or by failure. Agent activity never moves a session out of these.
@@ -80,12 +81,50 @@ def is_rotating(session_id: str) -> bool:
return session_id in _rotating
-def workspace_for(binding: dict) -> HerdrWorkspace:
- """One Herdr workspace pod per binding: ``main-0`` for the main thread, else ``workspace-0``."""
+def workspace_for(binding: dict) -> HerdrWorkspace | SubstrateWorkspace:
+ """Select the configured transport and map a native binding to its workspace.
+
+ In Substrate mode, the native kind selects its atespace, actor, and shim Secret from
+ ``SUBSTRATE_ACTOR_BINDINGS``. No actor identity is inferred from a session or binding.
+ Herdr's existing pod mapping remains the default and is unchanged.
+ """
+ if settings.workspace_runtime == "substrate":
+ agent = binding["kind"]
+ actor_binding = settings.substrate_actor_bindings.get(agent)
+ if actor_binding is None:
+ raise RuntimeError(
+ f"no Substrate actor binding is configured for native agent {agent}"
+ )
+ atespace = actor_binding.atespace
+ actor = actor_binding.actor
+ key = (
+ "substrate",
+ atespace,
+ actor,
+ actor_binding.shim_token_secret_name,
+ binding["kind"],
+ )
+ if key not in _workspaces:
+ _workspaces[key] = SubstrateWorkspace(
+ atespace=atespace,
+ actor=actor,
+ agent=agent,
+ shim_token_secret_name=actor_binding.shim_token_secret_name,
+ native_session_id=binding.get("native_session_id"),
+ )
+ workspace = _workspaces[key]
+ if not isinstance(workspace, SubstrateWorkspace):
+ raise RuntimeError("workspace cache has an incompatible Substrate entry")
+ workspace.set_native_session_id(binding.get("native_session_id"))
+ return workspace
pod = binding.get("pod") or settings.workspace_pod
- if pod not in _workspaces:
- _workspaces[pod] = HerdrWorkspace(pod=pod)
- return _workspaces[pod]
+ key = ("herdr", pod)
+ if key not in _workspaces:
+ _workspaces[key] = HerdrWorkspace(pod=pod)
+ workspace = _workspaces[key]
+ if not isinstance(workspace, HerdrWorkspace):
+ raise RuntimeError("workspace cache has an incompatible Herdr entry")
+ return workspace
def rotation_due(
@@ -768,7 +807,7 @@ async def identity(session_id: str) -> NativeSessionInfo | None:
ready, uid = pod.ready, pod.uid
if ready:
live = (await ws.agent_status(binding["agent_name"])) is not None
- except TransportError as exc:
+ except (TransportError, WorkspaceUnavailable) as exc:
note = f"workspace unreachable: {exc}"
if any(d.state == "uncertain" for d in deliveries):
note = "delivery unknown: the last prompt was not replayed; check the reply, then send again if needed"
diff --git a/backend/src/mainloop/runtime/substrate_workspace.py b/backend/src/mainloop/runtime/substrate_workspace.py
new file mode 100644
index 0000000..3690ad2
--- /dev/null
+++ b/backend/src/mainloop/runtime/substrate_workspace.py
@@ -0,0 +1,418 @@
+"""Native-session transport to a pre-created Substrate actor through its CONNECT router.
+
+The actor shim owns the native CLI turn and journal files. This adapter preserves the
+``HerdrWorkspace`` method contract used by ``native_sessions`` while treating delivery errors
+after ``POST /turn`` as unknown; callers must reconcile the journal and never replay blindly.
+"""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import http.client
+import json
+import logging
+import re
+import socket
+from dataclasses import dataclass
+from urllib.parse import urlencode, urlsplit
+
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+from mainloop.config import settings
+from mainloop.runtime.herdr import (
+ JournalSlice,
+ PodState,
+ TransportError,
+ WorkspaceUnavailable,
+)
+
+logger = logging.getLogger(__name__)
+
+_AGENT = re.compile(r"^(claude|codex)$")
+_DNS_LABEL = re.compile(r"^[a-z0-9](?:[-a-z0-9]{0,61}[a-z0-9])?$")
+_MAX_RESPONSE_BYTES = 2 * 1024 * 1024
+_JOURNAL_PAGE_SIZE = 200
+_ACTOR_PORT = 8090
+_SECRET_API: client.CoreV1Api | None = None
+
+
+@dataclass(frozen=True, slots=True)
+class _Response:
+ status: int
+ body: str
+
+
+def _secret_api() -> client.CoreV1Api:
+ global _SECRET_API
+ if _SECRET_API is None:
+ try:
+ config.load_incluster_config()
+ except config.ConfigException:
+ config.load_kube_config()
+ _SECRET_API = client.CoreV1Api()
+ return _SECRET_API
+
+
+def _read_secret_token(secret_name: str, namespace: str) -> str:
+ try:
+ secret = _secret_api().read_namespaced_secret(secret_name, namespace)
+ except ApiException as exc:
+ if exc.status == 404:
+ raise WorkspaceUnavailable(
+ "Substrate shim token Secret is unavailable"
+ ) from exc
+ raise TransportError(
+ f"Substrate shim token Secret read failed (status {exc.status})"
+ ) from exc
+ encoded = (secret.data or {}).get("token")
+ if not isinstance(encoded, str):
+ raise WorkspaceUnavailable("Substrate shim token Secret has no token key")
+ try:
+ token = base64.b64decode(encoded, validate=True).decode("utf-8").strip()
+ except (ValueError, UnicodeDecodeError) as exc:
+ raise WorkspaceUnavailable("Substrate shim token Secret is invalid") from exc
+ if not token:
+ raise WorkspaceUnavailable("Substrate shim token Secret is empty")
+ return token
+
+
+def _read_headers(reader) -> tuple[int, http.client.HTTPMessage]:
+ status_line = reader.readline(8192)
+ if not status_line.endswith(b"\r\n"):
+ raise ValueError("invalid HTTP status line")
+ parts = status_line.decode("latin1").strip().split(" ", 2)
+ if len(parts) < 2 or not parts[0].startswith("HTTP/"):
+ raise ValueError("invalid HTTP status line")
+ status = int(parts[1])
+ headers = http.client.parse_headers(reader)
+ return status, headers
+
+
+def _read_body(reader, headers: http.client.HTTPMessage) -> bytes:
+ transfer_encoding = headers.get("transfer-encoding", "").lower()
+ if "chunked" in transfer_encoding:
+ chunks: list[bytes] = []
+ size = 0
+ while True:
+ line = reader.readline(8192)
+ if not line:
+ raise ValueError("truncated chunked HTTP response")
+ chunk_size = int(line.split(b";", 1)[0].strip(), 16)
+ if chunk_size == 0:
+ while reader.readline(8192) not in (b"\r\n", b"\n", b""):
+ pass
+ return b"".join(chunks)
+ size += chunk_size
+ if size > _MAX_RESPONSE_BYTES:
+ raise ValueError("HTTP response exceeded the bounded size")
+ chunk = reader.read(chunk_size)
+ if len(chunk) != chunk_size or reader.read(2) != b"\r\n":
+ raise ValueError("truncated chunked HTTP response")
+ chunks.append(chunk)
+ length = headers.get("content-length")
+ if length is not None:
+ size = int(length)
+ if size < 0 or size > _MAX_RESPONSE_BYTES:
+ raise ValueError("HTTP response exceeded the bounded size")
+ body = reader.read(size)
+ if len(body) != size:
+ raise ValueError("truncated HTTP response")
+ return body
+ body = reader.read(_MAX_RESPONSE_BYTES + 1)
+ if len(body) > _MAX_RESPONSE_BYTES:
+ raise ValueError("HTTP response exceeded the bounded size")
+ return body
+
+
+def _router_request(
+ *,
+ host: str,
+ port: int,
+ actor: str,
+ atespace: str,
+ actor_port: int,
+ timeout: float,
+ method: str,
+ path: str,
+ token: str | None,
+ body: dict | None,
+) -> _Response:
+ target = f"actor-upstream:{actor_port}"
+ sock = socket.create_connection((host, port), timeout=timeout)
+ sock.settimeout(timeout)
+ reader = sock.makefile("rb")
+ try:
+ connect = (
+ f"CONNECT {target} HTTP/1.1\r\n"
+ f"Host: {target}\r\n"
+ f"ate-target-actor: {atespace}/{actor}\r\n"
+ "Connection: keep-alive\r\n\r\n"
+ ).encode("ascii")
+ sock.sendall(connect)
+ connect_status, _ = _read_headers(reader)
+ if connect_status != 200:
+ return _Response(connect_status, "")
+
+ encoded_body = (
+ json.dumps(body, separators=(",", ":")).encode()
+ if body is not None
+ else b""
+ )
+ headers = [f"Host: {target}", "Connection: close"]
+ if token is not None:
+ headers.append(f"Authorization: Bearer {token}")
+ if body is not None:
+ headers.extend(
+ [
+ "Content-Type: application/json",
+ f"Content-Length: {len(encoded_body)}",
+ ]
+ )
+ request = (
+ f"{method} {path} HTTP/1.1\r\n" + "\r\n".join(headers) + "\r\n\r\n"
+ ).encode("ascii")
+ sock.sendall(request + encoded_body)
+ status, response_headers = _read_headers(reader)
+ response_body = _read_body(reader, response_headers)
+ return _Response(status, response_body.decode("utf-8", errors="replace"))
+ finally:
+ reader.close()
+ sock.close()
+
+
+class SubstrateWorkspace:
+ """Drive one native agent in a pre-created Substrate actor."""
+
+ def __init__(
+ self,
+ *,
+ atespace: str,
+ actor: str,
+ agent: str,
+ shim_token_secret_name: str,
+ native_session_id: str | None = None,
+ router_address: str | None = None,
+ timeout: float | None = None,
+ ):
+ if not _DNS_LABEL.fullmatch(atespace) or not _DNS_LABEL.fullmatch(actor):
+ raise ValueError("Substrate atespace and actor must be DNS labels")
+ if not _AGENT.fullmatch(agent):
+ raise ValueError("Substrate native agent must be claude or codex")
+ self.atespace = atespace
+ self.actor = actor
+ self.pod = actor # The existing native-session view exposes this display field.
+ self.agent = agent
+ self.native_session_id = native_session_id
+ address = urlsplit(router_address or settings.substrate_router_address)
+ if (
+ address.scheme != "http"
+ or not address.hostname
+ or address.username is not None
+ or address.password is not None
+ or address.path not in ("", "/")
+ or address.query
+ or address.fragment
+ ):
+ raise ValueError("Substrate router address must be an HTTP origin")
+ self.router_host = address.hostname
+ self.router_port = address.port or 80
+ self.actor_port = _ACTOR_PORT
+ self.timeout = timeout or settings.substrate_resume_timeout_seconds
+ self.secret_namespace = settings.substrate_shim_secret_namespace
+ self.secret_name = shim_token_secret_name
+ if not _DNS_LABEL.fullmatch(self.secret_name):
+ raise ValueError("Substrate shim Secret name must be a DNS label")
+ self._token_value: str | None = None
+
+ def set_native_session_id(self, native_session_id: str | None) -> None:
+ self.native_session_id = native_session_id
+
+ async def _token(self) -> str:
+ if self._token_value is None:
+ self._token_value = await asyncio.to_thread(
+ _read_secret_token, self.secret_name, self.secret_namespace
+ )
+ return self._token_value
+
+ async def _request(
+ self,
+ method: str,
+ path: str,
+ *,
+ body: dict | None = None,
+ authenticated: bool = True,
+ ) -> _Response:
+ token = await self._token() if authenticated else None
+ response = await self._exchange(method, path, token=token, body=body)
+ if response.status == 401 and authenticated:
+ self._token_value = None
+ if method == "POST" and path == "/turn":
+ raise RuntimeError(
+ "Substrate shim rejected its bearer token (HTTP 401); prompt was not retried"
+ )
+ token = await self._token()
+ response = await self._exchange(method, path, token=token, body=body)
+ if response.status == 401:
+ self._token_value = None
+ if response.status == 503:
+ raise WorkspaceUnavailable(
+ "Substrate router or actor capacity is unavailable"
+ )
+ return response
+
+ async def _exchange(
+ self, method: str, path: str, *, token: str | None, body: dict | None
+ ) -> _Response:
+ try:
+ response = await asyncio.to_thread(
+ _router_request,
+ host=self.router_host,
+ port=self.router_port,
+ actor=self.actor,
+ atespace=self.atespace,
+ actor_port=self.actor_port,
+ timeout=self.timeout,
+ method=method,
+ path=path,
+ token=token,
+ body=body,
+ )
+ except (OSError, TimeoutError, ValueError, http.client.HTTPException) as exc:
+ raise TransportError(
+ f"Substrate router request failed: {type(exc).__name__}"
+ ) from exc
+ return response
+
+ def _json(self, response: _Response, *, method: str) -> dict:
+ if response.status == 401:
+ raise RuntimeError("Substrate shim rejected its bearer token (HTTP 401)")
+ if response.status == 409:
+ raise RuntimeError("Substrate shim rejected the concurrent turn (HTTP 409)")
+ if not 200 <= response.status < 300:
+ raise RuntimeError(
+ f"Substrate shim {method} failed (HTTP {response.status})"
+ )
+ try:
+ document = json.loads(response.body)
+ except json.JSONDecodeError as exc:
+ raise TransportError("Substrate shim returned invalid JSON") from exc
+ if not isinstance(document, dict):
+ raise TransportError("Substrate shim returned an invalid response")
+ return document
+
+ async def pod_state(self) -> PodState:
+ try:
+ response = await self._request("GET", "/healthz", authenticated=False)
+ except WorkspaceUnavailable:
+ return PodState(self.actor, None, False)
+ if response.status == 503:
+ return PodState(self.actor, None, False)
+ if response.status != 200:
+ raise TransportError(
+ f"Substrate health check failed (HTTP {response.status})"
+ )
+ return PodState(self.actor, None, True)
+
+ async def require_ready(self) -> PodState:
+ state = await self.pod_state()
+ if not state.ready:
+ raise WorkspaceUnavailable(
+ f"Substrate actor {self.atespace}/{self.actor} is not ready"
+ )
+ return state
+
+ async def agent_status(self, name: str) -> dict | None:
+ if not name:
+ raise ValueError("native agent name is required")
+ query = urlencode({"agent": self.agent})
+ response = await self._request("GET", f"/turn/status?{query}")
+ if response.status == 404:
+ return None
+ return self._json(response, method="GET /turn/status")
+
+ async def start(
+ self,
+ binding: str,
+ name: str,
+ *,
+ native_id: str | None,
+ resume: bool,
+ extra: dict[str, str] | None = None,
+ ) -> dict:
+ del binding, name, resume, extra
+ self.native_session_id = native_id
+ await self.require_ready()
+ query = urlencode({"agent": self.agent})
+ response = await self._request("GET", f"/agent/ready?{query}")
+ self._json(response, method="GET /agent/ready")
+ # The actor image and its provider credentials are provisioned before binding. A start
+ # verifies them without spawning another CLI process or creating/resuming an actor.
+ return {"actor": self.actor}
+
+ async def send(self, name: str, text: str) -> None:
+ if not name:
+ raise ValueError("native agent name is required")
+ payload = {"agent": self.agent, "prompt": text}
+ if self.native_session_id:
+ payload["session_id"] = self.native_session_id
+ response = await self._request("POST", "/turn", body=payload)
+ self._json(response, method="POST /turn")
+
+ async def stop(self, name: str) -> None:
+ if not name:
+ raise ValueError("native agent name is required")
+ response = await self._request("POST", "/turn/stop", body={"agent": self.agent})
+ self._json(response, method="POST /turn/stop")
+
+ async def _latest_turn(self) -> dict | None:
+ query = urlencode({"agent": self.agent})
+ response = await self._request("GET", f"/turn/status?{query}")
+ if response.status == 404:
+ return None
+ return self._json(response, method="GET /turn/status")
+
+ async def native_id(self, name: str) -> str | None:
+ if not name:
+ raise ValueError("native agent name is required")
+ turn = await self._latest_turn()
+ native_id = turn.get("native_session_id") if turn else None
+ if isinstance(native_id, str) and native_id:
+ self.native_session_id = native_id
+ return native_id
+ return None
+
+ async def journal(self, name: str, native_id: str, from_line: int) -> JournalSlice:
+ if not name:
+ raise ValueError("native agent name is required")
+ query = urlencode(
+ {
+ "agent": self.agent,
+ "id": native_id,
+ "from": max(0, from_line),
+ "limit": _JOURNAL_PAGE_SIZE,
+ }
+ )
+ response = await self._request("GET", f"/journal?{query}")
+ if response.status == 404:
+ return JournalSlice(None, 0, [])
+ document = self._json(response, method="GET /journal")
+ file = document.get("file")
+ total = document.get("total_lines")
+ raw_lines = document.get("lines")
+ if (file is not None and not isinstance(file, str)) or not isinstance(
+ total, int
+ ):
+ raise TransportError("Substrate journal response has invalid metadata")
+ if not isinstance(raw_lines, list):
+ raise TransportError("Substrate journal response has invalid lines")
+ lines: list[tuple[int, str]] = []
+ for item in raw_lines:
+ if (
+ not isinstance(item, dict)
+ or not isinstance(item.get("line"), int)
+ or not isinstance(item.get("text"), str)
+ ):
+ raise TransportError("Substrate journal response has an invalid line")
+ lines.append((item["line"], item["text"]))
+ return JournalSlice(file, total, lines)
diff --git a/backend/tests/runtime/test_substrate_workspace.py b/backend/tests/runtime/test_substrate_workspace.py
new file mode 100644
index 0000000..43a1bd2
--- /dev/null
+++ b/backend/tests/runtime/test_substrate_workspace.py
@@ -0,0 +1,375 @@
+"""In-process fake-router coverage; tests never open sockets or access Kubernetes."""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import unittest
+from unittest.mock import AsyncMock, patch
+from urllib.parse import parse_qs, urlsplit
+from uuid import uuid4
+
+from mainloop.config import SubstrateActorBinding, settings
+from mainloop.runtime import native_sessions
+from mainloop.runtime.herdr import WorkspaceUnavailable
+from mainloop.runtime.substrate_workspace import SubstrateWorkspace, _Response
+
+FIXTURE_VALUE = "fixture-shim-value-one"
+ROTATED_FIXTURE_VALUE = "fixture-shim-value-two"
+FAKE_SHIM_NAME = "shim-fixture"
+
+
+class FakeRouterAndShim:
+ """In-process CONNECT/router and authenticated shim model for transport contracts."""
+
+ def __init__(self):
+ self.suspended = False
+ self.capacity = False
+ self.expected_token = FIXTURE_VALUE
+ self.inflight: set[str] = set()
+ self.turns: dict[str, dict] = {}
+ self.journal_lines = [
+ '{"type":"user"}',
+ '{"type":"assistant"}',
+ '{"type":"system","subtype":"turn_duration"}',
+ ]
+ self.connects: list[tuple[str, str]] = []
+ self.requests: list[tuple[str, str, dict]] = []
+
+ def request(
+ self,
+ *,
+ actor: str,
+ atespace: str,
+ method: str,
+ path: str,
+ token: str | None,
+ body: dict | None,
+ **_unused,
+ ) -> _Response:
+ self.connects.append(("CONNECT", f"{atespace}/{actor}"))
+ if self.capacity:
+ return _Response(503, "capacity unavailable")
+ body = body or {}
+ self.requests.append((method, path, body))
+ if (method, path) != ("GET", "/healthz") and token != self.expected_token:
+ return _Response(401, "unauthorized")
+ if method == "GET" and path == "/healthz":
+ if self.suspended:
+ self.suspended = False
+ return _Response(200, "ok")
+ parsed = urlsplit(path)
+ query = parse_qs(parsed.query)
+ if method == "GET" and parsed.path == "/agent/ready":
+ return _Response(
+ 200, json.dumps({"agent": query["agent"][0], "configured": True})
+ )
+ if method == "GET" and parsed.path == "/turn/status":
+ turn = self.turns.get(query.get("agent", [""])[0])
+ return (
+ _Response(200, json.dumps(turn)) if turn else _Response(404, "no turn")
+ )
+ if method == "POST" and parsed.path == "/turn":
+ agent = body.get("agent")
+ if agent in self.inflight:
+ return _Response(409, "turn already in flight")
+ turn = {
+ "id": str(uuid4()),
+ "agent": agent,
+ "status": "running",
+ "native_session_id": body.get("session_id") or "native-fixture-id",
+ "events": [],
+ }
+ self.turns[agent] = turn
+ self.inflight.add(agent)
+ return _Response(202, json.dumps({"id": turn["id"], "status": "running"}))
+ if method == "POST" and parsed.path == "/turn/stop":
+ agent = body.get("agent")
+ self.inflight.discard(agent)
+ turn = self.turns.get(agent)
+ if turn:
+ turn["status"] = "interrupted"
+ return _Response(200, json.dumps({"status": "interrupted"}))
+ if method == "GET" and parsed.path == "/journal":
+ start = int(query.get("from", ["0"])[0])
+ limit = int(query.get("limit", ["200"])[0])
+ document = {
+ "file": "/fake/fixture-session.jsonl",
+ "total_lines": len(self.journal_lines),
+ "lines": [
+ {"line": n, "text": line}
+ for n, line in enumerate(self.journal_lines, 1)
+ if n > start
+ ][:limit],
+ }
+ return _Response(200, json.dumps(document))
+ return _Response(404, "not found")
+
+
+class FakeSubstrateWorkspace(SubstrateWorkspace):
+ def __init__(self, router: FakeRouterAndShim, **kwargs):
+ super().__init__(**kwargs)
+ self.router = router
+ self.fixture_value = FIXTURE_VALUE
+ self.fixture_reads = 0
+
+ async def _token(self) -> str:
+ if self._token_value is None:
+ self.fixture_reads += 1
+ self._token_value = self.fixture_value
+ return self._token_value
+
+ async def _exchange(
+ self, method: str, path: str, *, token: str | None, body: dict | None
+ ):
+ return self.router.request(
+ actor=self.actor,
+ atespace=self.atespace,
+ method=method,
+ path=path,
+ token=token,
+ body=body,
+ )
+
+
+def fake_workspace(router: FakeRouterAndShim, *, shim_name=FAKE_SHIM_NAME):
+ return FakeSubstrateWorkspace(
+ router,
+ atespace="atespace-fixture",
+ actor="actor-fixture",
+ agent="claude",
+ shim_token_secret_name=shim_name,
+ router_address="http://router-fixture:8081",
+ timeout=2,
+ )
+
+
+class SubstrateWorkspaceTests(unittest.TestCase):
+ def test_start_send_status_native_id_and_journal_pages(self):
+ async def exercise():
+ router = FakeRouterAndShim()
+ router.suspended = True
+ workspace = fake_workspace(router)
+ ident = await workspace.start(
+ "claude", "agent-fixture", native_id=None, resume=False
+ )
+ self.assertEqual(ident["actor"], "actor-fixture")
+ self.assertFalse(router.suspended)
+
+ await workspace.send("agent-fixture", "fixture prompt")
+ status = await workspace.agent_status("agent-fixture")
+ self.assertEqual(status["status"], "running")
+ self.assertEqual(
+ await workspace.native_id("agent-fixture"), "native-fixture-id"
+ )
+ first = await workspace.journal("agent-fixture", "native-fixture-id", 0)
+ next_page = await workspace.journal("agent-fixture", "native-fixture-id", 1)
+ self.assertEqual(first.file, "/fake/fixture-session.jsonl")
+ self.assertEqual(first.total_lines, 3)
+ self.assertEqual(first.lines[0], (1, '{"type":"user"}'))
+ self.assertEqual(next_page.lines[0], (2, '{"type":"assistant"}'))
+ self.assertIn(
+ ("CONNECT", "atespace-fixture/actor-fixture"), router.connects
+ )
+ sent = [
+ body
+ for method, path, body in router.requests
+ if method == "POST" and path == "/turn"
+ ]
+ self.assertEqual(sent, [{"agent": "claude", "prompt": "fixture prompt"}])
+
+ asyncio.run(exercise())
+
+ def test_capacity_503_maps_to_workspace_unavailable(self):
+ async def exercise():
+ router = FakeRouterAndShim()
+ router.capacity = True
+ with self.assertRaises(WorkspaceUnavailable):
+ await fake_workspace(router).require_ready()
+
+ asyncio.run(exercise())
+
+ def test_concurrent_turn_and_bad_token_are_rejected(self):
+ async def exercise():
+ router = FakeRouterAndShim()
+ workspace = fake_workspace(router)
+ await workspace.send("agent-fixture", "first fixture prompt")
+ with self.assertRaisesRegex(RuntimeError, "409"):
+ await workspace.send("agent-fixture", "second fixture prompt")
+
+ router.expected_token = ROTATED_FIXTURE_VALUE
+ with self.assertRaisesRegex(RuntimeError, "401"):
+ await workspace.agent_status("agent-fixture")
+
+ asyncio.run(exercise())
+
+ def test_401_refreshes_cached_secret_for_safe_request(self):
+ async def exercise():
+ router = FakeRouterAndShim()
+ workspace = fake_workspace(router)
+ await workspace.send("agent-fixture", "initial fixture prompt")
+ router.expected_token = ROTATED_FIXTURE_VALUE
+ workspace.fixture_value = ROTATED_FIXTURE_VALUE
+
+ status = await workspace.agent_status("agent-fixture")
+
+ self.assertEqual(status["status"], "running")
+ self.assertEqual(workspace.fixture_reads, 2)
+ status_requests = [
+ request
+ for request in router.requests
+ if request[0] == "GET" and request[1].startswith("/turn/status?")
+ ]
+ self.assertEqual(len(status_requests), 2)
+
+ asyncio.run(exercise())
+
+ def test_401_clears_secret_without_replaying_turn(self):
+ async def exercise():
+ router = FakeRouterAndShim()
+ workspace = fake_workspace(router)
+ await workspace.send("agent-fixture", "initial fixture prompt")
+ router.expected_token = ROTATED_FIXTURE_VALUE
+ workspace.fixture_value = ROTATED_FIXTURE_VALUE
+ previous_turn_count = sum(
+ 1
+ for request in router.requests
+ if request[0] == "POST" and request[1] == "/turn"
+ )
+
+ with self.assertRaisesRegex(RuntimeError, "401"):
+ await workspace.send("agent-fixture", "one-shot fixture prompt")
+
+ turn_requests = [
+ request
+ for request in router.requests
+ if request[0] == "POST" and request[1] == "/turn"
+ ]
+ self.assertEqual(len(turn_requests), previous_turn_count + 1)
+ self.assertEqual(workspace.fixture_reads, 1)
+ self.assertIsNone(workspace._token_value)
+
+ asyncio.run(exercise())
+
+ def test_native_sessions_delivery_uses_configured_actor(self):
+ async def exercise():
+ router = FakeRouterAndShim()
+ binding = {
+ "session_id": "session-fixture",
+ "kind": "claude",
+ "role": "agent",
+ "agent_name": "agent-fixture",
+ "native_session_id": "native-fixture-id",
+ "journal_cursor": 0,
+ "generation": 1,
+ "journal_ref": None,
+ "model": None,
+ }
+ workspace_binding = SubstrateActorBinding(
+ atespace="atespace-configured",
+ actor="actor-configured",
+ shim_token_secret_name=FAKE_SHIM_NAME,
+ )
+ key = (
+ "substrate",
+ workspace_binding.atespace,
+ workspace_binding.actor,
+ workspace_binding.shim_token_secret_name,
+ "claude",
+ )
+
+ async def fake_exchange(workspace, method, path, *, token, body):
+ return router.request(
+ actor=workspace.actor,
+ atespace=workspace.atespace,
+ method=method,
+ path=path,
+ token=token,
+ body=body,
+ )
+
+ async def fake_token(_workspace):
+ return FIXTURE_VALUE
+
+ with (
+ patch.object(settings, "workspace_runtime", "substrate"),
+ patch.object(
+ settings,
+ "substrate_router_address",
+ "http://router-fixture:8081",
+ ),
+ patch.object(
+ settings,
+ "substrate_shim_secret_namespace",
+ "namespace-fixture",
+ ),
+ patch.object(
+ settings,
+ "substrate_actor_bindings",
+ {"claude": workspace_binding},
+ ),
+ patch.object(SubstrateWorkspace, "_exchange", fake_exchange),
+ patch.object(SubstrateWorkspace, "_token", fake_token),
+ patch.object(
+ native_sessions,
+ "get_binding",
+ new=AsyncMock(return_value=binding),
+ ),
+ patch.object(native_sessions, "_update_binding", new=AsyncMock()),
+ patch.object(
+ native_sessions, "_set_delivery", new=AsyncMock()
+ ) as set_delivery,
+ patch.object(native_sessions, "sync", new=AsyncMock()) as sync,
+ ):
+ native_sessions._workspaces.pop(key, None)
+ self.assertEqual(settings.workspace_runtime, "substrate")
+ self.assertIsInstance(
+ native_sessions.workspace_for(binding), SubstrateWorkspace
+ )
+ await asyncio.wait_for(
+ native_sessions._deliver(
+ "session-fixture",
+ "message-fixture",
+ "configured fixture prompt",
+ ),
+ timeout=2,
+ )
+ workspace = native_sessions.workspace_for(binding)
+ self.assertIsInstance(workspace, SubstrateWorkspace)
+ self.assertEqual(
+ (workspace.atespace, workspace.actor, workspace.secret_name),
+ ("atespace-configured", "actor-configured", FAKE_SHIM_NAME),
+ )
+ self.assertEqual(workspace.secret_namespace, "namespace-fixture")
+ self.assertIn(
+ ("CONNECT", "atespace-configured/actor-configured"), router.connects
+ )
+ self.assertEqual(
+ [
+ request
+ for request in router.requests
+ if request[0] == "POST" and request[1] == "/turn"
+ ],
+ [
+ (
+ "POST",
+ "/turn",
+ {
+ "agent": "claude",
+ "prompt": "configured fixture prompt",
+ "session_id": "native-fixture-id",
+ },
+ )
+ ],
+ )
+ set_delivery.assert_awaited_once_with(
+ "message-fixture", "sending", cursor_before=3
+ )
+ sync.assert_awaited_once_with("session-fixture")
+ native_sessions._workspaces.pop(key, None)
+
+ asyncio.run(exercise())
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 5293a1f..69bd1e3 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -46,6 +46,23 @@ native session/thread id and final message. It permits one in-flight turn per ag
output file; `GET /run/:id` reports its status and bounded output. `/healthz` and `/readyz` check
only the shim and workspace. The actor image and these routes still need live proof.
+## Substrate runtime
+
+`WORKSPACE_RUNTIME=substrate` selects the native-session transport; `herdr` remains the default.
+`SUBSTRATE_ROUTER_ADDRESS` configures the HTTP CONNECT listener. `SUBSTRATE_ACTOR_BINDINGS` is a
+JSON object keyed by `claude` and `codex`; each entry supplies `atespace`, `actor`, and
+`shim_token_secret_name`. `SUBSTRATE_SHIM_SECRET_NAMESPACE` selects the Secret namespace and
+defaults to `mainloop-control`. The backend reads the Secret's `token` key there and keeps the
+value out of logs and config. Actor names and Secret names stay in deployment config; Mainloop
+attaches only to the named, pre-created actor. A 401 refreshes the cached token and retries
+read/status operations once; a rejected `POST /turn` clears the cache and is never replayed.
+
+The adapter routes turns and status through the authenticated shim, verifies the selected CLI
+credential is installed before starting a session, then mirrors replies and completion from native
+session journals. The shim's bounded `GET /journal` endpoint pages Claude transcripts and Codex
+rollout files. This is fixture-backed transport behavior; it does not add live-proof claims for the
+headless image or its current actors.
+
## Earlier real-versus-stand-in inventory (before Round 3)
| Layer | Status |
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
index d71d199..4b575cf 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
+++ b/spikes/substrate-workspace-adapter/live-agent-image/Dockerfile
@@ -4,7 +4,9 @@
# No credentials are baked in or fetched during startup; the golden actor reaches readiness
# without credentials or external network access.
FROM node:22-bookworm-slim
-RUN apt-get update && apt-get install -y --no-install-recommends jq ca-certificates git curl procps ripgrep \
+RUN apt-get update \
+ && apt-get install -y --no-install-recommends \
+ jq ca-certificates git curl procps ripgrep util-linux \
&& rm -rf /var/lib/apt/lists/* \
&& useradd -m -u 10001 agent
# The run-specific Substrate MITM CA is a public trust anchor. BuildKit mounts
@@ -26,7 +28,7 @@ COPY agent-config/mainloop-system.txt /etc/agent-config/mainloop-system.txt
COPY exec-shim.js /usr/local/bin/exec-shim.js
COPY entrypoint.sh /usr/local/bin/entrypoint.sh
RUN chmod +x /usr/local/bin/entrypoint.sh /usr/local/bin/start-native-agent \
- && mkdir -p /work && chown -R agent:agent /work
+ && mkdir -p /work && chown 10001:10001 /work
ENV EXEC_SHIM=/usr/local/bin/exec-shim.js
ENV NATIVE_AGENT_LAUNCHER=/usr/local/bin/start-native-agent
ENV HOME=/home/agent
@@ -36,5 +38,6 @@ ENV EXEC_SHIM_STATE_DIR=/work/repo/.mainloop/exec-shim
ENV AGENT_SYSTEM_PROMPT_FILE=/etc/agent-config/mainloop-system.txt
ENV NODE_EXTRA_CA_CERTS=/etc/ssl/certs/ca-certificates.crt
ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt
-USER 10001:10001
+# Substrate currently starts actors as root regardless of this image's USER setting.
+USER 0:0
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
index 3ecbb62..309ec8a 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
+++ b/spikes/substrate-workspace-adapter/live-agent-image/entrypoint.sh
@@ -2,6 +2,42 @@
# Start only the actor-local shim. Native CLIs run headlessly once per delivered turn.
set -eu
+AGENT_UID=10001
+AGENT_GID=10001
+CURRENT_UID="$(id -u)"
+export HOME=/home/agent
+WORKSPACE_PATH="${WORKSPACE_PATH:-/work/repo}"
+EXEC_SHIM_STATE_DIR="${EXEC_SHIM_STATE_DIR:-${WORKSPACE_PATH}/.mainloop/exec-shim}"
+export WORKSPACE_PATH EXEC_SHIM_STATE_DIR
+
+if [[ ${CURRENT_UID} -eq 0 && ${1-} != "--runtime-user" ]]; then
+ if ! command -v setpriv >/dev/null 2>&1; then
+ echo 'setpriv is required to run the actor shim without root' >&2
+ exit 1
+ fi
+ EXEC_SHIM_STATE_DIR="$(realpath -m "${EXEC_SHIM_STATE_DIR}")"
+ case "${EXEC_SHIM_STATE_DIR}" in
+ /work/*) ;;
+ *)
+ echo 'EXEC_SHIM_STATE_DIR must be under /work' >&2
+ exit 1
+ ;;
+ esac
+ export EXEC_SHIM_STATE_DIR
+ mkdir -p "${HOME}" /work "${EXEC_SHIM_STATE_DIR}"
+ chown -R "${AGENT_UID}:${AGENT_GID}" "${HOME}" /work
+ exec setpriv --reuid "${AGENT_UID}" --regid "${AGENT_GID}" --init-groups \
+ --bounding-set=-all --no-new-privs -- "$0" --runtime-user
+fi
+
+if [[ ${1-} == "--runtime-user" ]]; then
+ shift
+fi
+if [[ ${CURRENT_UID} -eq 0 ]]; then
+ echo 'entrypoint refused to continue as UID 0' >&2
+ exit 1
+fi
+
node /usr/local/bin/prepare-native-agent-config.cjs
mkdir -p "${WORKSPACE_PATH}"
[[ -d "${WORKSPACE_PATH}/.git" ]] || git -C "${WORKSPACE_PATH}" init -q
diff --git a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
index c92b73b..9119a35 100644
--- a/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
+++ b/spikes/substrate-workspace-adapter/live-agent-image/exec-shim.js
@@ -13,6 +13,10 @@ const MAX_CREDENTIAL_BYTES = 64 * 1024;
const MAX_JOB_OUTPUT_BYTES = 2 * 1024 * 1024;
const MAX_RESPONSE_OUTPUT_BYTES = 32 * 1024;
const MAX_RETURN_EVENTS = 256;
+const MAX_JOURNAL_LINES = 200;
+const MAX_JOURNAL_LINE_BYTES = 1024 * 1024;
+const MAX_JOURNAL_RESPONSE_BYTES = 2 * 1024 * 1024;
+const MAX_JOURNAL_SEARCH_ENTRIES = 100_000;
const DEFAULT_RUN_TIMEOUT_MS = 60_000;
const DEFAULT_TURN_TIMEOUT_MS = 10 * 60_000;
const MAX_TIMEOUT_MS = 10 * 60_000;
@@ -26,6 +30,15 @@ const RUN_DIR = path.join(STATE_DIR, 'runs');
const TURN_DIR = path.join(STATE_DIR, 'turns');
const LAUNCHER = process.env.NATIVE_AGENT_LAUNCHER || '/usr/local/bin/start-native-agent';
+if (
+ typeof process.getuid === 'function' &&
+ process.getuid() === 0 &&
+ process.env.EXEC_SHIM_TEST_ALLOW_ROOT !== '1'
+) {
+ process.stderr.write('exec-shim refuses to start as UID 0\n');
+ process.exit(1);
+}
+
for (const directory of [STATE_DIR, RUN_DIR, TURN_DIR]) {
fs.mkdirSync(directory, { recursive: true, mode: 0o700 });
fs.chmodSync(directory, 0o700);
@@ -45,6 +58,8 @@ try {
const jobs = new Map();
const activeTurns = new Map();
+const latestTurns = new Map();
+const turnProcesses = new Map();
function json(res, status, document) {
res.writeHead(status, { 'content-type': 'application/json' }).end(JSON.stringify(document));
@@ -171,8 +186,12 @@ function loadJobs(directory, kind) {
atomicJsonWrite(path.join(directory, name), record);
}
jobs.set(record.id, { ...record, kind });
- if (kind === 'turn' && record.blocking && record.agent) {
- activeTurns.set(record.agent, record.id);
+ if (kind === 'turn' && record.agent) {
+ const previous = latestTurns.get(record.agent);
+ if (!previous || String(record.created_at) > String(previous.created_at)) {
+ latestTurns.set(record.agent, record);
+ }
+ if (record.blocking) activeTurns.set(record.agent, record.id);
}
} catch {
// Ignore an incomplete or corrupt record; it cannot safely be resumed.
@@ -296,6 +315,7 @@ function runChild(job, child, timeoutMs, onStart) {
if (finalized) return;
finalized = true;
if (timedOut) job.status = 'timed_out';
+ else if (job.stop_requested) job.status = 'interrupted';
else if (spawnError) job.status = 'failed';
else job.status = code === 0 ? 'completed' : 'failed';
job.exit_code = code;
@@ -305,6 +325,8 @@ function runChild(job, child, timeoutMs, onStart) {
if (job.kind === 'turn' && activeTurns.get(job.agent) === job.id) {
activeTurns.delete(job.agent);
}
+ if (job.kind === 'turn') turnProcesses.delete(job.id);
+ if (job.finishTurn) job.finishTurn();
};
const timer = setTimeout(() => {
timedOut = true;
@@ -385,6 +407,118 @@ function turnResponse(job) {
};
}
+function findJournal(agent, id) {
+ const root =
+ agent === 'claude'
+ ? path.join(process.env.CLAUDE_CONFIG_DIR || path.join(HOME_PATH, '.claude'), 'projects')
+ : path.join(CODEX_HOME_PATH, 'sessions');
+ const expected = agent === 'claude' ? `${id}.jsonl` : null;
+ const stack = [root];
+ let visited = 0;
+ while (stack.length > 0 && visited < MAX_JOURNAL_SEARCH_ENTRIES) {
+ const directory = stack.pop();
+ let entries;
+ try {
+ entries = fs.readdirSync(directory, { withFileTypes: true });
+ } catch (err) {
+ if (err.code === 'ENOENT' || err.code === 'ENOTDIR' || err.code === 'EACCES') continue;
+ throw err;
+ }
+ for (const entry of entries) {
+ visited += 1;
+ if (visited > MAX_JOURNAL_SEARCH_ENTRIES) break;
+ const candidate = path.join(directory, entry.name);
+ if (entry.isDirectory()) stack.push(candidate);
+ else if (
+ entry.isFile() &&
+ (agent === 'claude'
+ ? entry.name === expected
+ : entry.name.startsWith('rollout-') && entry.name.endsWith(`-${id}.jsonl`))
+ )
+ return candidate;
+ }
+ }
+ return null;
+}
+
+async function journalPage(file, from, limit) {
+ const lines = [];
+ let totalLines = 0;
+ let responseBytes = 0;
+ let pending = Buffer.alloc(0);
+ const stream = fs.createReadStream(file);
+ for await (const chunk of stream) {
+ let offset = 0;
+ let boundary;
+ while ((boundary = chunk.indexOf(0x0a, offset)) !== -1) {
+ const part = chunk.subarray(offset, boundary);
+ const line = pending.length ? Buffer.concat([pending, part]) : part;
+ pending = Buffer.alloc(0);
+ if (line.length > MAX_JOURNAL_LINE_BYTES) throw new RangeError('journal line too large');
+ totalLines += 1;
+ const cost = line.length + 24;
+ if (
+ totalLines > from &&
+ totalLines <= from + limit &&
+ responseBytes + cost <= MAX_JOURNAL_RESPONSE_BYTES
+ ) {
+ lines.push({ line: totalLines, text: line.toString('utf8') });
+ responseBytes += cost;
+ }
+ offset = boundary + 1;
+ }
+ if (offset < chunk.length) {
+ const rest = chunk.subarray(offset);
+ pending = pending.length ? Buffer.concat([pending, rest]) : rest;
+ if (pending.length > MAX_JOURNAL_LINE_BYTES) throw new RangeError('journal line too large');
+ }
+ }
+ return { file, total_lines: totalLines, lines };
+}
+
+async function getJournal(req, res) {
+ const query = new URL(req.url, 'http://exec-shim.invalid').searchParams;
+ const agent = query.get('agent');
+ const id = query.get('id');
+ const fromText = query.get('from') || '0';
+ const limitText = query.get('limit') || String(MAX_JOURNAL_LINES);
+ if (
+ (agent !== 'claude' && agent !== 'codex') ||
+ typeof id !== 'string' ||
+ !/^[A-Za-z0-9._:-]{1,256}$/.test(id) ||
+ !/^\d{1,12}$/.test(fromText) ||
+ !/^\d{1,4}$/.test(limitText)
+ ) {
+ res.writeHead(400).end('invalid journal query');
+ return;
+ }
+ const from = Number(fromText);
+ const limit = Number(limitText);
+ if (
+ !Number.isSafeInteger(from) ||
+ !Number.isInteger(limit) ||
+ limit < 1 ||
+ limit > MAX_JOURNAL_LINES
+ ) {
+ res.writeHead(400).end('invalid journal page');
+ return;
+ }
+ const file = findJournal(agent, id);
+ if (!file) {
+ json(res, 200, { file: null, total_lines: 0, lines: [] });
+ return;
+ }
+ try {
+ json(res, 200, await journalPage(file, from, limit));
+ } catch (err) {
+ if (err instanceof RangeError) {
+ res.writeHead(413).end('journal line too large');
+ return;
+ }
+ res.writeHead(500).end('journal could not be read');
+ }
+}
+
function workspaceReady() {
try {
const stat = fs.statSync(WORKSPACE_PATH);
@@ -498,7 +632,11 @@ function startTurn(document, res) {
native_session_id: sessionId || null,
blocking: true
});
+ job.completion = new Promise((resolve) => {
+ job.finishTurn = resolve;
+ });
activeTurns.set(agent, job.id);
+ latestTurns.set(agent, job);
try {
const child = spawn(LAUNCHER, [agent, ...(sessionId ? [sessionId] : [])], {
cwd: WORKSPACE_PATH,
@@ -507,6 +645,7 @@ function startTurn(document, res) {
detached: true
});
runChild(job, child, timeoutMs, (processChild) => {
+ turnProcesses.set(job.id, processChild);
processChild.stdout.on('data', (chunk) => appendBounded(job, turnEventsPath(job), chunk));
processChild.stderr.on('data', (chunk) => appendBounded(job, turnStderrPath(job), chunk));
processChild.stdin.on('error', () => {});
@@ -523,6 +662,70 @@ function startTurn(document, res) {
}
}
+function currentTurn(agent, res) {
+ if (agent !== 'claude' && agent !== 'codex') {
+ res.writeHead(400).end('unsupported agent');
+ return;
+ }
+ const job = latestTurns.get(agent);
+ if (!job) {
+ res.writeHead(404).end('no turn');
+ return;
+ }
+ json(res, 200, turnResponse(job));
+}
+
+function agentReady(agent, res) {
+ if (agent !== 'claude' && agent !== 'codex') {
+ res.writeHead(400).end('unsupported agent');
+ return;
+ }
+ const credential = credentialPaths.get(agent === 'claude' ? 'claude-token' : 'codex-auth');
+ try {
+ const stat = fs.statSync(credential);
+ if (stat.isFile() && stat.size > 0) {
+ json(res, 200, { agent, configured: true });
+ return;
+ }
+ } catch {
+ // A missing credential is a readiness failure; never include file contents or paths.
+ }
+ res.writeHead(503).end('agent credential unavailable');
+}
+
+async function stopTurn(document, res) {
+ const agent = document.agent;
+ if (agent !== 'claude' && agent !== 'codex') {
+ res.writeHead(400).end('unsupported agent');
+ return;
+ }
+ const id = activeTurns.get(agent);
+ const child = id && turnProcesses.get(id);
+ const job = id && jobs.get(id);
+ if (!job) {
+ json(res, 200, { status: 'not_running' });
+ return;
+ }
+ if (!child) {
+ // A restored actor can retain an interrupted job record without a live process. The caller
+ // explicitly requested stop, so release the per-agent turn lock without replaying anything.
+ job.status = 'interrupted';
+ job.blocking = false;
+ saveJob(job);
+ activeTurns.delete(agent);
+ json(res, 200, { id, status: job.status });
+ return;
+ }
+ job.stop_requested = true;
+ saveJob(job);
+ killProcessGroup(child, 'SIGTERM');
+ const killTimer = setTimeout(() => killProcessGroup(child, 'SIGKILL'), 500);
+ killTimer.unref();
+ await job.completion;
+ clearTimeout(killTimer);
+ json(res, 200, { id, status: job.status });
+}
+
function getJob(kind, id, res) {
if (!/^[0-9a-f-]{36}$/i.test(id)) {
res.writeHead(404).end('not found');
@@ -618,6 +821,28 @@ const server = http.createServer((req, res) => {
handleBody(req, res, (document) => startTurn(document, res));
return;
}
+ if (req.method === 'POST' && req.url === '/turn/stop') {
+ if (!authorized(req)) return unauthorized(res);
+ handleBody(req, res, (document) => void stopTurn(document, res));
+ return;
+ }
+ if (req.method === 'GET' && req.url.startsWith('/turn/status?')) {
+ if (!authorized(req)) return unauthorized(res);
+ const query = new URL(req.url, 'http://exec-shim.invalid').searchParams;
+ currentTurn(query.get('agent'), res);
+ return;
+ }
+ if (req.method === 'GET' && req.url.startsWith('/agent/ready?')) {
+ if (!authorized(req)) return unauthorized(res);
+ const query = new URL(req.url, 'http://exec-shim.invalid').searchParams;
+ agentReady(query.get('agent'), res);
+ return;
+ }
+ if (req.method === 'GET' && req.url.startsWith('/journal?')) {
+ if (!authorized(req)) return unauthorized(res);
+ void getJournal(req, res);
+ return;
+ }
const match = req.method === 'GET' && req.url.match(/^\/(run|turn)\/([^/?]+)$/);
if (match) {
if (!authorized(req)) return unauthorized(res);
diff --git a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
index 63c0f46..8740ee0 100644
--- a/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
+++ b/spikes/substrate-workspace-adapter/tests/exec-shim.test.js
@@ -5,7 +5,7 @@ import fs from 'node:fs';
import http from 'node:http';
import os from 'node:os';
import path from 'node:path';
-import { spawn } from 'node:child_process';
+import { spawn, spawnSync } from 'node:child_process';
import { once } from 'node:events';
import { test } from 'node:test';
import { fileURLToPath } from 'node:url';
@@ -14,6 +14,8 @@ const __dirname = path.dirname(fileURLToPath(import.meta.url));
const image = path.resolve(__dirname, '../live-agent-image');
const shim = path.join(image, 'exec-shim.js');
const launcher = path.join(image, 'bin/start-native-agent');
+const dockerfile = path.join(image, 'Dockerfile');
+const entrypoint = path.join(image, 'entrypoint.sh');
const fixtures = path.join(__dirname, 'fixtures/native');
const token = 'fixture-only-shim-token-long-enough-for-testing';
@@ -30,6 +32,7 @@ async function startShim(root, extraEnv = {}) {
const child = spawn(process.execPath, [shimCopy], {
env: {
...process.env,
+ ...(process.getuid() === 0 ? { EXEC_SHIM_TEST_ALLOW_ROOT: '1' } : {}),
...extraEnv,
HOME: home,
CODEX_HOME: path.join(home, '.codex'),
@@ -62,6 +65,43 @@ async function startShim(root, extraEnv = {}) {
return { child, port, output: () => output, home, workspace, fakeBin, state };
}
+test('exec shim refuses UID 0 unless its explicit test-only override is set', () => {
+ const source = `Object.defineProperty(process, 'getuid', { value: () => 0 }); require(${JSON.stringify(shim)});`;
+ const result = spawnSync(process.execPath, ['-e', source], {
+ encoding: 'utf8',
+ env: { ...process.env, EXEC_SHIM_TEST_ALLOW_ROOT: '0' }
+ });
+ assert.equal(result.status, 1);
+ assert.match(result.stderr, /exec-shim refuses to start as UID 0/);
+});
+
+test('actor image prepares its writable paths before dropping root privileges', () => {
+ const imageDockerfile = fs.readFileSync(dockerfile, 'utf8');
+ const imageEntrypoint = fs.readFileSync(entrypoint, 'utf8');
+ assert.ok(imageDockerfile.includes('util-linux'));
+ assert.match(imageDockerfile, /^USER 0:0$/m);
+ const statePreparation = imageEntrypoint.indexOf(
+ 'mkdir -p "${HOME}" /work "${EXEC_SHIM_STATE_DIR}"'
+ );
+ const ownership = imageEntrypoint.indexOf('chown -R "${AGENT_UID}:${AGENT_GID}" "${HOME}" /work');
+ const privilegeDrop = imageEntrypoint.indexOf('exec setpriv');
+ const shimStart = imageEntrypoint.indexOf('node "${EXEC_SHIM}"');
+ assert.ok(statePreparation >= 0 && statePreparation < ownership);
+ assert.ok(ownership >= 0 && ownership < privilegeDrop);
+ assert.ok(privilegeDrop >= 0 && privilegeDrop < shimStart);
+ for (const option of [
+ '--reuid "${AGENT_UID}"',
+ '--regid "${AGENT_GID}"',
+ '--init-groups',
+ '--bounding-set=-all',
+ '--no-new-privs'
+ ]) {
+ assert.ok(imageEntrypoint.includes(option), `entrypoint is missing ${option}`);
+ }
+ assert.equal(imageEntrypoint.includes('IS_SANDBOX'), false);
+ assert.equal(imageDockerfile.includes('EXEC_SHIM_TEST_ALLOW_ROOT'), false);
+});
+
function request(port, method, route, { body, bearer } = {}) {
return new Promise((resolve, reject) => {
const headers = {};
@@ -328,6 +368,148 @@ test('a second concurrent turn for the same agent receives 409 and is not queued
assert.equal(result.status, 'completed');
});
+test('turn status locates the latest turn after it completes', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-status-'));
+ const running = await startShim(root, {
+ CLAUDE_EVENTS: path.join(fixtures, 'claude-stream-json.jsonl')
+ });
+ fakeCli(
+ path.join(running.fakeBin, 'claude'),
+ '#!/bin/sh\ncat >/dev/null\ncat "$CLAUDE_EVENTS"\n'
+ );
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+ await installToken(running);
+ await installCredential(running, 'claude-token', 'fixture-claude-token');
+
+ const submitted = await request(running.port, 'POST', '/turn', {
+ bearer: token,
+ body: { agent: 'claude', prompt: 'fixture status prompt' }
+ });
+ assert.equal(submitted.status, 202, submitted.body);
+ const { id } = JSON.parse(submitted.body);
+ const current = await waitForJob(running, 'turn', id);
+ const status = await request(running.port, 'GET', '/turn/status?agent=claude', {
+ bearer: token
+ });
+ assert.equal(status.status, 200, status.body);
+ assert.equal(JSON.parse(status.body).id, id);
+ assert.equal(JSON.parse(status.body).status, 'completed');
+ assert.equal(JSON.parse(status.body).native_session_id, current.native_session_id);
+});
+
+test('agent readiness requires its credential and rejects unauthenticated checks', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-agent-ready-'));
+ const running = await startShim(root);
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+ await installToken(running);
+
+ const missing = await request(running.port, 'GET', '/agent/ready?agent=codex', {
+ bearer: token
+ });
+ assert.equal(missing.status, 503);
+ const unauthorized = await request(running.port, 'GET', '/agent/ready?agent=codex');
+ assert.equal(unauthorized.status, 401);
+
+ await installCredential(running, 'codex-auth', '{"auth_mode":"fixture"}');
+ const configured = await request(running.port, 'GET', '/agent/ready?agent=codex', {
+ bearer: token
+ });
+ assert.equal(configured.status, 200, configured.body);
+ assert.deepEqual(JSON.parse(configured.body), { agent: 'codex', configured: true });
+});
+
+test('journal returns bounded numbered pages for Claude and Codex and rejects a bad token', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-journal-'));
+ const running = await startShim(root);
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+ await installToken(running);
+
+ const claudeId = 'fixture-claude-session';
+ const claudeFile = path.join(running.home, '.claude', 'projects', 'p1', `${claudeId}.jsonl`);
+ fs.mkdirSync(path.dirname(claudeFile), { recursive: true });
+ fs.writeFileSync(claudeFile, '{"type":"system"}\n{"type":"user"}\n{"type":"assistant"}\npartial');
+ const firstPage = await request(
+ running.port,
+ 'GET',
+ `/journal?agent=claude&id=${claudeId}&from=1&limit=1`,
+ { bearer: token }
+ );
+ assert.equal(firstPage.status, 200, firstPage.body);
+ assert.deepEqual(JSON.parse(firstPage.body), {
+ file: claudeFile,
+ total_lines: 3,
+ lines: [{ line: 2, text: '{"type":"user"}' }]
+ });
+ const wrongToken = await request(
+ running.port,
+ 'GET',
+ `/journal?agent=claude&id=${claudeId}&from=0`,
+ { bearer: `${token}-wrong` }
+ );
+ assert.equal(wrongToken.status, 401);
+
+ const codexId = 'fixture-codex-thread';
+ const codexFile = path.join(
+ running.home,
+ '.codex',
+ 'sessions',
+ '2026',
+ '09',
+ 'rollout-2026-09-24T00-00-00-fixture-codex-thread.jsonl'
+ );
+ fs.mkdirSync(path.dirname(codexFile), { recursive: true });
+ fs.writeFileSync(codexFile, '{"type":"thread.started"}\n{"type":"turn.completed"}\n');
+ const codexPage = await request(
+ running.port,
+ 'GET',
+ `/journal?agent=codex&id=${codexId}&from=0&limit=1`,
+ { bearer: token }
+ );
+ assert.equal(codexPage.status, 200, codexPage.body);
+ assert.deepEqual(JSON.parse(codexPage.body), {
+ file: codexFile,
+ total_lines: 2,
+ lines: [{ line: 1, text: '{"type":"thread.started"}' }]
+ });
+ assert.equal(running.output().includes(token), false);
+});
+
+test('stop interrupts an in-flight turn and releases its slot', async (t) => {
+ const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-stop-'));
+ const running = await startShim(root);
+ fakeCli(path.join(running.fakeBin, 'claude'), '#!/bin/sh\ncat >/dev/null\nsleep 5\n');
+ t.after(async () => {
+ await stop(running.child);
+ fs.rmSync(root, { recursive: true, force: true });
+ });
+ await installToken(running);
+ await installCredential(running, 'claude-token', 'fixture-claude-token');
+
+ const submitted = await request(running.port, 'POST', '/turn', {
+ bearer: token,
+ body: { agent: 'claude', prompt: 'stop fixture turn' }
+ });
+ assert.equal(submitted.status, 202, submitted.body);
+ const { id } = JSON.parse(submitted.body);
+ const stopped = await request(running.port, 'POST', '/turn/stop', {
+ bearer: token,
+ body: { agent: 'claude' }
+ });
+ assert.equal(stopped.status, 200, stopped.body);
+ assert.equal(JSON.parse(stopped.body).status, 'interrupted');
+ const result = await waitForJob(running, 'turn', id);
+ assert.equal(result.status, 'interrupted');
+});
+
test('/run executes a command, stores bounded output, and reports timeout', async (t) => {
const root = fs.mkdtempSync(path.join(os.tmpdir(), 'exec-shim-run-'));
const running = await startShim(root);
From 5d276462181c957c94483d8274d2dd622ba97608 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Thu, 24 Sep 2026 01:42:01 +0000
Subject: [PATCH 16/30] feat: add local Substrate preview deployment
Add a portable Kustomize overlay for Mainloop's control namespace, backend,
frontend, PostgreSQL, and config-driven headless actor bindings. Add a bounded
helper to build and preflight local-registry images, record their digests, and
deploy a temporary digest-pinned render only to the pinned preview context. The
deploy helper creates a random PostgreSQL Secret when absent, inspects status
and logs, and opens local port-forwards. Document the preview workflow and
teardown, including cluster-lane ownership of shim Secrets.
---
docs/spikes/substrate-preview-quickstart.md | 61 ++++
.../substrate-preview/backend-rbac.yaml | 33 ++
.../overlays/substrate-preview/backend.yaml | 69 ++++
.../overlays/substrate-preview/configmap.yaml | 15 +
.../overlays/substrate-preview/database.yaml | 73 ++++
.../overlays/substrate-preview/frontend.yaml | 64 ++++
.../substrate-preview/kustomization.yaml | 13 +
.../overlays/substrate-preview/namespace.yaml | 6 +
.../overlays/substrate-preview/services.yaml | 38 +++
scripts/substrate-preview.sh | 320 ++++++++++++++++++
10 files changed, 692 insertions(+)
create mode 100644 docs/spikes/substrate-preview-quickstart.md
create mode 100644 k8s/apps/mainloop/overlays/substrate-preview/backend-rbac.yaml
create mode 100644 k8s/apps/mainloop/overlays/substrate-preview/backend.yaml
create mode 100644 k8s/apps/mainloop/overlays/substrate-preview/configmap.yaml
create mode 100644 k8s/apps/mainloop/overlays/substrate-preview/database.yaml
create mode 100644 k8s/apps/mainloop/overlays/substrate-preview/frontend.yaml
create mode 100644 k8s/apps/mainloop/overlays/substrate-preview/kustomization.yaml
create mode 100644 k8s/apps/mainloop/overlays/substrate-preview/namespace.yaml
create mode 100644 k8s/apps/mainloop/overlays/substrate-preview/services.yaml
create mode 100755 scripts/substrate-preview.sh
diff --git a/docs/spikes/substrate-preview-quickstart.md b/docs/spikes/substrate-preview-quickstart.md
new file mode 100644
index 0000000..73bd0ef
--- /dev/null
+++ b/docs/spikes/substrate-preview-quickstart.md
@@ -0,0 +1,61 @@
+# Mainloop Substrate preview
+
+This local Kind preview runs Mainloop's backend, frontend, and PostgreSQL in `mainloop-control`. The backend uses the headless Claude and Codex actors already provisioned in the preview cluster. This overlay does not create actors or include shim-token values.
+
+## Prerequisites
+
+- The `kind-substrate-preview` cluster, `localhost:5001` registry, Substrate router ingress policy, and headless actors are ready. The cluster lane creates the two referenced `mainloop-shim-*` Secrets in `mainloop-control`, each with a `token` key; the deploy script does not create them.
+- Docker, `kubectl`, `kubectl-ate`, `curl`, and `openssl` are installed. Docker can reach the local registry.
+- The preview kubeconfig is available at `/tmp/substrate-preview-kubeconfig`.
+
+Set the kubeconfig path once in the shell:
+
+```bash
+export SUBSTRATE_PREVIEW_KUBECONFIG=/tmp/substrate-preview-kubeconfig
+```
+
+## Build and deploy
+
+```bash
+scripts/substrate-preview.sh build
+scripts/substrate-preview.sh deploy
+scripts/substrate-preview.sh status
+```
+
+`build` pushes the backend, frontend, and mirrored PostgreSQL images to `localhost:5001`, checks each pushed manifest by digest, and records those digests in `$XDG_STATE_HOME/mainloop/substrate-preview/image-digests` (or `$HOME/.local/state/mainloop/substrate-preview/image-digests` when `XDG_STATE_HOME` is unset). Run `build` before `deploy`; deploy fails if that state file is missing or invalid. `deploy` renders a temporary copy of the overlay with the recorded digests, then applies it to context `kind-substrate-preview`; the tracked overlay remains tag-based. On first deploy, it creates `mainloop-db-app` with a random password if that Secret is absent. The password is not printed, and subsequent deploys keep the existing Secret.
+
+## Open Mainloop
+
+Run this in a terminal and leave it running; Ctrl+C stops both port-forwards:
+
+```bash
+scripts/substrate-preview.sh open
+```
+
+Open the printed frontend URL. The backend API docs are at `http://127.0.0.1:8000/docs`.
+
+## Watch actors and logs
+
+The installed `kubectl-ate get actor` command does not provide a watch flag. Poll both atespaces every two seconds to see actors suspend and resume:
+
+```bash
+watch -n 2 'kubectl-ate --kubeconfig "$SUBSTRATE_PREVIEW_KUBECONFIG" --context kind-substrate-preview get actor --all-atespaces'
+```
+
+In another terminal, follow backend logs (or choose `frontend` or `postgres`):
+
+```bash
+scripts/substrate-preview.sh logs
+scripts/substrate-preview.sh logs frontend
+```
+
+`scripts/substrate-preview.sh status` prints Mainloop pods, Substrate actors, and WorkerPools.
+
+## Tear down
+
+This deletes the `mainloop-control` namespace, its local PostgreSQL data, and shim-token Secrets there. It leaves Substrate actors and cluster-level services alone.
+
+```bash
+kubectl --kubeconfig "$SUBSTRATE_PREVIEW_KUBECONFIG" --context kind-substrate-preview \
+ delete -k k8s/apps/mainloop/overlays/substrate-preview
+```
diff --git a/k8s/apps/mainloop/overlays/substrate-preview/backend-rbac.yaml b/k8s/apps/mainloop/overlays/substrate-preview/backend-rbac.yaml
new file mode 100644
index 0000000..f9f5e1d
--- /dev/null
+++ b/k8s/apps/mainloop/overlays/substrate-preview/backend-rbac.yaml
@@ -0,0 +1,33 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+ name: mainloop-backend
+ namespace: mainloop-control
+automountServiceAccountToken: true
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+ name: mainloop-read-shim-tokens
+ namespace: mainloop-control
+rules:
+ - apiGroups: ['']
+ resources: [secrets]
+ resourceNames:
+ - mainloop-shim-live-agent-gate-headless-claude-reproof
+ - mainloop-shim-native-codex-headless-codex-reproof
+ verbs: [get]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+ name: mainloop-read-shim-tokens
+ namespace: mainloop-control
+subjects:
+ - kind: ServiceAccount
+ name: mainloop-backend
+ namespace: mainloop-control
+roleRef:
+ apiGroup: rbac.authorization.k8s.io
+ kind: Role
+ name: mainloop-read-shim-tokens
diff --git a/k8s/apps/mainloop/overlays/substrate-preview/backend.yaml b/k8s/apps/mainloop/overlays/substrate-preview/backend.yaml
new file mode 100644
index 0000000..acc6a8b
--- /dev/null
+++ b/k8s/apps/mainloop/overlays/substrate-preview/backend.yaml
@@ -0,0 +1,69 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: mainloop-backend
+ namespace: mainloop-control
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: mainloop-backend
+ template:
+ metadata:
+ labels:
+ app: mainloop-backend
+ spec:
+ serviceAccountName: mainloop-backend
+ containers:
+ - name: backend
+ image: localhost:5001/mainloop-backend:substrate-preview
+ imagePullPolicy: Always
+ ports:
+ - name: http
+ containerPort: 8000
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: 1000
+ runAsGroup: 1000
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: [ALL]
+ seccompProfile:
+ type: RuntimeDefault
+ envFrom:
+ - configMapRef:
+ name: mainloop-config
+ env:
+ - name: DB_USER
+ valueFrom:
+ secretKeyRef:
+ name: mainloop-db-app
+ key: username
+ - name: DB_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: mainloop-db-app
+ key: password
+ resources:
+ requests:
+ memory: 1Gi
+ cpu: 250m
+ limits:
+ memory: 2Gi
+ cpu: 1000m
+ livenessProbe:
+ httpGet:
+ path: /health
+ port: http
+ initialDelaySeconds: 90
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 3
+ readinessProbe:
+ httpGet:
+ path: /health
+ port: http
+ initialDelaySeconds: 30
+ periodSeconds: 5
+ timeoutSeconds: 3
+ failureThreshold: 3
diff --git a/k8s/apps/mainloop/overlays/substrate-preview/configmap.yaml b/k8s/apps/mainloop/overlays/substrate-preview/configmap.yaml
new file mode 100644
index 0000000..b923b9d
--- /dev/null
+++ b/k8s/apps/mainloop/overlays/substrate-preview/configmap.yaml
@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: ConfigMap
+metadata:
+ name: mainloop-config
+ namespace: mainloop-control
+data:
+ HOST: 0.0.0.0
+ PORT: '8000'
+ DB_HOST: mainloop-db-pooler
+ DB_PORT: '5432'
+ DB_NAME: mainloop
+ FRONTEND_DOMAIN: localhost:3000
+ WORKSPACE_RUNTIME: substrate
+ SUBSTRATE_ROUTER_ADDRESS: http://atenet-router.ate-system.svc.cluster.local:8081
+ SUBSTRATE_ACTOR_BINDINGS: '{"claude":{"atespace":"live-agent-gate","actor":"headless-claude-reproof","shim_token_secret_name":"mainloop-shim-live-agent-gate-headless-claude-reproof"},"codex":{"atespace":"native-codex","actor":"headless-codex-reproof","shim_token_secret_name":"mainloop-shim-native-codex-headless-codex-reproof"}}'
diff --git a/k8s/apps/mainloop/overlays/substrate-preview/database.yaml b/k8s/apps/mainloop/overlays/substrate-preview/database.yaml
new file mode 100644
index 0000000..fda2932
--- /dev/null
+++ b/k8s/apps/mainloop/overlays/substrate-preview/database.yaml
@@ -0,0 +1,73 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: postgres
+ namespace: mainloop-control
+spec:
+ clusterIP: None
+ selector:
+ app: postgres
+ ports:
+ - name: postgres
+ port: 5432
+ targetPort: 5432
+---
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+ name: postgres
+ namespace: mainloop-control
+spec:
+ serviceName: postgres
+ replicas: 1
+ selector:
+ matchLabels:
+ app: postgres
+ template:
+ metadata:
+ labels:
+ app: postgres
+ spec:
+ containers:
+ - name: postgres
+ image: localhost:5001/postgres:16-alpine
+ imagePullPolicy: Always
+ ports:
+ - name: postgres
+ containerPort: 5432
+ env:
+ - name: POSTGRES_USER
+ valueFrom:
+ secretKeyRef:
+ name: mainloop-db-app
+ key: username
+ - name: POSTGRES_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: mainloop-db-app
+ key: password
+ - name: POSTGRES_DB
+ value: mainloop
+ volumeMounts:
+ - name: postgres-data
+ mountPath: /var/lib/postgresql/data
+ resources:
+ requests:
+ memory: 256Mi
+ cpu: 100m
+ limits:
+ memory: 512Mi
+ cpu: 500m
+ readinessProbe:
+ exec:
+ command: [pg_isready, -U, mainloop]
+ initialDelaySeconds: 5
+ periodSeconds: 5
+ volumeClaimTemplates:
+ - metadata:
+ name: postgres-data
+ spec:
+ accessModes: [ReadWriteOnce]
+ resources:
+ requests:
+ storage: 1Gi
diff --git a/k8s/apps/mainloop/overlays/substrate-preview/frontend.yaml b/k8s/apps/mainloop/overlays/substrate-preview/frontend.yaml
new file mode 100644
index 0000000..5031c06
--- /dev/null
+++ b/k8s/apps/mainloop/overlays/substrate-preview/frontend.yaml
@@ -0,0 +1,64 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+ name: mainloop-frontend
+ namespace: mainloop-control
+spec:
+ replicas: 1
+ selector:
+ matchLabels:
+ app: mainloop-frontend
+ template:
+ metadata:
+ labels:
+ app: mainloop-frontend
+ spec:
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: 1001
+ runAsGroup: 1001
+ fsGroup: 1001
+ seccompProfile:
+ type: RuntimeDefault
+ containers:
+ - name: frontend
+ image: localhost:5001/mainloop-frontend:substrate-preview
+ imagePullPolicy: Always
+ ports:
+ - name: http
+ containerPort: 3000
+ securityContext:
+ runAsNonRoot: true
+ runAsUser: 1001
+ runAsGroup: 1001
+ allowPrivilegeEscalation: false
+ capabilities:
+ drop: [ALL]
+ seccompProfile:
+ type: RuntimeDefault
+ env:
+ - name: ORIGIN
+ value: http://localhost:3000
+ resources:
+ requests:
+ memory: 128Mi
+ cpu: 100m
+ limits:
+ memory: 256Mi
+ cpu: 250m
+ livenessProbe:
+ httpGet:
+ path: /
+ port: http
+ initialDelaySeconds: 30
+ periodSeconds: 10
+ timeoutSeconds: 5
+ failureThreshold: 3
+ readinessProbe:
+ httpGet:
+ path: /
+ port: http
+ initialDelaySeconds: 10
+ periodSeconds: 5
+ timeoutSeconds: 3
+ failureThreshold: 3
diff --git a/k8s/apps/mainloop/overlays/substrate-preview/kustomization.yaml b/k8s/apps/mainloop/overlays/substrate-preview/kustomization.yaml
new file mode 100644
index 0000000..b5ad33e
--- /dev/null
+++ b/k8s/apps/mainloop/overlays/substrate-preview/kustomization.yaml
@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+namespace: mainloop-control
+
+resources:
+ - namespace.yaml
+ - configmap.yaml
+ - database.yaml
+ - backend-rbac.yaml
+ - backend.yaml
+ - frontend.yaml
+ - services.yaml
diff --git a/k8s/apps/mainloop/overlays/substrate-preview/namespace.yaml b/k8s/apps/mainloop/overlays/substrate-preview/namespace.yaml
new file mode 100644
index 0000000..2ddd664
--- /dev/null
+++ b/k8s/apps/mainloop/overlays/substrate-preview/namespace.yaml
@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+ name: mainloop-control
+ labels:
+ mainloop.dev/role: control
diff --git a/k8s/apps/mainloop/overlays/substrate-preview/services.yaml b/k8s/apps/mainloop/overlays/substrate-preview/services.yaml
new file mode 100644
index 0000000..81d4843
--- /dev/null
+++ b/k8s/apps/mainloop/overlays/substrate-preview/services.yaml
@@ -0,0 +1,38 @@
+apiVersion: v1
+kind: Service
+metadata:
+ name: mainloop-db-pooler
+ namespace: mainloop-control
+spec:
+ selector:
+ app: postgres
+ ports:
+ - name: postgres
+ port: 5432
+ targetPort: 5432
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mainloop-backend
+ namespace: mainloop-control
+spec:
+ selector:
+ app: mainloop-backend
+ ports:
+ - name: http
+ port: 8000
+ targetPort: 8000
+---
+apiVersion: v1
+kind: Service
+metadata:
+ name: mainloop-frontend
+ namespace: mainloop-control
+spec:
+ selector:
+ app: mainloop-frontend
+ ports:
+ - name: http
+ port: 3000
+ targetPort: 3000
diff --git a/scripts/substrate-preview.sh b/scripts/substrate-preview.sh
new file mode 100755
index 0000000..39497e9
--- /dev/null
+++ b/scripts/substrate-preview.sh
@@ -0,0 +1,320 @@
+#!/usr/bin/env bash
+set -Eeuo pipefail
+
+CONTEXT=kind-substrate-preview
+NAMESPACE=mainloop-control
+REGISTRY=localhost:5001
+IMAGE_TAG=substrate-preview
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd -- "${SCRIPT_DIR}/.." && pwd)"
+PREVIEW_KUBECONFIG=
+PREFLIGHT_DIGEST=
+
+usage() {
+ cat <<'EOF'
+Usage: scripts/substrate-preview.sh [component]
+
+Commands:
+ build Build app images, mirror PostgreSQL, push and preflight manifests
+ deploy Apply the Kustomize overlay to kind-substrate-preview
+ open Port-forward frontend and backend; Ctrl+C stops both forwards
+ status Show Mainloop pods, Substrate actors, and WorkerPools
+ logs [component] Follow backend, frontend, or postgres logs (default: backend)
+
+Cluster commands require SUBSTRATE_PREVIEW_KUBECONFIG to name the preview kubeconfig.
+EOF
+}
+
+fail() {
+ printf 'Error: %s\n' "$*" >&2
+ exit 1
+}
+
+require_command() {
+ command -v "$1" >/dev/null 2>&1 || fail "required command not found: $1"
+}
+
+require_preview_kubeconfig() {
+ [[ -n ${SUBSTRATE_PREVIEW_KUBECONFIG-} ]] ||
+ fail 'SUBSTRATE_PREVIEW_KUBECONFIG is unset; set it to the preview kubeconfig path.'
+ [[ -f ${SUBSTRATE_PREVIEW_KUBECONFIG} && -r ${SUBSTRATE_PREVIEW_KUBECONFIG} ]] ||
+ fail "preview kubeconfig is not a readable file: ${SUBSTRATE_PREVIEW_KUBECONFIG}"
+ PREVIEW_KUBECONFIG="${SUBSTRATE_PREVIEW_KUBECONFIG}"
+}
+
+kube() {
+ kubectl --kubeconfig "${PREVIEW_KUBECONFIG}" --context "${CONTEXT}" "$@"
+}
+
+image_digest_file() {
+ local state_root="${XDG_STATE_HOME:-${HOME-}}"
+ [[ -n ${state_root} ]] || fail 'HOME is unset and XDG_STATE_HOME is unset; cannot locate the image digest state file.'
+ [[ ${state_root} == /* ]] || fail 'XDG_STATE_HOME must be an absolute path.'
+ printf '%s/mainloop/substrate-preview/image-digests\n' "${state_root%/}"
+}
+
+validate_image_digest() {
+ local image_name="$1"
+ local digest="$2"
+ [[ ${digest} =~ ^sha256:[a-f0-9]{64}$ ]] ||
+ fail "recorded ${image_name} digest is missing or invalid; run scripts/substrate-preview.sh build first."
+}
+
+load_image_digests() {
+ local digest_file
+ digest_file="$(image_digest_file)"
+ [[ -f ${digest_file} && -r ${digest_file} ]] ||
+ fail "no recorded image digests at ${digest_file}; run scripts/substrate-preview.sh build first."
+
+ BACKEND_DIGEST="$(sed -n 's/^backend=//p' "${digest_file}")"
+ FRONTEND_DIGEST="$(sed -n 's/^frontend=//p' "${digest_file}")"
+ POSTGRES_DIGEST="$(sed -n 's/^postgres=//p' "${digest_file}")"
+ validate_image_digest backend "${BACKEND_DIGEST}"
+ validate_image_digest frontend "${FRONTEND_DIGEST}"
+ validate_image_digest postgres "${POSTGRES_DIGEST}"
+}
+
+record_image_digests() {
+ local backend_digest="$1"
+ local frontend_digest="$2"
+ local postgres_digest="$3"
+ local digest_file
+ local digest_dir
+ local temporary_file
+
+ digest_file="$(image_digest_file)"
+ digest_dir="${digest_file%/*}"
+ mkdir -p -- "${digest_dir}"
+ chmod 700 "${digest_dir}"
+ temporary_file="$(mktemp "${digest_dir}/.image-digests.XXXXXX")"
+ chmod 600 "${temporary_file}"
+ {
+ printf 'backend=%s\n' "${backend_digest}"
+ printf 'frontend=%s\n' "${frontend_digest}"
+ printf 'postgres=%s\n' "${postgres_digest}"
+ } >"${temporary_file}"
+ mv -f -- "${temporary_file}" "${digest_file}"
+ printf 'Recorded preflighted image digests in %s\n' "${digest_file}"
+}
+
+ate() {
+ kubectl-ate --kubeconfig "${PREVIEW_KUBECONFIG}" --context "${CONTEXT}" "$@"
+}
+
+preflight_image() {
+ local image_ref="$1"
+ local repository="$2"
+ local push_log="${BUILD_TMPDIR}/${repository//\//_}.push.log"
+ local digest
+
+ docker push "${image_ref}" 2>&1 | tee "${push_log}"
+ digest="$(sed -nE 's/.*digest: (sha256:[a-f0-9]{64}).*/\1/p' "${push_log}" | tail -n 1)"
+ [[ ${digest} =~ ^sha256:[a-f0-9]{64}$ ]] ||
+ fail "could not read a sha256 digest from docker push output for ${image_ref}"
+
+ printf 'Preflighting %s at %s\n' "${image_ref}" "${digest}"
+ curl -fsI \
+ -H 'Accept: application/vnd.oci.image.index.v1+json, application/vnd.oci.image.manifest.v1+json, application/vnd.docker.distribution.manifest.list.v2+json, application/vnd.docker.distribution.manifest.v2+json' \
+ "http://${REGISTRY}/v2/${repository}/manifests/${digest}" >/dev/null
+ PREFLIGHT_DIGEST="${digest}"
+}
+
+build_images() {
+ require_command docker
+ require_command curl
+ BUILD_TMPDIR="$(mktemp -d)"
+ trap 'rm -rf -- "$BUILD_TMPDIR"' EXIT
+
+ docker build -f backend/Dockerfile -t "${REGISTRY}/mainloop-backend:${IMAGE_TAG}" .
+ docker build -f frontend/Dockerfile \
+ --build-arg VITE_API_URL=http://localhost:8000 \
+ -t "${REGISTRY}/mainloop-frontend:${IMAGE_TAG}" .
+ docker pull postgres:16-alpine
+ docker tag postgres:16-alpine "${REGISTRY}/postgres:16-alpine"
+
+ preflight_image "${REGISTRY}/mainloop-backend:${IMAGE_TAG}" mainloop-backend
+ local backend_digest="${PREFLIGHT_DIGEST}"
+ preflight_image "${REGISTRY}/mainloop-frontend:${IMAGE_TAG}" mainloop-frontend
+ local frontend_digest="${PREFLIGHT_DIGEST}"
+ preflight_image "${REGISTRY}/postgres:16-alpine" postgres
+ local postgres_digest="${PREFLIGHT_DIGEST}"
+ record_image_digests "${backend_digest}" "${frontend_digest}" "${postgres_digest}"
+}
+
+apply_digest_pinned_overlay() (
+ set -Eeuo pipefail
+ local overlay_dir="${REPO_ROOT}/k8s/apps/mainloop/overlays/substrate-preview"
+ local render_dir
+ render_dir="$(mktemp -d)"
+ trap 'rm -rf -- "${render_dir}"' EXIT
+ cp -R -- "${overlay_dir}/." "${render_dir}/"
+ cat >>"${render_dir}/kustomization.yaml" </dev/null
+ unset database_password
+ printf 'Created mainloop-db-app with a random password.\n'
+}
+
+deploy_overlay() {
+ load_image_digests
+ require_command kubectl
+ require_preview_kubeconfig
+ kube apply -f "${REPO_ROOT}/k8s/apps/mainloop/overlays/substrate-preview/namespace.yaml"
+ ensure_database_secret
+ apply_digest_pinned_overlay
+}
+
+show_status() {
+ require_command kubectl
+ require_command kubectl-ate
+ require_preview_kubeconfig
+
+ printf '%s\n' '== Mainloop pods =='
+ kube get pods -n "${NAMESPACE}" -o wide
+ printf '\n%s\n' '== Substrate actors =='
+ ate get actor --all-atespaces
+ printf '\n%s\n' '== Substrate WorkerPools =='
+ kube get workerpools --all-namespaces -o wide
+}
+
+follow_logs() {
+ local component="${1:-backend}"
+ local target
+
+ case "${component}" in
+ backend) target=deployment/mainloop-backend ;;
+ frontend) target=deployment/mainloop-frontend ;;
+ postgres) target=statefulset/postgres ;;
+ *) fail "unknown log component '${component}' (choose backend, frontend, or postgres)" ;;
+ esac
+
+ require_command kubectl
+ require_preview_kubeconfig
+ kube logs --follow --tail=200 -n "${NAMESPACE}" "${target}"
+}
+
+open_preview() {
+ require_command kubectl
+ require_preview_kubeconfig
+
+ local forward_dir
+ local frontend_pid
+ local backend_pid
+ local frontend_ready=0
+ local backend_ready=0
+ local attempt
+
+ forward_dir="$(mktemp -d)"
+ cleanup_forwards() {
+ local result=$?
+ trap - EXIT
+ for child_pid in "${frontend_pid-}" "${backend_pid-}"; do
+ if [[ -n ${child_pid} ]]; then
+ kill "${child_pid}" 2>/dev/null || true
+ wait "${child_pid}" 2>/dev/null || true
+ fi
+ done
+ if [[ ${result} -ne 0 ]]; then
+ for forward_log in "${forward_dir}"/*.log; do
+ [[ -s ${forward_log} ]] && cat "${forward_log}" >&2
+ done
+ fi
+ rm -rf -- "${forward_dir}"
+ return "${result}"
+ }
+ trap cleanup_forwards EXIT
+ trap 'exit 130' INT
+ trap 'exit 143' TERM
+
+ kube port-forward --address 127.0.0.1 -n "${NAMESPACE}" service/mainloop-frontend 3000:3000 \
+ >"${forward_dir}/frontend.log" 2>&1 &
+ frontend_pid=$!
+ kube port-forward --address 127.0.0.1 -n "${NAMESPACE}" service/mainloop-backend 8000:8000 \
+ >"${forward_dir}/backend.log" 2>&1 &
+ backend_pid=$!
+
+ attempt=0
+ while ((attempt < 60)); do
+ attempt=$((attempt + 1))
+ if ! kill -0 "${frontend_pid}" 2>/dev/null || ! kill -0 "${backend_pid}" 2>/dev/null; then
+ fail 'a port-forward exited before both local ports became ready'
+ fi
+ if (echo >/dev/tcp/127.0.0.1/3000) >/dev/null 2>&1; then
+ frontend_ready=1
+ fi
+ if (echo >/dev/tcp/127.0.0.1/8000) >/dev/null 2>&1; then
+ backend_ready=1
+ fi
+ [[ ${frontend_ready} -eq 1 && ${backend_ready} -eq 1 ]] && break
+ sleep 0.5
+ done
+ [[ ${frontend_ready} -eq 1 && ${backend_ready} -eq 1 ]] ||
+ fail 'timed out waiting for frontend and backend port-forwards'
+
+ printf 'Mainloop: http://127.0.0.1:3000\n'
+ printf 'Backend API: http://127.0.0.1:8000/docs\n'
+ printf 'Press Ctrl+C to stop both port-forwards.\n'
+ wait -n "${frontend_pid}" "${backend_pid}"
+}
+
+main() {
+ local command="${1-}"
+ case "${command}" in
+ build)
+ [[ $# -eq 1 ]] || fail 'build takes no additional arguments'
+ cd "${REPO_ROOT}"
+ build_images
+ ;;
+ deploy)
+ [[ $# -eq 1 ]] || fail 'deploy takes no additional arguments'
+ deploy_overlay
+ ;;
+ open)
+ [[ $# -eq 1 ]] || fail 'open takes no additional arguments'
+ open_preview
+ ;;
+ status)
+ [[ $# -eq 1 ]] || fail 'status takes no additional arguments'
+ show_status
+ ;;
+ logs)
+ [[ $# -le 2 ]] || fail 'logs accepts at most one component'
+ follow_logs "${2:-backend}"
+ ;;
+ -h | --help | help)
+ usage
+ ;;
+ *)
+ usage >&2
+ fail "unknown command '${command-}'"
+ ;;
+ esac
+}
+
+main "$@"
From 63c5f2b11995920dc0489e3ee2e0f1b2a0246fa0 Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Thu, 24 Sep 2026 01:42:46 +0000
Subject: [PATCH 17/30] feat(workspaces): add lifecycle state and controls
Persist a strict declarative workspace manifest and separate desired and observed lifecycle
state, conditions, transitions, operation IDs, snapshots, and ownership generations from session
and delivery state. Add owner-scoped lifecycle and refresh APIs, row-locked suspend reservations
with a second pre-call delivery fence, and workspace SSE updates. Show lifecycle badges across
session views and add a workspace detail page with conditions, manifest, snapshot status, and
pending/error-aware controls. Document the user-visible behavior and cover lifecycle policy,
fence ordering, and the API with fake-backed tests.
---
backend/src/mainloop/api.py | 2 +
backend/src/mainloop/db/postgres.py | 17 +-
.../src/mainloop/runtime/workspace_adapter.py | 948 +++++++++++++++++-
backend/src/mainloop/runtime/workspace_api.py | 101 ++
backend/src/mainloop/sse.py | 12 +
backend/tests/runtime/test_workspace_api.py | 150 +++
.../tests/runtime/test_workspace_lifecycle.py | 364 +++++++
docs/specs/sessions.md | 5 +
docs/specs/workspaces.md | 55 +
docs/spikes/substrate-workspace-adapter.md | 20 +
frontend/src/lib/api.ts | 84 ++
.../src/lib/components/SessionBlock.svelte | 8 +
.../src/lib/components/SessionListItem.svelte | 8 +
.../src/lib/components/SessionPicker.svelte | 8 +-
.../components/WorkspaceLifecycleBadge.svelte | 38 +
frontend/src/lib/sse.ts | 2 +
frontend/src/lib/stores/workspaces.ts | 61 ++
frontend/src/routes/+layout.svelte | 9 +
.../src/routes/sessions/[id]/+page.svelte | 18 +
.../src/routes/workspaces/[id]/+page.svelte | 284 ++++++
models/src/models/__init__.py | 18 +
models/src/models/workspace.py | 183 ++++
22 files changed, 2370 insertions(+), 25 deletions(-)
create mode 100644 backend/src/mainloop/runtime/workspace_api.py
create mode 100644 backend/tests/runtime/test_workspace_api.py
create mode 100644 backend/tests/runtime/test_workspace_lifecycle.py
create mode 100644 docs/specs/workspaces.md
create mode 100644 frontend/src/lib/components/WorkspaceLifecycleBadge.svelte
create mode 100644 frontend/src/lib/stores/workspaces.ts
create mode 100644 frontend/src/routes/workspaces/[id]/+page.svelte
create mode 100644 models/src/models/workspace.py
diff --git a/backend/src/mainloop/api.py b/backend/src/mainloop/api.py
index d1b6e6c..8a6380f 100644
--- a/backend/src/mainloop/api.py
+++ b/backend/src/mainloop/api.py
@@ -17,6 +17,7 @@
ConversationResponse,
)
from mainloop.runtime.agent_api import router as agent_api_router
+from mainloop.runtime.workspace_api import router as workspace_api_router
from mainloop.services.chat_handler import process_message
from mainloop.services.github_pr import (
CommitSummary,
@@ -395,6 +396,7 @@ async def list_topics(user_id: str = Header(alias="X-User-ID", default=None)):
app.include_router(agent_api_router)
+app.include_router(workspace_api_router)
# ============= Conversation Endpoints =============
diff --git a/backend/src/mainloop/db/postgres.py b/backend/src/mainloop/db/postgres.py
index 810f1b7..8683c64 100644
--- a/backend/src/mainloop/db/postgres.py
+++ b/backend/src/mainloop/db/postgres.py
@@ -250,6 +250,21 @@ def _parse_json_field(value: Any) -> list | dict | None:
UNIQUE (atespace, actor_name)
);
+-- Workspace lifecycle belongs to the workspace, separate from task/session, agent and delivery
+-- state. The manifest is declarative intent; only actor observations establish runtime state.
+CREATE TABLE IF NOT EXISTS workspace_lifecycles (
+ workspace_id TEXT PRIMARY KEY REFERENCES workspace_bindings(workspace_id) ON DELETE CASCADE,
+ desired_state TEXT NOT NULL DEFAULT 'running',
+ observed_state TEXT NOT NULL DEFAULT 'unknown',
+ manifest JSONB NOT NULL,
+ conditions JSONB NOT NULL DEFAULT '[]'::jsonb,
+ last_transition JSONB,
+ operation_id TEXT,
+ snapshot_ref TEXT,
+ created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
+);
+
-- Topics are durable records (not sessions). Supervisors (next slice) attach to a topic.
CREATE TABLE IF NOT EXISTS topics (
id TEXT PRIMARY KEY,
@@ -933,7 +948,7 @@ async def list_queue_items(
query = f"""
SELECT * FROM queue_items
- WHERE {' AND '.join(conditions)}
+ WHERE {" AND ".join(conditions)}
ORDER BY
CASE priority
WHEN 'urgent' THEN 1
diff --git a/backend/src/mainloop/runtime/workspace_adapter.py b/backend/src/mainloop/runtime/workspace_adapter.py
index aa3eb64..091190d 100644
--- a/backend/src/mainloop/runtime/workspace_adapter.py
+++ b/backend/src/mainloop/runtime/workspace_adapter.py
@@ -17,7 +17,9 @@
from __future__ import annotations
import asyncio
+import json
import logging
+import uuid
from datetime import UTC, datetime
from mainloop.config import settings
@@ -31,12 +33,39 @@
TransportError,
)
-from models import CapabilityResult, CapabilityState, WorkspaceBinding
+from models import (
+ CapabilityResult,
+ CapabilityState,
+ WorkspaceAgentKind,
+ WorkspaceBinding,
+ WorkspaceCondition,
+ WorkspaceConditionStatus,
+ WorkspaceDesiredState,
+ WorkspaceLifecycle,
+ WorkspaceManifest,
+ WorkspaceObservedState,
+ WorkspaceTransition,
+)
logger = logging.getLogger(__name__)
_locks: dict[str, asyncio.Lock] = {}
+LIFECYCLE_STATE = {
+ ActorState.RUNNING: WorkspaceObservedState.RUNNING,
+ ActorState.SUSPENDING: WorkspaceObservedState.SUSPENDING,
+ ActorState.SUSPENDED: WorkspaceObservedState.SUSPENDED,
+ ActorState.RESUMING: WorkspaceObservedState.RESUMING,
+ ActorState.CRASHED: WorkspaceObservedState.FAILED,
+ ActorState.DELETING: WorkspaceObservedState.FAILED,
+ ActorState.PAUSED: WorkspaceObservedState.UNKNOWN,
+ ActorState.PAUSING: WorkspaceObservedState.UNKNOWN,
+ ActorState.REVERTING: WorkspaceObservedState.UNKNOWN,
+ ActorState.UNSPECIFIED: WorkspaceObservedState.UNKNOWN,
+}
+
+BLOCKING_DELIVERY_STATES = {"recorded", "queued", "sending", "delivered", "uncertain"}
+
def _lock(session_id: str) -> asyncio.Lock:
return _locks.setdefault(session_id, asyncio.Lock())
@@ -139,6 +168,8 @@ async def _update_observed(
actor.external_snapshot_uri,
last_error,
)
+ await ensure_workspace_lifecycle(session_id)
+ await _record_observation(session_id, actor=actor, binding_error=last_error)
async def _mark_missing(session_id: str, now: datetime) -> None:
@@ -210,6 +241,8 @@ async def ensure_workspace(
if action == "create":
actor = await control.create_actor(atespace, name, template=template)
await _insert_row(session_id, atespace, name, template, actor, now)
+ await ensure_workspace_lifecycle(session_id)
+ await _record_observation(session_id, actor=actor)
else:
await _update_observed(session_id, actor, now)
return _binding_from_row(await get_workspace(session_id)) # type: ignore[arg-type]
@@ -238,32 +271,18 @@ async def observe_workspace(
async def resume_workspace(
session_id: str, *, control: SubstrateControl | None = None
-) -> WorkspaceBinding:
- control = control or _control()
- async with _lock(session_id):
- row = await get_workspace(session_id)
- if row is None:
- raise ContractError(f"no workspace binding for session {session_id}")
- actor = await control.resume_actor(row["atespace"], row["actor_name"])
- now = datetime.now(UTC)
- await _update_observed(session_id, actor, now)
- await _bump_generation(session_id, row["ownership_generation"])
- return _binding_from_row(await get_workspace(session_id)) # type: ignore[arg-type]
+) -> WorkspaceLifecycle:
+ return await _request_workspace_state(
+ session_id, WorkspaceDesiredState.RUNNING, control=control
+ )
async def suspend_workspace(
session_id: str, *, control: SubstrateControl | None = None
-) -> WorkspaceBinding:
- control = control or _control()
- async with _lock(session_id):
- row = await get_workspace(session_id)
- if row is None:
- raise ContractError(f"no workspace binding for session {session_id}")
- actor = await control.suspend_actor(row["atespace"], row["actor_name"])
- now = datetime.now(UTC)
- await _update_observed(session_id, actor, now)
- await _bump_generation(session_id, row["ownership_generation"])
- return _binding_from_row(await get_workspace(session_id)) # type: ignore[arg-type]
+) -> WorkspaceLifecycle:
+ return await _request_workspace_state(
+ session_id, WorkspaceDesiredState.SUSPENDED, control=control
+ )
async def revert_workspace(
@@ -300,3 +319,886 @@ def is_crashed(binding_row: dict) -> bool:
and binding_row.get("last_error") is not None
and "CRASHED" in (binding_row.get("last_error") or "").upper()
)
+
+
+def lifecycle_state(actor_state: ActorState) -> WorkspaceObservedState:
+ """Project Substrate actor status without treating unknown states as healthy."""
+ return LIFECYCLE_STATE[actor_state]
+
+
+def suspend_fence_reason(delivery_states: set[str]) -> str | None:
+ """Return why parking is unsafe while the durable delivery ledger is open."""
+ if "recorded" in delivery_states:
+ return "DeliveryRecorded"
+ if delivery_states & {"sending", "delivered"}:
+ return "TurnInFlight"
+ if "uncertain" in delivery_states:
+ return "DeliveryUncertain"
+ if "queued" in delivery_states:
+ return "DeliveryQueued"
+ return None
+
+
+def _suspend_fence_detail(reason: str) -> str:
+ return {
+ "DeliveryRecorded": "A recorded delivery must be reconciled before the workspace can be suspended.",
+ "TurnInFlight": "A native turn is still in flight; wait for it to finish before suspending.",
+ "DeliveryUncertain": "A delivery outcome is uncertain; reconcile it before suspending.",
+ "DeliveryQueued": "A queued delivery must be handled before the workspace can be suspended.",
+ }[reason]
+
+
+def _json_value(value):
+ if isinstance(value, str):
+ return json.loads(value)
+ return value
+
+
+def _condition(
+ condition_type: str,
+ status: WorkspaceConditionStatus,
+ reason: str,
+ message: str,
+ at: datetime,
+) -> WorkspaceCondition:
+ return WorkspaceCondition(
+ type=condition_type,
+ status=status,
+ reason=reason,
+ message=message,
+ last_transition_time=at,
+ )
+
+
+def _conditions_for(
+ *,
+ observed: WorkspaceObservedState,
+ desired: WorkspaceDesiredState,
+ snapshot_ref: str | None,
+ reason: str,
+ message: str,
+ at: datetime,
+ previous: tuple[WorkspaceCondition, ...] = (),
+ operation_status: WorkspaceConditionStatus | None = None,
+ operation_reason: str | None = None,
+ operation_message: str | None = None,
+ suspend_allowed: tuple[bool, str, str] | None = None,
+) -> tuple[WorkspaceCondition, ...]:
+ available_status = {
+ WorkspaceObservedState.RUNNING: WorkspaceConditionStatus.TRUE,
+ WorkspaceObservedState.SUSPENDED: WorkspaceConditionStatus.FALSE,
+ WorkspaceObservedState.FAILED: WorkspaceConditionStatus.FALSE,
+ }.get(observed, WorkspaceConditionStatus.UNKNOWN)
+ parked_status = (
+ WorkspaceConditionStatus.TRUE
+ if observed == WorkspaceObservedState.SUSPENDED and snapshot_ref
+ else (
+ WorkspaceConditionStatus.FALSE
+ if observed
+ in (WorkspaceObservedState.RUNNING, WorkspaceObservedState.FAILED)
+ or observed == WorkspaceObservedState.SUSPENDED
+ else WorkspaceConditionStatus.UNKNOWN
+ )
+ )
+ desired_status = (
+ WorkspaceConditionStatus.TRUE
+ if observed.value == desired.value
+ else (
+ WorkspaceConditionStatus.FALSE
+ if observed
+ in (WorkspaceObservedState.RUNNING, WorkspaceObservedState.SUSPENDED)
+ else WorkspaceConditionStatus.UNKNOWN
+ )
+ )
+ specs = [
+ ("Available", available_status, reason, message),
+ (
+ "Parked",
+ parked_status,
+ (
+ "SnapshotConfirmed"
+ if parked_status == WorkspaceConditionStatus.TRUE
+ else (
+ "SnapshotReferenceMissing"
+ if observed == WorkspaceObservedState.SUSPENDED
+ else reason
+ )
+ ),
+ (
+ "The suspended workspace has a recorded snapshot."
+ if parked_status == WorkspaceConditionStatus.TRUE
+ else (
+ "Substrate reported suspension without a snapshot reference."
+ if observed == WorkspaceObservedState.SUSPENDED
+ else message
+ )
+ ),
+ ),
+ (
+ "DesiredState",
+ desired_status,
+ (
+ "DesiredStateObserved"
+ if desired_status == WorkspaceConditionStatus.TRUE
+ else reason
+ ),
+ f"Desired {desired.value}; observed {observed.value}.",
+ ),
+ (
+ "ControlOperation",
+ operation_status
+ or (
+ WorkspaceConditionStatus.TRUE
+ if observed
+ in (WorkspaceObservedState.RUNNING, WorkspaceObservedState.SUSPENDED)
+ and desired_status == WorkspaceConditionStatus.TRUE
+ else WorkspaceConditionStatus.UNKNOWN
+ ),
+ operation_reason or reason,
+ operation_message or message,
+ ),
+ ]
+ if suspend_allowed is not None:
+ allowed, allowed_reason, allowed_message = suspend_allowed
+ specs.append(
+ (
+ "SuspendAllowed",
+ (
+ WorkspaceConditionStatus.TRUE
+ if allowed
+ else WorkspaceConditionStatus.FALSE
+ ),
+ allowed_reason,
+ allowed_message,
+ )
+ )
+
+ old = {item.type: item for item in previous}
+ result = []
+ for condition_type, status, item_reason, item_message in specs:
+ prior = old.get(condition_type)
+ changed = (
+ prior is None
+ or prior.status != status
+ or prior.reason != item_reason
+ or prior.message != item_message
+ )
+ result.append(
+ _condition(
+ condition_type,
+ status,
+ item_reason,
+ item_message,
+ at if changed else prior.last_transition_time,
+ )
+ )
+ return tuple(result)
+
+
+def _transition(
+ previous: WorkspaceLifecycle | None,
+ state: WorkspaceObservedState,
+ reason: str,
+ at: datetime,
+) -> WorkspaceTransition | None:
+ if previous is not None and previous.observed_state == state:
+ return previous.last_transition
+ return WorkspaceTransition(
+ from_state=previous.observed_state if previous else None,
+ to_state=state,
+ reason=reason,
+ occurred_at=at,
+ )
+
+
+def _actor_reason(state: ActorState) -> tuple[str, str]:
+ messages = {
+ ActorState.RUNNING: (
+ "ActorRunning",
+ "Substrate reports the workspace actor running.",
+ ),
+ ActorState.RESUMING: (
+ "ActorResuming",
+ "Substrate is restoring the workspace actor.",
+ ),
+ ActorState.SUSPENDING: (
+ "ActorSuspending",
+ "Substrate is suspending the workspace actor.",
+ ),
+ ActorState.SUSPENDED: (
+ "ActorSuspended",
+ "Substrate reports the workspace actor suspended.",
+ ),
+ ActorState.CRASHED: (
+ "ActorCrashed",
+ "Substrate reports the workspace actor crashed.",
+ ),
+ ActorState.DELETING: (
+ "ActorDeleting",
+ "Substrate reports the workspace actor deleting.",
+ ),
+ ActorState.PAUSED: (
+ "UnexpectedPaused",
+ "Substrate reported PAUSED; it is not confirmed parked.",
+ ),
+ ActorState.PAUSING: (
+ "UnexpectedPausing",
+ "Substrate reported PAUSING; lifecycle is not confirmed.",
+ ),
+ ActorState.REVERTING: (
+ "ActorReverting",
+ "Substrate is reverting the workspace actor.",
+ ),
+ ActorState.UNSPECIFIED: (
+ "ActorStateUnknown",
+ "Substrate did not report a recognized actor state.",
+ ),
+ }
+ return messages[state]
+
+
+def _lifecycle_from_row(row: dict) -> WorkspaceLifecycle:
+ payload = {
+ "workspace_id": row["workspace_id"],
+ "session_id": row["workspace_id"],
+ "desired_state": row["desired_state"],
+ "observed_state": row["observed_state"],
+ "manifest": _json_value(row["manifest"]),
+ "conditions": _json_value(row["conditions"]),
+ "last_transition": (
+ _json_value(row["last_transition"]) if row.get("last_transition") else None
+ ),
+ "operation_id": row.get("operation_id"),
+ "snapshot_ref": row.get("snapshot_ref"),
+ "ownership_generation": row["ownership_generation"],
+ "updated_at": row["updated_at"],
+ }
+ return WorkspaceLifecycle.model_validate_json(
+ json.dumps(payload, default=lambda value: value.isoformat())
+ )
+
+
+def _manifest_from_session(row: dict) -> WorkspaceManifest:
+ raw_kind = row.get("agent_kind")
+ agent_kinds = (
+ (WorkspaceAgentKind(raw_kind),)
+ if raw_kind in {kind.value for kind in WorkspaceAgentKind}
+ else ()
+ )
+ return WorkspaceManifest(
+ repo_url=row.get("repo_url"),
+ branch=row.get("branch_name") or row.get("base_branch") or "main",
+ agent_kinds=agent_kinds,
+ skills=(),
+ mcp_servers=(),
+ egress_allowlist=(),
+ resource_class="default",
+ )
+
+
+async def _get_lifecycle_row(workspace_id: str) -> dict | None:
+ async with db.connection() as conn:
+ row = await conn.fetchrow(
+ """SELECT l.*, b.ownership_generation, b.external_snapshot_uri
+ FROM workspace_lifecycles l
+ JOIN workspace_bindings b USING (workspace_id)
+ WHERE l.workspace_id=$1""",
+ workspace_id,
+ )
+ return dict(row) if row else None
+
+
+async def get_workspace_lifecycle(workspace_id: str) -> WorkspaceLifecycle | None:
+ row = await _get_lifecycle_row(workspace_id)
+ return _lifecycle_from_row(row) if row else None
+
+
+async def ensure_workspace_lifecycle(workspace_id: str) -> WorkspaceLifecycle | None:
+ """Backfill a declarative manifest and lifecycle record for an existing actor binding."""
+ async with db.connection() as conn:
+ row = await conn.fetchrow(
+ """SELECT b.*, s.repo_url, s.branch_name, s.base_branch, n.kind AS agent_kind
+ FROM workspace_bindings b
+ JOIN sessions s ON s.id=b.workspace_id
+ LEFT JOIN native_bindings n ON n.session_id=b.workspace_id
+ WHERE b.workspace_id=$1""",
+ workspace_id,
+ )
+ if row is None:
+ return None
+ binding = dict(row)
+ existing = await get_workspace_lifecycle(workspace_id)
+ if existing:
+ return existing
+
+ at = datetime.now(UTC)
+ initial_state = (
+ WorkspaceObservedState.RUNNING
+ if binding["observed_state"] == "ready"
+ else WorkspaceObservedState.UNKNOWN
+ )
+ reason = (
+ "LifecycleInitialized"
+ if initial_state == WorkspaceObservedState.UNKNOWN
+ else "ActorRunning"
+ )
+ message = (
+ "Lifecycle tracking was initialized; refresh status to inspect Substrate."
+ if initial_state == WorkspaceObservedState.UNKNOWN
+ else "Substrate previously reported the workspace ready."
+ )
+ manifest = _manifest_from_session(binding)
+ conditions = _conditions_for(
+ observed=initial_state,
+ desired=WorkspaceDesiredState.RUNNING,
+ snapshot_ref=binding.get("external_snapshot_uri"),
+ reason=reason,
+ message=message,
+ at=at,
+ )
+ async with db.connection() as conn:
+ await conn.execute(
+ """INSERT INTO workspace_lifecycles
+ (workspace_id, desired_state, observed_state, manifest, conditions, snapshot_ref,
+ updated_at)
+ VALUES ($1,$2,$3,$4::jsonb,$5::jsonb,$6,$7)
+ ON CONFLICT (workspace_id) DO NOTHING""",
+ workspace_id,
+ WorkspaceDesiredState.RUNNING.value,
+ initial_state.value,
+ json.dumps(manifest.model_dump(mode="json")),
+ json.dumps([condition.model_dump(mode="json") for condition in conditions]),
+ binding.get("external_snapshot_uri"),
+ at,
+ )
+ return await get_workspace_lifecycle(workspace_id)
+
+
+async def list_workspace_lifecycles(user_id: str) -> list[WorkspaceLifecycle]:
+ async with db.connection() as conn:
+ rows = await conn.fetch(
+ """SELECT b.workspace_id FROM workspace_bindings b
+ JOIN sessions s ON s.id=b.workspace_id
+ WHERE s.user_id=$1 ORDER BY s.created_at DESC""",
+ user_id,
+ )
+ for row in rows:
+ await ensure_workspace_lifecycle(row["workspace_id"])
+ async with db.connection() as conn:
+ lifecycle_rows = await conn.fetch(
+ """SELECT l.*, b.ownership_generation, b.external_snapshot_uri
+ FROM workspace_lifecycles l
+ JOIN workspace_bindings b USING (workspace_id)
+ JOIN sessions s ON s.id=b.workspace_id
+ WHERE s.user_id=$1 ORDER BY s.created_at DESC""",
+ user_id,
+ )
+ return [_lifecycle_from_row(dict(row)) for row in lifecycle_rows]
+
+
+async def _save_lifecycle(conn, lifecycle: WorkspaceLifecycle) -> None:
+ await conn.execute(
+ """UPDATE workspace_lifecycles
+ SET desired_state=$2, observed_state=$3, conditions=$4::jsonb,
+ last_transition=$5::jsonb, operation_id=$6, snapshot_ref=$7, updated_at=$8
+ WHERE workspace_id=$1""",
+ lifecycle.workspace_id,
+ lifecycle.desired_state.value,
+ lifecycle.observed_state.value,
+ json.dumps([item.model_dump(mode="json") for item in lifecycle.conditions]),
+ (
+ json.dumps(lifecycle.last_transition.model_dump(mode="json"))
+ if lifecycle.last_transition
+ else None
+ ),
+ lifecycle.operation_id,
+ lifecycle.snapshot_ref,
+ lifecycle.updated_at,
+ )
+
+
+async def _record_observation(
+ workspace_id: str,
+ *,
+ actor: ActorRecord | None = None,
+ failure: tuple[WorkspaceObservedState, str, str] | None = None,
+ binding_error: str | None = None,
+ desired_state: WorkspaceDesiredState | None = None,
+ operation_status: WorkspaceConditionStatus | None = None,
+ operation_reason: str | None = None,
+ operation_message: str | None = None,
+ suspend_allowed: tuple[bool, str, str] | None = None,
+) -> WorkspaceLifecycle:
+ previous = await ensure_workspace_lifecycle(workspace_id)
+ if previous is None:
+ raise ContractError(f"no workspace binding for {workspace_id}")
+ at = datetime.now(UTC)
+ if actor is not None:
+ state = lifecycle_state(actor.state)
+ reason, message = _actor_reason(actor.state)
+ snapshot_ref = (
+ actor.external_snapshot_uri
+ if state == WorkspaceObservedState.SUSPENDED
+ else actor.external_snapshot_uri or previous.snapshot_ref
+ )
+ observed_state_text = OBSERVED_STATE[actor.state]
+ last_error = (
+ binding_error
+ if binding_error is not None
+ else CRASHED_NOTE if actor.state == ActorState.CRASHED else None
+ )
+ if actor.state in (ActorState.PAUSED, ActorState.DELETING):
+ last_error = reason
+ async with db.connection() as conn:
+ await conn.execute(
+ """UPDATE workspace_bindings
+ SET observed_state=$2, observed_at=$3, external_snapshot_uri=$4,
+ last_error=$5, updated_at=NOW() WHERE workspace_id=$1""",
+ workspace_id,
+ observed_state_text,
+ at,
+ actor.external_snapshot_uri,
+ last_error,
+ )
+ else:
+ if failure is None:
+ raise ValueError("actor or failure observation is required")
+ state, reason, message = failure
+ snapshot_ref = previous.snapshot_ref
+
+ desired = desired_state or previous.desired_state
+ stable = state in (
+ WorkspaceObservedState.RUNNING,
+ WorkspaceObservedState.SUSPENDED,
+ WorkspaceObservedState.FAILED,
+ )
+ operation_id = None if stable else previous.operation_id
+ conditions = _conditions_for(
+ observed=state,
+ desired=desired,
+ snapshot_ref=snapshot_ref,
+ reason=reason,
+ message=message,
+ at=at,
+ previous=previous.conditions,
+ operation_status=operation_status,
+ operation_reason=operation_reason,
+ operation_message=operation_message,
+ suspend_allowed=suspend_allowed,
+ )
+ updated = WorkspaceLifecycle(
+ workspace_id=workspace_id,
+ session_id=workspace_id,
+ desired_state=desired,
+ observed_state=state,
+ manifest=previous.manifest,
+ conditions=conditions,
+ last_transition=_transition(previous, state, reason, at),
+ operation_id=operation_id,
+ snapshot_ref=snapshot_ref,
+ ownership_generation=previous.ownership_generation,
+ updated_at=at,
+ )
+ async with db.connection() as conn:
+ await _save_lifecycle(conn, updated)
+ return await get_workspace_lifecycle(workspace_id) or updated
+
+
+async def refresh_workspace_lifecycle(
+ workspace_id: str, *, control: SubstrateControl | None = None
+) -> WorkspaceLifecycle:
+ """Observe the actor once; transport errors remain unknown until a later refresh."""
+ control = control or _control()
+ async with _lock(workspace_id):
+ row = await get_workspace(workspace_id)
+ if row is None:
+ raise ContractError(f"no workspace binding for {workspace_id}")
+ await ensure_workspace_lifecycle(workspace_id)
+ try:
+ actor = await control.get_actor(row["atespace"], row["actor_name"])
+ except Exception:
+ return await _record_observation(
+ workspace_id,
+ failure=(
+ WorkspaceObservedState.UNKNOWN,
+ "ObservationUncertain",
+ "Substrate could not confirm workspace status. Refresh again before retrying an operation.",
+ ),
+ )
+ if actor is None:
+ return await _record_observation(
+ workspace_id,
+ failure=(
+ WorkspaceObservedState.FAILED,
+ "ActorMissing",
+ "The workspace binding exists but Substrate returned no actor.",
+ ),
+ )
+ return await _record_observation(workspace_id, actor=actor)
+
+
+async def _record_operation_failure(
+ workspace_id: str,
+ *,
+ reason: str,
+ message: str,
+ uncertain: bool,
+) -> WorkspaceLifecycle:
+ previous = await ensure_workspace_lifecycle(workspace_id)
+ if previous is None:
+ raise ContractError(f"no workspace binding for {workspace_id}")
+ at = datetime.now(UTC)
+ state = WorkspaceObservedState.UNKNOWN if uncertain else previous.observed_state
+ conditions = _conditions_for(
+ observed=state,
+ desired=previous.desired_state,
+ snapshot_ref=previous.snapshot_ref,
+ reason=reason,
+ message=message,
+ at=at,
+ previous=previous.conditions,
+ operation_status=(
+ WorkspaceConditionStatus.UNKNOWN
+ if uncertain
+ else WorkspaceConditionStatus.FALSE
+ ),
+ operation_reason=reason,
+ operation_message=message,
+ )
+ updated = WorkspaceLifecycle(
+ workspace_id=previous.workspace_id,
+ session_id=previous.session_id,
+ desired_state=previous.desired_state,
+ observed_state=state,
+ manifest=previous.manifest,
+ conditions=conditions,
+ last_transition=_transition(previous, state, reason, at),
+ operation_id=previous.operation_id if uncertain else None,
+ snapshot_ref=previous.snapshot_ref,
+ ownership_generation=previous.ownership_generation,
+ updated_at=at,
+ )
+ async with db.connection() as conn:
+ await _save_lifecycle(conn, updated)
+ return await get_workspace_lifecycle(workspace_id) or updated
+
+
+async def _delivery_states(workspace_id: str, *, conn=None) -> set[str]:
+ if conn is None:
+ async with db.connection() as connection:
+ rows = await connection.fetch(
+ """SELECT state FROM native_deliveries
+ WHERE session_id=$1 AND state = ANY($2)""",
+ workspace_id,
+ list(BLOCKING_DELIVERY_STATES),
+ )
+ else:
+ rows = await conn.fetch(
+ """SELECT state FROM native_deliveries
+ WHERE session_id=$1 AND state = ANY($2)""",
+ workspace_id,
+ list(BLOCKING_DELIVERY_STATES),
+ )
+ return {row["state"] for row in rows}
+
+
+async def _record_suspend_fence(
+ workspace_id: str,
+ reason: str,
+ message: str,
+ *,
+ previous: WorkspaceLifecycle | None = None,
+ conn=None,
+) -> WorkspaceLifecycle:
+ previous = previous or await ensure_workspace_lifecycle(workspace_id)
+ if previous is None:
+ raise ContractError(f"no workspace binding for {workspace_id}")
+ at = datetime.now(UTC)
+ conditions = _conditions_for(
+ observed=previous.observed_state,
+ desired=previous.desired_state,
+ snapshot_ref=previous.snapshot_ref,
+ reason=reason,
+ message=message,
+ at=at,
+ previous=previous.conditions,
+ operation_status=WorkspaceConditionStatus.FALSE,
+ operation_reason=reason,
+ operation_message=message,
+ suspend_allowed=(False, reason, message),
+ )
+ updated = WorkspaceLifecycle(
+ **{
+ **previous.model_dump(),
+ "conditions": conditions,
+ "updated_at": at,
+ }
+ )
+ if conn is None:
+ async with db.connection() as connection:
+ await _save_lifecycle(connection, updated)
+ else:
+ await _save_lifecycle(conn, updated)
+ if conn is not None:
+ return updated
+ return await get_workspace_lifecycle(workspace_id) or updated
+
+
+async def _reserve_operation(
+ previous: WorkspaceLifecycle,
+ desired_state: WorkspaceDesiredState,
+) -> WorkspaceLifecycle:
+ at = datetime.now(UTC)
+ transitional = (
+ WorkspaceObservedState.SUSPENDING
+ if desired_state == WorkspaceDesiredState.SUSPENDED
+ else WorkspaceObservedState.RESUMING
+ )
+ operation_id = str(uuid.uuid4())
+ reason = (
+ "SuspendRequested"
+ if desired_state == WorkspaceDesiredState.SUSPENDED
+ else "ResumeRequested"
+ )
+ message = (
+ f"Mainloop recorded a request to make the workspace {desired_state.value}."
+ )
+ conditions = _conditions_for(
+ observed=transitional,
+ desired=desired_state,
+ snapshot_ref=previous.snapshot_ref,
+ reason=reason,
+ message=message,
+ at=at,
+ previous=previous.conditions,
+ operation_status=WorkspaceConditionStatus.UNKNOWN,
+ operation_reason=reason,
+ operation_message=message,
+ suspend_allowed=(
+ (
+ True,
+ "NoOpenDelivery",
+ "No open or uncertain delivery blocks suspension.",
+ )
+ if desired_state == WorkspaceDesiredState.SUSPENDED
+ else None
+ ),
+ )
+ updated = WorkspaceLifecycle(
+ workspace_id=previous.workspace_id,
+ session_id=previous.session_id,
+ desired_state=desired_state,
+ observed_state=transitional,
+ manifest=previous.manifest,
+ conditions=conditions,
+ last_transition=_transition(previous, transitional, reason, at),
+ operation_id=operation_id,
+ snapshot_ref=previous.snapshot_ref,
+ ownership_generation=previous.ownership_generation,
+ updated_at=at,
+ )
+ fence_detail: str | None = None
+ async with db.connection() as conn:
+ async with conn.transaction():
+ binding = await conn.fetchrow(
+ """SELECT ownership_generation FROM workspace_bindings
+ WHERE workspace_id=$1 FOR UPDATE""",
+ previous.workspace_id,
+ )
+ if binding is None:
+ raise ContractError(f"no workspace binding for {previous.workspace_id}")
+ if binding["ownership_generation"] != previous.ownership_generation:
+ raise StaleOwnership(
+ f"workspace {previous.workspace_id} changed during lifecycle request"
+ )
+ if desired_state == WorkspaceDesiredState.SUSPENDED:
+ fence_reason = suspend_fence_reason(
+ await _delivery_states(previous.workspace_id, conn=conn)
+ )
+ if fence_reason:
+ fence_detail = _suspend_fence_detail(fence_reason)
+ await _record_suspend_fence(
+ previous.workspace_id,
+ fence_reason,
+ fence_detail,
+ previous=previous,
+ conn=conn,
+ )
+ if fence_detail is None:
+ tag = await conn.execute(
+ """UPDATE workspace_bindings
+ SET ownership_generation=ownership_generation+1, updated_at=NOW()
+ WHERE workspace_id=$1 AND ownership_generation=$2""",
+ previous.workspace_id,
+ previous.ownership_generation,
+ )
+ if tag.endswith(" 0"):
+ raise StaleOwnership(
+ f"workspace {previous.workspace_id} changed during lifecycle request"
+ )
+ await _save_lifecycle(conn, updated)
+ if fence_detail is not None:
+ raise ContractError(fence_detail)
+ return await get_workspace_lifecycle(previous.workspace_id) or updated
+
+
+async def _request_workspace_state(
+ workspace_id: str,
+ desired_state: WorkspaceDesiredState,
+ *,
+ control: SubstrateControl | None = None,
+) -> WorkspaceLifecycle:
+ control = control or _control()
+ async with _lock(workspace_id):
+ previous = await ensure_workspace_lifecycle(workspace_id)
+ row = await get_workspace(workspace_id)
+ if previous is None or row is None:
+ raise ContractError(f"no workspace binding for {workspace_id}")
+ if (
+ previous.desired_state == desired_state
+ and previous.observed_state.value == desired_state.value
+ and previous.operation_id is None
+ ):
+ return previous
+
+ # A pending operation means a previous control call may have timed out. Its original
+ # intent is already durable; inspect that actor before deciding whether a new operation
+ # can be recorded. First attempts persist intent before the first Substrate call.
+ had_pending_operation = previous.operation_id is not None
+ if not had_pending_operation:
+ previous = await _reserve_operation(previous, desired_state)
+
+ try:
+ actor = await control.get_actor(row["atespace"], row["actor_name"])
+ except Exception:
+ return await _record_operation_failure(
+ workspace_id,
+ reason="ObservationUncertain",
+ message="Substrate status is unknown. Refresh status before retrying the operation.",
+ uncertain=True,
+ )
+ if actor is None:
+ return await _record_observation(
+ workspace_id,
+ failure=(
+ WorkspaceObservedState.FAILED,
+ "ActorMissing",
+ "The workspace binding exists but Substrate returned no actor.",
+ ),
+ desired_state=desired_state,
+ )
+
+ observed = lifecycle_state(actor.state)
+ if observed.value == desired_state.value:
+ if had_pending_operation and previous.desired_state != desired_state:
+ # Finish reconciling the old intent, then persist the new request. The observed
+ # actor already matches it, so no second control mutation is needed.
+ previous = await _record_observation(
+ workspace_id, actor=actor, desired_state=previous.desired_state
+ )
+ await _reserve_operation(previous, desired_state)
+ return await _record_observation(
+ workspace_id, actor=actor, desired_state=desired_state
+ )
+ if observed in (
+ WorkspaceObservedState.SUSPENDING,
+ WorkspaceObservedState.RESUMING,
+ WorkspaceObservedState.UNKNOWN,
+ WorkspaceObservedState.FAILED,
+ ):
+ reason, message = _actor_reason(actor.state)
+ return await _record_observation(
+ workspace_id,
+ actor=actor,
+ operation_status=WorkspaceConditionStatus.UNKNOWN,
+ operation_reason=reason,
+ operation_message=message,
+ desired_state=(
+ previous.desired_state
+ if had_pending_operation and previous.desired_state != desired_state
+ else desired_state
+ ),
+ )
+
+ if had_pending_operation:
+ # The inspected actor is stable in the opposite state. Retire the uncertain
+ # operation record before persisting a fresh attempt; this is the required
+ # inspect-before-retry fence.
+ previous = await _record_observation(
+ workspace_id, actor=actor, desired_state=previous.desired_state
+ )
+ previous = await _reserve_operation(previous, desired_state)
+
+ if desired_state == WorkspaceDesiredState.SUSPENDED:
+ fence_reason = suspend_fence_reason(await _delivery_states(workspace_id))
+ if fence_reason:
+ detail = _suspend_fence_detail(fence_reason)
+ await _record_observation(
+ workspace_id,
+ actor=actor,
+ desired_state=desired_state,
+ operation_status=WorkspaceConditionStatus.FALSE,
+ operation_reason=fence_reason,
+ operation_message=detail,
+ suspend_allowed=(False, fence_reason, detail),
+ )
+ raise ContractError(detail)
+
+ try:
+ if desired_state == WorkspaceDesiredState.SUSPENDED:
+ actor = await control.suspend_actor(row["atespace"], row["actor_name"])
+ else:
+ actor = await control.resume_actor(row["atespace"], row["actor_name"])
+ except TransportError:
+ return await _record_operation_failure(
+ workspace_id,
+ reason="OperationOutcomeUnknown",
+ message="Substrate did not confirm the operation. Refresh status before retrying; Mainloop will not replay it automatically.",
+ uncertain=True,
+ )
+ except RuntimeError:
+ return await _record_operation_failure(
+ workspace_id,
+ reason="OperationRejected",
+ message="Substrate rejected the lifecycle request. Refresh status before retrying.",
+ uncertain=False,
+ )
+ except Exception:
+ return await _record_operation_failure(
+ workspace_id,
+ reason="OperationOutcomeUnknown",
+ message="Substrate did not confirm the operation. Refresh status before retrying; Mainloop will not replay it automatically.",
+ uncertain=True,
+ )
+ observed = lifecycle_state(actor.state)
+ if observed.value == desired_state.value and (
+ desired_state != WorkspaceDesiredState.SUSPENDED
+ or actor.external_snapshot_uri is not None
+ ):
+ operation_status = WorkspaceConditionStatus.TRUE
+ operation_reason = "OperationConfirmed"
+ operation_message = (
+ f"Substrate confirmed the workspace {desired_state.value}."
+ )
+ elif observed in (
+ WorkspaceObservedState.RUNNING,
+ WorkspaceObservedState.SUSPENDED,
+ WorkspaceObservedState.FAILED,
+ ):
+ operation_status = WorkspaceConditionStatus.FALSE
+ operation_reason = "DesiredStateNotReached"
+ operation_message = f"Substrate observed {observed.value}; desired {desired_state.value} was not confirmed."
+ else:
+ operation_status = WorkspaceConditionStatus.UNKNOWN
+ operation_reason = "ActorTransitionInProgress"
+ operation_message = (
+ "Substrate has not reached a stable state; refresh status to reconcile."
+ )
+ return await _record_observation(
+ workspace_id,
+ actor=actor,
+ desired_state=desired_state,
+ operation_status=operation_status,
+ operation_reason=operation_reason,
+ operation_message=operation_message,
+ )
diff --git a/backend/src/mainloop/runtime/workspace_api.py b/backend/src/mainloop/runtime/workspace_api.py
new file mode 100644
index 0000000..9cec084
--- /dev/null
+++ b/backend/src/mainloop/runtime/workspace_api.py
@@ -0,0 +1,101 @@
+"""Workspace lifecycle endpoints for manifest and actor state."""
+
+from fastapi import APIRouter, Header, HTTPException
+from mainloop.db import db
+from mainloop.runtime import workspace_adapter
+from mainloop.runtime.contracts import ContractError
+from mainloop.sse import notify_workspace_updated
+
+from models import WorkspaceLifecycle
+
+router = APIRouter(prefix="/workspaces", tags=["workspaces"])
+
+
+def _user_id(value: str | None) -> str:
+ return value or "local-dev-user"
+
+
+async def _require_owned_workspace(workspace_id: str, user_id: str) -> None:
+ async with db.connection() as conn:
+ exists = await conn.fetchval(
+ """SELECT EXISTS (
+ SELECT 1 FROM workspace_bindings b
+ JOIN sessions s ON s.id=b.workspace_id
+ WHERE b.workspace_id=$1 AND s.user_id=$2
+ )""",
+ workspace_id,
+ user_id,
+ )
+ if not exists:
+ raise HTTPException(status_code=404, detail="Workspace not found")
+
+
+async def _publish(user_id: str, lifecycle: WorkspaceLifecycle) -> None:
+ await notify_workspace_updated(user_id, lifecycle.model_dump(mode="json"))
+
+
+@router.get("", response_model=list[WorkspaceLifecycle])
+async def list_workspaces(
+ user_id: str | None = Header(default=None, alias="X-User-ID"),
+):
+ return await workspace_adapter.list_workspace_lifecycles(_user_id(user_id))
+
+
+@router.get("/{workspace_id}", response_model=WorkspaceLifecycle)
+async def get_workspace(
+ workspace_id: str,
+ user_id: str | None = Header(default=None, alias="X-User-ID"),
+):
+ owner = _user_id(user_id)
+ await _require_owned_workspace(workspace_id, owner)
+ lifecycle = await workspace_adapter.ensure_workspace_lifecycle(workspace_id)
+ if lifecycle is None:
+ raise HTTPException(status_code=404, detail="Workspace not found")
+ return lifecycle
+
+
+async def _run_operation(
+ workspace_id: str,
+ owner: str,
+ operation,
+) -> WorkspaceLifecycle:
+ await _require_owned_workspace(workspace_id, owner)
+ try:
+ lifecycle = await operation(workspace_id)
+ except ContractError as exc:
+ current = await workspace_adapter.get_workspace_lifecycle(workspace_id)
+ if current is not None:
+ await _publish(owner, current)
+ raise HTTPException(status_code=409, detail=str(exc)) from exc
+ await _publish(owner, lifecycle)
+ return lifecycle
+
+
+@router.post("/{workspace_id}/suspend", response_model=WorkspaceLifecycle)
+async def suspend_workspace(
+ workspace_id: str,
+ user_id: str | None = Header(default=None, alias="X-User-ID"),
+):
+ return await _run_operation(
+ workspace_id, _user_id(user_id), workspace_adapter.suspend_workspace
+ )
+
+
+@router.post("/{workspace_id}/resume", response_model=WorkspaceLifecycle)
+async def resume_workspace(
+ workspace_id: str,
+ user_id: str | None = Header(default=None, alias="X-User-ID"),
+):
+ return await _run_operation(
+ workspace_id, _user_id(user_id), workspace_adapter.resume_workspace
+ )
+
+
+@router.post("/{workspace_id}/refresh", response_model=WorkspaceLifecycle)
+async def refresh_workspace(
+ workspace_id: str,
+ user_id: str | None = Header(default=None, alias="X-User-ID"),
+):
+ return await _run_operation(
+ workspace_id, _user_id(user_id), workspace_adapter.refresh_workspace_lifecycle
+ )
diff --git a/backend/src/mainloop/sse.py b/backend/src/mainloop/sse.py
index 4098773..7571a97 100644
--- a/backend/src/mainloop/sse.py
+++ b/backend/src/mainloop/sse.py
@@ -23,6 +23,7 @@ class EventType(str, Enum):
SESSION_UPDATED = "session:updated"
SESSION_NEEDS_INPUT = "session:needs_input"
SESSION_MESSAGE = "session:message"
+ WORKSPACE_UPDATED = "workspace:updated"
HEARTBEAT = "heartbeat"
@@ -212,3 +213,14 @@ async def notify_session_message(
},
),
)
+
+
+async def notify_workspace_updated(user_id: str, lifecycle: dict[str, Any]) -> None:
+ """Publish the current durable workspace lifecycle projection."""
+ await event_bus.publish_to_user(
+ user_id,
+ SSEEvent(
+ event=EventType.WORKSPACE_UPDATED,
+ data={"workspace": lifecycle},
+ ),
+ )
diff --git a/backend/tests/runtime/test_workspace_api.py b/backend/tests/runtime/test_workspace_api.py
new file mode 100644
index 0000000..1c1d834
--- /dev/null
+++ b/backend/tests/runtime/test_workspace_api.py
@@ -0,0 +1,150 @@
+"""Workspace API projection and ownership behavior with adapter fakes."""
+
+import unittest
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, patch
+
+from fastapi import HTTPException
+from mainloop.runtime import workspace_api
+from mainloop.runtime.contracts import ContractError
+
+from models import (
+ WorkspaceAgentKind,
+ WorkspaceDesiredState,
+ WorkspaceLifecycle,
+ WorkspaceManifest,
+ WorkspaceObservedState,
+)
+
+
+def lifecycle() -> WorkspaceLifecycle:
+ return WorkspaceLifecycle(
+ workspace_id="session-1",
+ session_id="session-1",
+ desired_state=WorkspaceDesiredState.RUNNING,
+ observed_state=WorkspaceObservedState.RUNNING,
+ manifest=WorkspaceManifest(
+ repo_url="https://github.com/example/repo",
+ branch="main",
+ agent_kinds=(WorkspaceAgentKind.CODEX,),
+ skills=("skill://review",),
+ mcp_servers=("mcp://docs",),
+ egress_allowlist=("github.com",),
+ resource_class="small",
+ ),
+ ownership_generation=4,
+ updated_at=datetime(2026, 9, 24, tzinfo=UTC),
+ )
+
+
+class WorkspaceApiTests(unittest.IsolatedAsyncioTestCase):
+ def test_routes_expose_the_workspace_lifecycle_api(self):
+ route_methods = {
+ (route.path, method)
+ for route in workspace_api.router.routes
+ for method in route.methods or ()
+ }
+ self.assertTrue(
+ {
+ ("/workspaces", "GET"),
+ ("/workspaces/{workspace_id}", "GET"),
+ ("/workspaces/{workspace_id}/suspend", "POST"),
+ ("/workspaces/{workspace_id}/resume", "POST"),
+ ("/workspaces/{workspace_id}/refresh", "POST"),
+ }.issubset(route_methods)
+ )
+
+ async def test_list_returns_lifecycle_and_manifest(self):
+ with patch.object(
+ workspace_api.workspace_adapter,
+ "list_workspace_lifecycles",
+ new=AsyncMock(return_value=[lifecycle()]),
+ ) as list_workspaces:
+ response = await workspace_api.list_workspaces(user_id="owner-1")
+
+ self.assertEqual(response[0].observed_state, WorkspaceObservedState.RUNNING)
+ self.assertEqual(response[0].manifest.agent_kinds, (WorkspaceAgentKind.CODEX,))
+ list_workspaces.assert_awaited_once_with("owner-1")
+
+ async def test_get_detail_and_lifecycle_actions_publish_the_lifecycle(self):
+ current = lifecycle()
+ with (
+ patch.object(workspace_api, "_require_owned_workspace", new=AsyncMock()),
+ patch.object(
+ workspace_api.workspace_adapter,
+ "ensure_workspace_lifecycle",
+ new=AsyncMock(return_value=current),
+ ),
+ patch.object(
+ workspace_api.workspace_adapter,
+ "suspend_workspace",
+ new=AsyncMock(return_value=current),
+ ),
+ patch.object(
+ workspace_api.workspace_adapter,
+ "resume_workspace",
+ new=AsyncMock(return_value=current),
+ ),
+ patch.object(
+ workspace_api.workspace_adapter,
+ "refresh_workspace_lifecycle",
+ new=AsyncMock(return_value=current),
+ ),
+ patch.object(workspace_api, "_publish", new=AsyncMock()) as publish,
+ ):
+ detail = await workspace_api.get_workspace("session-1", user_id="owner-1")
+ suspended = await workspace_api.suspend_workspace(
+ "session-1", user_id="owner-1"
+ )
+ resumed = await workspace_api.resume_workspace(
+ "session-1", user_id="owner-1"
+ )
+ refreshed = await workspace_api.refresh_workspace(
+ "session-1", user_id="owner-1"
+ )
+
+ self.assertEqual(detail.workspace_id, "session-1")
+ self.assertEqual(suspended.workspace_id, "session-1")
+ self.assertEqual(resumed.workspace_id, "session-1")
+ self.assertEqual(refreshed.workspace_id, "session-1")
+ self.assertEqual(suspended.manifest.resource_class, "small")
+ self.assertEqual(publish.await_count, 3)
+
+ async def test_fence_error_is_a_conflict_and_preserves_reason(self):
+ current = lifecycle()
+ with (
+ patch.object(workspace_api, "_require_owned_workspace", new=AsyncMock()),
+ patch.object(
+ workspace_api.workspace_adapter,
+ "suspend_workspace",
+ new=AsyncMock(
+ side_effect=ContractError("A recorded delivery is still open.")
+ ),
+ ),
+ patch.object(
+ workspace_api.workspace_adapter,
+ "get_workspace_lifecycle",
+ new=AsyncMock(return_value=current),
+ ),
+ patch.object(workspace_api, "_publish", new=AsyncMock()),
+ ):
+ with self.assertRaises(HTTPException) as raised:
+ await workspace_api.suspend_workspace("session-1", user_id="owner-1")
+
+ self.assertEqual(raised.exception.status_code, 409)
+ self.assertEqual(raised.exception.detail, "A recorded delivery is still open.")
+
+ async def test_owner_check_hides_other_users_workspaces(self):
+ with patch.object(
+ workspace_api,
+ "_require_owned_workspace",
+ new=AsyncMock(side_effect=HTTPException(404, "Workspace not found")),
+ ):
+ with self.assertRaises(HTTPException) as raised:
+ await workspace_api.get_workspace("session-1", user_id="other")
+
+ self.assertEqual(raised.exception.status_code, 404)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/backend/tests/runtime/test_workspace_lifecycle.py b/backend/tests/runtime/test_workspace_lifecycle.py
new file mode 100644
index 0000000..fea49fe
--- /dev/null
+++ b/backend/tests/runtime/test_workspace_lifecycle.py
@@ -0,0 +1,364 @@
+"""Workspace lifecycle policy and orchestration tests; no Substrate cluster is contacted."""
+
+import asyncio
+import unittest
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, patch
+
+from mainloop.runtime import workspace_adapter as adapter
+from mainloop.runtime.contracts import ContractError
+from mainloop.runtime.substrate import ActorRecord, ActorState, TransportError
+
+from models import (
+ WorkspaceDesiredState,
+ WorkspaceLifecycle,
+ WorkspaceManifest,
+ WorkspaceObservedState,
+)
+
+NOW = datetime(2026, 9, 24, tzinfo=UTC)
+
+
+def lifecycle(
+ *,
+ desired: WorkspaceDesiredState = WorkspaceDesiredState.RUNNING,
+ observed: WorkspaceObservedState = WorkspaceObservedState.RUNNING,
+ operation_id: str | None = None,
+) -> WorkspaceLifecycle:
+ return WorkspaceLifecycle(
+ workspace_id="session-1",
+ session_id="session-1",
+ desired_state=desired,
+ observed_state=observed,
+ manifest=WorkspaceManifest(
+ repo_url="https://github.com/example/repo",
+ branch="main",
+ resource_class="default",
+ ),
+ operation_id=operation_id,
+ ownership_generation=3,
+ updated_at=NOW,
+ )
+
+
+def actor(state: ActorState, snapshot: str | None = None) -> ActorRecord:
+ return ActorRecord(
+ atespace="workspaces",
+ name="actor-1",
+ uid="uid-1",
+ state=state,
+ external_snapshot_uri=snapshot,
+ current_actor_template_uid="template-1",
+ raw={},
+ )
+
+
+class FakeTransaction:
+ def __init__(self, connection):
+ self.connection = connection
+
+ async def __aenter__(self):
+ self.connection.in_transaction = True
+ self.connection.events.append("begin")
+ return self.connection
+
+ async def __aexit__(self, exc_type, exc, traceback):
+ self.connection.events.append("rollback" if exc_type else "commit")
+ self.connection.in_transaction = False
+
+
+class FakePostgresConnection:
+ def __init__(self, delivery_states=()):
+ self.delivery_states = delivery_states
+ self.events = []
+ self.in_transaction = False
+ self.lock_query = None
+
+ def transaction(self):
+ return FakeTransaction(self)
+
+ async def fetchrow(self, query, *_args):
+ if not self.in_transaction:
+ raise AssertionError(
+ "workspace binding lock must be inside the transaction"
+ )
+ self.events.append("lock")
+ self.lock_query = query
+ return {"ownership_generation": 3}
+
+ async def fetch(self, query, *_args):
+ if not self.in_transaction:
+ raise AssertionError("delivery fence must be inside the transaction")
+ if "native_deliveries" not in query:
+ raise AssertionError("expected native delivery state query")
+ self.events.append("delivery-check")
+ return [{"state": state} for state in self.delivery_states]
+
+ async def execute(self, query, *_args):
+ if not self.in_transaction:
+ raise AssertionError("workspace reservation must be inside the transaction")
+ if "UPDATE workspace_bindings" in query:
+ self.events.append("reserve")
+ elif "UPDATE workspace_lifecycles" in query:
+ self.events.append("lifecycle")
+ return "UPDATE 1"
+
+
+def fake_db_connection(connection):
+ @asynccontextmanager
+ async def connect():
+ yield connection
+
+ return connect
+
+
+class WorkspaceStateMappingTests(unittest.TestCase):
+ def test_actor_states_map_without_guessing_transitions(self):
+ expected = {
+ ActorState.RUNNING: WorkspaceObservedState.RUNNING,
+ ActorState.SUSPENDING: WorkspaceObservedState.SUSPENDING,
+ ActorState.SUSPENDED: WorkspaceObservedState.SUSPENDED,
+ ActorState.RESUMING: WorkspaceObservedState.RESUMING,
+ ActorState.CRASHED: WorkspaceObservedState.FAILED,
+ ActorState.DELETING: WorkspaceObservedState.FAILED,
+ ActorState.PAUSED: WorkspaceObservedState.UNKNOWN,
+ ActorState.PAUSING: WorkspaceObservedState.UNKNOWN,
+ ActorState.REVERTING: WorkspaceObservedState.UNKNOWN,
+ ActorState.UNSPECIFIED: WorkspaceObservedState.UNKNOWN,
+ }
+ for actor_state, workspace_state in expected.items():
+ with self.subTest(actor_state=actor_state):
+ self.assertEqual(adapter.lifecycle_state(actor_state), workspace_state)
+
+ def test_suspend_fence_blocks_open_and_uncertain_deliveries(self):
+ self.assertIsNone(adapter.suspend_fence_reason(set()))
+ self.assertEqual(adapter.suspend_fence_reason({"recorded"}), "DeliveryRecorded")
+ self.assertEqual(adapter.suspend_fence_reason({"sending"}), "TurnInFlight")
+ self.assertEqual(adapter.suspend_fence_reason({"delivered"}), "TurnInFlight")
+ self.assertEqual(
+ adapter.suspend_fence_reason({"uncertain"}), "DeliveryUncertain"
+ )
+ self.assertEqual(adapter.suspend_fence_reason({"queued"}), "DeliveryQueued")
+
+
+class WorkspaceOperationTests(unittest.IsolatedAsyncioTestCase):
+ async def test_resume_is_idempotent_when_actor_is_already_running(self):
+ stored = lifecycle(
+ desired=WorkspaceDesiredState.SUSPENDED,
+ observed=WorkspaceObservedState.SUSPENDED,
+ )
+ resumed = lifecycle()
+ control = AsyncMock()
+ control.get_actor.return_value = actor(ActorState.RUNNING)
+
+ with (
+ patch.object(adapter, "_lock", return_value=asyncio.Lock()),
+ patch.object(
+ adapter,
+ "ensure_workspace_lifecycle",
+ new=AsyncMock(return_value=stored),
+ ),
+ patch.object(
+ adapter,
+ "get_workspace",
+ new=AsyncMock(
+ return_value={
+ "atespace": "workspaces",
+ "actor_name": "actor-1",
+ }
+ ),
+ ),
+ patch.object(
+ adapter, "_reserve_operation", new=AsyncMock(return_value=stored)
+ ),
+ patch.object(
+ adapter, "_record_observation", new=AsyncMock(return_value=resumed)
+ ),
+ ):
+ result = await adapter.resume_workspace("session-1", control=control)
+
+ self.assertEqual(result.observed_state, WorkspaceObservedState.RUNNING)
+ control.get_actor.assert_awaited_once_with("workspaces", "actor-1")
+ control.resume_actor.assert_not_awaited()
+
+ async def test_recorded_delivery_refuses_suspend_before_control_call(self):
+ stored = lifecycle()
+ control = AsyncMock()
+ connection = FakePostgresConnection({"recorded"})
+ with (
+ patch.object(adapter, "_lock", return_value=asyncio.Lock()),
+ patch.object(adapter.db, "connection", new=fake_db_connection(connection)),
+ patch.object(
+ adapter,
+ "ensure_workspace_lifecycle",
+ new=AsyncMock(return_value=stored),
+ ),
+ patch.object(
+ adapter,
+ "get_workspace",
+ new=AsyncMock(
+ return_value={
+ "atespace": "workspaces",
+ "actor_name": "actor-1",
+ }
+ ),
+ ),
+ ):
+ with self.assertRaisesRegex(ContractError, "recorded delivery"):
+ await adapter.suspend_workspace("session-1", control=control)
+
+ self.assertIn("FOR UPDATE", connection.lock_query)
+ self.assertEqual(
+ connection.events,
+ ["begin", "lock", "delivery-check", "lifecycle", "commit"],
+ )
+ control.get_actor.assert_not_awaited()
+ control.suspend_actor.assert_not_awaited()
+
+ async def test_reservation_and_delivery_fence_share_the_locked_transaction(self):
+ connection = FakePostgresConnection()
+ with (
+ patch.object(adapter.db, "connection", new=fake_db_connection(connection)),
+ patch.object(
+ adapter,
+ "get_workspace_lifecycle",
+ new=AsyncMock(return_value=None),
+ ),
+ ):
+ result = await adapter._reserve_operation(
+ lifecycle(), WorkspaceDesiredState.SUSPENDED
+ )
+
+ self.assertEqual(result.desired_state, WorkspaceDesiredState.SUSPENDED)
+ self.assertIn("FOR UPDATE", connection.lock_query)
+ self.assertEqual(
+ connection.events,
+ ["begin", "lock", "delivery-check", "reserve", "lifecycle", "commit"],
+ )
+
+ async def test_delivery_recorded_after_reservation_blocks_suspend_call(self):
+ stored = lifecycle()
+ reserved = lifecycle(
+ desired=WorkspaceDesiredState.SUSPENDED,
+ observed=WorkspaceObservedState.SUSPENDING,
+ operation_id="op-1",
+ )
+ control = AsyncMock()
+ events = []
+ delivery_recorded = False
+
+ async def reserve_operation(_previous, _desired_state):
+ nonlocal delivery_recorded
+ events.append("reserved")
+ delivery_recorded = True
+ return reserved
+
+ async def inspect_actor(*_args):
+ events.append("actor-inspected")
+ return actor(ActorState.RUNNING)
+
+ async def delivery_states(_workspace_id, **_kwargs):
+ events.append("pre-suspend-fence")
+ return {"recorded"} if delivery_recorded else set()
+
+ control.get_actor.side_effect = inspect_actor
+ record_observation = AsyncMock(return_value=stored)
+ with (
+ patch.object(adapter, "_lock", return_value=asyncio.Lock()),
+ patch.object(
+ adapter,
+ "ensure_workspace_lifecycle",
+ new=AsyncMock(return_value=stored),
+ ),
+ patch.object(
+ adapter,
+ "get_workspace",
+ new=AsyncMock(
+ return_value={
+ "atespace": "workspaces",
+ "actor_name": "actor-1",
+ }
+ ),
+ ),
+ patch.object(
+ adapter,
+ "_reserve_operation",
+ new=AsyncMock(side_effect=reserve_operation),
+ ),
+ patch.object(
+ adapter,
+ "_delivery_states",
+ new=AsyncMock(side_effect=delivery_states),
+ ),
+ patch.object(adapter, "_record_observation", new=record_observation),
+ ):
+ with self.assertRaisesRegex(ContractError, "recorded delivery"):
+ await adapter.suspend_workspace("session-1", control=control)
+
+ self.assertEqual(
+ events,
+ ["reserved", "actor-inspected", "pre-suspend-fence"],
+ )
+ self.assertEqual(
+ record_observation.await_args.kwargs["operation_reason"],
+ "DeliveryRecorded",
+ )
+ control.suspend_actor.assert_not_awaited()
+
+ async def test_suspend_timeout_persists_unknown_and_keeps_operation_id(self):
+ stored = lifecycle()
+ reserved = lifecycle(
+ desired=WorkspaceDesiredState.SUSPENDED,
+ observed=WorkspaceObservedState.SUSPENDING,
+ operation_id="op-1",
+ )
+ uncertain = lifecycle(
+ desired=WorkspaceDesiredState.SUSPENDED,
+ observed=WorkspaceObservedState.UNKNOWN,
+ operation_id="op-1",
+ )
+ control = AsyncMock()
+ control.get_actor.return_value = actor(ActorState.RUNNING)
+ control.suspend_actor.side_effect = TransportError("timed out")
+
+ with (
+ patch.object(adapter, "_lock", return_value=asyncio.Lock()),
+ patch.object(
+ adapter,
+ "ensure_workspace_lifecycle",
+ new=AsyncMock(return_value=stored),
+ ),
+ patch.object(
+ adapter,
+ "get_workspace",
+ new=AsyncMock(
+ return_value={
+ "atespace": "workspaces",
+ "actor_name": "actor-1",
+ }
+ ),
+ ),
+ patch.object(
+ adapter, "_delivery_states", new=AsyncMock(return_value=set())
+ ),
+ patch.object(
+ adapter, "_reserve_operation", new=AsyncMock(return_value=reserved)
+ ),
+ patch.object(
+ adapter,
+ "_record_operation_failure",
+ new=AsyncMock(return_value=uncertain),
+ ) as record_failure,
+ ):
+ result = await adapter.suspend_workspace("session-1", control=control)
+
+ self.assertEqual(result.observed_state, WorkspaceObservedState.UNKNOWN)
+ self.assertEqual(result.operation_id, "op-1")
+ record_failure.assert_awaited_once()
+ self.assertTrue(record_failure.await_args.kwargs["uncertain"])
+ control.suspend_actor.assert_awaited_once_with("workspaces", "actor-1")
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docs/specs/sessions.md b/docs/specs/sessions.md
index 09e9550..a4822d4 100644
--- a/docs/specs/sessions.md
+++ b/docs/specs/sessions.md
@@ -16,6 +16,11 @@ When sessions exist:
- Each session shows title and status badge
- Active count shown in header (e.g., "2 active")
- Clicking a session opens its detail view
+- Sessions with a workspace show a separate lifecycle badge; it does not change the session status
+- Workspace details and controls are available from the session detail view
+
+Workspace states are specified in [Workspaces](workspaces.md). A parked workspace can still have
+an active, completed, or waiting session status; these are separate records.
## Status Badges
diff --git a/docs/specs/workspaces.md b/docs/specs/workspaces.md
new file mode 100644
index 0000000..2103331
--- /dev/null
+++ b/docs/specs/workspaces.md
@@ -0,0 +1,55 @@
+# Workspaces
+
+Workspaces are runtime resources attached to sessions. Workspace lifecycle is separate from the
+session's task status, native-agent activity, message delivery, user attention, and publication
+state.
+
+## Lifecycle
+
+Mainloop records desired state (`running` or `suspended`), observed state, conditions, the last
+observed transition, an operation ID, and the last known snapshot reference. The lifecycle is
+owned by Mainloop; Substrate is the source of actor observations.
+
+| Observed state | User label | Meaning |
+| ---------------------------------------- | ---------------------------- | -------------------------------------------------------------------------- |
+| `running` | RUNNING | Substrate reports the actor running. |
+| `suspending` | SUSPENDING | Substrate is suspending the actor. |
+| `suspended` with a snapshot reference | PARKED | Substrate reports suspension and Mainloop has a snapshot reference. |
+| `suspended` without a snapshot reference | SUSPENDED · SNAPSHOT UNKNOWN | Actor suspension is observed, but parking is not confirmed. |
+| `resuming` | RESUMING | Substrate is restoring the actor. |
+| `failed` | FAILED | Substrate reports a crashed/deleting actor, or the bound actor is missing. |
+| `unknown` | UNKNOWN | A transport or unrecognized actor state prevents a reliable conclusion. |
+
+The UI shows workspace state wherever sessions are listed and links from the session detail to
+`/workspaces/{id}`. The workspace page shows the manifest, conditions, transition time and
+snapshot reference, and offers suspend, resume, and status refresh controls. Session badges and
+session status are not changed by workspace operations.
+
+## API
+
+- `GET /workspaces` lists the current user's workspace lifecycle records.
+- `GET /workspaces/{id}` returns one workspace lifecycle and manifest.
+- `POST /workspaces/{id}/suspend` records the desired state and requests suspension.
+- `POST /workspaces/{id}/resume` records the desired state and requests resumption.
+- `POST /workspaces/{id}/refresh` reads Substrate status without changing desired state.
+- Lifecycle changes are published through the existing event stream as `workspace:updated`.
+
+Suspend is refused while the native delivery ledger contains a recorded, queued, sending,
+delivered-but-incomplete, or uncertain delivery. The API reports `409` with the reason. A
+transport timeout is stored as `unknown`; the UI asks the owner to refresh status, and Mainloop
+does not replay the operation without inspecting the actor first. The durable workspace
+generation is advanced with a compare-and-swap before a lifecycle control call.
+
+## Declarative manifest
+
+Each workspace exposes repository URL and branch, allowed agent kinds (`claude` and `codex`),
+skill and MCP references, an egress host allowlist, and a resource class. These values describe
+intent only. This slice stores and displays them; it does not provision repositories, tools,
+network policy, or resources. A missing repository URL is reported as undeclared rather than
+inferred.
+
+## Scope and evidence
+
+These endpoints operate on existing Substrate workspace bindings. They do not provision an
+actor. Runtime behavior is covered by fake-backed tests; this specification does not claim a
+live cluster integration proof.
diff --git a/docs/spikes/substrate-workspace-adapter.md b/docs/spikes/substrate-workspace-adapter.md
index 69bd1e3..73f3c5b 100644
--- a/docs/spikes/substrate-workspace-adapter.md
+++ b/docs/spikes/substrate-workspace-adapter.md
@@ -498,3 +498,23 @@ sources match the pinned Substrate checkout. The Codex file-write route now requ
shim token even on a tokenless golden and validates a JSON object. That source hardening is
fixture-tested, but the retained actor image digest predates the change; rebuild before using
that route in another run.
+
+## Design only: owner-audited operator execution
+
+The shim's `POST /run` endpoint can execute a command and retains bounded output and timeout
+status. It is not exposed to clients in this slice. A future operator path should authorize the
+workspace owner in Mainloop, keep the per-actor shim token server-side, and persist an audit
+record before dispatch with the operator, workspace, command string, working directory,
+timeout, and a result reference. The owner action should be explicitly scoped and bounded; the
+shim's bearer token alone is not Mainloop owner authorization. This is a design proposal only:
+there is no operator-exec API, UI, audit record, or runtime change here.
+
+## Follow-up for the transport lane: serialize delivery with suspend
+
+Suspend reservation takes a `FOR UPDATE` lock on the workspace binding and checks the delivery
+ledger in the same transaction, then checks the ledger again immediately before calling
+Substrate. The delivery-recording path does not take that row lock yet, so a delivery can still
+race after the last check and before Substrate applies suspension, including across backend
+replicas. The transport lane should make delivery recording take the same lock (or an agreed
+workspace advisory lock) and prove the handoff with a concurrency test. Until then, the second
+check narrows this race but does not eliminate it.
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index f8be1e9..ba57597 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -199,6 +199,54 @@ export interface NativeDelivery {
detail: string | null;
}
+export type WorkspaceDesiredState = 'running' | 'suspended';
+export type WorkspaceObservedState =
+ | 'running'
+ | 'suspending'
+ | 'suspended'
+ | 'resuming'
+ | 'failed'
+ | 'unknown';
+
+export interface WorkspaceManifest {
+ repo_url: string | null;
+ branch: string;
+ agent_kinds: ('claude' | 'codex')[];
+ skills: string[];
+ mcp_servers: string[];
+ egress_allowlist: string[];
+ resource_class: string;
+}
+
+export interface WorkspaceCondition {
+ type: string;
+ status: 'True' | 'False' | 'Unknown';
+ reason: string;
+ message: string;
+ last_transition_time: string;
+}
+
+export interface WorkspaceTransition {
+ from_state: WorkspaceObservedState | null;
+ to_state: WorkspaceObservedState;
+ reason: string;
+ occurred_at: string;
+}
+
+export interface WorkspaceLifecycle {
+ workspace_id: string;
+ session_id: string;
+ desired_state: WorkspaceDesiredState;
+ observed_state: WorkspaceObservedState;
+ manifest: WorkspaceManifest;
+ conditions: WorkspaceCondition[];
+ last_transition: WorkspaceTransition | null;
+ operation_id: string | null;
+ snapshot_ref: string | null;
+ ownership_generation: number;
+ updated_at: string;
+}
+
export interface TopicLine {
name: string;
status_line: string;
@@ -411,6 +459,42 @@ export const api = {
},
// Session endpoints
+ async listWorkspaces(): Promise {
+ const response = await apiFetch(`${API_URL}/workspaces`);
+ if (!response.ok) throw new Error('Failed to list workspaces');
+ return response.json();
+ },
+
+ async getWorkspace(workspaceId: string): Promise {
+ const response = await apiFetch(`${API_URL}/workspaces/${workspaceId}`);
+ if (!response.ok) throw new Error(await errorDetail(response, 'Failed to get workspace'));
+ return response.json();
+ },
+
+ async suspendWorkspace(workspaceId: string): Promise {
+ const response = await apiFetch(`${API_URL}/workspaces/${workspaceId}/suspend`, {
+ method: 'POST'
+ });
+ if (!response.ok) throw new Error(await errorDetail(response, 'Failed to suspend workspace'));
+ return response.json();
+ },
+
+ async resumeWorkspace(workspaceId: string): Promise {
+ const response = await apiFetch(`${API_URL}/workspaces/${workspaceId}/resume`, {
+ method: 'POST'
+ });
+ if (!response.ok) throw new Error(await errorDetail(response, 'Failed to resume workspace'));
+ return response.json();
+ },
+
+ async refreshWorkspace(workspaceId: string): Promise {
+ const response = await apiFetch(`${API_URL}/workspaces/${workspaceId}/refresh`, {
+ method: 'POST'
+ });
+ if (!response.ok) throw new Error(await errorDetail(response, 'Failed to refresh workspace'));
+ return response.json();
+ },
+
async listSessions(options?: { status?: string }): Promise {
const params = new URLSearchParams();
if (options?.status) params.set('status', options.status);
diff --git a/frontend/src/lib/components/SessionBlock.svelte b/frontend/src/lib/components/SessionBlock.svelte
index a452c9f..f2130e1 100644
--- a/frontend/src/lib/components/SessionBlock.svelte
+++ b/frontend/src/lib/components/SessionBlock.svelte
@@ -1,6 +1,8 @@
+
+
+ WS {label}
+
diff --git a/frontend/src/lib/sse.ts b/frontend/src/lib/sse.ts
index 5b5147e..faf148c 100644
--- a/frontend/src/lib/sse.ts
+++ b/frontend/src/lib/sse.ts
@@ -13,6 +13,7 @@ export type SSEEventType =
| 'session:updated'
| 'session:needs_input'
| 'session:message'
+ | 'workspace:updated'
| 'heartbeat'
| 'log'
| 'status'
@@ -88,6 +89,7 @@ export class SSEClient {
'session:updated',
'session:needs_input',
'session:message',
+ 'workspace:updated',
'heartbeat',
'log',
'status',
diff --git a/frontend/src/lib/stores/workspaces.ts b/frontend/src/lib/stores/workspaces.ts
new file mode 100644
index 0000000..e4710f3
--- /dev/null
+++ b/frontend/src/lib/stores/workspaces.ts
@@ -0,0 +1,61 @@
+import { writable } from 'svelte/store';
+import { api, type WorkspaceLifecycle } from '$lib/api';
+
+interface WorkspacesState {
+ workspaces: WorkspaceLifecycle[];
+ loading: boolean;
+ error: string | null;
+}
+
+function createWorkspacesStore() {
+ const { subscribe, set, update } = writable({
+ workspaces: [],
+ loading: false,
+ error: null
+ });
+
+ return {
+ subscribe,
+
+ async fetchWorkspaces() {
+ update((state) => ({ ...state, loading: true, error: null }));
+ try {
+ const workspaces = await api.listWorkspaces();
+ update((state) => ({ ...state, workspaces, loading: false }));
+ } catch (error) {
+ update((state) => ({
+ ...state,
+ loading: false,
+ error: error instanceof Error ? error.message : 'Failed to fetch workspaces'
+ }));
+ }
+ },
+
+ upsert(workspace: WorkspaceLifecycle) {
+ update((state) => {
+ const index = state.workspaces.findIndex(
+ (item) => item.workspace_id === workspace.workspace_id
+ );
+ const workspaces = [...state.workspaces];
+ if (index === -1) workspaces.push(workspace);
+ else workspaces[index] = workspace;
+ return { ...state, workspaces };
+ });
+ },
+
+ get(workspaceId: string): WorkspaceLifecycle | undefined {
+ let result: WorkspaceLifecycle | undefined;
+ const unsubscribe = subscribe((state) => {
+ result = state.workspaces.find((item) => item.workspace_id === workspaceId);
+ });
+ unsubscribe();
+ return result;
+ },
+
+ reset() {
+ set({ workspaces: [], loading: false, error: null });
+ }
+ };
+}
+
+export const workspaces = createWorkspacesStore();
diff --git a/frontend/src/routes/+layout.svelte b/frontend/src/routes/+layout.svelte
index 6330c4b..82b0618 100644
--- a/frontend/src/routes/+layout.svelte
+++ b/frontend/src/routes/+layout.svelte
@@ -1,9 +1,11 @@
+
+
+ {workspace ? `Workspace ${workspace.workspace_id}` : 'Workspace'} - mainloop
+
+
+{#if pageError}
+
+
- Starts a real agent in the workspace pod, under Herdr, in bypass-permissions mode. Replies are read from the
+ Starts a real agent in a Substrate workspace, in bypass-permissions mode. Replies are read from the
agent's native journal.
diff --git a/k8s/apps/mainloop/base/deployment-agent-controller.yaml b/k8s/apps/mainloop/base/deployment-agent-controller.yaml
deleted file mode 100644
index b6c46e5..0000000
--- a/k8s/apps/mainloop/base/deployment-agent-controller.yaml
+++ /dev/null
@@ -1,79 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: mainloop-agent-controller
- namespace: mainloop
-spec:
- replicas: 1
- selector:
- matchLabels:
- app: mainloop-agent-controller
- template:
- metadata:
- labels:
- app: mainloop-agent-controller
- spec:
- securityContext:
- runAsNonRoot: true
- runAsUser: 1000
- runAsGroup: 1000
- fsGroup: 1000
- seccompProfile:
- type: RuntimeDefault
- imagePullSecrets:
- - name: ghcr-secret
- containers:
- - name: agent-controller
- image: ghcr.io/oldsj/mainloop-agent-controller:latest
- ports:
- - containerPort: 8001
- securityContext:
- runAsNonRoot: true
- runAsUser: 1000
- runAsGroup: 1000
- allowPrivilegeEscalation: false
- capabilities:
- drop:
- - ALL
- seccompProfile:
- type: RuntimeDefault
- env:
- - name: CLAUDE_CODE_OAUTH_TOKEN
- valueFrom:
- secretKeyRef:
- name: claude-credentials
- key: oauth-token
- - name: GH_TOKEN
- valueFrom:
- secretKeyRef:
- name: mainloop-secrets
- key: github-token
- volumeMounts:
- - name: workspace
- mountPath: /workspace
- resources:
- requests:
- memory: 512Mi
- cpu: 500m
- limits:
- memory: 1Gi
- cpu: 1000m
- livenessProbe:
- httpGet:
- path: /health
- port: 8001
- initialDelaySeconds: 30
- periodSeconds: 10
- timeoutSeconds: 5
- failureThreshold: 3
- readinessProbe:
- httpGet:
- path: /health
- port: 8001
- initialDelaySeconds: 10
- periodSeconds: 5
- timeoutSeconds: 3
- failureThreshold: 3
- volumes:
- - name: workspace
- emptyDir: {}
diff --git a/k8s/apps/mainloop/base/deployment-backend.yaml b/k8s/apps/mainloop/base/deployment-backend.yaml
index 141d65e..0e9f8fd 100644
--- a/k8s/apps/mainloop/base/deployment-backend.yaml
+++ b/k8s/apps/mainloop/base/deployment-backend.yaml
@@ -47,11 +47,6 @@ spec:
secretKeyRef:
name: mainloop-secrets
key: db-password
- - name: CLAUDE_CODE_OAUTH_TOKEN
- valueFrom:
- secretKeyRef:
- name: mainloop-secrets
- key: claude-secret-token
- name: GITHUB_TOKEN
valueFrom:
secretKeyRef:
diff --git a/k8s/apps/mainloop/base/kustomization.yaml b/k8s/apps/mainloop/base/kustomization.yaml
index c73fc1e..403b9a5 100644
--- a/k8s/apps/mainloop/base/kustomization.yaml
+++ b/k8s/apps/mainloop/base/kustomization.yaml
@@ -6,11 +6,8 @@ namespace: mainloop
resources:
- namespace.yaml
- rbac-backend.yaml
- - networkpolicy.yaml
- deployment-backend.yaml
- - deployment-agent-controller.yaml
- deployment-frontend.yaml
- service-backend.yaml
- service-frontend.yaml
- - service-agent-controller.yaml
- configmap.yaml
diff --git a/k8s/apps/mainloop/base/networkpolicy-task-namespace.yaml b/k8s/apps/mainloop/base/networkpolicy-task-namespace.yaml
deleted file mode 100644
index 2ae3613..0000000
--- a/k8s/apps/mainloop/base/networkpolicy-task-namespace.yaml
+++ /dev/null
@@ -1,60 +0,0 @@
-# Network policies for task namespaces (worker agents)
-# These should be applied when creating a new task namespace
-# Workers get minimal network access: DNS + internet only, no cluster internal
----
-# Default deny-all for task namespace
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: default-deny-all
- # namespace: - set dynamically when creating namespace
-spec:
- podSelector: {}
- policyTypes:
- - Ingress
- - Egress
----
-# Allow DNS queries (required for internet access)
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: allow-dns
- # namespace: - set dynamically when creating namespace
-spec:
- podSelector: {}
- policyTypes:
- - Egress
- egress:
- - to:
- - namespaceSelector:
- matchLabels:
- kubernetes.io/metadata.name: kube-system
- ports:
- - protocol: UDP
- port: 53
----
-# Allow egress to internet ONLY (block all cluster internal)
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: allow-internet-only
- # namespace: - set dynamically when creating namespace
-spec:
- podSelector: {}
- policyTypes:
- - Egress
- egress:
- # Allow to public internet only (block RFC1918 private networks)
- - to:
- - ipBlock:
- cidr: 0.0.0.0/0
- except:
- # Block all private/internal networks
- - 10.0.0.0/8 # Private class A
- - 172.16.0.0/12 # Private class B
- - 192.168.0.0/16 # Private class C
- - 169.254.0.0/16 # Link-local
- - 127.0.0.0/8 # Loopback
- - fc00::/7 # IPv6 private
- - fe80::/10 # IPv6 link-local
- - ::1/128 # IPv6 loopback
diff --git a/k8s/apps/mainloop/base/networkpolicy.yaml b/k8s/apps/mainloop/base/networkpolicy.yaml
deleted file mode 100644
index 657813e..0000000
--- a/k8s/apps/mainloop/base/networkpolicy.yaml
+++ /dev/null
@@ -1,32 +0,0 @@
----
-# Restrict agent-controller to internet-only access (no internal cluster access)
-apiVersion: networking.k8s.io/v1
-kind: NetworkPolicy
-metadata:
- name: restrict-agent-controller
- namespace: mainloop
-spec:
- podSelector:
- matchLabels:
- app: mainloop-agent-controller
- policyTypes:
- - Egress
- egress:
- # Allow DNS
- - to:
- - namespaceSelector:
- matchLabels:
- kubernetes.io/metadata.name: kube-system
- ports:
- - protocol: UDP
- port: 53
- - protocol: TCP
- port: 53
- # Allow internet only (block private IP ranges)
- - to:
- - ipBlock:
- cidr: 0.0.0.0/0
- except:
- - 10.0.0.0/8 # Private class A
- - 172.16.0.0/12 # Private class B
- - 192.168.0.0/16 # Private class C
diff --git a/k8s/apps/mainloop/base/rbac-backend.yaml b/k8s/apps/mainloop/base/rbac-backend.yaml
index 36611cc..e7dff99 100644
--- a/k8s/apps/mainloop/base/rbac-backend.yaml
+++ b/k8s/apps/mainloop/base/rbac-backend.yaml
@@ -1,4 +1,3 @@
----
apiVersion: v1
kind: ServiceAccount
metadata:
@@ -8,118 +7,21 @@ metadata:
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
- name: mainloop-backend-cluster-role
+ name: mainloop-backend-secret-reader
rules:
- # Namespace management - create/delete task namespaces
- - apiGroups: ['']
- resources: [namespaces]
- verbs: [create, delete, get, list, watch]
-
- # Secret management - read secrets from mainloop, create in task namespaces
- apiGroups: ['']
resources: [secrets]
- verbs: [create, get, list, delete]
-
- # ServiceAccount management - create worker service accounts in task namespaces
- - apiGroups: ['']
- resources: [serviceaccounts]
- verbs: [create, get, delete]
-
- # RoleBinding management - bind worker role in task namespaces
- - apiGroups: [rbac.authorization.k8s.io]
- resources: [rolebindings]
- verbs: [create, get, delete]
-
- # ClusterRole binding - allow binding the worker-role to service accounts
- # This is required for RBAC escalation prevention - we must explicitly allow binding this role
- - apiGroups: [rbac.authorization.k8s.io]
- resources: [clusterroles]
- resourceNames: [mainloop-worker-role]
- verbs: [bind]
-
- # Job management - create/monitor/delete worker jobs
- - apiGroups: [batch]
- resources: [jobs]
- verbs: [create, get, list, watch, delete]
-
- # Pod management - for job monitoring
- - apiGroups: ['']
- resources: [pods, pods/log]
- verbs: [get, list, watch]
-
- # NetworkPolicy management - isolate task namespaces
- - apiGroups: [networking.k8s.io]
- resources: [networkpolicies]
- verbs: [create, get, delete]
+ verbs: [get]
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
- name: mainloop-backend-cluster-role-binding
+ name: mainloop-backend-secret-reader
subjects:
- kind: ServiceAccount
name: mainloop-backend
namespace: mainloop
roleRef:
kind: ClusterRole
- name: mainloop-backend-cluster-role
+ name: mainloop-backend-secret-reader
apiGroup: rbac.authorization.k8s.io
----
-# ClusterRole for worker jobs - grants permissions needed for Claude Code agents
-# This is bound to the worker ServiceAccount via RoleBinding in each task namespace
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
- name: mainloop-worker-role
-rules:
- # Core resources - pods, services, configmaps, etc.
- - apiGroups: ['']
- resources:
- - pods
- - services
- - configmaps
- - persistentvolumeclaims
- - serviceaccounts
- - endpoints
- verbs: [create, get, list, watch, update, patch, delete]
-
- # Pod logs - read only
- - apiGroups: ['']
- resources:
- - pods/log
- verbs: [get, list, watch]
-
- # Secrets - full access for app credentials
- - apiGroups: ['']
- resources:
- - secrets
- verbs: [create, get, list, watch, update, patch, delete]
-
- # Apps - deployments, replicasets, statefulsets, daemonsets
- - apiGroups: [apps]
- resources:
- - deployments
- - replicasets
- - statefulsets
- - daemonsets
- verbs: [create, get, list, watch, update, patch, delete]
-
- # Batch - jobs, cronjobs
- - apiGroups: [batch]
- resources:
- - jobs
- - cronjobs
- verbs: [create, get, list, watch, update, patch, delete]
-
- # Networking - ingresses, networkpolicies
- - apiGroups: [networking.k8s.io]
- resources:
- - ingresses
- - networkpolicies
- verbs: [create, get, list, watch, update, patch, delete]
-
- # Events - for debugging
- - apiGroups: ['']
- resources:
- - events
- verbs: [get, list, watch]
diff --git a/k8s/apps/mainloop/base/service-agent-controller.yaml b/k8s/apps/mainloop/base/service-agent-controller.yaml
deleted file mode 100644
index 12ad891..0000000
--- a/k8s/apps/mainloop/base/service-agent-controller.yaml
+++ /dev/null
@@ -1,11 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
- name: mainloop-agent-controller
- namespace: mainloop
-spec:
- selector:
- app: mainloop-agent-controller
- ports:
- - port: 8001
- targetPort: 8001
diff --git a/k8s/apps/mainloop/overlays/dev/agent-controller-patch.yaml b/k8s/apps/mainloop/overlays/dev/agent-controller-patch.yaml
deleted file mode 100644
index ef6e867..0000000
--- a/k8s/apps/mainloop/overlays/dev/agent-controller-patch.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-# Patch agent-controller deployment for local testing
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: mainloop-agent-controller
- namespace: mainloop
-spec:
- template:
- spec:
- # Remove GHCR image pull secrets (using local images)
- imagePullSecrets: []
- containers:
- - name: agent-controller
- # Use locally loaded image, never try to pull
- imagePullPolicy: Never
diff --git a/k8s/apps/mainloop/overlays/dev/configmap-patch.yaml b/k8s/apps/mainloop/overlays/dev/configmap-patch.yaml
index 293a858..2a324c1 100644
--- a/k8s/apps/mainloop/overlays/dev/configmap-patch.yaml
+++ b/k8s/apps/mainloop/overlays/dev/configmap-patch.yaml
@@ -10,8 +10,3 @@ data:
DB_PORT: '5432'
DB_NAME: mainloop
FRONTEND_DOMAIN: localhost:3000
- # Backend internal URL for K8s Job callbacks
- BACKEND_INTERNAL_URL: http://mainloop-backend.mainloop.svc.cluster.local:8000
- # Use local worker image for dev (loaded into Kind)
- WORKER_IMAGE: mainloop-agent-controller:dev
- WORKER_IMAGE_PULL_POLICY: Never
diff --git a/k8s/apps/mainloop/overlays/dev/kustomization.yaml b/k8s/apps/mainloop/overlays/dev/kustomization.yaml
index 02315ce..9a5d741 100644
--- a/k8s/apps/mainloop/overlays/dev/kustomization.yaml
+++ b/k8s/apps/mainloop/overlays/dev/kustomization.yaml
@@ -24,11 +24,6 @@ patches:
kind: Deployment
name: mainloop-frontend
- - path: agent-controller-patch.yaml
- target:
- kind: Deployment
- name: mainloop-agent-controller
-
# Override images to use local dev-tagged images (loaded into Kind, no registry)
images:
- name: ghcr.io/oldsj/mainloop-backend
@@ -37,6 +32,3 @@ images:
- name: ghcr.io/oldsj/mainloop-frontend
newName: mainloop-frontend
newTag: dev
- - name: ghcr.io/oldsj/mainloop-agent-controller
- newName: mainloop-agent-controller
- newTag: dev
diff --git a/k8s/apps/mainloop/overlays/prod/kustomization.yaml b/k8s/apps/mainloop/overlays/prod/kustomization.yaml
index e2f4c80..da47838 100644
--- a/k8s/apps/mainloop/overlays/prod/kustomization.yaml
+++ b/k8s/apps/mainloop/overlays/prod/kustomization.yaml
@@ -15,4 +15,3 @@ patches:
target:
kind: Cluster
name: mainloop-db
- - path: shutdown-patch.yaml
diff --git a/k8s/apps/mainloop/overlays/prod/personal-config-patch.yaml.example b/k8s/apps/mainloop/overlays/prod/personal-config-patch.yaml.example
index 81f1158..31edbe0 100644
--- a/k8s/apps/mainloop/overlays/prod/personal-config-patch.yaml.example
+++ b/k8s/apps/mainloop/overlays/prod/personal-config-patch.yaml.example
@@ -52,14 +52,3 @@ spec:
- name: FRONTEND_DOMAIN
value: "mainloop.example.com" # Your frontend domain (for CORS)
---
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: mainloop-agent-controller
- namespace: mainloop
-spec:
- template:
- spec:
- containers:
- - name: agent-controller
- image: ghcr.io/yourusername/mainloop-agent-controller:latest # Your GHCR username
diff --git a/k8s/apps/mainloop/overlays/prod/shutdown-patch.yaml b/k8s/apps/mainloop/overlays/prod/shutdown-patch.yaml
deleted file mode 100644
index ed15000..0000000
--- a/k8s/apps/mainloop/overlays/prod/shutdown-patch.yaml
+++ /dev/null
@@ -1,39 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: mainloop-backend
- namespace: mainloop
-spec:
- replicas: 0
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: mainloop-frontend
- namespace: mainloop
-spec:
- replicas: 0
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: mainloop-agent-controller
- namespace: mainloop
-spec:
- replicas: 0
----
-apiVersion: postgresql.cnpg.io/v1
-kind: Cluster
-metadata:
- name: mainloop-db
- namespace: mainloop
- annotations:
- cnpg.io/hibernation: "on"
----
-apiVersion: postgresql.cnpg.io/v1
-kind: Pooler
-metadata:
- name: mainloop-db-pooler
- namespace: mainloop
-spec:
- instances: 0
diff --git a/k8s/apps/mainloop/overlays/spike-herdr/kustomization.yaml b/k8s/apps/mainloop/overlays/spike-herdr/kustomization.yaml
deleted file mode 100644
index ed49c61..0000000
--- a/k8s/apps/mainloop/overlays/spike-herdr/kustomization.yaml
+++ /dev/null
@@ -1,43 +0,0 @@
-# Local-kind overlay for the native-agent slice: the `test` overlay (backend, frontend, Postgres)
-# without the Claude Agent SDK controller, which this slice does not use. The workspace pod
-# (Herdr + real claude/codex), its exec Role and the credential Secrets live in
-# spikes/k8s-herdr-agents (applied by build-real-agents.sh); Secret values are created by path
-# and never appear in manifests.
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-
-resources:
- - ../test
-
-patches:
- - target:
- kind: Deployment
- name: mainloop-agent-controller
- patch: |-
- - op: replace
- path: /spec/replicas
- value: 0
-
- # Native main thread (plan r7): Claude under Herdr in pod main-0 replaces the SDK chat path.
- # The SDK path stays available behind MAIN_THREAD_MODE=sdk. Model/effort/rotation use defaults
- # (sonnet, medium, 20k tokens above baseline or 12 turns).
- - target:
- kind: Deployment
- name: mainloop-backend
- patch: |-
- - op: add
- path: /spec/template/spec/containers/0/env/-
- value: { name: MAIN_THREAD_MODE, value: native }
-
-# The test overlay remaps ghcr.io/yourusername/*, but the base uses ghcr.io/oldsj/*; remap those
-# to the locally built, kind-loaded images (imagePullPolicy is Never).
-images:
- - name: ghcr.io/oldsj/mainloop-backend
- newName: mainloop-backend
- newTag: test
- - name: ghcr.io/oldsj/mainloop-frontend
- newName: mainloop-frontend
- newTag: test
- - name: ghcr.io/oldsj/mainloop-agent-controller
- newName: mainloop-agent-controller
- newTag: test
diff --git a/k8s/apps/mainloop/overlays/substrate-preview/configmap.yaml b/k8s/apps/mainloop/overlays/substrate-preview/configmap.yaml
index b923b9d..57b703d 100644
--- a/k8s/apps/mainloop/overlays/substrate-preview/configmap.yaml
+++ b/k8s/apps/mainloop/overlays/substrate-preview/configmap.yaml
@@ -10,6 +10,5 @@ data:
DB_PORT: '5432'
DB_NAME: mainloop
FRONTEND_DOMAIN: localhost:3000
- WORKSPACE_RUNTIME: substrate
SUBSTRATE_ROUTER_ADDRESS: http://atenet-router.ate-system.svc.cluster.local:8081
SUBSTRATE_ACTOR_BINDINGS: '{"claude":{"atespace":"live-agent-gate","actor":"headless-claude-reproof","shim_token_secret_name":"mainloop-shim-live-agent-gate-headless-claude-reproof"},"codex":{"atespace":"native-codex","actor":"headless-codex-reproof","shim_token_secret_name":"mainloop-shim-native-codex-headless-codex-reproof"}}'
diff --git a/k8s/apps/mainloop/overlays/test/agent-controller-patch.yaml b/k8s/apps/mainloop/overlays/test/agent-controller-patch.yaml
deleted file mode 100644
index ef6e867..0000000
--- a/k8s/apps/mainloop/overlays/test/agent-controller-patch.yaml
+++ /dev/null
@@ -1,15 +0,0 @@
-# Patch agent-controller deployment for local testing
-apiVersion: apps/v1
-kind: Deployment
-metadata:
- name: mainloop-agent-controller
- namespace: mainloop
-spec:
- template:
- spec:
- # Remove GHCR image pull secrets (using local images)
- imagePullSecrets: []
- containers:
- - name: agent-controller
- # Use locally loaded image, never try to pull
- imagePullPolicy: Never
diff --git a/k8s/apps/mainloop/overlays/test/configmap-patch.yaml b/k8s/apps/mainloop/overlays/test/configmap-patch.yaml
index b016a33..2a324c1 100644
--- a/k8s/apps/mainloop/overlays/test/configmap-patch.yaml
+++ b/k8s/apps/mainloop/overlays/test/configmap-patch.yaml
@@ -10,8 +10,3 @@ data:
DB_PORT: '5432'
DB_NAME: mainloop
FRONTEND_DOMAIN: localhost:3000
- # Backend internal URL for K8s Job callbacks
- BACKEND_INTERNAL_URL: http://mainloop-backend.mainloop.svc.cluster.local:8000
- # Use local worker image for test (loaded into Kind)
- WORKER_IMAGE: mainloop-agent-controller:test
- WORKER_IMAGE_PULL_POLICY: Never
diff --git a/k8s/apps/mainloop/overlays/test/kustomization.yaml b/k8s/apps/mainloop/overlays/test/kustomization.yaml
index c9b93c4..1b13f1b 100644
--- a/k8s/apps/mainloop/overlays/test/kustomization.yaml
+++ b/k8s/apps/mainloop/overlays/test/kustomization.yaml
@@ -27,12 +27,6 @@ patches:
kind: Deployment
name: mainloop-frontend
- # Patch agent-controller deployment for test environment
- - path: agent-controller-patch.yaml
- target:
- kind: Deployment
- name: mainloop-agent-controller
-
# Disable HTTPRoutes that don't exist in test (prevent errors)
# Note: These resources don't exist in base anymore, no patching needed
@@ -44,6 +38,3 @@ images:
- name: ghcr.io/oldsj/mainloop-frontend
newName: mainloop-frontend
newTag: test
- - name: ghcr.io/oldsj/mainloop-agent-controller
- newName: mainloop-agent-controller
- newTag: test
diff --git a/models/README.md b/models/README.md
index 111a4dd..5478a93 100644
--- a/models/README.md
+++ b/models/README.md
@@ -5,15 +5,12 @@ Shared Pydantic models for the mainloop project.
This package contains all shared data models used across:
- Backend API
-- Claude agent integration
- BigQuery schemas
## Models
- `Conversation`: Conversation metadata and state
- `Message`: Individual messages in a conversation
-- `AgentTask`: Claude agent task definitions
-- `AgentResponse`: Claude agent responses
## Usage
diff --git a/models/src/models/__init__.py b/models/src/models/__init__.py
index b86dcbc..000cd85 100644
--- a/models/src/models/__init__.py
+++ b/models/src/models/__init__.py
@@ -1,6 +1,5 @@
"""Shared Pydantic models for mainloop."""
-from models.agent import AgentResponse, AgentTask
from models.conversation import Conversation, Message
from models.native_agent import (
AttentionItem,
@@ -53,8 +52,6 @@
# Existing
"Conversation",
"Message",
- "AgentTask",
- "AgentResponse",
# Session models
"Session",
"SessionCreate",
diff --git a/models/src/models/agent.py b/models/src/models/agent.py
deleted file mode 100644
index 9d86921..0000000
--- a/models/src/models/agent.py
+++ /dev/null
@@ -1,31 +0,0 @@
-"""Claude agent task and response models."""
-
-from datetime import datetime
-from typing import Any
-
-from pydantic import BaseModel, Field
-
-
-class AgentTask(BaseModel):
- """A task for the Claude agent."""
-
- id: str = Field(..., description="Unique task ID")
- conversation_id: str = Field(..., description="Parent conversation ID")
- prompt: str = Field(..., description="Task prompt for Claude")
- context: dict[str, Any] | None = Field(None, description="Additional context")
- created_at: datetime = Field(
- default_factory=datetime.now, description="Creation timestamp"
- )
-
-
-class AgentResponse(BaseModel):
- """Response from the Claude agent."""
-
- task_id: str = Field(..., description="Parent task ID")
- content: str = Field(..., description="Response content")
- tool_uses: list[dict[str, Any]] | None = Field(
- None, description="Tool uses during execution"
- )
- created_at: datetime = Field(
- default_factory=datetime.now, description="Creation timestamp"
- )
diff --git a/models/src/models/native_agent.py b/models/src/models/native_agent.py
index 588839c..a6faacb 100644
--- a/models/src/models/native_agent.py
+++ b/models/src/models/native_agent.py
@@ -63,8 +63,6 @@ class NativeBinding(ContractModel):
provider: Identifier
runtime_type: Identifier
native_session_id: Identifier
- herdr_session_id: Identifier
- herdr_agent_id: Identifier
creation_mode: Literal["created", "attached", "discovered"]
ownership_generation: Generation
observed: ProviderExtension | None = None
diff --git a/models/src/models/session.py b/models/src/models/session.py
index 8c54157..8dd26cf 100644
--- a/models/src/models/session.py
+++ b/models/src/models/session.py
@@ -149,9 +149,9 @@ class SessionCreate(BaseModel):
None, description="Main thread message ID to anchor this session to"
)
- # Optional: run a real native agent under Herdr in the workspace pod
+ # Optional native runtime selection. Omitted sessions use Claude Code by default.
agent_kind: Literal["claude", "codex"] | None = Field(
- None, description="Native agent kind; omit for the existing session worker"
+ None, description="Native agent kind; defaults to Claude Code"
)
@@ -183,7 +183,7 @@ class NativeDeliveryInfo(BaseModel):
class NativeSessionInfo(BaseModel):
- """Identity strip for a session bound to a native agent under Herdr."""
+ """Identity strip for a session bound to a native agent in Substrate."""
session_id: str
kind: Literal["claude", "codex"]
@@ -194,11 +194,7 @@ class NativeSessionInfo(BaseModel):
native_session_id: str | None = None
model: str | None = None
approval_policy: str
- herdr_pane_id: str | None = None
- herdr_terminal_id: str | None = None
- herdr_workspace_id: str | None = None
- workspace_pod: str | None = None
- workspace_pod_uid: str | None = None
+ workspace_name: str | None = None
workspace_ready: bool = False
agent_live: bool | None = None
generation: int = 1
diff --git a/scripts/kind/create-secrets.sh b/scripts/kind/create-secrets.sh
index 95784e4..fb7eca7 100755
--- a/scripts/kind/create-secrets.sh
+++ b/scripts/kind/create-secrets.sh
@@ -25,18 +25,10 @@ set +a
# Create mainloop namespace if not exists
kubectl --context="${KIND_CONTEXT}" create namespace mainloop --dry-run=client -o yaml | kubectl --context="${KIND_CONTEXT}" apply -f -
-# Create claude-credentials secret (for agent-controller)
-echo "Creating claude-credentials..."
-kubectl --context="${KIND_CONTEXT}" create secret generic claude-credentials \
- --namespace mainloop \
- --from-literal=oauth-token="${CLAUDE_CODE_OAUTH_TOKEN-}" \
- --dry-run=client -o yaml | kubectl --context="${KIND_CONTEXT}" apply -f -
-
-# Create mainloop-secrets secret (for backend)
+# Create mainloop-secrets secret (for backend control-plane access)
echo "Creating mainloop-secrets..."
kubectl --context="${KIND_CONTEXT}" create secret generic mainloop-secrets \
--namespace mainloop \
- --from-literal=claude-secret-token="${CLAUDE_CODE_OAUTH_TOKEN-}" \
--from-literal=github-token="${GITHUB_TOKEN-}" \
--from-literal=db-username=mainloop \
--from-literal=db-password=mainloop \
diff --git a/scripts/kind/deploy.sh b/scripts/kind/deploy.sh
index ade4e75..c1da51a 100755
--- a/scripts/kind/deploy.sh
+++ b/scripts/kind/deploy.sh
@@ -12,8 +12,8 @@ echo "Using context: ${KIND_CONTEXT}"
# Delete old deployments and wait for pods to terminate
echo "Cleaning up old deployments..."
-kubectl --context="${KIND_CONTEXT}" delete deployment mainloop-frontend mainloop-backend mainloop-agent-controller -n mainloop --ignore-not-found=true --wait=true
-kubectl --context="${KIND_CONTEXT}" wait --for=delete pod -l 'app in (mainloop-frontend, mainloop-backend, mainloop-agent-controller)' -n mainloop --timeout=60s 2>/dev/null || true
+kubectl --context="${KIND_CONTEXT}" delete deployment mainloop-frontend mainloop-backend -n mainloop --ignore-not-found=true --wait=true
+kubectl --context="${KIND_CONTEXT}" wait --for=delete pod -l 'app in (mainloop-frontend, mainloop-backend)' -n mainloop --timeout=60s 2>/dev/null || true
# Apply test overlay
echo "Applying manifests..."
@@ -23,7 +23,6 @@ kubectl --context="${KIND_CONTEXT}" apply -k "${REPO_ROOT}/k8s/apps/mainloop/ove
echo "Waiting for deployments..."
kubectl --context="${KIND_CONTEXT}" rollout status deployment/mainloop-backend -n mainloop --timeout=120s
kubectl --context="${KIND_CONTEXT}" rollout status deployment/mainloop-frontend -n mainloop --timeout=120s
-kubectl --context="${KIND_CONTEXT}" rollout status deployment/mainloop-agent-controller -n mainloop --timeout=120s
kubectl --context="${KIND_CONTEXT}" rollout status statefulset/postgres -n mainloop --timeout=120s
echo "=== Deployment complete ==="
diff --git a/scripts/kind/load-images.sh b/scripts/kind/load-images.sh
index c4bd45d..a4870e0 100755
--- a/scripts/kind/load-images.sh
+++ b/scripts/kind/load-images.sh
@@ -19,15 +19,10 @@ docker build -f frontend/Dockerfile \
--build-arg VITE_API_URL=http://localhost:8081 \
-t mainloop-frontend:test .
-echo "Building agent-controller..."
-docker build -f claude-agent/Dockerfile \
- -t mainloop-agent-controller:test ./claude-agent
-
# Load images into Kind
echo "Loading images into Kind cluster..."
kind load docker-image mainloop-backend:test --name "${CLUSTER_NAME}"
kind load docker-image mainloop-frontend:test --name "${CLUSTER_NAME}"
-kind load docker-image mainloop-agent-controller:test --name "${CLUSTER_NAME}"
echo "=== Images loaded ==="
docker exec "${CLUSTER_NAME}-control-plane" crictl images | grep mainloop || true
diff --git a/scripts/kind/reset-data.sh b/scripts/kind/reset-data.sh
index fe992c2..6e5c1d4 100755
--- a/scripts/kind/reset-data.sh
+++ b/scripts/kind/reset-data.sh
@@ -1,5 +1,5 @@
#!/usr/bin/env bash
-# Reset database and k8s task namespaces
+# Reset the local development database
set -euo pipefail
CLUSTER_NAME="${KIND_CLUSTER_NAME:-mainloop-test}"
@@ -7,19 +7,6 @@ CONTEXT="kind-${CLUSTER_NAME}"
echo "=== Using context: ${CONTEXT} ==="
-echo "=== Cleaning up k8s task namespaces ==="
-# Delete all task-* namespaces (legacy worker workflows)
-for ns in $(kubectl --context "${CONTEXT}" get ns -o name 2>/dev/null | grep "^namespace/task-" | cut -d/ -f2); do
- echo "Deleting namespace: ${ns}"
- kubectl --context "${CONTEXT}" delete ns "${ns}" --wait=false 2>/dev/null || true
-done
-
-# Delete all mainloop-session-* namespaces (session workers)
-for ns in $(kubectl --context "${CONTEXT}" get ns -o name 2>/dev/null | grep "^namespace/mainloop-session-" | cut -d/ -f2); do
- echo "Deleting namespace: ${ns}"
- kubectl --context "${CONTEXT}" delete ns "${ns}" --wait=false 2>/dev/null || true
-done
-
echo "=== Resetting database ==="
# Drop both public and dbos schemas to fully reset state
kubectl --context "${CONTEXT}" exec -n mainloop postgres-0 -- psql -U mainloop -d mainloop -c "
diff --git a/spikes/k8s-herdr-agents/Dockerfile b/spikes/k8s-herdr-agents/Dockerfile
deleted file mode 100644
index b4c0cda..0000000
--- a/spikes/k8s-herdr-agents/Dockerfile
+++ /dev/null
@@ -1,17 +0,0 @@
-# Spike image: real Herdr + real claude/codex CLIs (+ deterministic stand-in agents). Non-root.
-# herdr, claude and codex binaries are copied from the host into the build context by the
-# build script (never committed). No credentials are baked in: they arrive as Kubernetes Secrets.
-FROM debian:bookworm-slim
-RUN apt-get update && apt-get install -y --no-install-recommends jq ca-certificates git ripgrep curl \
- && rm -rf /var/lib/apt/lists/* \
- && useradd -m -u 10001 agent
-COPY herdr /usr/local/bin/herdr
-COPY claude /usr/local/bin/claude
-COPY codex /usr/local/bin/codex
-COPY codex-code-mode-host /usr/local/bin/codex-code-mode-host
-COPY bin/standin-agent /usr/local/bin/standin-agent
-# One stand-in implementation installed under two Herdr-recognised kind names.
-RUN ln -s standin-agent /usr/local/bin/pi && ln -s standin-agent /usr/local/bin/qwen
-COPY bin/agentctl bin/entrypoint.sh bin/mainloop /usr/local/bin/
-USER 10001:10001
-ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/spikes/k8s-herdr-agents/bin/agentctl b/spikes/k8s-herdr-agents/bin/agentctl
deleted file mode 100755
index 35f944b..0000000
--- a/spikes/k8s-herdr-agents/bin/agentctl
+++ /dev/null
@@ -1,235 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
-# Pod-side operations, same verbs for every agent kind. Kind, args and resume syntax come from
-# /etc/agent-config/.env (ConfigMap), not from this script.
-# agentctl start [--name N] [--new-id ID | --resume ID] start under Herdr
-# context-model options: --cwd-rel D (scratch cwd under the workspace root), --model M,
-# --effort E, --standing-b64 B (standing context file), --token T (per-binding CLI token)
-# agentctl send deliver one prompt (Herdr input only; never reads a reply)
-# agentctl native-id discover the native session id (from the native journal)
-# agentctl journal print native journal lines after line
-# agentctl status Herdr liveness/state hint (JSON)
-# agentctl stop
-# agentctl prompt stand-in only: deliver and grep the reply from the pane
-# agentctl identity
-# Replies for real agents are read from the native journals via `journal`, never from the pane.
-set -eu
-cmd="${1:?usage: agentctl start|send|native-id|journal|status|stop|prompt|identity ...}"
-shift
-H=(herdr --session "${HERDR_SESSION}")
-STATE="${WORKSPACE_PATH}/.mainloop"
-conf_dir="${AGENT_CONFIG_DIR:-/etc/agent-config}"
-
-load_conf() { #
- [[ -f "${conf_dir}/$1.env" ]] || {
- echo "no binding config: ${conf_dir}/$1.env" >&2
- exit 2
- }
- # shellcheck disable=SC1090
- . "${conf_dir}/$1.env"
-}
-
-cwd="${WORKSPACE_PATH}"
-
-trust_cwd() { # : pre-accept the trust dialog for a scratch cwd (no human at the TUI)
- case "$1" in
- claude)
- local tmp
- tmp="$(mktemp)"
- jq --arg p "$2" '.projects[$p] = ((.projects[$p] // {}) + {hasTrustDialogAccepted: true, hasCompletedProjectOnboarding: true})' "${HOME}/.claude.json" >"${tmp}" &&
- cat "${tmp}" >"${HOME}/.claude.json"
- rm -f "${tmp}"
- ;;
- codex)
- grep -qF "[projects.\"$2\"]" "${CODEX_HOME}/config.toml" || printf '\n[projects."%s"]\ntrust_level = "trusted"\n' "$2" >>"${CODEX_HOME}/config.toml"
- ;;
- esac
-}
-
-pane_for_name() { # : one Herdr workspace (labelled with the name) per agent
- local ws
- ws="$("${H[@]}" workspace list | jq -r --arg b "$1" '.result.workspaces[] | select(.label==$b) | .workspace_id' | head -n1)"
- if [[ -z ${ws} ]]; then
- ws="$("${H[@]}" workspace create --label "$1" --cwd "${cwd}" | jq -r .result.workspace.workspace_id)"
- fi
- "${H[@]}" pane list --workspace "${ws}" | jq -r '.result.panes[0].pane_id'
-}
-
-journal_file() { #
- case "$1" in
- claude) find "${CLAUDE_CONFIG_DIR:-${HOME}/.claude}/projects" -name "$2.jsonl" 2>/dev/null | head -n1 ;;
- codex) find "${CODEX_HOME}/sessions" -name "rollout-*-$2.jsonl" 2>/dev/null | head -n1 ;;
- *)
- echo "no journal for kind $1" >&2
- return 1
- ;;
- esac
-}
-
-case "${cmd}" in
-start)
- binding="${1:?binding required}"
- shift
- name="${binding}"
- mode=new
- nid=""
- cwd_rel=""
- model=""
- effort=""
- standing_b64=""
- token=""
- while [[ $# -gt 0 ]]; do
- case "$1" in
- --name)
- name="$2"
- shift 2
- ;;
- --new-id)
- mode=new
- nid="$2"
- shift 2
- ;;
- --resume)
- mode=resume
- nid="$2"
- shift 2
- ;;
- --cwd-rel)
- cwd_rel="$2"
- shift 2
- ;;
- --model)
- model="$2"
- shift 2
- ;;
- --effort)
- effort="$2"
- shift 2
- ;;
- --standing-b64)
- standing_b64="$2"
- shift 2
- ;;
- --token)
- token="$2"
- shift 2
- ;;
- *)
- echo "unknown option $1" >&2
- exit 2
- ;;
- esac
- done
- load_conf "${binding}"
- ident="${STATE}/${name}.identity.json"
- if "${H[@]}" agent get "${name}" >/dev/null 2>&1; then
- echo "agent ${name} already live"
- exit 0
- fi
- if [[ ${mode} == resume ]]; then args="${AGENT_RESUME_ARGS:-${AGENT_ARGS}}"; else args="${AGENT_NEW_ARGS:-${AGENT_ARGS}}"; fi
- if [[ -n ${cwd_rel} ]]; then
- cwd="$(dirname "${WORKSPACE_PATH}")/${cwd_rel}"
- mkdir -p "${cwd}/.mainloop"
- chmod 700 "${cwd}/.mainloop"
- # Secrets and generated context go to files on the PVC (0600), never into the pane command.
- [[ -z ${token} ]] || (
- umask 077
- printf '%s' "${token}" >"${cwd}/.mainloop/token"
- )
- [[ -z ${standing_b64} ]] || printf '%s' "${standing_b64}" | base64 -d >"${cwd}/.mainloop/standing.md"
- [[ ! -f "${conf_dir}/${binding}.settings.json" ]] || cp "${conf_dir}/${binding}.settings.json" "${cwd}/.mainloop/settings.json"
- trust_cwd "${AGENT_KIND}" "${cwd}"
- fi
- args="${args//\{id\}/${nid}}"
- args="${args//\{model\}/${model}}"
- args="${args//\{effort\}/${effort}}"
- args="${args//\{standing\}/${cwd}/.mainloop/standing.md}"
- args="${args//\{settings\}/${cwd}/.mainloop/settings.json}"
- mkdir -p "${STATE}"
- touch "${STATE}/${name}.started"
- pane="$(pane_for_name "${name}")"
- # $args is intentionally word-split: it is the native executable's argument list.
- # shellcheck disable=SC2086
- "${H[@]}" agent start "${name}" --kind "${AGENT_KIND}" --pane "${pane}" --timeout 60000 -- ${args} >/dev/null
- "${H[@]}" agent get "${name}" | jq -c --arg b "${binding}" --arg k "${AGENT_KIND}" --arg args "${args}" --arg mode "${mode}" --arg nid "${nid}" \
- '.result.agent | {binding:$b, kind:$k, args:$args, mode:$mode, native_session_id:(if $nid=="" then null else $nid end), herdr_agent:.agent, herdr_name:.name, pane_id, terminal_id, workspace_id, status:.agent_status}' >"${ident}"
- cat "${ident}"
- ;;
-send)
- name="${1:?name required}"
- text="${2:?text required}"
- # One delivery, no --wait retries: Herdr status is a hint, the journal is the receipt.
- "${H[@]}" agent prompt "${name}" "${text}" >/dev/null
- echo sent
- ;;
-native-id)
- name="${1:?name required}"
- ident="${STATE}/${name}.identity.json"
- kind="$(jq -r .kind "${ident}")"
- known="$(jq -r '.native_session_id // empty' "${ident}")"
- if [[ -n ${known} ]]; then
- echo "${known}"
- exit 0
- fi
- case "${kind}" in
- codex)
- f="$(find "${CODEX_HOME}/sessions" -name 'rollout-*.jsonl' -newer "${STATE}/${name}.started" 2>/dev/null | sort | head -n1)"
- [[ -n ${f} ]] || exit 1
- id="$(basename "${f}" .jsonl | sed -E 's/^rollout-[0-9T:-]+-//')"
- ;;
- *) exit 1 ;;
- esac
- tmp="$(mktemp)"
- jq --arg id "${id}" '.native_session_id=$id' "${ident}" >"${tmp}" && cat "${tmp}" >"${ident}" && rm -f "${tmp}"
- echo "${id}"
- ;;
-journal)
- kind="$(jq -r .kind "${STATE}/${1:?name required}.identity.json")"
- id="${2:?native id required}"
- from="${3:-0}"
- f="$(journal_file "${kind}" "${id}")"
- [[ -n ${f} ]] || {
- echo "#nofile"
- exit 0
- }
- n="$(wc -l <"${f}")" # complete (newline-terminated) lines only
- printf '#file\t%s\t%s\n' "${f}" "${n}"
- if [[ ${n} -gt ${from} ]]; then sed -n "$((from + 1)),${n}p" "${f}" | awk -v s="${from}" '{print (NR + s) "\t" $0}'; fi
- ;;
-status)
- # A failed `agent get` must fail the verb (a pipeline would report jq's exit status).
- out="$("${H[@]}" agent get "${1:?name required}")" || exit 1
- printf '%s' "${out}" | jq -c '.result.agent | {name, agent, pane_id, terminal_id, status: .agent_status}'
- ;;
-prompt) # stand-in agents only
- binding="${1:?binding required}"
- text="${2:?prompt text required}"
- "${H[@]}" agent prompt "${binding}" "${text}" --wait --timeout 60000 >/dev/null
- "${H[@]}" agent read "${binding}" | grep 'STANDIN-REPLY' | grep -F "echo=${text}" | tail -n1
- ;;
-stop)
- name="${1:?name required}"
- kind="$(jq -r '.kind // empty' "${STATE}/${name}.identity.json" 2>/dev/null || true)"
- if [[ ${kind} == claude ]]; then
- # A pasted "/exit" is text, and the restricted main thread has slash commands disabled:
- # two quick Ctrl-C key presses exit Claude Code (measured).
- "${H[@]}" agent send-keys "${name}" ctrl+c ctrl+c >/dev/null
- else
- "${H[@]}" agent prompt "${name}" "/exit" >/dev/null
- fi
- for _ in $(seq 1 20); do
- "${H[@]}" agent get "${name}" >/dev/null 2>&1 || {
- echo "agent ${name} stopped"
- exit 0
- }
- sleep 0.5
- done
- echo "agent ${name} still live after stop" >&2
- exit 1
- ;;
-identity) cat "${STATE}/${1:?name required}.identity.json" ;;
-*)
- echo "unknown command ${cmd}" >&2
- exit 2
- ;;
-esac
diff --git a/spikes/k8s-herdr-agents/bin/entrypoint.sh b/spikes/k8s-herdr-agents/bin/entrypoint.sh
deleted file mode 100755
index a9ab0be..0000000
--- a/spikes/k8s-herdr-agents/bin/entrypoint.sh
+++ /dev/null
@@ -1,35 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
-# Pod entrypoint: real Herdr headless server, with all state on the PVC.
-# Seeds agent trust/onboarding state so agents start without a human at a dialog, and copies
-# the read-only Codex auth Secret to a writable CODEX_HOME on the PVC. Never prints credentials.
-set -eu
-mkdir -p "${HOME}" "${WORKSPACE_PATH}" "${STANDIN_STATE_DIR}" "${CODEX_HOME}" "${HOME}/.claude"
-[[ -d "${WORKSPACE_PATH}/.git" ]] || git -C "${WORKSPACE_PATH}" init -q
-
-# Claude Code: onboarding done, workspace trusted, bypass-permissions warning accepted.
-# CLAUDE_CODE_OAUTH_TOKEN comes from a Secret-backed env var (subscription token).
-if [[ ! -s "${HOME}/.claude.json" ]]; then
- jq -n --arg p "${WORKSPACE_PATH}" '{
- hasCompletedOnboarding: true,
- numStartups: 1,
- theme: "dark",
- projects: {($p): {hasTrustDialogAccepted: true, hasCompletedProjectOnboarding: true, allowedTools: []}}
- }' >"${HOME}/.claude.json"
-fi
-[[ -s "${HOME}/.claude/settings.json" ]] || echo '{"skipDangerousModePermissionPrompt": true}' >"${HOME}/.claude/settings.json"
-
-# Codex: copy auth from the read-only Secret once (Codex rewrites auth.json on refresh, so the
-# writable copy on the PVC is authoritative afterwards); trust the workspace.
-if [[ -f /etc/agent-secrets/codex/auth.json ]] && [[ ! -s "${CODEX_HOME}/auth.json" ]]; then
- install -m 600 /etc/agent-secrets/codex/auth.json "${CODEX_HOME}/auth.json"
-fi
-if [[ ! -s "${CODEX_HOME}/config.toml" ]]; then
- printf '[projects."%s"]\ntrust_level = "trusted"\n' "${WORKSPACE_PATH}" >"${CODEX_HOME}/config.toml"
-fi
-# Codex shows an "Approaching rate limits - switch model?" modal after a turn, which swallows the next
-# prompt. Hide only that nudge (Codex's own "keep current model, never show again"); no model change.
-grep -q '^\[notice\]' "${CODEX_HOME}/config.toml" || printf '\n[notice]\nhide_rate_limit_model_nudge = true\n' >>"${CODEX_HOME}/config.toml"
-
-echo "herdr $(herdr --version) server starting (HOME=${HOME} session=${HERDR_SESSION})"
-exec herdr --session "${HERDR_SESSION}" server
diff --git a/spikes/k8s-herdr-agents/bin/mainloop b/spikes/k8s-herdr-agents/bin/mainloop
deleted file mode 100755
index 03b0ca0..0000000
--- a/spikes/k8s-herdr-agents/bin/mainloop
+++ /dev/null
@@ -1,159 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
-# `mainloop`: the agents' thin client for the Mainloop control plane. It holds no policy.
-# Identity is the per-binding token in .mainloop/token (found by walking up from $PWD, written
-# by `agentctl start`); the server decides what this token may do and answers in plain text.
-set -u
-API="${MAINLOOP_API:-http://mainloop-backend.mainloop.svc.cluster.local:8000}"
-
-find_token() {
- local d="${PWD}"
- while :; do
- [[ -f "${d}/.mainloop/token" ]] && {
- cat "${d}/.mainloop/token"
- return 0
- }
- [[ ${d} == / ]] && return 1
- d="$(dirname "${d}")"
- done
-}
-TOKEN="${MAINLOOP_TOKEN:-$(find_token)}" || {
- echo "mainloop: no agent token found from ${PWD}" >&2
- exit 2
-}
-
-call() { # [json body] ; query params via Q=(--data-urlencode k=v ...)
- local out code body
- out="$(curl -sS -m 30 -w '\n%{http_code}' -X "$1" -H "Authorization: Bearer ${TOKEN}" \
- -H 'Content-Type: application/json' ${Q[@]+"${Q[@]}"} ${3:+-d "$3"} -G "${API}$2" 2>&1)" ||
- {
- echo "mainloop: control plane unreachable" >&2
- exit 3
- }
- code="${out##*$'\n'}"
- body="${out%$'\n'*}"
- if [[ ${code} -ge 200 ]] 2>/dev/null && [[ ${code} -lt 300 ]]; then
- printf '%s' "${body}" | jq -r '.text // .'
- else
- echo "mainloop: refused (${code}): $(printf '%s' "${body}" | jq -r '.detail // .' 2>/dev/null || printf '%s' "${body}")" >&2
- exit 1
- fi
-}
-Q=()
-# POST/GET with a body use -d, which makes curl POST; -G turns -d into a query string, so bodies
-# are sent with --json-style separately:
-post() { #
- local out code body
- out="$(curl -sS -m 30 -w '\n%{http_code}' -X POST -H "Authorization: Bearer ${TOKEN}" \
- -H 'Content-Type: application/json' --data-binary "$2" "${API}$1" 2>&1)" ||
- {
- echo "mainloop: control plane unreachable" >&2
- exit 3
- }
- code="${out##*$'\n'}"
- body="${out%$'\n'*}"
- if [[ ${code} -ge 200 ]] 2>/dev/null && [[ ${code} -lt 300 ]]; then
- printf '%s' "${body}" | jq -r '.text // .'
- else
- echo "mainloop: refused (${code}): $(printf '%s' "${body}" | jq -r '.detail // .' 2>/dev/null || printf '%s' "${body}")" >&2
- exit 1
- fi
-}
-usage() {
- sed -n '2,3p' "$0"
- echo "verbs: whoami topics topic note decide pending delegate status read cancel clear report standing"
-}
-
-verb="${1:-help}"
-[[ $# -gt 0 ]] && shift
-case "${verb}" in
-help | -h | --help) usage ;;
-whoami) call GET /agent-api/whoami ;;
-topics) call GET /agent-api/topics ;;
-standing) call GET /agent-api/standing ;;
-topic)
- [[ ${1-} == open ]] || {
- echo "usage: mainloop topic open [--status ]" >&2
- exit 2
- }
- shift
- name="${1:?topic name required}"
- shift
- status=""
- [[ ${1-} == --status ]] && status="${2-}"
- post /agent-api/topics "$(jq -n --arg n "${name}" --arg s "${status}" 'if $s=="" then {name:$n} else {name:$n,status:$s} end')"
- ;;
-note | decide | pending)
- kind="${verb}"
- [[ ${verb} == decide ]] && kind=decision
- if [[ ${verb} == pending ]] && [[ ${1-} == --done ]]; then
- post "/agent-api/records/${2:?id required}/done" '{}'
- exit
- fi
- text="${1:?text required}"
- shift
- topic=""
- [[ ${1-} == --topic ]] && topic="${2-}"
- post /agent-api/records "$(jq -n --arg k "${kind}" --arg t "${text}" --arg p "${topic}" 'if $p=="" then {kind:$k,text:$t} else {kind:$k,text:$t,topic:$p} end')"
- ;;
-delegate)
- topic=inbox
- kind=""
- title=""
- brief=""
- while [[ $# -gt 0 ]]; do
- case "$1" in
- --topic)
- topic="$2"
- shift 2
- ;;
- --kind)
- kind="$2"
- shift 2
- ;;
- --title)
- title="$2"
- shift 2
- ;;
- *)
- brief="$1"
- shift
- ;;
- esac
- done
- [[ -n ${kind} ]] && [[ -n ${brief} ]] || {
- echo 'usage: mainloop delegate --topic --kind claude|codex --title "" ""' >&2
- exit 2
- }
- post /agent-api/delegate "$(jq -n --arg t "${topic}" --arg k "${kind}" --arg ti "${title}" --arg b "${brief}" '{topic:$t,kind:$k,title:$ti,brief:$b}')"
- ;;
-status)
- [[ -n ${1-} ]] && Q=(--data-urlencode "session=$1")
- call GET /agent-api/status
- ;;
-read)
- id="${1:?session id required}"
- shift
- since=0
- [[ ${1-} == --since ]] && since="${2:-0}"
- Q=(--data-urlencode "session=${id}" --data-urlencode "since=${since}")
- call GET /agent-api/read
- ;;
-cancel)
- post /agent-api/cancel "$(jq -n --arg s "${1:?session id required}" '{session:$s}')"
- ;;
-clear)
- post /agent-api/clear "$(jq -n --arg s "${1-}" 'if $s=="" then {} else {session:$s} end')"
- ;;
-report)
- [[ ${1-} == --summary ]] || {
- echo 'usage: mainloop report --summary ""' >&2
- exit 2
- }
- post /agent-api/report "$(jq -n --arg s "${2:?summary required}" '{summary:$s}')"
- ;;
-*)
- usage >&2
- exit 2
- ;;
-esac
diff --git a/spikes/k8s-herdr-agents/bin/standin-agent b/spikes/k8s-herdr-agents/bin/standin-agent
deleted file mode 100755
index 43cced8..0000000
--- a/spikes/k8s-herdr-agents/bin/standin-agent
+++ /dev/null
@@ -1,63 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
-# Deterministic STAND-IN for a coding-agent CLI. Not a real provider agent.
-# Installed under the executable names Herdr recognises (pi, qwen), so Herdr
-# detects the kind from the process name and its bundled screen rules.
-# Native state (session id + transcript) lives on the workspace volume and is
-# resumed when the same --session-name starts again.
-set -u
-kind="$(basename "$0")"
-session_name="default"
-label=""
-while [[ $# -gt 0 ]]; do
- case "$1" in
- --session-name)
- session_name="$2"
- shift 2
- ;;
- --label)
- label="$2"
- shift 2
- ;;
- *) shift ;;
- esac
-done
-
-state_dir="${STANDIN_STATE_DIR:-${HOME}/.standin}/${kind}"
-mkdir -p "${state_dir}"
-meta="${state_dir}/${session_name}.meta"
-log="${state_dir}/${session_name}.jsonl"
-if [[ -f ${meta} ]]; then
- session_id="$(cat "${meta}")"
- resumed="resumed"
-else
- session_id="${kind}-$(od -An -N4 -tx1 /dev/urandom | tr -d ' \n')"
- printf '%s' "${session_id}" >"${meta}"
- : >"${log}"
- resumed="new"
-fi
-turns="$(wc -l <"${log}")"
-last_prompt="none"
-[[ ${turns} -gt 0 ]] && last_prompt="$(tail -n1 "${log}" | cut -f2)"
-
-echo "STAND-IN agent kind=${kind} label=${label:-none} (deterministic, no provider)"
-echo "native session: ${session_id} (${resumed}, ${turns} prior turns)"
-
-while true; do
- printf '\n> '
- IFS= read -r line || exit 0
- [[ ${line} == "/exit" ]] && exit 0
- printf '\r\033[2K'
- # Working signals differ per kind, mirroring how Herdr detects each agent.
- case "${kind}" in
- qwen) printf '\033]0;\xe2\x97\x90 working\a' ;;
- *) printf '\xe2\xa0\x8b Working... (esc to interrupt)' ;;
- esac
- sleep "${STANDIN_WORK_SECONDS:-2}"
- printf '\r\033[2K'
- [[ ${kind} == qwen ]] && printf '\033]0;\xe2\x97\x87 ready\a'
- turns=$((turns + 1))
- printf '%s\t%s\n' "${turns}" "${line}" >>"${log}"
- echo "STANDIN-REPLY kind=${kind} session=${session_id} turn=${turns} prior=${last_prompt} echo=${line}"
- last_prompt="${line}"
-done
diff --git a/spikes/k8s-herdr-agents/build-real-agents.sh b/spikes/k8s-herdr-agents/build-real-agents.sh
deleted file mode 100755
index ac6c928..0000000
--- a/spikes/k8s-herdr-agents/build-real-agents.sh
+++ /dev/null
@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
-# Build the real-agent workspace image (host claude/codex/herdr copied into a transient build
-# context, never committed), load it into kind-mainloop-test, create credential Secrets BY PATH
-# (values are never printed), and apply the workspace manifest. No cleanup, nothing deleted.
-set -euo pipefail
-SPIKE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-RUN_ID="${RUN_ID:?RUN_ID required}"
-KUBECONFIG_FILE="${KUBECONFIG_FILE:?run-owned kubeconfig required}"
-CTX=kind-mainloop-test
-NS=herdr-spike
-IMAGE="mainloop-spike-herdr:real-${RUN_ID}${IMAGE_SUFFIX-}"
-k() { kubectl --kubeconfig "${KUBECONFIG_FILE}" --context "${CTX}" "$@"; }
-B="$(mktemp -d)"
-trap 'rm -rf "$B"' EXIT
-cp "$(readlink -f "$(command -v herdr)")" "${B}/herdr"
-cp "$(readlink -f "$(command -v claude)")" "${B}/claude"
-cp "$(readlink -f "$(command -v codex)")" "${B}/codex"
-# Codex shell tools need its companion helper (without it: "codex-code-mode-host is missing").
-cp "$(dirname "$(readlink -f "$(command -v codex)")")/codex-code-mode-host" "${B}/codex-code-mode-host"
-cp -r "${SPIKE_DIR}/bin" "${SPIKE_DIR}/Dockerfile" "${B}/"
-sudo -n docker build -q -t "${IMAGE}" "${B}"
-sudo -n docker image inspect "${IMAGE}" --format 'image {{.Id}} user={{.Config.User}}'
-sudo -n "$(command -v kind)" load docker-image "${IMAGE}" --name mainloop-test
-# Secrets by path. Claude token: whitespace stripped through a process substitution, never echoed.
-k create ns "${NS}" --dry-run=client -o yaml | k apply -f - >/dev/null
-k -n "${NS}" create secret generic claude-oauth --from-file=oauth-token=<(tr -d ' \r\n' <"${HOME}/.claude-token") --dry-run=client -o yaml | k apply -f - >/dev/null
-k -n "${NS}" create secret generic codex-auth --from-file=auth.json="${HOME}/.codex/auth.json" --dry-run=client -o yaml | k apply -f - >/dev/null
-sed "s#__IMAGE__#${IMAGE}#" "${SPIKE_DIR}/k8s/workspace.yaml" | k apply -f -
-k -n "${NS}" rollout status statefulset/workspace --timeout=240s
diff --git a/spikes/k8s-herdr-agents/demo.sh b/spikes/k8s-herdr-agents/demo.sh
deleted file mode 100755
index 93d1216..0000000
--- a/spikes/k8s-herdr-agents/demo.sh
+++ /dev/null
@@ -1,122 +0,0 @@
-#!/usr/bin/env bash
-# shellcheck disable=SC2312,SC2310,SC2311,SC2249 # pedantic optional checks; spike scripts
-# Bounded local demo: kind + real Herdr + two stand-in agent kinds.
-# Never cleans up: cluster, PVC, images and evidence are left for inspection.
-set -euo pipefail
-
-SPIKE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-RUN_ID="${RUN_ID:-20260920T022900Z}"
-EVIDENCE="${EVIDENCE_DIR:-${SPIKE_DIR}/../../.tasknotes/runs/${RUN_ID}/spike-evidence}"
-CLUSTER=mainloop-test
-CONTEXT=kind-mainloop-test
-IMAGE="mainloop-spike-herdr:${RUN_ID}"
-KUBECONFIG_FILE="${KUBECONFIG_FILE:-${EVIDENCE}/kubeconfig-${CLUSTER}}" # run-owned, never committed
-NS=herdr-spike
-SUDO="${SUDO:-sudo -n}" # docker is root-only on this host
-KIND_BIN="$(command -v kind)"
-HERDR_BIN="$(readlink -f "$(command -v herdr)")"
-
-mkdir -p "${EVIDENCE}"
-LOG="${EVIDENCE}/demo.log"
-BUILD_CTX="$(mktemp -d)"
-trap 'rm -rf "$BUILD_CTX"' EXIT # only the transient build context is removed
-
-say() { printf '%s\n' "$*" | tee -a "${LOG}"; }
-run() {
- say "\$ $*"
- "$@" 2>&1 | tee -a "${LOG}"
-}
-k() { kubectl --kubeconfig "${KUBECONFIG_FILE}" --context "${CONTEXT}" "$@"; }
-kx() { k -n "${NS}" exec workspace-0 -c workspace -- "$@"; }
-
-say "== Mainloop spike: Kubernetes + Herdr + arbitrary agents (${RUN_ID}) =="
-say "REAL: kind cluster ${CLUSTER}, StatefulSet/PVC, non-root pod, Herdr $(herdr --version | cut -d' ' -f2) server + agent detection"
-say "STAND-IN: 'pi' and 'qwen' executables are one deterministic script (no provider, no credentials)"
-
-# --- static checks ---
-bash -n "${SPIKE_DIR}/demo.sh" "${SPIKE_DIR}"/bin/*
-say "static: bash -n ok"
-
-# --- cluster (explicit run-owned kubeconfig; create only if absent) ---
-if ! ${SUDO} "${KIND_BIN}" get clusters 2>/dev/null | grep -qx "${CLUSTER}"; then
- run ${SUDO} "${KIND_BIN}" create cluster --name "${CLUSTER}" --kubeconfig "${KUBECONFIG_FILE}" --wait 120s
- ${SUDO} chown "$(id -u):$(id -g)" "${KUBECONFIG_FILE}"
-elif [[ ! -s ${KUBECONFIG_FILE} ]]; then
- ${SUDO} "${KIND_BIN}" get kubeconfig --name "${CLUSTER}" >"${KUBECONFIG_FILE}"
-fi
-chmod 600 "${KUBECONFIG_FILE}"
-[[ "$(k config current-context)" == "${CONTEXT}" ]] || {
- say "wrong context"
- exit 1
-}
-say "context: $(k config current-context)"
-
-# --- image ---
-cp "${HERDR_BIN}" "${BUILD_CTX}/herdr"
-cp -r "${SPIKE_DIR}/bin" "${SPIKE_DIR}/Dockerfile" "${BUILD_CTX}/"
-run ${SUDO} docker build -q -t "${IMAGE}" "${BUILD_CTX}"
-IMAGE_ID="$(${SUDO} docker image inspect "${IMAGE}" --format '{{.Id}}')"
-say "image: ${IMAGE} id=${IMAGE_ID}"
-say "image user: $(${SUDO} docker image inspect "${IMAGE}" --format '{{.Config.User}}')"
-say "image herdr: $(${SUDO} docker run --rm --entrypoint herdr "${IMAGE}" --version)"
-run ${SUDO} "${KIND_BIN}" load docker-image "${IMAGE}" --name "${CLUSTER}"
-
-# --- deploy ---
-sed "s#__IMAGE__#${IMAGE}#" "${SPIKE_DIR}/k8s/workspace.yaml" >"${EVIDENCE}/workspace.rendered.yaml"
-run kubectl --kubeconfig "${KUBECONFIG_FILE}" --context "${CONTEXT}" apply -f "${EVIDENCE}/workspace.rendered.yaml"
-k -n "${NS}" rollout status statefulset/workspace --timeout=180s | tee -a "${LOG}"
-POD1_UID="$(k -n "${NS}" get pod workspace-0 -o jsonpath='{.metadata.uid}')"
-PVC="$(k -n "${NS}" get pod workspace-0 -o jsonpath='{.spec.volumes[?(@.name=="workspace")].persistentVolumeClaim.claimName}')"
-say "pod1 uid=${POD1_UID} pvc=${PVC}"
-say "pod user: $(kx id)"
-say "pod herdr: $(kx herdr --version)"
-say "sa token mounted: $(kx sh -c 'ls /var/run/secrets/kubernetes.io 2>&1 | head -1')"
-for _ in $(seq 1 20); do
- kx herdr --session mainloop-spike status server >/dev/null 2>&1 && break
- sleep 1
-done
-
-# --- journey 1: both kinds, same operation ---
-NONCE1="n1-${RANDOM}${RANDOM}"
-for b in alpha beta; do
- say "-- start ${b} (config: $(kx sh -c "tr '\n' ' ' /dev/null 2>&1 && break
- sleep 1
-done
-say "persisted identities: $(kx sh -c 'cat /workspace/repo/.mainloop/*.identity.json')"
-say "persisted native state: $(kx sh -c 'ls /workspace/.standin/*')"
-say "herdr live agents after restart (agent processes do not survive): $(kx herdr --session mainloop-spike agent list | jq -c '.result.agents|length')"
-
-# --- journey 2: restart agents, resume native state, follow-up prompt ---
-NONCE2="n2-${RANDOM}${RANDOM}"
-for b in alpha beta; do
- kx agentctl start "${b}" | tee -a "${LOG}"
- say "-- follow-up prompt ${b} nonce=${NONCE2}-${b}"
- REPLY="$(kx agentctl prompt "${b}" "${NONCE2}-${b}")"
- say "${REPLY}"
- case "${REPLY}" in *"turn=2"*"prior=${NONCE1}-${b}"*) say "OK ${b} resumed native session (turn 2, prior=${NONCE1}-${b})" ;; *)
- say "FAIL ${b} did not resume"
- exit 1
- ;;
- esac
-done
-say "pod2 herdr identities: $(kx herdr --session mainloop-spike agent list | jq -c '[.result.agents[]|{name,agent,pane_id,terminal_id}]')"
-say "== PASS. Left running: cluster ${CLUSTER}, ns ${NS}, PVC ${PVC}, image ${IMAGE}, evidence ${EVIDENCE} =="
-say "inspect: kubectl --kubeconfig ${KUBECONFIG_FILE} --context ${CONTEXT} -n ${NS} exec -it workspace-0 -- herdr --session mainloop-spike"
diff --git a/spikes/k8s-herdr-agents/k8s/workspace.yaml b/spikes/k8s-herdr-agents/k8s/workspace.yaml
deleted file mode 100644
index 06611e9..0000000
--- a/spikes/k8s-herdr-agents/k8s/workspace.yaml
+++ /dev/null
@@ -1,281 +0,0 @@
-# Disposable spike workspace: one non-root pod, one retained PVC, real Herdr server.
-# Agent kind and args are configuration (ConfigMap), not image contents.
-apiVersion: v1
-kind: Namespace
-metadata:
- name: herdr-spike
- labels:
- mainloop.dev/spike: k8s-herdr-agents
----
-apiVersion: v1
-kind: ConfigMap
-metadata:
- name: agent-config
- namespace: herdr-spike
- labels:
- mainloop.dev/spike: k8s-herdr-agents
-data:
- # Two bindings, two Herdr-supported kinds. Change AGENT_KIND/AGENT_ARGS to reconfigure.
- alpha.env: |
- AGENT_KIND=pi
- AGENT_ARGS="--session-name alpha --label reviewer"
- beta.env: |
- AGENT_KIND=qwen
- AGENT_ARGS="--session-name beta --label implementer"
- # Real agents, bypass-permissions mode. {id} is the native session id.
- claude.env: |
- AGENT_KIND=claude
- AGENT_NEW_ARGS="--dangerously-skip-permissions --append-system-prompt-file /etc/agent-config/mainloop-system.md --session-id {id}"
- AGENT_RESUME_ARGS="--dangerously-skip-permissions --append-system-prompt-file /etc/agent-config/mainloop-system.md --resume {id}"
- APPROVAL_POLICY=bypass-permissions
- # Herdr delivers input as a terminal paste, which Claude Code wraps in , and the
- # model may treat that as untrusted data. This states who the author is.
- mainloop-system.md: |
- Messages in this session are relayed by the Mainloop control plane. The user typed each one in
- the Mainloop chat UI. Text that arrives wrapped in pasted-content markers is the user's own
- message: follow it as a direct instruction from the user.
- codex.env: |
- AGENT_KIND=codex
- AGENT_NEW_ARGS="--dangerously-bypass-approvals-and-sandbox"
- AGENT_RESUME_ARGS="resume {id} --dangerously-bypass-approvals-and-sandbox"
- APPROVAL_POLICY=bypass-permissions
- # --- Context model (plan r7) ---------------------------------------------------------------
- # Main thread: a dispatcher. Only Bash restricted to `mainloop ...` (E5); no repo (scratch cwd);
- # not bypass-permissions. Native auto-compaction is left at its default (knob semantics are
- # unverified, E3); no compaction was observed below the rotation budget, and Mainloop rotates first.
- # {model} {effort} {standing} {settings} are filled by `agentctl start`.
- claude-main.env: |
- AGENT_KIND=claude
- AGENT_NEW_ARGS="--model {model} --effort {effort} --tools Bash --strict-mcp-config --disable-slash-commands --settings {settings} --append-system-prompt-file {standing} --session-id {id}"
- AGENT_RESUME_ARGS="--model {model} --effort {effort} --tools Bash --strict-mcp-config --disable-slash-commands --settings {settings} --append-system-prompt-file {standing} --resume {id}"
- APPROVAL_POLICY=restricted
- claude-main.settings.json: |
- {"permissions": {"allow": ["Bash(mainloop:*)"], "defaultMode": "dontAsk"}}
- # Children: real workers in a scratch cwd, bypass-permissions inside the pod (the pod is the boundary).
- # After native compaction the SessionStart(compact) hook re-injects the standing context.
- claude-child.env: |
- AGENT_KIND=claude
- AGENT_NEW_ARGS="--dangerously-skip-permissions --settings {settings} --append-system-prompt-file {standing} --session-id {id}"
- AGENT_RESUME_ARGS="--dangerously-skip-permissions --settings {settings} --append-system-prompt-file {standing} --resume {id}"
- APPROVAL_POLICY=bypass-permissions
- claude-child.settings.json: |
- {"hooks": {"SessionStart": [{"matcher": "compact", "hooks": [{"type": "command", "command": "mainloop standing"}]}]}}
- codex-child.env: |
- AGENT_KIND=codex
- AGENT_NEW_ARGS="--dangerously-bypass-approvals-and-sandbox"
- AGENT_RESUME_ARGS="resume {id} --dangerously-bypass-approvals-and-sandbox"
- APPROVAL_POLICY=bypass-permissions
----
-apiVersion: v1
-kind: Service
-metadata:
- name: workspace
- namespace: herdr-spike
- labels:
- mainloop.dev/spike: k8s-herdr-agents
-spec:
- clusterIP: None
- selector:
- app: workspace
- ports:
- - name: none
- port: 1
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: workspace
- namespace: herdr-spike
- labels:
- mainloop.dev/spike: k8s-herdr-agents
-spec:
- serviceName: workspace
- replicas: 1
- selector:
- matchLabels:
- app: workspace
- persistentVolumeClaimRetentionPolicy:
- whenDeleted: Retain
- whenScaled: Retain
- template:
- metadata:
- labels:
- app: workspace
- mainloop.dev/spike: k8s-herdr-agents
- spec:
- automountServiceAccountToken: false
- securityContext:
- runAsNonRoot: true
- runAsUser: 10001
- runAsGroup: 10001
- fsGroup: 10001
- seccompProfile:
- type: RuntimeDefault
- containers:
- - name: workspace
- image: __IMAGE__
- imagePullPolicy: Never
- env:
- - { name: HOME, value: /workspace/.home }
- - { name: WORKSPACE_PATH, value: /workspace/repo }
- - { name: STANDIN_STATE_DIR, value: /workspace/.standin }
- - { name: HERDR_SESSION, value: mainloop-spike }
- - { name: AGENT_CONFIG_DIR, value: /etc/agent-config }
- - { name: CODEX_HOME, value: /workspace/.codex }
- - name: CLAUDE_CODE_OAUTH_TOKEN
- valueFrom:
- secretKeyRef: { name: claude-oauth, key: oauth-token, optional: true }
- securityContext:
- allowPrivilegeEscalation: false
- readOnlyRootFilesystem: true
- capabilities:
- drop: [ALL]
- resources:
- requests: { cpu: 100m, memory: 256Mi }
- limits: { cpu: '2', memory: 2Gi }
- volumeMounts:
- - { name: workspace, mountPath: /workspace }
- - { name: tmp, mountPath: /tmp }
- - { name: agent-config, mountPath: /etc/agent-config, readOnly: true }
- - { name: codex-auth, mountPath: /etc/agent-secrets/codex, readOnly: true }
- volumes:
- - name: tmp
- emptyDir: {}
- - name: agent-config
- configMap:
- name: agent-config
- - name: codex-auth
- secret: { secretName: codex-auth, optional: true }
- volumeClaimTemplates:
- - metadata:
- name: workspace
- labels:
- mainloop.dev/spike: k8s-herdr-agents
- spec:
- accessModes: [ReadWriteOnce]
- resources:
- requests:
- storage: 1Gi
----
-# Main thread pod (owner decision 2): same image and shape as workspace-0, own PVC, no Codex auth.
-# The conversation agent runs in a scratch cwd (/workspace/main) with no repository.
-apiVersion: v1
-kind: Service
-metadata:
- name: main
- namespace: herdr-spike
- labels:
- mainloop.dev/spike: k8s-herdr-agents
-spec:
- clusterIP: None
- selector:
- app: main
- ports:
- - name: none
- port: 1
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
- name: main
- namespace: herdr-spike
- labels:
- mainloop.dev/spike: k8s-herdr-agents
-spec:
- serviceName: main
- replicas: 1
- selector:
- matchLabels:
- app: main
- persistentVolumeClaimRetentionPolicy:
- whenDeleted: Retain
- whenScaled: Retain
- template:
- metadata:
- labels:
- app: main
- mainloop.dev/spike: k8s-herdr-agents
- spec:
- automountServiceAccountToken: false
- securityContext:
- runAsNonRoot: true
- runAsUser: 10001
- runAsGroup: 10001
- fsGroup: 10001
- seccompProfile:
- type: RuntimeDefault
- containers:
- - name: workspace
- image: __IMAGE__
- imagePullPolicy: Never
- env:
- - { name: HOME, value: /workspace/.home }
- - { name: WORKSPACE_PATH, value: /workspace/repo }
- - { name: STANDIN_STATE_DIR, value: /workspace/.standin }
- - { name: HERDR_SESSION, value: mainloop-main }
- - { name: AGENT_CONFIG_DIR, value: /etc/agent-config }
- - { name: CODEX_HOME, value: /workspace/.codex }
- - name: CLAUDE_CODE_OAUTH_TOKEN
- valueFrom:
- secretKeyRef: { name: claude-oauth, key: oauth-token, optional: true }
- securityContext:
- allowPrivilegeEscalation: false
- readOnlyRootFilesystem: true
- capabilities:
- drop: [ALL]
- resources:
- requests: { cpu: 100m, memory: 256Mi }
- limits: { cpu: '1', memory: 1Gi }
- volumeMounts:
- - { name: workspace, mountPath: /workspace }
- - { name: tmp, mountPath: /tmp }
- - { name: agent-config, mountPath: /etc/agent-config, readOnly: true }
- volumes:
- - name: tmp
- emptyDir: {}
- - name: agent-config
- configMap:
- name: agent-config
- volumeClaimTemplates:
- - metadata:
- name: workspace
- labels:
- mainloop.dev/spike: k8s-herdr-agents
- spec:
- accessModes: [ReadWriteOnce]
- resources:
- requests:
- storage: 1Gi
----
-# Least-privilege exec access for the Mainloop backend (transport for the Herdr adapter):
-# get/list the workspace pod and exec into it, in this namespace only.
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
- name: workspace-exec
- namespace: herdr-spike
- labels:
- mainloop.dev/spike: k8s-herdr-agents
-rules:
- - apiGroups: ['']
- resources: [pods]
- verbs: [get, list]
- - apiGroups: ['']
- resources: [pods/exec]
- verbs: [create, get] # the client's WebSocket exec is a GET upgrade
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
- name: mainloop-backend-workspace-exec
- namespace: herdr-spike
- labels:
- mainloop.dev/spike: k8s-herdr-agents
-subjects:
- - kind: ServiceAccount
- name: mainloop-backend
- namespace: mainloop
-roleRef:
- kind: Role
- name: workspace-exec
- apiGroup: rbac.authorization.k8s.io
From 96716feea501256c04e55956860c67cce676dfed Mon Sep 17 00:00:00 2001
From: James Olds <12104969+oldsj@users.noreply.github.com>
Date: Thu, 24 Sep 2026 03:58:07 +0000
Subject: [PATCH 19/30] feat(workspaces): add per-branch dev lifecycle
---
backend/src/mainloop/api.py | 6 +
backend/src/mainloop/db/postgres.py | 4 +
.../src/mainloop/runtime/actor_provisioner.py | 163 +++++++++++
.../src/mainloop/runtime/native_sessions.py | 21 +-
.../src/mainloop/runtime/workspace_adapter.py | 132 ++++++++-
backend/src/mainloop/runtime/workspace_api.py | 255 +++++++++++++++++-
.../runtime/test_delivery_suspend_fence.py | 3 +
backend/tests/runtime/test_workspace_api.py | 3 +
.../runtime/test_workspace_dev_manifest.py | 144 ++++++++++
backend/tests/runtime/test_workspace_idle.py | 173 ++++++++++++
.../test_workspace_provisioning_api.py | 240 +++++++++++++++++
docs/specs/workspaces.md | 44 +--
examples/devenv-sample/Dockerfile | 14 +
examples/devenv-sample/README.md | 15 ++
examples/devenv-sample/mainloop.yaml | 21 ++
examples/devenv-sample/package.json | 13 +
examples/devenv-sample/server.mjs | 62 +++++
frontend/src/lib/api.ts | 43 +++
.../src/routes/projects/[id]/+page.svelte | 168 ++++++++++--
.../src/routes/workspaces/[id]/+page.svelte | 190 +++++++++----
models/src/models/__init__.py | 6 +
models/src/models/workspace.py | 85 +++++-
22 files changed, 1700 insertions(+), 105 deletions(-)
create mode 100644 backend/src/mainloop/runtime/actor_provisioner.py
create mode 100644 backend/tests/runtime/test_workspace_dev_manifest.py
create mode 100644 backend/tests/runtime/test_workspace_idle.py
create mode 100644 backend/tests/runtime/test_workspace_provisioning_api.py
create mode 100644 examples/devenv-sample/Dockerfile
create mode 100644 examples/devenv-sample/README.md
create mode 100644 examples/devenv-sample/mainloop.yaml
create mode 100644 examples/devenv-sample/package.json
create mode 100644 examples/devenv-sample/server.mjs
diff --git a/backend/src/mainloop/api.py b/backend/src/mainloop/api.py
index accaa95..a30af92 100644
--- a/backend/src/mainloop/api.py
+++ b/backend/src/mainloop/api.py
@@ -125,6 +125,12 @@ async def startup_event():
@app.on_event("shutdown")
async def shutdown_event():
"""Clean up on shutdown."""
+ import asyncio
+
+ task = getattr(app.state, "native_reconcile", None)
+ if task is not None:
+ task.cancel()
+ await asyncio.gather(task, return_exceptions=True)
await db.disconnect()
diff --git a/backend/src/mainloop/db/postgres.py b/backend/src/mainloop/db/postgres.py
index 4aa9137..c6b8f55 100644
--- a/backend/src/mainloop/db/postgres.py
+++ b/backend/src/mainloop/db/postgres.py
@@ -231,6 +231,7 @@ def _parse_json_field(value: Any) -> list | dict | None:
atespace TEXT NOT NULL,
actor_name TEXT NOT NULL,
actor_template TEXT NOT NULL,
+ shim_token_secret_name TEXT,
native_session_id TEXT,
preview_route TEXT,
runtime_endpoint TEXT,
@@ -256,9 +257,12 @@ def _parse_json_field(value: Any) -> list | dict | None:
last_transition JSONB,
operation_id TEXT,
snapshot_ref TEXT,
+ last_activity_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
created_at TIMESTAMPTZ NOT NULL DEFAULT NOW(),
updated_at TIMESTAMPTZ NOT NULL DEFAULT NOW()
);
+ALTER TABLE workspace_bindings ADD COLUMN IF NOT EXISTS shim_token_secret_name TEXT;
+ALTER TABLE workspace_lifecycles ADD COLUMN IF NOT EXISTS last_activity_at TIMESTAMPTZ NOT NULL DEFAULT NOW();
-- Topics are durable records (not sessions). Supervisors (next slice) attach to a topic.
CREATE TABLE IF NOT EXISTS topics (
diff --git a/backend/src/mainloop/runtime/actor_provisioner.py b/backend/src/mainloop/runtime/actor_provisioner.py
new file mode 100644
index 0000000..dbec165
--- /dev/null
+++ b/backend/src/mainloop/runtime/actor_provisioner.py
@@ -0,0 +1,163 @@
+"""Provision one branch workspace actor and its control-plane shim credential."""
+
+from __future__ import annotations
+
+import asyncio
+import base64
+import secrets
+from dataclasses import dataclass
+from typing import Protocol
+
+from kubernetes import client, config
+from kubernetes.client.rest import ApiException
+from mainloop.config import settings
+from mainloop.runtime.substrate import ActorRecord, SubstrateControl
+
+
+@dataclass(frozen=True, slots=True)
+class ProvisionedActor:
+ actor: ActorRecord
+ shim_token_secret_name: str
+
+
+class ActorProvisioner(Protocol):
+ async def create(
+ self,
+ *,
+ atespace: str,
+ actor_name: str,
+ template: str,
+ shim_token_secret_name: str,
+ ) -> ProvisionedActor: ...
+
+ async def delete(
+ self, *, atespace: str, actor_name: str, shim_token_secret_name: str | None
+ ) -> None: ...
+
+
+class SubstrateActorProvisioner:
+ """Uses the existing Substrate control adapter and a namespaced Secret."""
+
+ def __init__(self, control: SubstrateControl | None = None, core_api=None):
+ self.control = control or SubstrateControl()
+ if core_api is not None:
+ self.core_api = core_api
+ else:
+ try:
+ config.load_incluster_config()
+ except config.ConfigException:
+ config.load_kube_config(
+ config_file=settings.substrate_kubeconfig or None,
+ context=settings.substrate_context or None,
+ )
+ self.core_api = client.CoreV1Api()
+
+ async def create(
+ self,
+ *,
+ atespace: str,
+ actor_name: str,
+ template: str,
+ shim_token_secret_name: str,
+ ) -> ProvisionedActor:
+ token = secrets.token_urlsafe(32)
+ try:
+ await asyncio.to_thread(
+ self.core_api.create_namespaced_secret,
+ settings.substrate_shim_secret_namespace,
+ client.V1Secret(
+ metadata=client.V1ObjectMeta(name=shim_token_secret_name),
+ type="Opaque",
+ data={"token": base64.b64encode(token.encode()).decode()},
+ ),
+ )
+ except ApiException as exc:
+ if exc.status != 409:
+ raise RuntimeError(
+ f"workspace shim Secret creation failed (status {exc.status})"
+ ) from exc
+
+ # An existing actor after a timeout belongs to this persisted binding. Inspect first;
+ # never issue a second create for an uncertain outcome.
+ actor = await self.control.get_actor(atespace, actor_name)
+ if actor is None:
+ actor = await self.control.create_actor(
+ atespace, actor_name, template=template
+ )
+ return ProvisionedActor(
+ actor=actor, shim_token_secret_name=shim_token_secret_name
+ )
+
+ async def delete(
+ self, *, atespace: str, actor_name: str, shim_token_secret_name: str | None
+ ) -> None:
+ await self.control.delete_actor(atespace, actor_name, any_state=True)
+ if shim_token_secret_name:
+ try:
+ await asyncio.to_thread(
+ self.core_api.delete_namespaced_secret,
+ shim_token_secret_name,
+ settings.substrate_shim_secret_namespace,
+ )
+ except ApiException as exc:
+ if exc.status != 404:
+ raise RuntimeError(
+ f"workspace shim Secret deletion failed (status {exc.status})"
+ ) from exc
+
+
+class FakeActorProvisioner:
+ """In-memory provisioner for tests; it never accesses Kubernetes or Substrate."""
+
+ def __init__(self):
+ self.actors: dict[tuple[str, str], ActorRecord] = {}
+ self.secrets: set[str] = set()
+
+ async def create(
+ self,
+ *,
+ atespace: str,
+ actor_name: str,
+ template: str,
+ shim_token_secret_name: str,
+ ) -> ProvisionedActor:
+ from mainloop.runtime.substrate import ActorState
+
+ key = (atespace, actor_name)
+ actor = self.actors.get(key) or ActorRecord(
+ atespace=atespace,
+ name=actor_name,
+ uid=f"fake-{actor_name}",
+ state=ActorState.RUNNING,
+ external_snapshot_uri=None,
+ current_actor_template_uid=template,
+ raw={},
+ )
+ self.actors[key] = actor
+ self.secrets.add(shim_token_secret_name)
+ return ProvisionedActor(
+ actor=actor, shim_token_secret_name=shim_token_secret_name
+ )
+
+ async def delete(
+ self, *, atespace: str, actor_name: str, shim_token_secret_name: str | None
+ ) -> None:
+ self.actors.pop((atespace, actor_name), None)
+ if shim_token_secret_name:
+ self.secrets.discard(shim_token_secret_name)
+
+
+_provisioner: ActorProvisioner | None = None
+
+
+def get_actor_provisioner() -> ActorProvisioner:
+ global _provisioner
+ if _provisioner is None:
+ _provisioner = SubstrateActorProvisioner()
+ return _provisioner
+
+
+def set_actor_provisioner(provisioner: ActorProvisioner | None) -> None:
+ """Replace the provisioner in tests or during application assembly."""
+ global _provisioner
+ _provisioner = provisioner
diff --git a/backend/src/mainloop/runtime/native_sessions.py b/backend/src/mainloop/runtime/native_sessions.py
index 9526049..8d19151 100644
--- a/backend/src/mainloop/runtime/native_sessions.py
+++ b/backend/src/mainloop/runtime/native_sessions.py
@@ -27,6 +27,7 @@
from mainloop.config import settings
from mainloop.db import db
+from mainloop.runtime import workspace_adapter
from mainloop.runtime.agent_api import hash_token, token_for
from mainloop.runtime.journal import completed_turns, parse_journal
from mainloop.runtime.standing import content_hash
@@ -238,10 +239,14 @@ async def _record_delivery_message(
async with db.connection() as conn:
async with conn.transaction():
binding = await conn.fetchrow(
- """SELECT workspace_id FROM workspace_bindings
+ """SELECT workspace_id,desired_state FROM workspace_bindings
WHERE workspace_id=$1 FOR UPDATE""",
session_id,
)
+ if binding and binding.get("desired_state") == "deleting":
+ raise ValueError(
+ "The workspace is being deleted; start another workspace."
+ )
lifecycle = (
await conn.fetchrow(
"""SELECT desired_state, observed_state FROM workspace_lifecycles
@@ -272,6 +277,11 @@ async def _record_delivery_message(
state,
source,
)
+ if binding:
+ await conn.execute(
+ "UPDATE workspace_lifecycles SET last_activity_at=NOW(), updated_at=NOW() WHERE workspace_id=$1",
+ session_id,
+ )
return message.id
@@ -283,6 +293,10 @@ async def submit_message(session_id: str, text: str, *, source: str = "user") ->
task brief to a fresh child). A ``queued`` delivery is sent by ``sync`` once the agent is idle.
"""
session = await db.get_session(session_id)
+ # Branch workspace turns touch and wake their actor before the delivery is recorded. Static
+ # agent bindings have no workspace_bindings row and continue through their existing path.
+ if await workspace_adapter.get_workspace(session_id) is not None:
+ await workspace_adapter.touch_workspace(session_id, reason="turn")
if source == "user" and session.status in ENDED_STATUSES:
raise ValueError(f"This session is {session.status.value}; start a new one.")
if source == "user" and session_id in _rotating:
@@ -761,6 +775,7 @@ async def rotate(
async def reconcile_loop(interval: float = 3.0) -> None:
"""Background mirror for sessions with open work, so replies, reports and rotation do not
depend on a browser polling."""
+ next_idle_check = 0.0
while True:
try:
async with db.connection() as conn:
@@ -775,6 +790,10 @@ async def reconcile_loop(interval: float = 3.0) -> None:
for sid in ids:
if sid not in _rotating:
await sync(sid)
+ loop = asyncio.get_running_loop()
+ if loop.time() >= next_idle_check:
+ await workspace_adapter.suspend_idle_workspaces()
+ next_idle_check = loop.time() + 60.0
except Exception:
logger.exception("reconcile loop iteration failed")
await asyncio.sleep(interval)
diff --git a/backend/src/mainloop/runtime/workspace_adapter.py b/backend/src/mainloop/runtime/workspace_adapter.py
index 091190d..9d6ad34 100644
--- a/backend/src/mainloop/runtime/workspace_adapter.py
+++ b/backend/src/mainloop/runtime/workspace_adapter.py
@@ -20,7 +20,7 @@
import json
import logging
import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from mainloop.config import settings
from mainloop.db import db
@@ -277,6 +277,87 @@ async def resume_workspace(
)
+async def touch_workspace(workspace_id: str, *, reason: str) -> WorkspaceLifecycle:
+ """Record turn or preview activity and wake a parked workspace when needed."""
+ if reason not in {"turn", "delivery", "preview"}:
+ raise ValueError("reason must be turn, delivery, or preview")
+ lifecycle = await ensure_workspace_lifecycle(workspace_id)
+ if lifecycle is None:
+ raise ContractError(f"no workspace binding for {workspace_id}")
+ async with db.connection() as conn:
+ async with conn.transaction():
+ binding = await conn.fetchrow(
+ "SELECT workspace_id,desired_state FROM workspace_bindings WHERE workspace_id=$1 FOR UPDATE",
+ workspace_id,
+ )
+ if binding is None:
+ raise ContractError(f"no workspace binding for {workspace_id}")
+ if binding.get("desired_state") == "deleting":
+ raise ContractError("The workspace is being deleted.")
+ current = await conn.fetchrow(
+ """SELECT desired_state, observed_state FROM workspace_lifecycles
+ WHERE workspace_id=$1 FOR UPDATE""",
+ workspace_id,
+ )
+ if current is None:
+ raise ContractError(f"no workspace lifecycle for {workspace_id}")
+ await conn.execute(
+ "UPDATE workspace_lifecycles SET last_activity_at=NOW(), updated_at=NOW() WHERE workspace_id=$1",
+ workspace_id,
+ )
+ should_wake = current["desired_state"] == "suspended" or current[
+ "observed_state"
+ ] in {"suspending", "suspended"}
+ if should_wake:
+ return await resume_workspace(workspace_id)
+ return await get_workspace_lifecycle(workspace_id) or lifecycle
+
+
+async def suspend_idle_workspaces() -> int:
+ """Suspend idle dev workspaces via the normal generation and delivery fence."""
+ async with db.connection() as conn:
+ rows = await conn.fetch(
+ """SELECT l.workspace_id
+ FROM workspace_lifecycles l
+ JOIN sessions s ON s.id=l.workspace_id
+ LEFT JOIN LATERAL (
+ SELECT MAX(created_at) AS last_delivery_at
+ FROM native_deliveries WHERE session_id=l.workspace_id
+ ) d ON TRUE
+ WHERE l.desired_state='running' AND l.observed_state='running'
+ AND l.manifest->'dev' IS NOT NULL
+ AND COALESCE((l.manifest->'dev'->>'idle_timeout_minutes')::integer, 0) > 0
+ AND GREATEST(l.last_activity_at, COALESCE(d.last_delivery_at, l.last_activity_at))
+ < NOW() - ((l.manifest->'dev'->>'idle_timeout_minutes')::integer * INTERVAL '1 minute')"""
+ )
+ suspended = 0
+ for row in rows:
+ try:
+ result = await suspend_workspace_if_idle(row["workspace_id"])
+ if result is not None:
+ suspended += 1
+ except ContractError:
+ # The fenced path records the reason; open deliveries are expected to be skipped.
+ continue
+ except Exception:
+ logger.exception(
+ "Idle workspace suspend failed for %s", row["workspace_id"]
+ )
+ return suspended
+
+
+async def suspend_workspace_if_idle(
+ workspace_id: str, *, control: SubstrateControl | None = None
+) -> WorkspaceLifecycle | None:
+ """Recheck activity under the workspace row lock before reserving a suspend."""
+ return await _request_workspace_state(
+ workspace_id,
+ WorkspaceDesiredState.SUSPENDED,
+ control=control,
+ only_if_idle=True,
+ )
+
+
async def suspend_workspace(
session_id: str, *, control: SubstrateControl | None = None
) -> WorkspaceLifecycle:
@@ -570,6 +651,7 @@ def _lifecycle_from_row(row: dict) -> WorkspaceLifecycle:
),
"operation_id": row.get("operation_id"),
"snapshot_ref": row.get("snapshot_ref"),
+ "last_activity_at": row.get("last_activity_at"),
"ownership_generation": row["ownership_generation"],
"updated_at": row["updated_at"],
}
@@ -593,6 +675,7 @@ def _manifest_from_session(row: dict) -> WorkspaceManifest:
mcp_servers=(),
egress_allowlist=(),
resource_class="default",
+ dev=None,
)
@@ -947,7 +1030,9 @@ async def _record_suspend_fence(
async def _reserve_operation(
previous: WorkspaceLifecycle,
desired_state: WorkspaceDesiredState,
-) -> WorkspaceLifecycle:
+ *,
+ only_if_idle: bool = False,
+) -> WorkspaceLifecycle | None:
at = datetime.now(UTC)
transitional = (
WorkspaceObservedState.SUSPENDING
@@ -1001,16 +1086,45 @@ async def _reserve_operation(
async with db.connection() as conn:
async with conn.transaction():
binding = await conn.fetchrow(
- """SELECT ownership_generation FROM workspace_bindings
+ """SELECT ownership_generation,desired_state FROM workspace_bindings
WHERE workspace_id=$1 FOR UPDATE""",
previous.workspace_id,
)
if binding is None:
raise ContractError(f"no workspace binding for {previous.workspace_id}")
+ if binding.get("desired_state") == "deleting":
+ raise ContractError("The workspace is being deleted.")
if binding["ownership_generation"] != previous.ownership_generation:
raise StaleOwnership(
f"workspace {previous.workspace_id} changed during lifecycle request"
)
+ if only_if_idle:
+ if desired_state != WorkspaceDesiredState.SUSPENDED:
+ raise ValueError("only_if_idle applies to suspension")
+ dev = previous.manifest.dev
+ if dev is None:
+ return None
+ activity = await conn.fetchrow(
+ """SELECT last_activity_at,
+ (SELECT MAX(created_at) FROM native_deliveries
+ WHERE session_id=$1) AS last_delivery_at
+ FROM workspace_lifecycles WHERE workspace_id=$1 FOR UPDATE""",
+ previous.workspace_id,
+ )
+ if activity is None:
+ raise ContractError(
+ f"no workspace lifecycle for {previous.workspace_id}"
+ )
+ last_active = max(
+ value
+ for value in (
+ activity["last_activity_at"],
+ activity["last_delivery_at"],
+ )
+ if value is not None
+ )
+ if last_active > at - timedelta(minutes=dev.idle_timeout_minutes):
+ return None
if desired_state == WorkspaceDesiredState.SUSPENDED:
fence_reason = suspend_fence_reason(
await _delivery_states(previous.workspace_id, conn=conn)
@@ -1047,7 +1161,8 @@ async def _request_workspace_state(
desired_state: WorkspaceDesiredState,
*,
control: SubstrateControl | None = None,
-) -> WorkspaceLifecycle:
+ only_if_idle: bool = False,
+) -> WorkspaceLifecycle | None:
control = control or _control()
async with _lock(workspace_id):
previous = await ensure_workspace_lifecycle(workspace_id)
@@ -1066,7 +1181,14 @@ async def _request_workspace_state(
# can be recorded. First attempts persist intent before the first Substrate call.
had_pending_operation = previous.operation_id is not None
if not had_pending_operation:
- previous = await _reserve_operation(previous, desired_state)
+ reserved = (
+ await _reserve_operation(previous, desired_state, only_if_idle=True)
+ if only_if_idle
+ else await _reserve_operation(previous, desired_state)
+ )
+ if reserved is None:
+ return None
+ previous = reserved
try:
actor = await control.get_actor(row["atespace"], row["actor_name"])
diff --git a/backend/src/mainloop/runtime/workspace_api.py b/backend/src/mainloop/runtime/workspace_api.py
index 9cec084..b9c1af6 100644
--- a/backend/src/mainloop/runtime/workspace_api.py
+++ b/backend/src/mainloop/runtime/workspace_api.py
@@ -1,16 +1,44 @@
-"""Workspace lifecycle endpoints for manifest and actor state."""
+"""Workspace lifecycle endpoints for branch workspace actors."""
-from fastapi import APIRouter, Header, HTTPException
+import json
+import uuid
+from datetime import UTC, datetime
+from typing import Annotated
+
+from fastapi import APIRouter, Header, HTTPException, Response
+from fastapi.responses import JSONResponse
+from mainloop.config import settings
from mainloop.db import db
from mainloop.runtime import workspace_adapter
+from mainloop.runtime.actor_provisioner import get_actor_provisioner
from mainloop.runtime.contracts import ContractError
from mainloop.sse import notify_workspace_updated
+from pydantic import BaseModel, ConfigDict, Field, StrictStr, field_validator
-from models import WorkspaceLifecycle
+from models import (
+ WorkspaceDev,
+ WorkspaceLifecycle,
+ WorkspaceManifest,
+ WorkspaceObservedState,
+)
router = APIRouter(prefix="/workspaces", tags=["workspaces"])
+class CreateWorkspaceRequest(BaseModel):
+ project_id: Annotated[StrictStr, Field(min_length=1)]
+ branch: Annotated[StrictStr, Field(min_length=1)]
+ dev: WorkspaceDev
+
+ model_config = ConfigDict(extra="forbid", strict=True)
+
+ @field_validator("branch")
+ @classmethod
+ def validate_branch_name(cls, value: str) -> str:
+ WorkspaceManifest(branch=value, resource_class="default")
+ return value
+
+
def _user_id(value: str | None) -> str:
return value or "local-dev-user"
@@ -34,6 +62,122 @@ async def _publish(user_id: str, lifecycle: WorkspaceLifecycle) -> None:
await notify_workspace_updated(user_id, lifecycle.model_dump(mode="json"))
+@router.post("", response_model=WorkspaceLifecycle)
+async def create_workspace(
+ request: CreateWorkspaceRequest,
+ user_id: str | None = Header(default=None, alias="X-User-ID"),
+):
+ """Create a branch workspace and its independent Substrate actor."""
+ owner = _user_id(user_id)
+ workspace_id = str(uuid.uuid4())
+ actor_name = f"ml-{workspace_id[:16]}"
+ shim_token_secret_name = f"{actor_name}-shim"
+ atespace = settings.substrate_atespace
+ template = request.dev.actor_template or settings.substrate_actor_template
+
+ async with db.connection() as conn:
+ project = await conn.fetchrow(
+ "SELECT id, html_url FROM projects WHERE id=$1 AND user_id=$2",
+ request.project_id,
+ owner,
+ )
+ if project is None:
+ raise HTTPException(status_code=404, detail="Project not found")
+
+ manifest = WorkspaceManifest(
+ repo_url=project["html_url"],
+ branch=request.branch,
+ resource_class="default",
+ dev=request.dev,
+ )
+ conversation_id = str(uuid.uuid4())
+ now = datetime.now(UTC)
+ async with conn.transaction():
+ thread = await conn.fetchrow(
+ "SELECT id FROM main_threads WHERE user_id=$1 ORDER BY created_at LIMIT 1",
+ owner,
+ )
+ thread_id = thread["id"] if thread else str(uuid.uuid4())
+ if thread is None:
+ await conn.execute(
+ "INSERT INTO main_threads (id,user_id) VALUES ($1,$2)",
+ thread_id,
+ owner,
+ )
+ await conn.execute(
+ "INSERT INTO conversations (id,user_id,title) VALUES ($1,$2,$3)",
+ conversation_id,
+ owner,
+ f"{project['id']} · {request.branch}",
+ )
+ await conn.execute(
+ """INSERT INTO sessions
+ (id,user_id,main_thread_id,title,description,prompt,conversation_id,
+ status,created_at,repo_url,project_id,branch_name,base_branch)
+ VALUES ($1,$2,$3,$4,$5,$6,$7,'active',$8,$9,$10,$11,$12)""",
+ workspace_id,
+ owner,
+ thread_id,
+ f"{project['id']} · {request.branch}",
+ "Branch development workspace",
+ "Development workspace",
+ conversation_id,
+ now,
+ project["html_url"],
+ request.project_id,
+ request.branch,
+ request.branch,
+ )
+ await conn.execute(
+ """INSERT INTO workspace_bindings
+ (workspace_id,atespace,actor_name,actor_template,
+ shim_token_secret_name,observed_state,desired_state,created_at,updated_at)
+ VALUES ($1,$2,$3,$4,$5,'unknown','active',$6,$6)""",
+ workspace_id,
+ atespace,
+ actor_name,
+ template,
+ shim_token_secret_name,
+ now,
+ )
+ await conn.execute(
+ """INSERT INTO workspace_lifecycles
+ (workspace_id,desired_state,observed_state,manifest,conditions,
+ last_activity_at,updated_at)
+ VALUES ($1,'running','unknown',$2::jsonb,'[]'::jsonb,$3,$3)""",
+ workspace_id,
+ json.dumps(manifest.model_dump(mode="json")),
+ now,
+ )
+
+ try:
+ provisioner = get_actor_provisioner()
+ provisioned = await provisioner.create(
+ atespace=atespace,
+ actor_name=actor_name,
+ template=template,
+ shim_token_secret_name=shim_token_secret_name,
+ )
+ lifecycle = await workspace_adapter._record_observation(
+ workspace_id, actor=provisioned.actor
+ )
+ except Exception:
+ # The row and actor identity are durable before the external call. Keep them so a
+ # refresh can reconcile an outcome that timed out instead of creating a second actor.
+ lifecycle = await workspace_adapter._record_observation(
+ workspace_id,
+ failure=(
+ WorkspaceObservedState.UNKNOWN,
+ "ProvisioningUncertain",
+ "Actor provisioning did not return a confirmed result. Refresh status before retrying.",
+ ),
+ )
+ await _publish(owner, lifecycle)
+ return JSONResponse(status_code=202, content=lifecycle.model_dump(mode="json"))
+ await _publish(owner, lifecycle)
+ return lifecycle
+
+
@router.get("", response_model=list[WorkspaceLifecycle])
async def list_workspaces(
user_id: str | None = Header(default=None, alias="X-User-ID"),
@@ -99,3 +243,108 @@ async def refresh_workspace(
return await _run_operation(
workspace_id, _user_id(user_id), workspace_adapter.refresh_workspace_lifecycle
)
+
+
+@router.post("/{workspace_id}/touch", response_model=WorkspaceLifecycle)
+async def touch_workspace(
+ workspace_id: str,
+ reason: str,
+ user_id: str | None = Header(default=None, alias="X-User-ID"),
+):
+ owner = _user_id(user_id)
+ await _require_owned_workspace(workspace_id, owner)
+ try:
+ lifecycle = await workspace_adapter.touch_workspace(workspace_id, reason=reason)
+ except (ContractError, ValueError) as exc:
+ raise HTTPException(status_code=400, detail=str(exc)) from exc
+ await _publish(owner, lifecycle)
+ return lifecycle
+
+
+@router.delete("/{workspace_id}", status_code=204)
+async def delete_workspace(
+ workspace_id: str,
+ user_id: str | None = Header(default=None, alias="X-User-ID"),
+):
+ owner = _user_id(user_id)
+ await _require_owned_workspace(workspace_id, owner)
+ async with workspace_adapter._lock(workspace_id):
+ async with db.connection() as conn:
+ async with conn.transaction():
+ row = await conn.fetchrow(
+ """SELECT b.atespace,b.actor_name,b.shim_token_secret_name,
+ b.desired_state,s.conversation_id
+ FROM workspace_bindings b JOIN sessions s ON s.id=b.workspace_id
+ WHERE b.workspace_id=$1 FOR UPDATE OF b""",
+ workspace_id,
+ )
+ if row is None:
+ raise HTTPException(status_code=404, detail="Workspace not found")
+ open_deliveries = await workspace_adapter._delivery_states(
+ workspace_id, conn=conn
+ )
+ if open_deliveries:
+ raise HTTPException(
+ status_code=409,
+ detail="An open delivery must be reconciled before deleting this workspace.",
+ )
+ await conn.execute(
+ "UPDATE workspace_bindings SET desired_state='deleting',updated_at=NOW() WHERE workspace_id=$1",
+ workspace_id,
+ )
+ try:
+ await get_actor_provisioner().delete(
+ atespace=row["atespace"],
+ actor_name=row["actor_name"],
+ shim_token_secret_name=row["shim_token_secret_name"],
+ )
+ except Exception as exc:
+ async with db.connection() as conn:
+ await conn.execute(
+ """UPDATE workspace_bindings SET desired_state=$2,updated_at=NOW()
+ WHERE workspace_id=$1 AND desired_state='deleting'""",
+ workspace_id,
+ row["desired_state"],
+ )
+ raise HTTPException(
+ status_code=502,
+ detail="Substrate did not confirm workspace deletion; refresh before retrying.",
+ ) from exc
+ async with db.connection() as conn:
+ async with conn.transaction():
+ await conn.execute(
+ "DELETE FROM native_deliveries WHERE session_id=$1", workspace_id
+ )
+ await conn.execute(
+ "DELETE FROM native_events WHERE session_id=$1", workspace_id
+ )
+ await conn.execute(
+ "DELETE FROM native_lineage WHERE session_id=$1", workspace_id
+ )
+ await conn.execute(
+ "DELETE FROM native_bindings WHERE session_id=$1", workspace_id
+ )
+ await conn.execute(
+ "DELETE FROM workspace_lifecycles WHERE workspace_id=$1",
+ workspace_id,
+ )
+ await conn.execute(
+ "DELETE FROM workspace_bindings WHERE workspace_id=$1", workspace_id
+ )
+ await conn.execute("DELETE FROM sessions WHERE id=$1", workspace_id)
+ await conn.execute(
+ """DELETE FROM messages
+ WHERE conversation_id=$1
+ AND NOT EXISTS (
+ SELECT 1 FROM sessions s WHERE s.anchor_message_id=messages.id
+ )""",
+ row["conversation_id"],
+ )
+ await conn.execute(
+ """DELETE FROM conversations c WHERE c.id=$1
+ AND NOT EXISTS (
+ SELECT 1 FROM messages m WHERE m.conversation_id=c.id
+ )""",
+ row["conversation_id"],
+ )
+ return Response(status_code=204)
diff --git a/backend/tests/runtime/test_delivery_suspend_fence.py b/backend/tests/runtime/test_delivery_suspend_fence.py
index 630fd95..11fb8c0 100644
--- a/backend/tests/runtime/test_delivery_suspend_fence.py
+++ b/backend/tests/runtime/test_delivery_suspend_fence.py
@@ -44,6 +44,8 @@ async def fetchrow(self, query, *_args):
async def execute(self, query, *_args):
if "INSERT INTO native_deliveries" in query:
self.events.append("delivery-insert")
+ elif "UPDATE workspace_lifecycles" in query:
+ self.events.append("activity-touch")
def fake_connection(connection):
@@ -93,6 +95,7 @@ async def create_message(**_kwargs):
"lifecycle-read",
"message-insert",
"delivery-insert",
+ "activity-touch",
"commit",
],
)
diff --git a/backend/tests/runtime/test_workspace_api.py b/backend/tests/runtime/test_workspace_api.py
index 1c1d834..38b02ef 100644
--- a/backend/tests/runtime/test_workspace_api.py
+++ b/backend/tests/runtime/test_workspace_api.py
@@ -47,10 +47,13 @@ def test_routes_expose_the_workspace_lifecycle_api(self):
self.assertTrue(
{
("/workspaces", "GET"),
+ ("/workspaces", "POST"),
("/workspaces/{workspace_id}", "GET"),
+ ("/workspaces/{workspace_id}", "DELETE"),
("/workspaces/{workspace_id}/suspend", "POST"),
("/workspaces/{workspace_id}/resume", "POST"),
("/workspaces/{workspace_id}/refresh", "POST"),
+ ("/workspaces/{workspace_id}/touch", "POST"),
}.issubset(route_methods)
)
diff --git a/backend/tests/runtime/test_workspace_dev_manifest.py b/backend/tests/runtime/test_workspace_dev_manifest.py
new file mode 100644
index 0000000..fc56c7f
--- /dev/null
+++ b/backend/tests/runtime/test_workspace_dev_manifest.py
@@ -0,0 +1,144 @@
+"""Strict dev manifest validation and fake actor provisioning."""
+
+import base64
+import unittest
+from unittest.mock import AsyncMock, Mock, patch
+
+from mainloop.runtime.actor_provisioner import (
+ FakeActorProvisioner,
+ SubstrateActorProvisioner,
+)
+from mainloop.runtime.substrate import ActorRecord, ActorState
+from pydantic import ValidationError
+
+from models import WorkspaceManifest
+
+
+def manifest(dev: dict) -> WorkspaceManifest:
+ return WorkspaceManifest(
+ branch="feature/dev-env",
+ resource_class="default",
+ dev=dev,
+ )
+
+
+class WorkspaceDevManifestTests(unittest.TestCase):
+ def test_accepts_image_services_ports_and_timeout(self):
+ result = manifest(
+ {
+ "image": "node:22",
+ "actor_template": "sample",
+ "services": [
+ {
+ "name": "postgres",
+ "image": "postgres:16",
+ "env": {"POSTGRES_DB": "workspace"},
+ "ports": [5432],
+ }
+ ],
+ "ports": [{"name": "app", "number": 3000, "protocol": "http"}],
+ "idle_timeout_minutes": 45,
+ }
+ )
+
+ self.assertEqual(result.dev.image, "node:22")
+ self.assertEqual(result.dev.services[0].ports, (5432,))
+ self.assertEqual(result.dev.ports[0].number, 3000)
+ self.assertEqual(result.dev.idle_timeout_minutes, 45)
+
+ def test_requires_exactly_one_image_source(self):
+ for dev in ({}, {"image": "node:22", "devcontainer_ref": "ghcr.io/dev"}):
+ with self.subTest(dev=dev), self.assertRaises(ValidationError):
+ manifest(dev)
+
+ def test_rejects_unknown_fields_and_duplicate_ports(self):
+ with self.assertRaisesRegex(ValidationError, "extra_forbidden"):
+ manifest({"image": "node:22", "surprise": True})
+ with self.assertRaisesRegex(ValidationError, "unique"):
+ manifest(
+ {
+ "image": "node:22",
+ "ports": [
+ {"name": "app", "number": 3000},
+ {"name": "web", "number": 3000},
+ ],
+ }
+ )
+
+ def test_rejects_out_of_range_timeout_and_duplicate_service_names(self):
+ with self.assertRaises(ValidationError):
+ manifest({"image": "node:22", "idle_timeout_minutes": 0})
+ with self.assertRaisesRegex(ValidationError, "unique"):
+ manifest(
+ {
+ "image": "node:22",
+ "services": [
+ {"name": "db", "image": "postgres:16"},
+ {"name": "db", "image": "redis:7"},
+ ],
+ }
+ )
+
+ def test_rejects_non_strict_timeout(self):
+ with self.assertRaises(ValidationError):
+ manifest({"image": "node:22", "idle_timeout_minutes": "30"})
+
+
+class FakeProvisionerTests(unittest.IsolatedAsyncioTestCase):
+ async def test_fake_creates_and_deletes_actor_and_secret(self):
+ provisioner = FakeActorProvisioner()
+ # Test value is a Kubernetes Secret name, not credential material.
+ created = await provisioner.create( # nosec B106
+ atespace="workspaces",
+ actor_name="ml-feature-1",
+ template="sample-template",
+ shim_token_secret_name="ml-feature-1-shim",
+ )
+
+ self.assertEqual(created.actor.name, "ml-feature-1")
+ self.assertIn("ml-feature-1-shim", provisioner.secrets)
+ await provisioner.delete(
+ atespace="workspaces",
+ actor_name="ml-feature-1",
+ shim_token_secret_name=created.shim_token_secret_name,
+ )
+ self.assertFalse(provisioner.actors)
+ self.assertFalse(provisioner.secrets)
+
+ async def test_substrate_provisioner_stores_a_random_token_in_secret(self):
+ actor = ActorRecord(
+ atespace="workspaces",
+ name="ml-branch-1",
+ uid="actor-1",
+ state=ActorState.RUNNING,
+ external_snapshot_uri=None,
+ current_actor_template_uid="template-1",
+ raw={},
+ )
+ control = Mock()
+ control.get_actor = AsyncMock(return_value=actor)
+ control.create_actor = AsyncMock()
+ core_api = Mock()
+ provisioner = SubstrateActorProvisioner(control=control, core_api=core_api)
+
+ with patch(
+ "mainloop.runtime.actor_provisioner.secrets.token_urlsafe",
+ return_value="private-token",
+ ):
+ # Test value is a Kubernetes Secret name, not credential material.
+ result = await provisioner.create( # nosec B106
+ atespace="workspaces",
+ actor_name="ml-branch-1",
+ template="project-template",
+ shim_token_secret_name="ml-branch-1-shim",
+ )
+
+ secret = core_api.create_namespaced_secret.call_args.args[1]
+ self.assertEqual(base64.b64decode(secret.data["token"]), b"private-token")
+ self.assertEqual(result.actor, actor)
+ self.assertEqual(result.shim_token_secret_name, "ml-branch-1-shim")
+ control.create_actor.assert_not_awaited()
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/backend/tests/runtime/test_workspace_idle.py b/backend/tests/runtime/test_workspace_idle.py
new file mode 100644
index 0000000..7a21163
--- /dev/null
+++ b/backend/tests/runtime/test_workspace_idle.py
@@ -0,0 +1,173 @@
+"""Durable activity touches and the periodic idle policy remain fake-backed."""
+
+import asyncio
+import unittest
+from contextlib import asynccontextmanager
+from types import SimpleNamespace
+from unittest.mock import AsyncMock, patch
+
+from mainloop.runtime import native_sessions
+from mainloop.runtime import workspace_adapter as adapter
+from mainloop.runtime.contracts import ContractError
+
+from models import (
+ WorkspaceDesiredState,
+ WorkspaceLifecycle,
+ WorkspaceManifest,
+ WorkspaceObservedState,
+)
+
+
+def lifecycle(state=WorkspaceObservedState.SUSPENDED):
+ from datetime import UTC, datetime
+
+ return WorkspaceLifecycle(
+ workspace_id="workspace-1",
+ session_id="workspace-1",
+ desired_state=WorkspaceDesiredState.SUSPENDED,
+ observed_state=state,
+ manifest=WorkspaceManifest(
+ branch="feature/sample",
+ resource_class="default",
+ dev={"image": "node:22"},
+ ),
+ updated_at=datetime.now(UTC),
+ )
+
+
+class FakeConnection:
+ def __init__(self, rows=()):
+ self.rows = list(rows)
+ self.executed = []
+ self.query = ""
+
+ async def execute(self, query, *args):
+ self.executed.append((query, args))
+
+ @asynccontextmanager
+ async def transaction(self):
+ yield self
+
+ async def fetchrow(self, query, *_args):
+ if "FROM workspace_bindings" in query:
+ return {"workspace_id": "workspace-1", "ownership_generation": 3}
+ if "FROM workspace_lifecycles" in query and "last_activity_at" in query:
+ from datetime import UTC, datetime
+
+ return {
+ "last_activity_at": datetime.now(UTC),
+ "last_delivery_at": None,
+ }
+ if "FROM workspace_lifecycles" in query:
+ return {"desired_state": "suspended", "observed_state": "suspended"}
+ raise AssertionError(f"unexpected query: {query}")
+
+ async def fetch(self, query, *args):
+ self.query = query
+ return self.rows
+
+
+def fake_connection(connection):
+ @asynccontextmanager
+ async def connect():
+ yield connection
+
+ return connect
+
+
+class WorkspaceIdleTests(unittest.IsolatedAsyncioTestCase):
+ async def test_native_turn_touches_a_branch_workspace_before_recording(self):
+ with (
+ patch.object(
+ native_sessions.db,
+ "get_session",
+ new=AsyncMock(
+ return_value=SimpleNamespace(status="active", conversation_id="c")
+ ),
+ ),
+ patch.object(
+ adapter,
+ "get_workspace",
+ new=AsyncMock(return_value={"actor_name": "ml-ws"}),
+ ),
+ patch.object(adapter, "touch_workspace", new=AsyncMock()) as touch,
+ patch.object(native_sessions, "_lock", return_value=asyncio.Lock()),
+ patch.object(native_sessions, "_open_count", new=AsyncMock(return_value=1)),
+ patch.object(
+ native_sessions,
+ "_record_delivery_message",
+ new=AsyncMock(return_value="message-1"),
+ ),
+ ):
+ message_id = await native_sessions.submit_message(
+ "workspace-1", "work", source="report"
+ )
+
+ self.assertEqual(message_id, "message-1")
+ touch.assert_awaited_once_with("workspace-1", reason="turn")
+
+ async def test_turn_touch_records_activity_and_wakes_a_parked_workspace(self):
+ connection = FakeConnection()
+ current = lifecycle()
+ resumed = lifecycle(WorkspaceObservedState.RUNNING)
+ with (
+ patch.object(
+ adapter,
+ "ensure_workspace_lifecycle",
+ new=AsyncMock(return_value=current),
+ ),
+ patch.object(adapter.db, "connection", new=fake_connection(connection)),
+ patch.object(
+ adapter, "resume_workspace", new=AsyncMock(return_value=resumed)
+ ) as wake,
+ ):
+ result = await adapter.touch_workspace("workspace-1", reason="turn")
+
+ self.assertEqual(result.observed_state, WorkspaceObservedState.RUNNING)
+ self.assertIn("last_activity_at=NOW()", connection.executed[0][0])
+ wake.assert_awaited_once_with("workspace-1")
+
+ async def test_idle_reservation_rechecks_activity_under_the_binding_lock(self):
+ current = lifecycle(WorkspaceObservedState.RUNNING)
+ current = current.model_copy(
+ update={
+ "desired_state": WorkspaceDesiredState.RUNNING,
+ "ownership_generation": 3,
+ }
+ )
+ connection = FakeConnection()
+ with patch.object(adapter.db, "connection", new=fake_connection(connection)):
+ result = await adapter._reserve_operation(
+ current,
+ WorkspaceDesiredState.SUSPENDED,
+ only_if_idle=True,
+ )
+
+ self.assertIsNone(result)
+ self.assertFalse(connection.executed)
+
+ async def test_touch_rejects_unknown_activity_reasons(self):
+ with self.assertRaisesRegex(ValueError, "reason must be"):
+ await adapter.touch_workspace("workspace-1", reason="browser")
+
+ async def test_idle_scan_uses_durable_activity_and_fenced_suspend(self):
+ connection = FakeConnection(
+ [{"workspace_id": "workspace-1"}, {"workspace_id": "workspace-2"}]
+ )
+ suspend = AsyncMock(
+ side_effect=[lifecycle(), ContractError("delivery still open")]
+ )
+ with (
+ patch.object(adapter.db, "connection", new=fake_connection(connection)),
+ patch.object(adapter, "suspend_workspace_if_idle", new=suspend),
+ ):
+ count = await adapter.suspend_idle_workspaces()
+
+ self.assertIn("last_activity_at", connection.query)
+ self.assertIn("idle_timeout_minutes", connection.query)
+ self.assertEqual(count, 1)
+ self.assertEqual(suspend.await_count, 2)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/backend/tests/runtime/test_workspace_provisioning_api.py b/backend/tests/runtime/test_workspace_provisioning_api.py
new file mode 100644
index 0000000..0625b09
--- /dev/null
+++ b/backend/tests/runtime/test_workspace_provisioning_api.py
@@ -0,0 +1,240 @@
+"""Project ownership and actor provisioning orchestration use fakes."""
+
+import unittest
+from contextlib import asynccontextmanager
+from datetime import UTC, datetime
+from unittest.mock import AsyncMock, patch
+
+from fastapi import HTTPException
+from mainloop.runtime import workspace_api
+from mainloop.runtime.actor_provisioner import FakeActorProvisioner, ProvisionedActor
+from mainloop.runtime.substrate import ActorRecord, ActorState
+
+from models import (
+ WorkspaceDesiredState,
+ WorkspaceLifecycle,
+ WorkspaceManifest,
+ WorkspaceObservedState,
+)
+
+
+class FakeConnection:
+ def __init__(self, *, project=True):
+ self.project = project
+ self.statements = []
+ self.workspace_row = {
+ "atespace": "mainloop-workspaces",
+ "actor_name": "ml-workspace",
+ "shim_token_secret_name": "ml-workspace-shim",
+ "desired_state": "active",
+ "conversation_id": "conversation-1",
+ }
+
+ @asynccontextmanager
+ async def transaction(self):
+ yield
+
+ async def fetchrow(self, query, *_args):
+ if "FROM projects" in query:
+ return (
+ {"id": "project-1", "html_url": "https://github.com/example/repo"}
+ if self.project
+ else None
+ )
+ if "FROM main_threads" in query:
+ return None
+ if "FROM workspace_bindings" in query:
+ return self.workspace_row
+ raise AssertionError(f"unexpected query: {query}")
+
+ async def execute(self, query, *args):
+ self.statements.append((query, args))
+
+
+def fake_connection(connection):
+ @asynccontextmanager
+ async def connect():
+ yield connection
+
+ return connect
+
+
+def lifecycle(workspace_id: str, manifest: WorkspaceManifest) -> WorkspaceLifecycle:
+ return WorkspaceLifecycle(
+ workspace_id=workspace_id,
+ session_id=workspace_id,
+ desired_state=WorkspaceDesiredState.RUNNING,
+ observed_state=WorkspaceObservedState.RUNNING,
+ manifest=manifest,
+ updated_at=datetime.now(UTC),
+ )
+
+
+class WorkspaceProvisioningApiTests(unittest.IsolatedAsyncioTestCase):
+ async def test_create_persists_identity_and_provisions_one_actor(self):
+ connection = FakeConnection()
+ provisioner = FakeActorProvisioner()
+ manifest = WorkspaceManifest(
+ repo_url="https://github.com/example/repo",
+ branch="feature/one",
+ resource_class="default",
+ dev={"image": "node:22", "actor_template": "project-template"},
+ )
+ actor = ActorRecord(
+ atespace="mainloop-workspaces",
+ name="ml-workspace",
+ uid="actor-1",
+ state=ActorState.RUNNING,
+ external_snapshot_uri=None,
+ current_actor_template_uid="template-1",
+ raw={},
+ )
+ fake_create = AsyncMock(
+ return_value=ProvisionedActor(actor, "ml-workspace-shim")
+ )
+ provisioner.create = fake_create
+ with (
+ patch.object(
+ workspace_api.db, "connection", new=fake_connection(connection)
+ ),
+ patch.object(
+ workspace_api, "get_actor_provisioner", return_value=provisioner
+ ),
+ patch.object(
+ workspace_api.workspace_adapter,
+ "_record_observation",
+ new=AsyncMock(
+ side_effect=lambda workspace_id, **_kwargs: lifecycle(
+ workspace_id, manifest
+ )
+ ),
+ ),
+ patch.object(workspace_api, "_publish", new=AsyncMock()),
+ ):
+ result = await workspace_api.create_workspace(
+ workspace_api.CreateWorkspaceRequest(
+ project_id="project-1",
+ branch="feature/one",
+ dev={"image": "node:22", "actor_template": "project-template"},
+ ),
+ user_id="owner-1",
+ )
+
+ self.assertEqual(result.manifest.branch, "feature/one")
+ self.assertEqual(result.observed_state, WorkspaceObservedState.RUNNING)
+ self.assertTrue(
+ any(
+ "INSERT INTO workspace_bindings" in query
+ for query, _ in connection.statements
+ )
+ )
+ self.assertTrue(
+ any(
+ "INSERT INTO workspace_lifecycles" in query
+ for query, _ in connection.statements
+ )
+ )
+ kwargs = fake_create.await_args.kwargs
+ self.assertEqual(kwargs["template"], "project-template")
+ self.assertEqual(
+ kwargs["shim_token_secret_name"], f"{kwargs['actor_name']}-shim"
+ )
+
+ async def test_create_hides_projects_owned_by_another_user(self):
+ connection = FakeConnection(project=False)
+ with patch.object(
+ workspace_api.db, "connection", new=fake_connection(connection)
+ ):
+ with self.assertRaises(HTTPException) as raised:
+ await workspace_api.create_workspace(
+ workspace_api.CreateWorkspaceRequest(
+ project_id="project-1", branch="main", dev={"image": "node:22"}
+ ),
+ user_id="other-owner",
+ )
+
+ self.assertEqual(raised.exception.status_code, 404)
+ self.assertFalse(connection.statements)
+
+ async def test_delete_removes_actor_secret_and_workspace_records(self):
+ connection = FakeConnection()
+ provisioner = FakeActorProvisioner()
+ provisioner.actors[("mainloop-workspaces", "ml-workspace")] = ActorRecord(
+ atespace="mainloop-workspaces",
+ name="ml-workspace",
+ uid="actor-1",
+ state=ActorState.RUNNING,
+ external_snapshot_uri=None,
+ current_actor_template_uid="template-1",
+ raw={},
+ )
+ provisioner.secrets.add("ml-workspace-shim")
+ with (
+ patch.object(workspace_api, "_require_owned_workspace", new=AsyncMock()),
+ patch.object(
+ workspace_api.db, "connection", new=fake_connection(connection)
+ ),
+ patch.object(
+ workspace_api.workspace_adapter,
+ "_delivery_states",
+ new=AsyncMock(return_value=set()),
+ ),
+ patch.object(
+ workspace_api, "get_actor_provisioner", return_value=provisioner
+ ),
+ ):
+ response = await workspace_api.delete_workspace(
+ "workspace-1", user_id="owner-1"
+ )
+
+ self.assertEqual(response.status_code, 204)
+ self.assertFalse(provisioner.actors)
+ self.assertFalse(provisioner.secrets)
+ self.assertTrue(
+ any(
+ "DELETE FROM workspace_lifecycles" in query
+ for query, _ in connection.statements
+ )
+ )
+ self.assertTrue(
+ any(
+ "DELETE FROM workspace_bindings" in query
+ for query, _ in connection.statements
+ )
+ )
+ self.assertTrue(
+ any(
+ "DELETE FROM native_deliveries" in query
+ for query, _ in connection.statements
+ )
+ )
+ self.assertTrue(
+ any("DELETE FROM messages" in query for query, _ in connection.statements)
+ )
+
+ async def test_delete_refuses_open_deliveries_before_actor_mutation(self):
+ connection = FakeConnection()
+ provisioner = FakeActorProvisioner()
+ with (
+ patch.object(workspace_api, "_require_owned_workspace", new=AsyncMock()),
+ patch.object(
+ workspace_api.db, "connection", new=fake_connection(connection)
+ ),
+ patch.object(
+ workspace_api.workspace_adapter,
+ "_delivery_states",
+ new=AsyncMock(return_value={"sending"}),
+ ),
+ patch.object(
+ workspace_api, "get_actor_provisioner", return_value=provisioner
+ ),
+ ):
+ with self.assertRaises(HTTPException) as raised:
+ await workspace_api.delete_workspace("workspace-1", user_id="owner-1")
+
+ self.assertEqual(raised.exception.status_code, 409)
+ self.assertFalse(provisioner.actors)
+
+
+if __name__ == "__main__":
+ unittest.main()
diff --git a/docs/specs/workspaces.md b/docs/specs/workspaces.md
index 2103331..bc71e42 100644
--- a/docs/specs/workspaces.md
+++ b/docs/specs/workspaces.md
@@ -1,14 +1,14 @@
# Workspaces
-Workspaces are runtime resources attached to sessions. Workspace lifecycle is separate from the
-session's task status, native-agent activity, message delivery, user attention, and publication
-state.
+Workspaces are runtime resources attached to sessions. A project branch workspace has its own
+Substrate actor and lifecycle. Workspace lifecycle is separate from the session's task status,
+native-agent activity, message delivery, user attention, and publication state.
## Lifecycle
Mainloop records desired state (`running` or `suspended`), observed state, conditions, the last
-observed transition, an operation ID, and the last known snapshot reference. The lifecycle is
-owned by Mainloop; Substrate is the source of actor observations.
+observed transition, an operation ID, the last known snapshot reference, and the last activity
+time. Mainloop owns desired state; Substrate is the source of actor observations.
| Observed state | User label | Meaning |
| ---------------------------------------- | ---------------------------- | -------------------------------------------------------------------------- |
@@ -21,17 +21,23 @@ owned by Mainloop; Substrate is the source of actor observations.
| `unknown` | UNKNOWN | A transport or unrecognized actor state prevents a reliable conclusion. |
The UI shows workspace state wherever sessions are listed and links from the session detail to
-`/workspaces/{id}`. The workspace page shows the manifest, conditions, transition time and
-snapshot reference, and offers suspend, resume, and status refresh controls. Session badges and
-session status are not changed by workspace operations.
+`/workspaces/{id}`. The workspace page shows the manifest, conditions, transition time, snapshot
+reference, idle timeout, and last activity, with suspend, resume, refresh, and delete controls.
+Session badges and session status are not changed by workspace operations.
## API
- `GET /workspaces` lists the current user's workspace lifecycle records.
- `GET /workspaces/{id}` returns one workspace lifecycle and manifest.
+- `POST /workspaces` accepts a project ID, branch, and strict dev manifest, then provisions one
+ actor from its declared actor template or the configured default template.
- `POST /workspaces/{id}/suspend` records the desired state and requests suspension.
- `POST /workspaces/{id}/resume` records the desired state and requests resumption.
- `POST /workspaces/{id}/refresh` reads Substrate status without changing desired state.
+- `POST /workspaces/{id}/touch?reason=preview` records activity and wakes a suspended workspace.
+ The preview proxy can call the same `touch_workspace(workspace_id, reason)` service API.
+- `DELETE /workspaces/{id}` deletes the actor and its shim token Secret. Open deliveries return
+ `409`; an unconfirmed Substrate deletion keeps the durable workspace binding for reconciliation.
- Lifecycle changes are published through the existing event stream as `workspace:updated`.
Suspend is refused while the native delivery ledger contains a recorded, queued, sending,
@@ -43,13 +49,21 @@ generation is advanced with a compare-and-swap before a lifecycle control call.
## Declarative manifest
Each workspace exposes repository URL and branch, allowed agent kinds (`claude` and `codex`),
-skill and MCP references, an egress host allowlist, and a resource class. These values describe
-intent only. This slice stores and displays them; it does not provision repositories, tools,
-network policy, or resources. A missing repository URL is reported as undeclared rather than
-inferred.
+skill and MCP references, an egress host allowlist, a resource class, and an optional `dev`
+section. `dev` requires exactly one of `image` or `devcontainer_ref`; it can declare an actor
+template, sibling services (`name`, `image`, `env`, and numeric ports), HTTP preview ports
+(`name`, `number`, `protocol`), and an idle timeout from 1 to 1440 minutes. Unknown fields,
+duplicate service or port names, duplicate ports, and invalid timeouts are rejected by the
+shared strict model.
+
+The project page creates a workspace per branch and lists each workspace independently. Turn,
+delivery, and preview activity use the durable last-activity timestamp. Mainloop's existing
+reconcile loop checks idle workspaces once a minute and suspends expired workspaces through the
+fenced lifecycle operation; open deliveries still block suspension. Services and image values
+are declared by the project manifest and must match its configured actor template.
## Scope and evidence
-These endpoints operate on existing Substrate workspace bindings. They do not provision an
-actor. Runtime behavior is covered by fake-backed tests; this specification does not claim a
-live cluster integration proof.
+Runtime behavior is covered by fake-backed tests; this specification does not claim a live
+cluster integration proof. The sample under `examples/devenv-sample/` documents the intended
+Node plus Postgres project manifest shape.
diff --git a/examples/devenv-sample/Dockerfile b/examples/devenv-sample/Dockerfile
new file mode 100644
index 0000000..5dfab9f
--- /dev/null
+++ b/examples/devenv-sample/Dockerfile
@@ -0,0 +1,14 @@
+FROM node:22-bookworm-slim
+WORKDIR /workspace
+COPY package.json ./
+RUN npm install --omit=dev
+COPY server.mjs ./
+RUN groupadd --system app \
+ && useradd --system --gid app --create-home --home-dir /home/app app \
+ && chown -R app:app /workspace
+USER app:app
+ENV PORT=3000 PGHOST=postgres PGPORT=5432 PGUSER=postgres PGPASSWORD=dev-only PGDATABASE=workspace
+EXPOSE 3000
+HEALTHCHECK --interval=30s --timeout=3s --start-period=15s --retries=3 \
+ CMD ["node", "-e", "fetch('http://127.0.0.1:3000/').then((response) => process.exit(response.ok ? 0 : 1)).catch(() => process.exit(1))"]
+CMD ["npm", "start"]
diff --git a/examples/devenv-sample/README.md b/examples/devenv-sample/README.md
new file mode 100644
index 0000000..b9fe6c3
--- /dev/null
+++ b/examples/devenv-sample/README.md
@@ -0,0 +1,15 @@
+# Mainloop dev environment sample
+
+This small fixture demonstrates the shape of a branch workspace: one Node app, one sibling
+Postgres service, an HTTP preview port, a WebSocket echo endpoint, and a persistent counter.
+
+The app serves one page at `/`, echoes WebSocket messages at `/ws`, and reads or increments a
+Postgres row through `/api/counter`. `mainloop.yaml` declares the app image, service, preview
+port, actor template, and 30-minute idle timeout. Build the image from this directory with
+`docker build -t mainloop-devenv-sample:latest .` in an environment with Docker available.
+
+The image declares a real non-root `app` user and checks the app's `/` HTTP route with a
+container health check. The current Substrate actor path overrides the image's `USER` and runs
+actors as UID 0; that upstream runtime gap remains. The sample does not claim to validate
+non-root actor execution or live multi-container connectivity. Sample credentials are
+fixture-only; use secret-backed values for real projects.
diff --git a/examples/devenv-sample/mainloop.yaml b/examples/devenv-sample/mainloop.yaml
new file mode 100644
index 0000000..22734a4
--- /dev/null
+++ b/examples/devenv-sample/mainloop.yaml
@@ -0,0 +1,21 @@
+---
+repo_url: https://github.com/example/devenv-sample
+branch: main
+agent_kinds: [claude, codex]
+resource_class: default
+dev:
+ image: mainloop-devenv-sample:latest
+ actor_template: devenv-sample
+ services:
+ - name: postgres
+ image: postgres:16-alpine
+ env:
+ POSTGRES_DB: workspace
+ POSTGRES_USER: postgres
+ POSTGRES_PASSWORD: dev-only
+ ports: [5432]
+ ports:
+ - name: app
+ number: 3000
+ protocol: http
+ idle_timeout_minutes: 30
diff --git a/examples/devenv-sample/package.json b/examples/devenv-sample/package.json
new file mode 100644
index 0000000..f71d33a
--- /dev/null
+++ b/examples/devenv-sample/package.json
@@ -0,0 +1,13 @@
+{
+ "name": "mainloop-devenv-sample",
+ "version": "1.0.0",
+ "private": true,
+ "type": "module",
+ "scripts": {
+ "start": "node server.mjs"
+ },
+ "dependencies": {
+ "pg": "^8.13.1",
+ "ws": "^8.18.0"
+ }
+}
diff --git a/examples/devenv-sample/server.mjs b/examples/devenv-sample/server.mjs
new file mode 100644
index 0000000..ea957e7
--- /dev/null
+++ b/examples/devenv-sample/server.mjs
@@ -0,0 +1,62 @@
+import { createServer } from 'node:http';
+import { Pool } from 'pg';
+import { WebSocketServer } from 'ws';
+
+const port = Number(process.env.PORT ?? 3000);
+const pool = new Pool();
+const server = createServer(async (request, response) => {
+ if (request.method === 'GET' && request.url === '/') {
+ response.writeHead(200, { 'content-type': 'text/html; charset=utf-8' });
+ response.end(`
+
+Mainloop dev workspace
+
Branch workspace
Postgres counter:
+
WebSocket:
+`);
+ return;
+ }
+
+ if (request.url === '/api/counter' && request.method === 'GET') {
+ const result = await pool.query('SELECT value FROM sample_counter WHERE id=1');
+ response.writeHead(200, { 'content-type': 'application/json' });
+ response.end(JSON.stringify({ value: result.rows[0]?.value ?? 0 }));
+ return;
+ }
+
+ if (request.url === '/api/counter' && request.method === 'POST') {
+ const result = await pool.query(
+ 'INSERT INTO sample_counter (id,value) VALUES (1,1) ON CONFLICT (id) DO UPDATE SET value=sample_counter.value+1 RETURNING value'
+ );
+ response.writeHead(200, { 'content-type': 'application/json' });
+ response.end(JSON.stringify({ value: result.rows[0].value }));
+ return;
+ }
+
+ response.writeHead(404);
+ response.end('Not found');
+});
+
+const sockets = new WebSocketServer({ noServer: true });
+server.on('upgrade', (request, socket, head) => {
+ if (request.url !== '/ws') return socket.destroy();
+ sockets.handleUpgrade(request, socket, head, (websocket) => {
+ websocket.on('message', (message) => websocket.send(message.toString()));
+ });
+});
+
+await pool.query(`CREATE TABLE IF NOT EXISTS sample_counter (
+ id integer PRIMARY KEY,
+ value integer NOT NULL DEFAULT 0
+)`);
+await pool.query('INSERT INTO sample_counter (id,value) VALUES (1,0) ON CONFLICT (id) DO NOTHING');
+server.listen(port, '0.0.0.0', () => console.log(`sample app listening on ${port}`));
diff --git a/frontend/src/lib/api.ts b/frontend/src/lib/api.ts
index f1ce0f1..4270367 100644
--- a/frontend/src/lib/api.ts
+++ b/frontend/src/lib/api.ts
@@ -216,6 +216,29 @@ export interface WorkspaceManifest {
mcp_servers: string[];
egress_allowlist: string[];
resource_class: string;
+ dev: WorkspaceDev | null;
+}
+
+export interface WorkspacePort {
+ name: string;
+ number: number;
+ protocol: 'http';
+}
+
+export interface WorkspaceService {
+ name: string;
+ image: string;
+ env: Record;
+ ports: number[];
+}
+
+export interface WorkspaceDev {
+ image: string | null;
+ devcontainer_ref: string | null;
+ actor_template: string | null;
+ services: WorkspaceService[];
+ ports: WorkspacePort[];
+ idle_timeout_minutes: number;
}
export interface WorkspaceCondition {
@@ -243,6 +266,7 @@ export interface WorkspaceLifecycle {
last_transition: WorkspaceTransition | null;
operation_id: string | null;
snapshot_ref: string | null;
+ last_activity_at: string | null;
ownership_generation: number;
updated_at: string;
}
@@ -447,6 +471,25 @@ export const api = {
if (!response.ok) throw new Error('Failed to refresh project');
},
+ async createWorkspace(
+ projectId: string,
+ branch: string,
+ dev: WorkspaceDev
+ ): Promise {
+ const response = await apiFetch(`${API_URL}/workspaces`, {
+ method: 'POST',
+ headers: { 'Content-Type': 'application/json' },
+ body: JSON.stringify({ project_id: projectId, branch, dev })
+ });
+ if (!response.ok) throw new Error(await errorDetail(response, 'Failed to create workspace'));
+ return response.json();
+ },
+
+ async deleteWorkspace(workspaceId: string): Promise {
+ const response = await apiFetch(`${API_URL}/workspaces/${workspaceId}`, { method: 'DELETE' });
+ if (!response.ok) throw new Error(await errorDetail(response, 'Failed to delete workspace'));
+ },
+
/**
* Get the SSE endpoint URL for the global event stream.
*/
diff --git a/frontend/src/routes/projects/[id]/+page.svelte b/frontend/src/routes/projects/[id]/+page.svelte
index 985f870..4d39f99 100644
--- a/frontend/src/routes/projects/[id]/+page.svelte
+++ b/frontend/src/routes/projects/[id]/+page.svelte
@@ -3,17 +3,34 @@
import { projects, currentProject } from '$lib/stores/projects';
import { goto } from '$app/navigation';
import { statusLabel } from '$lib/sessionStatus';
+ import { api, type WorkspaceLifecycle } from '$lib/api';
// The route is reused when only [id] changes, so load per id rather than once on mount.
const projectId = $derived($page.params.id);
+ let branch = $state('');
+ let workspaceRows = $state([]);
+ let workspaceBusy = $state(false);
+ let workspaceError = $state(null);
$effect(() => {
- if (projectId) projects.fetchProjectDetail(projectId);
+ if (projectId) {
+ branch = '';
+ projects.fetchProjectDetail(projectId);
+ void api
+ .listWorkspaces()
+ .then((rows) => {
+ if (projectId === $page.params.id) workspaceRows = rows;
+ })
+ .catch(() => {
+ workspaceRows = [];
+ });
+ }
});
// The store keeps the last project until the next one arrives; don't show it under another id.
- const detail = $derived(
- $currentProject?.project.id === projectId ? $currentProject : null
+ const detail = $derived($currentProject?.project.id === projectId ? $currentProject : null);
+ const projectWorkspaces = $derived(
+ workspaceRows.filter((item) => item.manifest.repo_url === detail?.project.html_url)
);
function formatDate(dateStr: string): string {
@@ -48,25 +65,54 @@
return 'text-term-fg-muted';
}
}
+
+ async function createWorkspace(project: { id: string; default_branch: string }) {
+ workspaceBusy = true;
+ workspaceError = null;
+ try {
+ const workspace = await api.createWorkspace(
+ project.id,
+ branch.trim() || project.default_branch,
+ {
+ image: 'node:22-bookworm',
+ devcontainer_ref: null,
+ actor_template: null,
+ services: [],
+ ports: [],
+ idle_timeout_minutes: 30
+ }
+ );
+ await goto(`/workspaces/${workspace.workspace_id}`);
+ } catch (error) {
+ workspaceError = error instanceof Error ? error.message : 'Failed to create workspace';
+ } finally {
+ workspaceBusy = false;
+ }
+ }
{$currentProject?.project.full_name || 'Project'} - mainloop
-