feat(agents): add staging cluster diagnostic tools

.github/workflows/voltage-agent-images.yml

@@ -258,6 +258,7 @@ jobs:
             python3 "test-$test.py" "$IMAGE"
           done
           python3 -O test-profile.py "$IMAGE"
+          python3 test-cluster.py "$IMAGE"
 
   # main: build each platform on its native runner and push it by digest.
   publish:

voltage/images/buzz-agent/Dockerfile

@@ -2,13 +2,13 @@
 # Build from the repository root so the image packages the checked-out Buzz
 # source; Dockerfile.dockerignore next to this file selects the context.
 FROM rust:1.95-alpine3.22@sha256:064dfc925d68d1a63f4fd2871bd7dc6e6ea56692989a487185855d62885d90aa AS source
 RUN apk add --no-cache bash build-base cmake git musl-dev openssl-dev \
     openssl-libs-static perl pkgconf protoc python3 procps
 WORKDIR /build
 ENV CARGO_INCREMENTAL=0 CARGO_BUILD_JOBS=2
 COPY . .
 FROM source AS package-checks
 RUN apk add --no-cache tini \
     && tini -s -- cargo test --locked -p sprig -p buzz-acp -p buzz-agent -p buzz-dev-mcp -- --test-threads=1
 
 FROM source AS builder
@@ -19,7 +19,7 @@
 
 FROM node:22-trixie-slim@sha256:154ba2f4d6fec323d28e4f4bb86bba4677f1223391a1979cf521304e03a98dfa
 LABEL maintainer="Voltage <hello@voltage.cloud>"
 RUN apt-get update \
     && apt-get install -y --no-install-recommends bash ca-certificates curl git gh gnupg openssh-client socat tini \
         build-essential clang cmake pkg-config libssl-dev protobuf-compiler \
         python3 python3-venv python3-pip golang-go postgresql-client sqlite3 \
@@ -33,14 +33,16 @@
 # Release assets are public and pinned by checksum per architecture;
 # credentials enter only at runtime.
 ARG TARGETARCH
 RUN case "$TARGETARCH" in \
         amd64) via=x86_64 gitleaks=x64 sccache=x86_64 \
+            kubectl_sha=874d5e72dbb819f43cff16bcd1e4f8bac5b7f2361fe1e55049b0a6c676fb0cbf \
             via_sha=9e5a7f9384cacda29061d814e9397e7b6d90a72ee4663775362978ac9b65d8b7 \
             op_sha=198b05dcf9a0972778ce5a4e262c459979b0c837257b5da65e2fba6187734226 \
             gitleaks_sha=551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb \
             depot_sha=7a469c767bd3b7e67b3174f91af9c6315fea0ced4871948ee9d41a006fb2347e \
             sccache_sha=45f1447fbe231e3037bde351ef70677dd212216c8d62ae7ca409fecc4d6acc89 ;; \
         arm64) via=arm64 gitleaks=arm64 sccache=aarch64 \
+            kubectl_sha=cc749967b62f4422260bc9c0aa7a7c55f45175ae38cb8d95767b5d2b7e04c1fd \
             via_sha=26cc174802c875424b9846a398d4b2d890b911506f7c574dee96859620caa535 \
             op_sha=7b5c361a23e738a3969cf676960f4843af4aa9aa68dbec23d18a9134da428ba3 \
             gitleaks_sha=e4a487ee7ccd7d3a7f7ec08657610aa3606637dab924210b3aee62570fb4b080 \
@@ -65,6 +67,9 @@
     && curl -fsSL "https://github.com/mozilla/sccache/releases/download/v0.18.0/sccache-v0.18.0-$sccache-unknown-linux-musl.tar.gz" -o /tmp/sccache.tar.gz \
     && echo "$sccache_sha  /tmp/sccache.tar.gz" | sha256sum -c - \
     && tar -xzf /tmp/sccache.tar.gz -C /usr/local/bin --strip-components=1 "sccache-v0.18.0-$sccache-unknown-linux-musl/sccache" \
+    && curl -fsSL "https://dl.k8s.io/release/v1.35.8/bin/linux/${TARGETARCH}/kubectl" -o /usr/local/bin/kubectl \
+    && echo "$kubectl_sha  /usr/local/bin/kubectl" | sha256sum -c - \
+    && chmod 0755 /usr/local/bin/kubectl \
     && rm /tmp/via.tar.gz /tmp/op.zip /tmp/gitleaks.tar.gz /tmp/depot.tar.gz /tmp/sccache.tar.gz
 COPY voltage/images/buzz-agent/checkpoint-gitleaks.toml /opt/agent-tools/checkpoint-gitleaks.toml
 WORKDIR /opt/buzz-runtimes
@@ -85,6 +90,7 @@
 # a 0644 /etc/buzz-agent cannot be entered, and every Sandbox with sub-agents failed to start.
 RUN install -d -m 0755 /etc/buzz-agent
 COPY --chmod=0644 voltage/images/buzz-agent/subagents.md /etc/buzz-agent/subagents.md
+COPY --chmod=0644 voltage/images/buzz-agent/cluster.md /etc/buzz-agent/cluster.md
 COPY --chmod=0755 voltage/images/buzz-agent/heavy-bin/ /opt/buzz-heavy/bin/
 COPY --chmod=0755 voltage/images/buzz-agent/gh.sh /usr/local/bin/gh
 COPY --chmod=0755 voltage/images/buzz-agent/via.sh /usr/local/bin/via

voltage/images/buzz-agent/Dockerfile.dockerignore

@@ -97,3 +97,4 @@ voltage/
 !voltage/images/buzz-agent/worker.py
 !voltage/images/buzz-agent/subagent.py
 !voltage/images/buzz-agent/subagents.md
+!voltage/images/buzz-agent/cluster.md

voltage/images/buzz-agent/README.md

@@ -5,7 +5,7 @@ Build Linux `arm64` and `amd64` images for the five initial sessions owned by `t
 Staging agent workers are Graviton (`arm64`); the tool downloads in the Dockerfile are pinned per architecture.
 Each developer supplies their own accounts at runtime.
 
-The image contains Codex and Claude ACP adapters, Rust, Node, Go, Python, Git, GitHub CLI, `via`, and SQL clients.
+The image contains Codex and Claude ACP adapters, Rust, Node, Go, Python, Git, GitHub CLI, `via`, `kubectl`, and SQL clients.
 Buzz selects the runtime, model, persona, and arguments for each session.
 The image builds the Buzz source of the commit it is built from; npm dependencies use the committed lockfile.
 
@@ -40,6 +40,7 @@ python3 test-gate.py buzz-agent:local
 python3 test-worker.py buzz-agent:local
 python3 test-cache.py buzz-agent:local
 python3 -O test-profile.py buzz-agent:local
+python3 test-cluster.py buzz-agent:local
 ```
 
 These checks use synthetic tasks and accounts. Live GitHub, signing, `via`, cache, and staging behavior require end-to-end validation.
@@ -86,6 +87,25 @@ Codex uses `BUZZ_ACP_PERMISSION_MODE=agent-full-access` inside the restricted Ku
 The harness requires the adapter to advertise and confirm this mode before sending a prompt.
 Developer tools and configured MCP servers remain available; the previous text-only restriction is removed.
 
+## Staging cluster diagnostics
+
+The image includes checksum-pinned `kubectl` 1.35.8 for staging Kubernetes 1.35.
+During `buzz-agent-profile run` and `work`, the bootstrap configures staging access when projected credentials are mounted.
+Primary sessions reuse `/var/run/secrets/buzz-subagents`; workers use `/var/run/secrets/buzz-cluster`.
+The generated kubeconfig references the mounted token file and CA without copying token contents.
+Kubelet rotates the 600-second token.
+
+The bootstrap exports `KUBECONFIG` and appends [the diagnostic guide](cluster.md) to Claude Code and Codex instructions.
+It skips configuration when neither token is mounted, and rejects credentials for another developer namespace.
+It performs no cluster API calls during startup.
+
+The staging Terraform root owns the observer RBAC, worker projection, and API egress policy.
+Diagnostic permissions cover reads across staging namespaces, including logs, events, metrics, routing, and Argo CD state.
+They exclude Secrets, exec, attach, port forwarding, node/service proxies, and mutations.
+Existing orchestrator permissions continue to manage worker claims only in the developer namespace.
+Publish this image and promote its manager and worker pins after reviewing the Terraform change.
+Existing Pods retain their current tools until their normal replacement.
+
 ## Developer accounts
 
 Each developer enrolls one profile from the Buzz app (GPUI), through the Sandbox manager over the relay. Every one of their Sandboxes and workers loads it.

voltage/images/buzz-agent/cluster.md

@@ -0,0 +1,29 @@
+
+## Staging cluster diagnostics
+
+`kubectl` uses this Pod's temporary identity in `voltage-staging`.
+The `KUBECONFIG` environment variable selects the generated configuration.
+Kubelet rotates the mounted token; the configuration references its file.
+No personal AWS profile or production credentials are needed.
+
+Use these commands to investigate requests, scheduling, rollouts, and failures:
+
+```sh
+kubectl get pods -A -o wide
+kubectl get events -A --sort-by=.metadata.creationTimestamp
+kubectl get deployments,statefulsets,daemonsets -A
+kubectl describe pod -n <namespace> <pod>
+kubectl logs -n <namespace> <pod> -c <container> --since=30m --timestamps
+kubectl logs -n <namespace> <pod> -c <container> --previous --tail=200
+kubectl top nodes
+kubectl top pods -A
+kubectl get applications.argoproj.io -A
+kubectl get sandboxes.agents.x-k8s.io -A
+```
+
+Cluster diagnostic permissions allow reads across staging namespaces.
+They exclude Secrets, exec, attach, port forwarding, and node/service proxies.
+Existing `buzz-subagent` permissions remain limited to this developer's worker claims.
+Logs and manifests can contain sensitive information; quote only what explains the issue.
+For older logs and historical metrics, use the enrolled Grafana service through `via`.
+Report timestamps, namespaces, Pod names, and request/session IDs when explaining a failure.

voltage/images/buzz-agent/profile.py

@@ -220,6 +220,38 @@ def restore_checkpoint():
 
 SUBAGENT_CREDENTIALS = Path('/var/run/secrets/buzz-subagents')
 SUBAGENT_GUIDE = Path('/etc/buzz-agent/subagents.md')
+CLUSTER_CREDENTIALS = Path('/var/run/secrets/buzz-cluster')
+CLUSTER_GUIDE = Path('/etc/buzz-agent/cluster.md')
+
+
+def install_cluster_access(env):
+    """Configure kubectl with the mounted, rotating token; never copy its value."""
+    credentials = next((path for path in (CLUSTER_CREDENTIALS, SUBAGENT_CREDENTIALS)
+                        if (path / 'token').is_file()), None)
+    if credentials is None:
+        return
+    require(all((credentials / name).is_file() for name in ('ca.crt', 'namespace')),
+            'Staging cluster credentials are incomplete')
+    namespace = (credentials / 'namespace').read_text().strip()
+    require(namespace == 'buzz-agent-staging-' + env['BUZZ_DEVELOPER_ID'],
+            'Staging cluster credentials do not match the developer namespace')
+    config = {
+        'apiVersion': 'v1', 'kind': 'Config', 'current-context': 'voltage-staging',
+        'clusters': [{'name': 'voltage-staging', 'cluster': {
+            'server': 'https://kubernetes.default.svc',
+            'certificate-authority': str(credentials / 'ca.crt')}}],
+        'users': [{'name': 'agent-observer', 'user': {'tokenFile': str(credentials / 'token')}}],
+        'contexts': [{'name': 'voltage-staging', 'context': {
+            'cluster': 'voltage-staging', 'user': 'agent-observer', 'namespace': namespace}}],
+    }
+    path = DIRECTORY / 'staging-kubeconfig.json'
+    write_file(path, json.dumps(config))
+    env['KUBECONFIG'] = str(path)
+    guide = CLUSTER_GUIDE.read_text()
+    for target in (Path('/home/agent/.claude/CLAUDE.md'), Path(env.get('CODEX_HOME', '/home/agent/.codex')) / 'AGENTS.md'):
+        target.parent.mkdir(parents=True, exist_ok=True)
+        with open(target, 'a') as handle:
+            handle.write(guide)
 
 
 def install_subagent_guide(env):
@@ -234,6 +266,7 @@ def install_subagent_guide(env):
 
 
 def main():
+    """Load the owner profile and cluster access before starting a session or worker."""
     parser = argparse.ArgumentParser(description=__doc__)
     actions = parser.add_subparsers(dest='action', required=True)
     run_parser = actions.add_parser('run', help='Load the verified owner profile, then start the harness')
@@ -242,13 +275,17 @@ def main():
     args = parser.parse_args()
     if args.action == 'work':
         env = bootstrap()
+        step('configuring staging cluster diagnostics')
+        install_cluster_access(env)
         step('starting the worker')
         worker = ['/usr/local/bin/buzz-agent-cache', '/usr/local/bin/buzz-agent-worker']
         os.execvpe(worker[0], worker, env)
     else:
         env = bootstrap()
         step('restoring the checkpoint')
         restore_checkpoint()
+        step('configuring staging cluster diagnostics')
+        install_cluster_access(env)
         step('installing the sub-agent guide')
         install_subagent_guide(env)
         step('starting the harness')

voltage/images/buzz-agent/test-cluster.py

@@ -0,0 +1,102 @@
+"""Exercise staging kubeconfig generation with real kubectl and a local TLS API."""
+import subprocess
+import sys
+
+SCRIPT = r'''
+import http.server, json, os, runpy, ssl, subprocess, tempfile, threading
+from pathlib import Path
+
+profile = runpy.run_path('/usr/local/bin/buzz-agent-profile', run_name='probe')
+configure = profile['install_cluster_access']
+settings = configure.__globals__
+root = Path(tempfile.mkdtemp(prefix='cluster-test-', dir='/home/agent'))
+os.umask(0o077)
+key, cert = root / 'key.pem', root / 'ca.crt'
+subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1',
+                '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost',
+                '-keyout', str(key), '-out', str(cert)], check=True, capture_output=True)
+expected = ['synthetic-first-token']
+seen = []
+class API(http.server.BaseHTTPRequestHandler):
+    def log_message(self, *args): pass
+    def do_GET(self):
+        seen.append((self.path, self.headers.get('Authorization')))
+        if self.headers.get('Authorization') != 'Bearer ' + expected[0]:
+            self.send_error(401); return
+        body = json.dumps({'kind': 'PodList', 'items': []}).encode()
+        self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.send_header('Content-Length', str(len(body)))
+        self.end_headers(); self.wfile.write(body)
+server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), API)
+context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER)
+context.load_cert_chain(cert, key)
+server.socket = context.wrap_socket(server.socket, server_side=True)
+threading.Thread(target=server.serve_forever, daemon=True).start()
+
+def require(condition, message):
+    if not condition: raise RuntimeError(message)
+
+def setup(mode):
+    home = root / mode
+    home.mkdir(mode=0o700)
+    settings['DIRECTORY'] = home / 'config'
+    settings['CLUSTER_CREDENTIALS'] = home / 'worker'
+    settings['SUBAGENT_CREDENTIALS'] = home / 'primary'
+    env = {**os.environ, 'BUZZ_DEVELOPER_ID': 'fixture', 'CODEX_HOME': str(home / 'codex')}
+    env.pop('KUBECONFIG', None)
+    return home, env
+
+home, env = setup('no-mount')
+configure(env)
+require('KUBECONFIG' not in env and not settings['DIRECTORY'].exists(), 'unconfigured image gained cluster access')
+for mode, selected in [('primary', 'SUBAGENT_CREDENTIALS'), ('worker', 'CLUSTER_CREDENTIALS')]:
+    home, env = setup(mode)
+    creds = settings[selected]
+    creds.mkdir(mode=0o700)
+    (creds / 'token').write_text(expected[0])
+    (creds / 'ca.crt').write_bytes(cert.read_bytes())
+    (creds / 'namespace').write_text('buzz-agent-staging-fixture\n')
+    configure(env)
+    config_path = Path(env['KUBECONFIG'])
+    raw = config_path.read_text()
+    config = json.loads(raw)
+    require(config['current-context'] == 'voltage-staging', 'wrong default cluster context')
+    require(config['clusters'][0]['cluster']['server'] == 'https://kubernetes.default.svc', 'wrong cluster server')
+    require(config['users'][0]['user'] == {'tokenFile': str(creds / 'token')}, 'token was copied instead of referenced')
+    require(expected[0] not in raw and config_path.stat().st_mode & 0o777 == 0o600, 'unsafe kubeconfig')
+    require('Staging cluster diagnostics' in (home / 'codex/AGENTS.md').read_text(), 'runtime guide missing')
+    command = ['kubectl', '--context=voltage-staging', '--server=https://127.0.0.1:' + str(server.server_port),
+               '--tls-server-name=localhost', '--request-timeout=5s', 'get', '--raw=/api/v1/pods']
+    first = subprocess.run(command, env=env, text=True, capture_output=True)
+    require(first.returncode == 0 and json.loads(first.stdout)['kind'] == 'PodList', 'authenticated read failed: ' + first.stderr)
+    expected[0] = 'synthetic-rotated-' + mode
+    (creds / 'token').write_text(expected[0])
+    second = subprocess.run(command, env=env, text=True, capture_output=True)
+    require(second.returncode == 0 and seen[-1][1] == 'Bearer ' + expected[0], 'rotated token was not reread')
+    wrong_peer = command.copy()
+    wrong_peer[3] = '--tls-server-name=untrusted.example.invalid'
+    refused = subprocess.run(wrong_peer, env=env, text=True, capture_output=True)
+    require(refused.returncode != 0, 'TLS peer verification was bypassed')
+    require('synthetic-' not in first.stdout + first.stderr + second.stdout + second.stderr + refused.stderr,
+            'credential reached command output')
+    print('PASS: ' + mode + ' TLS reads and token rotation')
+home, env = setup('wrong-namespace')
+creds = settings['CLUSTER_CREDENTIALS']; creds.mkdir()
+for name, value in [('token', 'synthetic-other'), ('ca.crt', cert.read_text()), ('namespace', 'production')]:
+    (creds / name).write_text(value)
+try:
+    configure(env)
+except profile['Refused']:
+    pass
+else:
+    raise RuntimeError('accepted another namespace as staging')
+server.shutdown()
+print('PASS: unavailable credentials and namespace mismatch')
+'''
+
+if __name__ == '__main__':
+    result = subprocess.run(['docker', 'run', '--rm', '-i', '--network', 'none',
+                             '--entrypoint', 'python3', sys.argv[1] if len(sys.argv) > 1 else 'buzz-agent:local', '-'],
+                            input=SCRIPT, text=True, timeout=90)
+    raise SystemExit(result.returncode)

voltage/images/buzz-agent/test-image.sh

@@ -8,15 +8,17 @@ docker run --rm --entrypoint /bin/bash "$IMAGE" -ceu '
   test "$(id -g)" = 10001
   test "$HOME" = /home/agent
   test "$(pwd)" = /home/agent
-  for binary in buzz-acp buzz-dev-mcp codex-acp claude-agent-acp codex claude; do
+  for binary in buzz-acp buzz-dev-mcp codex-acp claude-agent-acp codex claude kubectl; do
     command -v "$binary" >/dev/null
   done
   codex --version
   claude --version
+  kubectl version --client
   codex login --help | grep -q -- --device-auth
   claude setup-token --help >/dev/null
   # Startup appends the sub-agent guide to the harness instructions as this user.
   test -r /etc/buzz-agent/subagents.md
+  test -r /etc/buzz-agent/cluster.md
   ! test -e /home/agent/.codex/auth.json
   ! test -e /home/agent/.claude/.credentials.json
 '