From 53eb056810640acfb1a01f598b59bcf8631d0727 Mon Sep 17 00:00:00 2001 From: Tee8z Date: Mon, 26 Oct 2026 11:02:05 -0400 Subject: [PATCH] feat(agents): add staging cluster diagnostic tools Signed-off-by: Tee8z --- .github/workflows/voltage-agent-images.yml | 1 + voltage/images/buzz-agent/Dockerfile | 6 ++ .../images/buzz-agent/Dockerfile.dockerignore | 1 + voltage/images/buzz-agent/README.md | 22 +++- voltage/images/buzz-agent/cluster.md | 29 +++++ voltage/images/buzz-agent/profile.py | 37 +++++++ voltage/images/buzz-agent/test-cluster.py | 102 ++++++++++++++++++ voltage/images/buzz-agent/test-image.sh | 4 +- 8 files changed, 200 insertions(+), 2 deletions(-) create mode 100644 voltage/images/buzz-agent/cluster.md create mode 100644 voltage/images/buzz-agent/test-cluster.py diff --git a/.github/workflows/voltage-agent-images.yml b/.github/workflows/voltage-agent-images.yml index e4f5ea6fd6a..b28027f087e 100644 --- a/.github/workflows/voltage-agent-images.yml +++ b/.github/workflows/voltage-agent-images.yml @@ -258,6 +258,7 @@ jobs: python3 "test-$test.py" "$IMAGE" done python3 -O test-profile.py "$IMAGE" + python3 test-cluster.py "$IMAGE" # main: build each platform on its native runner and push it by digest. publish: diff --git a/voltage/images/buzz-agent/Dockerfile b/voltage/images/buzz-agent/Dockerfile index a7e744934d6..e15c2ba8e86 100644 --- a/voltage/images/buzz-agent/Dockerfile +++ b/voltage/images/buzz-agent/Dockerfile @@ -35,12 +35,14 @@ ENV RUSTUP_HOME=/opt/rustup CARGO_HOME=/home/agent/.cargo CARGO_BUILD_JOBS=2 ARG TARGETARCH RUN case "$TARGETARCH" in \ amd64) via=x86_64 gitleaks=x64 sccache=x86_64 \ + kubectl_sha=874d5e72dbb819f43cff16bcd1e4f8bac5b7f2361fe1e55049b0a6c676fb0cbf \ via_sha=9e5a7f9384cacda29061d814e9397e7b6d90a72ee4663775362978ac9b65d8b7 \ op_sha=198b05dcf9a0972778ce5a4e262c459979b0c837257b5da65e2fba6187734226 \ gitleaks_sha=551f6fc83ea457d62a0d98237cbad105af8d557003051f41f3e7ca7b3f2470eb \ depot_sha=7a469c767bd3b7e67b3174f91af9c6315fea0ced4871948ee9d41a006fb2347e \ sccache_sha=45f1447fbe231e3037bde351ef70677dd212216c8d62ae7ca409fecc4d6acc89 ;; \ arm64) via=arm64 gitleaks=arm64 sccache=aarch64 \ + kubectl_sha=cc749967b62f4422260bc9c0aa7a7c55f45175ae38cb8d95767b5d2b7e04c1fd \ via_sha=26cc174802c875424b9846a398d4b2d890b911506f7c574dee96859620caa535 \ op_sha=7b5c361a23e738a3969cf676960f4843af4aa9aa68dbec23d18a9134da428ba3 \ gitleaks_sha=e4a487ee7ccd7d3a7f7ec08657610aa3606637dab924210b3aee62570fb4b080 \ @@ -65,6 +67,9 @@ RUN case "$TARGETARCH" in \ && curl -fsSL "https://github.com/mozilla/sccache/releases/download/v0.18.0/sccache-v0.18.0-$sccache-unknown-linux-musl.tar.gz" -o /tmp/sccache.tar.gz \ && echo "$sccache_sha /tmp/sccache.tar.gz" | sha256sum -c - \ && tar -xzf /tmp/sccache.tar.gz -C /usr/local/bin --strip-components=1 "sccache-v0.18.0-$sccache-unknown-linux-musl/sccache" \ + && curl -fsSL "https://dl.k8s.io/release/v1.35.8/bin/linux/${TARGETARCH}/kubectl" -o /usr/local/bin/kubectl \ + && echo "$kubectl_sha /usr/local/bin/kubectl" | sha256sum -c - \ + && chmod 0755 /usr/local/bin/kubectl \ && rm /tmp/via.tar.gz /tmp/op.zip /tmp/gitleaks.tar.gz /tmp/depot.tar.gz /tmp/sccache.tar.gz COPY voltage/images/buzz-agent/checkpoint-gitleaks.toml /opt/agent-tools/checkpoint-gitleaks.toml WORKDIR /opt/buzz-runtimes @@ -85,6 +90,7 @@ COPY --chmod=0755 voltage/images/buzz-agent/subagent.py /usr/local/bin/buzz-suba # a 0644 /etc/buzz-agent cannot be entered, and every Sandbox with sub-agents failed to start. RUN install -d -m 0755 /etc/buzz-agent COPY --chmod=0644 voltage/images/buzz-agent/subagents.md /etc/buzz-agent/subagents.md +COPY --chmod=0644 voltage/images/buzz-agent/cluster.md /etc/buzz-agent/cluster.md COPY --chmod=0755 voltage/images/buzz-agent/heavy-bin/ /opt/buzz-heavy/bin/ COPY --chmod=0755 voltage/images/buzz-agent/gh.sh /usr/local/bin/gh COPY --chmod=0755 voltage/images/buzz-agent/via.sh /usr/local/bin/via diff --git a/voltage/images/buzz-agent/Dockerfile.dockerignore b/voltage/images/buzz-agent/Dockerfile.dockerignore index d9fba31564e..33c0f53e17e 100644 --- a/voltage/images/buzz-agent/Dockerfile.dockerignore +++ b/voltage/images/buzz-agent/Dockerfile.dockerignore @@ -97,3 +97,4 @@ voltage/ !voltage/images/buzz-agent/worker.py !voltage/images/buzz-agent/subagent.py !voltage/images/buzz-agent/subagents.md +!voltage/images/buzz-agent/cluster.md diff --git a/voltage/images/buzz-agent/README.md b/voltage/images/buzz-agent/README.md index 957a7b15647..8168d9975f2 100644 --- a/voltage/images/buzz-agent/README.md +++ b/voltage/images/buzz-agent/README.md @@ -5,7 +5,7 @@ Build Linux `arm64` and `amd64` images for the five initial sessions owned by `t Staging agent workers are Graviton (`arm64`); the tool downloads in the Dockerfile are pinned per architecture. Each developer supplies their own accounts at runtime. -The image contains Codex and Claude ACP adapters, Rust, Node, Go, Python, Git, GitHub CLI, `via`, and SQL clients. +The image contains Codex and Claude ACP adapters, Rust, Node, Go, Python, Git, GitHub CLI, `via`, `kubectl`, and SQL clients. Buzz selects the runtime, model, persona, and arguments for each session. The image builds the Buzz source of the commit it is built from; npm dependencies use the committed lockfile. @@ -40,6 +40,7 @@ python3 test-gate.py buzz-agent:local python3 test-worker.py buzz-agent:local python3 test-cache.py buzz-agent:local python3 -O test-profile.py buzz-agent:local +python3 test-cluster.py buzz-agent:local ``` These checks use synthetic tasks and accounts. Live GitHub, signing, `via`, cache, and staging behavior require end-to-end validation. @@ -86,6 +87,25 @@ Codex uses `BUZZ_ACP_PERMISSION_MODE=agent-full-access` inside the restricted Ku The harness requires the adapter to advertise and confirm this mode before sending a prompt. Developer tools and configured MCP servers remain available; the previous text-only restriction is removed. +## Staging cluster diagnostics + +The image includes checksum-pinned `kubectl` 1.35.8 for staging Kubernetes 1.35. +During `buzz-agent-profile run` and `work`, the bootstrap configures staging access when projected credentials are mounted. +Primary sessions reuse `/var/run/secrets/buzz-subagents`; workers use `/var/run/secrets/buzz-cluster`. +The generated kubeconfig references the mounted token file and CA without copying token contents. +Kubelet rotates the 600-second token. + +The bootstrap exports `KUBECONFIG` and appends [the diagnostic guide](cluster.md) to Claude Code and Codex instructions. +It skips configuration when neither token is mounted, and rejects credentials for another developer namespace. +It performs no cluster API calls during startup. + +The staging Terraform root owns the observer RBAC, worker projection, and API egress policy. +Diagnostic permissions cover reads across staging namespaces, including logs, events, metrics, routing, and Argo CD state. +They exclude Secrets, exec, attach, port forwarding, node/service proxies, and mutations. +Existing orchestrator permissions continue to manage worker claims only in the developer namespace. +Publish this image and promote its manager and worker pins after reviewing the Terraform change. +Existing Pods retain their current tools until their normal replacement. + ## Developer accounts Each developer enrolls one profile from the Buzz app (GPUI), through the Sandbox manager over the relay. Every one of their Sandboxes and workers loads it. diff --git a/voltage/images/buzz-agent/cluster.md b/voltage/images/buzz-agent/cluster.md new file mode 100644 index 00000000000..b10442402c5 --- /dev/null +++ b/voltage/images/buzz-agent/cluster.md @@ -0,0 +1,29 @@ + +## Staging cluster diagnostics + +`kubectl` uses this Pod's temporary identity in `voltage-staging`. +The `KUBECONFIG` environment variable selects the generated configuration. +Kubelet rotates the mounted token; the configuration references its file. +No personal AWS profile or production credentials are needed. + +Use these commands to investigate requests, scheduling, rollouts, and failures: + +```sh +kubectl get pods -A -o wide +kubectl get events -A --sort-by=.metadata.creationTimestamp +kubectl get deployments,statefulsets,daemonsets -A +kubectl describe pod -n +kubectl logs -n -c --since=30m --timestamps +kubectl logs -n -c --previous --tail=200 +kubectl top nodes +kubectl top pods -A +kubectl get applications.argoproj.io -A +kubectl get sandboxes.agents.x-k8s.io -A +``` + +Cluster diagnostic permissions allow reads across staging namespaces. +They exclude Secrets, exec, attach, port forwarding, and node/service proxies. +Existing `buzz-subagent` permissions remain limited to this developer's worker claims. +Logs and manifests can contain sensitive information; quote only what explains the issue. +For older logs and historical metrics, use the enrolled Grafana service through `via`. +Report timestamps, namespaces, Pod names, and request/session IDs when explaining a failure. diff --git a/voltage/images/buzz-agent/profile.py b/voltage/images/buzz-agent/profile.py index 3717de5f8d2..aada8dd2ee7 100644 --- a/voltage/images/buzz-agent/profile.py +++ b/voltage/images/buzz-agent/profile.py @@ -220,6 +220,38 @@ def restore_checkpoint(): SUBAGENT_CREDENTIALS = Path('/var/run/secrets/buzz-subagents') SUBAGENT_GUIDE = Path('/etc/buzz-agent/subagents.md') +CLUSTER_CREDENTIALS = Path('/var/run/secrets/buzz-cluster') +CLUSTER_GUIDE = Path('/etc/buzz-agent/cluster.md') + + +def install_cluster_access(env): + """Configure kubectl with the mounted, rotating token; never copy its value.""" + credentials = next((path for path in (CLUSTER_CREDENTIALS, SUBAGENT_CREDENTIALS) + if (path / 'token').is_file()), None) + if credentials is None: + return + require(all((credentials / name).is_file() for name in ('ca.crt', 'namespace')), + 'Staging cluster credentials are incomplete') + namespace = (credentials / 'namespace').read_text().strip() + require(namespace == 'buzz-agent-staging-' + env['BUZZ_DEVELOPER_ID'], + 'Staging cluster credentials do not match the developer namespace') + config = { + 'apiVersion': 'v1', 'kind': 'Config', 'current-context': 'voltage-staging', + 'clusters': [{'name': 'voltage-staging', 'cluster': { + 'server': 'https://kubernetes.default.svc', + 'certificate-authority': str(credentials / 'ca.crt')}}], + 'users': [{'name': 'agent-observer', 'user': {'tokenFile': str(credentials / 'token')}}], + 'contexts': [{'name': 'voltage-staging', 'context': { + 'cluster': 'voltage-staging', 'user': 'agent-observer', 'namespace': namespace}}], + } + path = DIRECTORY / 'staging-kubeconfig.json' + write_file(path, json.dumps(config)) + env['KUBECONFIG'] = str(path) + guide = CLUSTER_GUIDE.read_text() + for target in (Path('/home/agent/.claude/CLAUDE.md'), Path(env.get('CODEX_HOME', '/home/agent/.codex')) / 'AGENTS.md'): + target.parent.mkdir(parents=True, exist_ok=True) + with open(target, 'a') as handle: + handle.write(guide) def install_subagent_guide(env): @@ -234,6 +266,7 @@ def install_subagent_guide(env): def main(): + """Load the owner profile and cluster access before starting a session or worker.""" parser = argparse.ArgumentParser(description=__doc__) actions = parser.add_subparsers(dest='action', required=True) run_parser = actions.add_parser('run', help='Load the verified owner profile, then start the harness') @@ -242,6 +275,8 @@ def main(): args = parser.parse_args() if args.action == 'work': env = bootstrap() + step('configuring staging cluster diagnostics') + install_cluster_access(env) step('starting the worker') worker = ['/usr/local/bin/buzz-agent-cache', '/usr/local/bin/buzz-agent-worker'] os.execvpe(worker[0], worker, env) @@ -249,6 +284,8 @@ def main(): env = bootstrap() step('restoring the checkpoint') restore_checkpoint() + step('configuring staging cluster diagnostics') + install_cluster_access(env) step('installing the sub-agent guide') install_subagent_guide(env) step('starting the harness') diff --git a/voltage/images/buzz-agent/test-cluster.py b/voltage/images/buzz-agent/test-cluster.py new file mode 100644 index 00000000000..8c274832335 --- /dev/null +++ b/voltage/images/buzz-agent/test-cluster.py @@ -0,0 +1,102 @@ +"""Exercise staging kubeconfig generation with real kubectl and a local TLS API.""" +import subprocess +import sys + +SCRIPT = r''' +import http.server, json, os, runpy, ssl, subprocess, tempfile, threading +from pathlib import Path + +profile = runpy.run_path('/usr/local/bin/buzz-agent-profile', run_name='probe') +configure = profile['install_cluster_access'] +settings = configure.__globals__ +root = Path(tempfile.mkdtemp(prefix='cluster-test-', dir='/home/agent')) +os.umask(0o077) +key, cert = root / 'key.pem', root / 'ca.crt' +subprocess.run(['openssl', 'req', '-x509', '-newkey', 'rsa:2048', '-nodes', '-days', '1', + '-subj', '/CN=localhost', '-addext', 'subjectAltName=DNS:localhost', + '-keyout', str(key), '-out', str(cert)], check=True, capture_output=True) +expected = ['synthetic-first-token'] +seen = [] +class API(http.server.BaseHTTPRequestHandler): + def log_message(self, *args): pass + def do_GET(self): + seen.append((self.path, self.headers.get('Authorization'))) + if self.headers.get('Authorization') != 'Bearer ' + expected[0]: + self.send_error(401); return + body = json.dumps({'kind': 'PodList', 'items': []}).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers(); self.wfile.write(body) +server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), API) +context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) +context.load_cert_chain(cert, key) +server.socket = context.wrap_socket(server.socket, server_side=True) +threading.Thread(target=server.serve_forever, daemon=True).start() + +def require(condition, message): + if not condition: raise RuntimeError(message) + +def setup(mode): + home = root / mode + home.mkdir(mode=0o700) + settings['DIRECTORY'] = home / 'config' + settings['CLUSTER_CREDENTIALS'] = home / 'worker' + settings['SUBAGENT_CREDENTIALS'] = home / 'primary' + env = {**os.environ, 'BUZZ_DEVELOPER_ID': 'fixture', 'CODEX_HOME': str(home / 'codex')} + env.pop('KUBECONFIG', None) + return home, env + +home, env = setup('no-mount') +configure(env) +require('KUBECONFIG' not in env and not settings['DIRECTORY'].exists(), 'unconfigured image gained cluster access') +for mode, selected in [('primary', 'SUBAGENT_CREDENTIALS'), ('worker', 'CLUSTER_CREDENTIALS')]: + home, env = setup(mode) + creds = settings[selected] + creds.mkdir(mode=0o700) + (creds / 'token').write_text(expected[0]) + (creds / 'ca.crt').write_bytes(cert.read_bytes()) + (creds / 'namespace').write_text('buzz-agent-staging-fixture\n') + configure(env) + config_path = Path(env['KUBECONFIG']) + raw = config_path.read_text() + config = json.loads(raw) + require(config['current-context'] == 'voltage-staging', 'wrong default cluster context') + require(config['clusters'][0]['cluster']['server'] == 'https://kubernetes.default.svc', 'wrong cluster server') + require(config['users'][0]['user'] == {'tokenFile': str(creds / 'token')}, 'token was copied instead of referenced') + require(expected[0] not in raw and config_path.stat().st_mode & 0o777 == 0o600, 'unsafe kubeconfig') + require('Staging cluster diagnostics' in (home / 'codex/AGENTS.md').read_text(), 'runtime guide missing') + command = ['kubectl', '--context=voltage-staging', '--server=https://127.0.0.1:' + str(server.server_port), + '--tls-server-name=localhost', '--request-timeout=5s', 'get', '--raw=/api/v1/pods'] + first = subprocess.run(command, env=env, text=True, capture_output=True) + require(first.returncode == 0 and json.loads(first.stdout)['kind'] == 'PodList', 'authenticated read failed: ' + first.stderr) + expected[0] = 'synthetic-rotated-' + mode + (creds / 'token').write_text(expected[0]) + second = subprocess.run(command, env=env, text=True, capture_output=True) + require(second.returncode == 0 and seen[-1][1] == 'Bearer ' + expected[0], 'rotated token was not reread') + wrong_peer = command.copy() + wrong_peer[3] = '--tls-server-name=untrusted.example.invalid' + refused = subprocess.run(wrong_peer, env=env, text=True, capture_output=True) + require(refused.returncode != 0, 'TLS peer verification was bypassed') + require('synthetic-' not in first.stdout + first.stderr + second.stdout + second.stderr + refused.stderr, + 'credential reached command output') + print('PASS: ' + mode + ' TLS reads and token rotation') +home, env = setup('wrong-namespace') +creds = settings['CLUSTER_CREDENTIALS']; creds.mkdir() +for name, value in [('token', 'synthetic-other'), ('ca.crt', cert.read_text()), ('namespace', 'production')]: + (creds / name).write_text(value) +try: + configure(env) +except profile['Refused']: + pass +else: + raise RuntimeError('accepted another namespace as staging') +server.shutdown() +print('PASS: unavailable credentials and namespace mismatch') +''' + +if __name__ == '__main__': + result = subprocess.run(['docker', 'run', '--rm', '-i', '--network', 'none', + '--entrypoint', 'python3', sys.argv[1] if len(sys.argv) > 1 else 'buzz-agent:local', '-'], + input=SCRIPT, text=True, timeout=90) + raise SystemExit(result.returncode) diff --git a/voltage/images/buzz-agent/test-image.sh b/voltage/images/buzz-agent/test-image.sh index 2bab2800d2a..d9d4528cb69 100755 --- a/voltage/images/buzz-agent/test-image.sh +++ b/voltage/images/buzz-agent/test-image.sh @@ -8,15 +8,17 @@ docker run --rm --entrypoint /bin/bash "$IMAGE" -ceu ' test "$(id -g)" = 10001 test "$HOME" = /home/agent test "$(pwd)" = /home/agent - for binary in buzz-acp buzz-dev-mcp codex-acp claude-agent-acp codex claude; do + for binary in buzz-acp buzz-dev-mcp codex-acp claude-agent-acp codex claude kubectl; do command -v "$binary" >/dev/null done codex --version claude --version + kubectl version --client codex login --help | grep -q -- --device-auth claude setup-token --help >/dev/null # Startup appends the sub-agent guide to the harness instructions as this user. test -r /etc/buzz-agent/subagents.md + test -r /etc/buzz-agent/cluster.md ! test -e /home/agent/.codex/auth.json ! test -e /home/agent/.claude/.credentials.json '