Refresh host secrets through the API

.github/workflows/on-pull-request.yml

@@ -187,6 +187,18 @@ jobs:
           docker logs drukbox-api-test
           exit 1
 
+      - name: Start the secrets exchange
+        run: |
+          docker exec --detach drukbox-api-test \
+            /bin/sh -c '.venv/bin/python -m secrets_exchange > /tmp/secrets_exchange.log 2>&1'
+          for _ in $(seq 1 60); do
+            curl -fsS http://127.0.0.1:8781/healthz >/dev/null 2>&1 && exit 0
+            sleep 1
+          done
+          echo "secrets exchange failed to start" >&2
+          docker exec drukbox-api-test cat /tmp/secrets_exchange.log
+          exit 1
+
       - name: Set up Node
         uses: actions/setup-node@v6
         with:
@@ -211,7 +223,9 @@ jobs:
 
       - name: Dump drukbox log on failure
         if: failure()
-        run: docker logs drukbox-api-test || true
+        run: |
+          docker logs drukbox-api-test || true
+          docker exec drukbox-api-test cat /tmp/secrets_exchange.log || true
 
       - name: Remove drukbox container
         if: always()

Makefile

@@ -7,4 +7,4 @@ LOCAL_ENV = \
 .PHONY: dev
 dev:
 	$(LOCAL_ENV) uv run alembic upgrade head
-	$(LOCAL_ENV) uv run uvicorn api.app:app
+	$(LOCAL_ENV) uv run python -m secrets_exchange & $(LOCAL_ENV) uv run uvicorn api.app:app

api-tests/tests/full-api.spec.js

@@ -21,6 +21,7 @@ const EXPECTED_OPENAPI_OPERATIONS = [
   "POST /http-proxies/{name}/hosts/{host_id}",
   "POST /hosts",
   "POST /hosts/{host_id}/renew",
+  "POST /hosts/{host_id}/secrets/{service}/refresh",
   "POST /templates",
   "POST /service-accounts",
 ];
@@ -136,6 +137,12 @@ test.describe("Drukbox API", () => {
     }
   });
 
+  test("doctor reports all dependencies healthy", async () => {
+    const report = await expectJson(await api.get("/doctor"), 200);
+    expect(report.ok).toBe(true);
+    expect(report.checks.find((check) => check.name === "exchange").status).toBe("ok");
+  });
+
   test("GET /hosts requires auth and returns hosts with service auth", async () => {
     await expectStatus(await publicApi.get("/hosts"), 401);
 
@@ -207,6 +214,14 @@ test.describe("Drukbox API", () => {
     expect(missing.detail).toBe("host not found");
   });
 
+  test("refresh requires auth and returns exchange errors", async () => {
+    const path = `/hosts/${createdHost.id}/secrets/anthropic/refresh`;
+    await expectStatus(await publicApi.post(path), 401);
+    await expectStatus(await badTokenApi.post(path), 403);
+    await expectStatus(await api.post(path), 409);
+    await expectStatus(await api.post(`/hosts/${createdHost.id}/secrets/missing/refresh`), 404);
+  });
+
   test("created host is observably active", async () => {
     test.setTimeout(config.hostActiveTimeoutMs);
 

deploy/proxy/swap.py

@@ -1,6 +1,6 @@
 """A mitmproxy addon: swaps a sandbox's placeholder for the real credential.
 
-    mitmdump -s /addon/swap.py --set exchange_url=http://exchange:8781
+    mitmdump -s /addon/swap.py --set exchange_url=http://127.0.0.1:8781
 
 TLS is terminated for hosts with a registered secret only. A loopback,
 private, link-local, or metadata destination is refused, for HTTP and CONNECT.

docs/api.md

@@ -31,6 +31,7 @@ migration, has no token, and cannot be removed. Admin keys act as it.
   `DELETE /templates/{id}`
 - `POST /http-proxies` · `DELETE /http-proxies/{name}` ·
   `POST|DELETE /http-proxies/{name}/hosts/{host_id}`
+- `POST /hosts/{host_id}/secrets/{service}/refresh` — refresh one secret
 - `GET /doctor` — read-only dependency diagnostics
 - `GET /healthz` — unauthenticated liveness probe
 
@@ -39,11 +40,24 @@ or claimed the host, `admin` for an admin key, or `null` for an unclaimed
 warm host. Callers cannot set it. An `Idempotency-Key` belongs to the
 service account that first used it. Another one reusing it gets `409`.
 
+## Refresh a host secret
+
+`POST /hosts/{host_id}/secrets/{service}/refresh` makes the exchange drop
+its value for that secret and fetch a new one. A provider that stores the
+value receives it at once. The response is `204` with no body. The API
+sends no `Authorization` header to the exchange.
+
+- `404` with `NOT_FOUND`: The host or the secret does not exist.
+- `409` with `SECRET_STATIC`: The secret has a static value.
+- `503` with `SECRET_REFRESH` and `Retry-After`: The exchange did not
+  answer, or the issuer or provider did not supply a value.
+
 ## The secrets exchange
 
-The exchange is a second process, `python -m secrets_exchange`, on a private
-port with no bearer token. Only the proxy and an issuer inside the deployment
-reach it. See [Architecture](architecture.md) for the flow.
+The exchange is a second process, `python -m secrets_exchange`, on loopback
+with no bearer token. The API and proxy share its network namespace.
+Remote callers use the API refresh route. See [Architecture](architecture.md)
+for the flow.
 
 - `GET /upstreams` — the hosts the proxy terminates TLS for
 - `GET /authorize` — the proxy's question: the header and the real credential

docs/architecture.md

@@ -131,10 +131,12 @@ A host that is gone is forgotten on the next pass.
 
 An issuer can end a value before its expiry, as an OAuth provider does when
 it revokes the previous token at a refresh. The issuer then orders a refresh
-at `POST /refresh/<host id>/<service>`. The exchange forgets the held value,
-fetches now, and on a provider that holds the value pushes at once. It answers
-`200` after that, and `503` with `Retry-After` when the issuer gave nothing
-usable. The order carries no value: the exchange only asks the issuer again.
+through the API at `POST /hosts/{host_id}/secrets/{service}/refresh` with a
+bearer token. The API passes the order to the exchange on loopback. The
+exchange forgets the held value, fetches now, and on a provider that holds
+the value pushes at once. The API answers `204` after that, and `503` with
+`Retry-After` when the issuer gave nothing usable. The order carries no
+value: the exchange only asks the issuer again.
 
 Provisioning mints a placeholder per secret. The placeholder names the host
 and the service, `drk.<host id>.<service>.<random>`. The entry keeps only a
@@ -204,7 +206,8 @@ customizes its host — `image`, `env`, `template`, `instance_type`, or
 ## Diagnostics
 
 `GET /doctor` runs one cheap, non-mutating probe per dependency —
-database, active provider, Tailscale when enabled — in parallel with a
+database, active provider, secrets exchange, Tailscale when enabled — in
+parallel with a
 per-probe timeout. Providers own their probe (`diagnose()`) and their
 remediation slug (`diagnose_hint`); the endpoint stays a thin
 orchestrator. It always returns 200; health is the `ok` field in the

docs/deploy.md

@@ -16,7 +16,10 @@ with the same tags.
 IMAGE=ghcr.io/czpython/drukbox:latest
 
 # API (port 8780; /healthz for liveness probes)
-docker run --rm -p 8780:8780 --env-file drukbox.env "$IMAGE"
+docker run --rm --name drukbox -p 8780:8780 --env-file drukbox.env "$IMAGE"
+
+# Secrets exchange (loopback, in the API network namespace)
+docker run --rm --network container:drukbox --env-file drukbox.env "$IMAGE" .venv/bin/python -m secrets_exchange
 
 # Migrations (one-off, before first start and on upgrades)
 docker run --rm --env-file drukbox.env "$IMAGE" .venv/bin/alembic upgrade head
@@ -296,45 +299,66 @@ placeholder for the real credential on the way out. Two pieces run this:
 - **The exchange process** runs as `python -m secrets_exchange` from this
   image. The proxy asks it which hosts to terminate, and, for a request with
   a placeholder, for the header the upstream reads and the real credential.
-  Its answer is the credential, so bind it where only the proxy can reach it.
+  Its answer contains a credential. Keep its listener on loopback. The API
+  carries remote refresh requests to it and checks its health.
+
+The API, exchange, and proxy must share a network namespace. This Compose
+example uses the API's namespace for the other two processes. Only the API
+and proxy ports are published:
 
 ```yaml
 services:
+  api:
+    image: ghcr.io/czpython/drukbox:latest
+    env_file: drukbox.env
+    environment:
+      SECRETS_PROXY_URL: http://proxy.example:8880
+      SECRETS_PROXY_CA_FILE: /secrets-proxy-ca/mitmproxy-ca-cert.pem
+    ports:
+      - "8780:8780"
+      - "8880:8880"
+    volumes:
+      - secrets-proxy-ca:/secrets-proxy-ca:ro
+
   exchange:
     image: ghcr.io/czpython/drukbox:latest
     command: [".venv/bin/python", "-m", "secrets_exchange"]
+    network_mode: "service:api"
     env_file: drukbox.env
     environment:
-      SECRETS_EXCHANGE_BIND_HOST: 0.0.0.0
+      SECRETS_EXCHANGE_BIND_HOST: 127.0.0.1
 
   proxy:
     image: ghcr.io/czpython/drukbox/proxy:latest
+    network_mode: "service:api"
     environment:
-      SECRETS_EXCHANGE_URL: http://exchange:8781
-    ports:
-      - "8880:8880"
+      SECRETS_EXCHANGE_URL: http://127.0.0.1:8781
     volumes:
       - secrets-proxy-ca:/home/mitmproxy/.mitmproxy
 
 volumes:
   secrets-proxy-ca:
 ```
 
-The exchange binds `0.0.0.0` inside the compose network and publishes no
-port, so only the proxy reaches it. The API reads the public certificate of
-the CA from the same volume, at `SECRETS_PROXY_CA_FILE`, and hands it to
-every sandbox with secrets:
+A recreated `api` container gets a new network namespace and the other two
+stay in the old one. After a change to `api`, recreate all three:
+`docker compose up -d --force-recreate api exchange proxy`.
 
-```yaml
-  api:
-    image: ghcr.io/czpython/drukbox:latest
-    env_file: drukbox.env
-    environment:
-      SECRETS_PROXY_URL: http://proxy.example:8880
-      SECRETS_PROXY_CA_FILE: /secrets-proxy-ca/mitmproxy-ca-cert.pem
-    volumes:
-      - secrets-proxy-ca:/secrets-proxy-ca:ro
-```
+Use Postgres for the shared database. Set `SECRETS_PROXY_URL` to the proxy
+address that sandboxes can contact. Apply the deployment's API and proxy
+access rules to the published ports. Do not publish port 8781 or bind the
+exchange to a public, bridge, or tailnet address.
+
+On a host-network deployment, all three processes use the host namespace.
+Keep `SECRETS_EXCHANGE_BIND_HOST=127.0.0.1`. The API reads
+`SECRETS_EXCHANGE_BIND_HOST` and `SECRETS_EXCHANGE_PORT` from the same env
+file as the exchange. The proxy reads `SECRETS_EXCHANGE_URL`. If you change
+the exchange port, set it in both places.
+
+Remote callers refresh a secret with
+`POST /hosts/{host_id}/secrets/{service}/refresh` on the API. They never
+connect to the exchange. The API reads the public CA certificate from the
+shared volume and gives it to each sandbox with secrets.
 
 A sandbox with secrets gets the certificate in `SECRETS_PROXY_CA`, base64,
 and installs it at boot with `update-ca-certificates`. `SSL_CERT_FILE`,
@@ -401,9 +425,9 @@ takes no secrets: a request with secrets always provisions a new sandbox.
 curl -fsS -H "Authorization: Bearer $TOKEN" http://localhost:8780/doctor
 ```
 
-`/doctor` runs one read-only probe per dependency (database, active
-provider, Tailscale when enabled) and reports per-check status,
-latency, and a remediation hint on failures. It always returns 200 —
+`/doctor` runs one read-only probe per dependency: database, active
+provider, secrets exchange, and Tailscale when enabled. It reports per-check
+status, latency, and a remediation hint on failures. It always returns 200 —
 health is the `ok` field. `GET /healthz` is the unauthenticated
 liveness probe.
 
@@ -449,8 +473,8 @@ Secrets exchange:
 | --- | --- | --- |
 | `SECRETS_PROXY_URL` | — | Proxy a sandbox sends its HTTPS through. Required to create a host with secrets on every provider but docker-sbx. |
 | `SECRETS_PROXY_CA_FILE` | — | Path of the proxy's public CA certificate, from the proxy's volume. Required with `SECRETS_PROXY_URL`. |
-| `SECRETS_EXCHANGE_BIND_HOST` | `127.0.0.1` | Interface the exchange process binds. Bind it where only the proxy can reach it. |
-| `SECRETS_EXCHANGE_PORT` | `8781` | Port the exchange process listens on. |
+| `SECRETS_EXCHANGE_BIND_HOST` | `127.0.0.1` | Loopback listener for the exchange. The API reads it to reach the exchange. |
+| `SECRETS_EXCHANGE_PORT` | `8781` | Port the exchange process listens on. The API reads it to reach the exchange. |
 
 Tailscale (required when `TAILSCALE_ENABLED=true`):
 

docs/security.md

@@ -105,10 +105,12 @@ API through it. It logs no credential.
 
 The real value is encrypted in the database. It passes through the exchange
 and the proxy for one request, and the exchange keeps an issuer's value in
-memory. An issuer that ends a value before its expiry orders a refresh with
-`POST /refresh/<host id>/<service>` on the exchange's private port. The order
-carries no value and no token. It makes the exchange ask the issuer again, so
-a stray order costs one fetch and nothing else. On docker-sbx the value lives
+memory. An issuer that ends a value before its expiry orders a refresh
+through the API at `POST /hosts/{host_id}/secrets/{service}/refresh` with an
+admin key or service account token. The exchange listens on loopback beside
+the API and takes the order from the API only. The order carries no value.
+It makes the exchange ask the issuer again, so a stray order costs one fetch
+and nothing else. On docker-sbx the value lives
 in sbx's own store, scoped to that sandbox, and
 drukbox runs no proxy there. Host deletion removes the sandbox's secrets and
 value files before the VM goes. The lease in `expires_at` schedules that

src/api/app.py

@@ -76,7 +76,7 @@ async def app_exception_handler(_request: Request, exc: AppException) -> JSONRes
     payload: dict[str, str] = {"detail": exc.detail}
     if exc.error_code:
         payload["error_code"] = exc.error_code
-    return JSONResponse(status_code=exc.status_code, content=payload)
+    return JSONResponse(status_code=exc.status_code, content=payload, headers=exc.headers)
 
 
 @app.get("/healthz", include_in_schema=False)

src/core/exceptions.py

@@ -12,6 +12,7 @@ class AppException(RuntimeError):
 
     status_code: ClassVar[int] = 500
     error_code: ClassVar[str | None] = None
+    headers: ClassVar[dict[str, str]] = {}
 
     def __init__(self, detail: str) -> None:
         super().__init__(detail)

src/diagnostics/api.py

@@ -2,7 +2,7 @@
 from typing import Annotated
 
 from fastapi import APIRouter, Depends, Request
-from pydantic import BaseModel
+from pydantic import BaseModel, ConfigDict
 from sqlalchemy import text
 from sqlalchemy.ext.asyncio import AsyncSession
 
@@ -12,11 +12,14 @@
 from hosts.auth import require_auth
 from networking.tailscale import Tailscale
 from providers.registry import get_default_vm_provider
+from secrets_exchange.client import SecretsExchange
 
 router = APIRouter(prefix="/doctor", tags=["doctor"], dependencies=[Depends(require_auth)])
 
 
 class CheckOut(BaseModel):
+    model_config = ConfigDict(from_attributes=True)
+
     name: str
     status: CheckStatus
     detail: str | None
@@ -69,6 +72,13 @@ async def _tailscale_probe() -> str:
         asyncio.ensure_future(
             run_check("provider", _provider_probe, hint=provider_hint, timeout=provider_timeout),
         ),
+        asyncio.ensure_future(
+            run_check(
+                "exchange",
+                SecretsExchange.from_settings().diagnose,
+                hint=SecretsExchange.diagnose_hint,
+            ),
+        ),
     ]
     if settings.tailscale_enabled:
         tasks.append(
@@ -82,16 +92,7 @@ async def _tailscale_probe() -> str:
         ok=ok,
         active_provider=settings.default_host_provider,
         tailscale_enabled=settings.tailscale_enabled,
-        checks=[
-            CheckOut(
-                name=check.name,
-                status=check.status,
-                detail=check.detail,
-                latency_ms=check.latency_ms,
-                hint=check.hint,
-            )
-            for check in checks
-        ],
+        checks=[CheckOut.model_validate(check) for check in checks],
     )