From ff1645171a6da3d7e6bdd4ad3ccf02b6c9889309 Mon Sep 17 00:00:00 2001 From: CSI Date: Tue, 15 Sep 2026 15:56:08 +0200 Subject: [PATCH 1/8] NsJail mode + parameterized secrets for Coolify deployment - sandbox-runner: target sandbox-build, privileged: true, cap_add, seccomp - sandbox-runner: KVM_ENABLED=false, SANDBOX_USE_CGROUPV2=false - Parameterize REDIS_PASSWORD, S3_ACCESS_KEY, S3_SECRET_KEY - No EGRESS_GATEWAY_URL change needed (NsJail shares host Docker DNS) --- docker-compose.yaml | 49 +++++++++++++++++++++++++++++---------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index ab37fd24..82b5fd6b 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -45,7 +45,7 @@ services: - CODEAPI_INTERNAL_SERVICE_TOKEN=${CODEAPI_INTERNAL_SERVICE_TOKEN:-localdev-internal-service-token} - REDIS_HOST=redis - REDIS_PORT=6379 - - REDIS_PASSWORD=localdev + - REDIS_PASSWORD=${REDIS_PASSWORD:-localdev} depends_on: - redis - tool_call_server @@ -77,7 +77,7 @@ services: - CODEAPI_INTERNAL_SERVICE_TOKEN=${CODEAPI_INTERNAL_SERVICE_TOKEN:-localdev-internal-service-token} - REDIS_HOST=redis - REDIS_PORT=6379 - - REDIS_PASSWORD=localdev + - REDIS_PASSWORD=${REDIS_PASSWORD:-localdev} - PYTHON_CONCURRENCY=${PYTHON_CONCURRENCY:-1} - OTHER_CONCURRENCY=${OTHER_CONCURRENCY:-8} - WORKER_HEALTH_PORT=3113 @@ -107,7 +107,7 @@ services: - CODEAPI_INTERNAL_SERVICE_TOKEN=${CODEAPI_INTERNAL_SERVICE_TOKEN:-localdev-internal-service-token} - REDIS_HOST=redis - REDIS_PORT=6379 - - REDIS_PASSWORD=localdev + - REDIS_PASSWORD=${REDIS_PASSWORD:-localdev} depends_on: - file_server - tool_call_server @@ -125,7 +125,7 @@ services: - TOOL_CALL_SESSION_EXPIRY=600 - REDIS_HOST=redis - REDIS_PORT=6379 - - REDIS_PASSWORD=localdev + - REDIS_PASSWORD=${REDIS_PASSWORD:-localdev} - CODEAPI_INTERNAL_SERVICE_TOKEN=${CODEAPI_INTERNAL_SERVICE_TOKEN:-localdev-internal-service-token} depends_on: - redis @@ -134,20 +134,35 @@ services: build: context: . dockerfile: api/Dockerfile - target: sandbox-runner-${KVM_ENABLED:-true} + target: sandbox-build container_name: sandbox-runner restart: always - privileged: false - devices: - - ${KVM_DEVICE_PATH:-/dev/kvm}:/dev/kvm + privileged: true + security_opt: + - seccomp=./seccomp/nsjail.json + cap_add: + - SYS_ADMIN + - SYS_CHROOT + - SYS_PTRACE + - SETUID + - SETGID + - NET_ADMIN + - DAC_OVERRIDE + - DAC_READ_SEARCH + - CHOWN + - FOWNER + - FSETID + - KILL + - SETFCAP + - MKNOD ports: - 2000:2000 volumes: - ${SANDBOX_PACKAGES_PATH:-./data/pkgs}:/host-packages:ro environment: - - KVM_ENABLED=${KVM_ENABLED:-true} - # The direct path uses /host-packages itself. The KVM path must not - # hand that directory to libkrun as a long-lived virtio-fs share. + - KVM_ENABLED=false + - SANDBOX_USE_CGROUPV2=false + - SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP=false - LAUNCHER_PACKAGES_HOST=/disabled-host-packages - CODEAPI_HARDENED_SANDBOX_MODE=${CODEAPI_HARDENED_SANDBOX_MODE:-true} - EGRESS_GATEWAY_URL=http://egress_gateway:3190 @@ -199,12 +214,12 @@ services: - MINIO_ENDPOINT=minio - MINIO_PORT=9000 - MINIO_USE_SSL=false - - MINIO_ACCESS_KEY=minioadmin - - MINIO_SECRET_KEY=minioadmin + - MINIO_ACCESS_KEY=${S3_ACCESS_KEY:-minioadmin} + - MINIO_SECRET_KEY=${S3_SECRET_KEY:-minioadmin} - FILE_SERVER_PORT=3000 - REDIS_HOST=redis - REDIS_PORT=6379 - - REDIS_PASSWORD=localdev + - REDIS_PASSWORD=${REDIS_PASSWORD:-localdev} - CODEAPI_INTERNAL_SERVICE_TOKEN=${CODEAPI_INTERNAL_SERVICE_TOKEN:-localdev-internal-service-token} depends_on: - minio @@ -213,7 +228,7 @@ services: redis: image: redis:7-alpine container_name: redis - command: redis-server --requirepass localdev + command: redis-server --requirepass ${REDIS_PASSWORD:-localdev} ports: - ${CODEAPI_REDIS_PORT:-16379}:6379 @@ -226,8 +241,8 @@ services: volumes: - minio_data:/data environment: - - MINIO_ROOT_USER=minioadmin - - MINIO_ROOT_PASSWORD=minioadmin + - MINIO_ROOT_USER=${S3_ACCESS_KEY:-minioadmin} + - MINIO_ROOT_PASSWORD=${S3_SECRET_KEY:-minioadmin} command: server /data --console-address ":9001" volumes: From 4ad5849e3e13e0877b97d0978d215f6cf2bbf433 Mon Sep 17 00:00:00 2001 From: CSI Date: Tue, 15 Sep 2026 16:10:56 +0200 Subject: [PATCH 2/8] Add entrypoint for sandbox-build target (NsJail direct mode) --- docker-compose.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yaml b/docker-compose.yaml index 82b5fd6b..076669b2 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -136,6 +136,7 @@ services: dockerfile: api/Dockerfile target: sandbox-build container_name: sandbox-runner + entrypoint: ["/sandbox_api/entrypoint.sh"] restart: always privileged: true security_opt: From 9c1600f78a2f1dee163f0a2707273fa0b5c16950 Mon Sep 17 00:00:00 2001 From: CSI Date: Tue, 15 Sep 2026 16:24:02 +0200 Subject: [PATCH 3/8] Fix healthcheck for sandbox-build target (Bun-based, from mac compose) --- docker-compose.yaml | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 076669b2..761fac49 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -196,11 +196,13 @@ services: egress_gateway: condition: service_healthy healthcheck: - test: ["CMD", "/usr/local/bin/sandbox-runner-healthcheck.sh"] - interval: 5s - timeout: 3s - retries: 60 - start_period: 20s + test: + - CMD-SHELL + - >- + bun -e "fetch('http://localhost:2000/api/v2/runtimes').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))" + interval: 10s + timeout: 5s + retries: 6 file_server: build: From ed9862f99cc8aab73778d3119d2f30c62ec873ed Mon Sep 17 00:00:00 2001 From: CSI Date: Tue, 15 Sep 2026 16:39:03 +0200 Subject: [PATCH 4/8] Add sandbox-nsjail Dockerfile target with baked-in packages Combines local-oci-runtime (NsJail direct mode) with package-builder runtime packages. No external package mount needed. --- api/Dockerfile | 6 ++++++ docker-compose.yaml | 3 +-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/api/Dockerfile b/api/Dockerfile index 66db9e3d..582e9379 100644 --- a/api/Dockerfile +++ b/api/Dockerfile @@ -190,6 +190,12 @@ ENV PORT=2000 \ EXPOSE 2000/tcp ENTRYPOINT ["/sandbox_api/entrypoint.sh"] +# Direct NsJail runtime with packages baked in (no external mount needed). +# Combines local-oci-runtime env with package-builder output. +FROM local-oci-runtime AS sandbox-nsjail + +COPY --from=package-builder /pkgs /pkgs + # Dedicated resident-process host. This image deliberately keeps the runner # control listener on 8080 and exposes one fixed user-app port separately. # `/execute` is disabled when hosted-app mode is enabled; the control plane diff --git a/docker-compose.yaml b/docker-compose.yaml index 761fac49..b9e9fd95 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -134,9 +134,8 @@ services: build: context: . dockerfile: api/Dockerfile - target: sandbox-build + target: sandbox-nsjail container_name: sandbox-runner - entrypoint: ["/sandbox_api/entrypoint.sh"] restart: always privileged: true security_opt: From 9d938428ef777e2ba3ffd60e67dc3b3f85266892 Mon Sep 17 00:00:00 2001 From: CSI Date: Tue, 15 Sep 2026 16:59:36 +0200 Subject: [PATCH 5/8] Revert to KVM mode with EGRESS_GATEWAY_URL localhost fix - target: sandbox-runner-true (KVM baked, packages included) - privileged: false, no cap_add needed (libkrun handles isolation) - EGRESS_GATEWAY_URL=http://localhost:3190 for sandbox-runner only (KVM microVM cannot resolve Docker DNS; uses SANDBOX_FORWARD_TARGET) - api/service-worker keep egress_gateway:3190 (Docker DNS works) --- docker-compose.yaml | 45 +++++++++++++++------------------------------ 1 file changed, 15 insertions(+), 30 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index b9e9fd95..ee6d7925 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -134,38 +134,25 @@ services: build: context: . dockerfile: api/Dockerfile - target: sandbox-nsjail + target: sandbox-runner-true container_name: sandbox-runner restart: always - privileged: true - security_opt: - - seccomp=./seccomp/nsjail.json - cap_add: - - SYS_ADMIN - - SYS_CHROOT - - SYS_PTRACE - - SETUID - - SETGID - - NET_ADMIN - - DAC_OVERRIDE - - DAC_READ_SEARCH - - CHOWN - - FOWNER - - FSETID - - KILL - - SETFCAP - - MKNOD + privileged: false + devices: + - ${KVM_DEVICE_PATH:-/dev/kvm}:/dev/kvm ports: - 2000:2000 volumes: - ${SANDBOX_PACKAGES_PATH:-./data/pkgs}:/host-packages:ro environment: - - KVM_ENABLED=false - - SANDBOX_USE_CGROUPV2=false - - SANDBOX_REMOVE_UMOUNT_AFTER_STARTUP=false + - KVM_ENABLED=true + # The direct path uses /host-packages itself. The KVM path must not + # hand that directory to libkrun as a long-lived virtio-fs share. - LAUNCHER_PACKAGES_HOST=/disabled-host-packages - CODEAPI_HARDENED_SANDBOX_MODE=${CODEAPI_HARDENED_SANDBOX_MODE:-true} - - EGRESS_GATEWAY_URL=http://egress_gateway:3190 + # KVM microVM cannot resolve Docker DNS names — use the local + # TCP forwarder set up by SANDBOX_FORWARD_TARGET instead. + - EGRESS_GATEWAY_URL=http://localhost:3190 - SANDBOX_EXECUTE_BODY_LIMIT=${SANDBOX_EXECUTE_BODY_LIMIT:-50mb} - SANDBOX_PACKAGES_DIRECTORY=/pkgs - SANDBOX_LOG_LEVEL=DEBUG @@ -195,13 +182,11 @@ services: egress_gateway: condition: service_healthy healthcheck: - test: - - CMD-SHELL - - >- - bun -e "fetch('http://localhost:2000/api/v2/runtimes').then((r) => process.exit(r.ok ? 0 : 1)).catch(() => process.exit(1))" - interval: 10s - timeout: 5s - retries: 6 + test: ["CMD", "/usr/local/bin/sandbox-runner-healthcheck.sh"] + interval: 5s + timeout: 3s + retries: 60 + start_period: 20s file_server: build: From 746524afbbff35e9b90c8766300c57bd93a10deb Mon Sep 17 00:00:00 2001 From: Christoph Simon | dotSource SE Date: Tue, 15 Sep 2026 18:09:22 +0200 Subject: [PATCH 6/8] fix: restore egress_gateway service URL for KVM sandbox-runner The KVM guest proxies sockets through TSI into the runner container's network namespace, and the launcher forwards Docker's resolver into the guest, so the guest resolves and reaches egress_gateway by name. Pointing EGRESS_GATEWAY_URL at localhost:3190 left nothing listening in the guest: every artifact upload failed with ConnectionRefused, responses were pruned to zero file refs, and no files could persist across executions (artifact_delivery_failed in runner logs). Reverts the localhost override introduced with the KVM revert. --- docker-compose.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index ee6d7925..642e8d04 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -150,9 +150,10 @@ services: # hand that directory to libkrun as a long-lived virtio-fs share. - LAUNCHER_PACKAGES_HOST=/disabled-host-packages - CODEAPI_HARDENED_SANDBOX_MODE=${CODEAPI_HARDENED_SANDBOX_MODE:-true} - # KVM microVM cannot resolve Docker DNS names — use the local - # TCP forwarder set up by SANDBOX_FORWARD_TARGET instead. - - EGRESS_GATEWAY_URL=http://localhost:3190 + # TSI opens guest sockets in this container's network namespace and + # the launcher forwards Docker's resolver into the guest, so the + # guest reaches the egress gateway by service name. + - EGRESS_GATEWAY_URL=http://egress_gateway:3190 - SANDBOX_EXECUTE_BODY_LIMIT=${SANDBOX_EXECUTE_BODY_LIMIT:-50mb} - SANDBOX_PACKAGES_DIRECTORY=/pkgs - SANDBOX_LOG_LEVEL=DEBUG From de36d4806e2f84243749f831da6709ab9b907d3d Mon Sep 17 00:00:00 2001 From: Christoph Simon | dotSource SE Date: Tue, 15 Sep 2026 20:40:50 +0200 Subject: [PATCH 7/8] fix: resolve service names to IPs before entering the KVM microVM libkrun's TSI networking cannot reach Docker's embedded DNS (127.0.0.11) from the guest: UDP resolver queries are refused, so EGRESS_GATEWAY_URL kept as a service name makes every artifact upload fail with ConnectionRefused (artifact_delivery_failed, files silently pruned). The resolv.conf handoff from #152 relies on that path and was never live-tested on a Docker host, regressing the original IP pinning. Resolve egress/file-server URLs and SANDBOX_FORWARD_TARGET in the container network namespace at boot and hand the guest literal IPs, restoring the pre-#152 behavior. The resolver handoff is kept for guest boot validation. Known tradeoff (same as before #152): a restarted egress gateway may move to a new IP until the runner is redeployed; compose deployments resolve at startup via depends_on. --- launcher/entrypoint.sh | 69 +++++++++++++++++++++++++++++++++++++++--- 1 file changed, 65 insertions(+), 4 deletions(-) diff --git a/launcher/entrypoint.sh b/launcher/entrypoint.sh index 304858e4..835e737d 100644 --- a/launcher/entrypoint.sh +++ b/launcher/entrypoint.sh @@ -1,10 +1,11 @@ #!/bin/bash set -e -# TSI opens guest sockets in this container's network namespace. Keep service -# names intact so new connections can resolve replacements after a restart. -# Forward the resolver and search domains supplied by Docker or Kubernetes, -# rather than pinning endpoint IPs or baking a deployment-specific nameserver. +# TSI opens guest sockets in this container's network namespace, but the +# guest cannot use Docker's embedded DNS (127.0.0.11) over TSI, so service +# names are resolved to IPs below, before the microVM boots. The resolver +# directives are still forwarded (SANDBOX_RESOLV_CONF) because guest-dns.sh +# requires a nameserver at boot. # # libkrun places every guest environment entry on the kernel command line, # which accepts only single-line printable ASCII and is truncated by the guest @@ -38,6 +39,66 @@ if [[ "$RESOLV_FIELD_SEPARATOR$SANDBOX_RESOLV_CONF" != *"${RESOLV_FIELD_SEPARATO exit 1 fi +# Resolve Docker Compose service names to IPs before entering the microVM. +# libkrun's TSI networking does not have access to Docker's embedded DNS +# (127.0.0.11), so DNS-based service discovery does not work inside the +# guest: UDP queries to it are refused and every artifact upload fails +# with artifact_delivery_failed. Resolve here, in the container's network +# namespace, and hand the guest literal IPs. + +resolve_url() { + local var_name="$1" + local url="${!var_name}" + [ -z "$url" ] && return + + local proto="${url%%://*}" + local rest="${url#*://}" + local host_port="${rest%%/*}" + local path="/${rest#*/}" + [ "$rest" = "$host_port" ] && path="" + local host="${host_port%%:*}" + local port="${host_port#*:}" + [ "$host" = "$port" ] && port="" + + echo "$host" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' && return + + local ip + ip=$(getent ahostsv4 "$host" 2>/dev/null | awk '{print $1}' | head -1) + if [ -n "$ip" ]; then + local new_url="${proto}://${ip}" + [ -n "$port" ] && new_url="${new_url}:${port}" + new_url="${new_url}${path}" + export "$var_name"="$new_url" + echo "[entrypoint] ${var_name}: ${host} -> ${ip}" + else + echo "[entrypoint] WARNING: could not resolve ${host}; passing hostname through to the guest" >&2 + fi +} + +resolve_host_port() { + local var_name="$1" + local val="${!var_name}" + [ -z "$val" ] && return + + local host="${val%%:*}" + local port="${val#*:}" + + echo "$host" | grep -qE '^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$' && return + + local ip + ip=$(getent ahostsv4 "$host" 2>/dev/null | awk '{print $1}' | head -1) + if [ -n "$ip" ]; then + export "$var_name"="${ip}:${port}" + echo "[entrypoint] ${var_name}: ${host} -> ${ip}" + else + echo "[entrypoint] WARNING: could not resolve ${host}; passing hostname through to the guest" >&2 + fi +} + +resolve_url EGRESS_GATEWAY_URL +resolve_url FILE_SERVER_URL +resolve_host_port SANDBOX_FORWARD_TARGET + if [ "${LAUNCHER_FILTER_VSOCK_ENOTCONN:-true}" = "true" ]; then # libkrun can emit this benign TSI/vsock teardown line after the guest has # already closed its side of the socket. It contains the word "error", so From 9f2235f8bd02d26500f5396a95cb181cfdb53b2b Mon Sep 17 00:00:00 2001 From: Christoph Simon | dotSource SE Date: Tue, 15 Sep 2026 21:01:05 +0200 Subject: [PATCH 8/8] fix: scope egress grant secret to the gateway under a Coolify-safe name Coolify's compose parser auto-creates an app-level variable for every ${VAR} reference and injects it into all services. With the reference named CODEAPI_EGRESS_GRANT_SECRET, hardened mode rejects it on the api and service-worker (must not be configured) and both crash-loop. Rename the overridable app variable to CODEAPI_GATEWAY_EGRESS_GRANT_SECRET; only the egress_gateway service maps it to CODEAPI_EGRESS_GRANT_SECRET. --- docker-compose.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/docker-compose.yaml b/docker-compose.yaml index 642e8d04..77e684d3 100644 --- a/docker-compose.yaml +++ b/docker-compose.yaml @@ -103,7 +103,11 @@ services: - EGRESS_GATEWAY_FILE_SERVER_URL=http://file_server:3000 - EGRESS_GATEWAY_TOOL_CALL_SERVER_URL=http://tool_call_server:3033 - EGRESS_GATEWAY_MAX_TOOL_CALL_BYTES=1048576 - - CODEAPI_EGRESS_GRANT_SECRET=${CODEAPI_EGRESS_GRANT_SECRET:-localdev-egress-grant-secret-change-me-32b} + # Scoped to the gateway only. Coolify injects app-level compose + # variables into every service, and CODEAPI_EGRESS_GRANT_SECRET is + # rejected on api/service-worker startup in hardened mode — so the + # overridable app var uses the gateway-specific name. + - CODEAPI_EGRESS_GRANT_SECRET=${CODEAPI_GATEWAY_EGRESS_GRANT_SECRET:-localdev-egress-grant-secret-change-me-32b} - CODEAPI_INTERNAL_SERVICE_TOKEN=${CODEAPI_INTERNAL_SERVICE_TOKEN:-localdev-internal-service-token} - REDIS_HOST=redis - REDIS_PORT=6379