Default Kubernetes image volumes when supported

README.md

@@ -226,10 +226,11 @@ backend:
 ```
 
   The custom image must have the harness binary reachable in the path that the Warp agent entrypoint scans (typically `/usr/local/bin` inside the sidecar image). `claude` must be in `PATH` when the harness process is invoked
-- by default, the Kubernetes backend materializes sidecars with root init containers into `emptyDir` volumes, matching the existing behavior
-- set `use_image_volumes: true` to opt into native image volumes for sidecars; in that mode, sidecar mounts are read-only and Kubernetes/runtime support for the built-in `ImageVolume` Pod volume source is required
+- when `use_image_volumes` is omitted, the worker probes native image-volume support with a real startup Job. A successful probe enables image volumes; any failure, including an inconclusive timeout or registry/network error, is logged and falls back to root init containers that materialize sidecars into `emptyDir` volumes
+- set `use_image_volumes: true` to require native image volumes and fail worker startup if the probe fails, or `use_image_volumes: false` to skip the image-volume probe and always use the legacy copy path
+- image-volume sidecar mounts are read-only. When image volumes are enabled, each read-only sidecar uses an image volume, while each read-write sidecar uses a root init container to copy its contents into an `emptyDir`. A task can mix both paths in automatic or forced-on mode; writable sidecars still require permission to run root init containers
 - Kubernetes `1.35+` is the recommended and tested target for `use_image_volumes: true`; Kubernetes `1.33`-`1.34` may work if `ImageVolume` is enabled and the container runtime supports image volumes
-- the worker runs a short-lived startup preflight Job for the configured sidecar-loading mode and waits for either preflight success or an early controller, mount, or admission failure, so incompatible cluster/runtime policy failures surface before the worker starts accepting tasks
+- the worker runs short-lived startup preflight Jobs and waits for either preflight success or an early controller, mount, or admission failure. Automatic mode reuses the image-volume preflight as its capability probe and then verifies the legacy path before falling back
 - `preflight_image` defaults to `busybox:1.36`; set it if your cluster only allows pulling startup-preflight images from an internal or allowlisted registry
 - `pod_template` accepts standard Kubernetes PodSpec YAML and is the declarative way to configure task pod scheduling, service accounts, image pull secrets, resources, and environment
 - when using `pod_template`, define a container named `task` if you want to customize the main task container directly; otherwise the worker appends its own `task` container to the PodSpec
@@ -327,7 +328,7 @@ Recommended namespace-scoped permissions for the worker are:
 When upgrading an existing installation, update its Role to grant `patch` on Jobs
 before deploying the worker; this permission is required to expire abandoned Jobs.
 
-The worker Deployment's `ServiceAccount` is separate from the task Job `serviceAccountName` you may set inside `backend.kubernetes.pod_template` / `kubernetesBackend.podTemplate`. The worker `Deployment` defaults to non-root. By default, task Jobs still materialize sidecars with root init containers; set `kubernetesBackend.useImageVolumes=true` to opt into native image volumes instead. Kubernetes `1.35+` is the recommended and tested target for that opt-in path, while Kubernetes `1.33`-`1.34` may work if `ImageVolume` is enabled and the container runtime supports image volumes. If your cluster restricts image sources for admission or policy reasons, set `kubernetesBackend.preflightImage` in the chart to an allowlisted image for the startup preflight Job, and configure task `imagePullSecrets` inside `podTemplate` when needed.
+The worker Deployment's `ServiceAccount` is separate from the task Job `serviceAccountName` you may set inside `backend.kubernetes.pod_template` / `kubernetesBackend.podTemplate`. The worker `Deployment` defaults to non-root. The chart leaves `kubernetesBackend.useImageVolumes` unset so the worker automatically enables image volumes when its real startup Job succeeds and otherwise falls back to root init containers, including when the probe is inconclusive. Set the value to `true` to require image-volume support or `false` to always use the legacy copy path. When image volumes are enabled, read-only sidecars use image volumes and read-write sidecars still use root init containers, including in forced-on mode. Kubernetes `1.35+` is the recommended and tested target, while Kubernetes `1.33`-`1.34` may work if `ImageVolume` is enabled and the container runtime supports image volumes. If your cluster restricts image sources for admission or policy reasons, set `kubernetesBackend.preflightImage` in the chart to an allowlisted image for the startup preflight Job, and configure task `imagePullSecrets` inside `podTemplate` when needed.
 
 ### Go Install
 

charts/oz-agent-worker/templates/configmap.yaml

@@ -21,7 +21,9 @@ data:
         default_image: {{ .Values.kubernetesBackend.defaultImage | quote }}
         {{- end }}
         image_pull_policy: {{ .Values.kubernetesBackend.imagePullPolicy | quote }}
+        {{- if ne .Values.kubernetesBackend.useImageVolumes nil }}
         use_image_volumes: {{ .Values.kubernetesBackend.useImageVolumes }}
+        {{- end }}
         unschedulable_timeout: {{ .Values.kubernetesBackend.unschedulableTimeout | quote }}
         {{- if .Values.kubernetesBackend.preflightImage }}
         preflight_image: {{ .Values.kubernetesBackend.preflightImage | quote }}

charts/oz-agent-worker/values.yaml

@@ -80,7 +80,9 @@ kubernetesBackend:
   namespace: ""
   defaultImage: ""
   imagePullPolicy: IfNotPresent
-  useImageVolumes: false
+  # null auto-detects image-volume support with the startup preflight. Set true
+  # to require image volumes or false to always use the legacy copy path.
+  useImageVolumes: null
   preflightImage: ""
   sidecarImage: ""
   setupCommand: ""

internal/config/config.go

@@ -74,11 +74,12 @@ type DirectConfig struct {
 
 // KubernetesConfig holds Kubernetes-backend-specific configuration.
 type KubernetesConfig struct {
-	Namespace             string            `yaml:"namespace"`
-	Kubeconfig            string            `yaml:"kubeconfig"`
-	DefaultImage          string            `yaml:"default_image" validate:"omitempty,no_whitespace"`
-	ImagePullPolicy       string            `yaml:"image_pull_policy" validate:"omitempty,oneof=Always Never IfNotPresent"`
-	UseImageVolumes       bool              `yaml:"use_image_volumes"`
+	Namespace       string `yaml:"namespace"`
+	Kubeconfig      string `yaml:"kubeconfig"`
+	DefaultImage    string `yaml:"default_image" validate:"omitempty,no_whitespace"`
+	ImagePullPolicy string `yaml:"image_pull_policy" validate:"omitempty,oneof=Always Never IfNotPresent"`
+	// UseImageVolumes is nil when support should be detected by the startup preflight.
+	UseImageVolumes       *bool             `yaml:"use_image_volumes"`
 	PreflightImage        string            `yaml:"preflight_image" validate:"omitempty,no_whitespace"`
 	SidecarImage          string            `yaml:"sidecar_image" validate:"omitempty,no_whitespace"`
 	SetupCommand          string            `yaml:"setup_command"`

internal/config/config_test.go

@@ -402,7 +402,7 @@ backend:
 	if cfg.Backend.Kubernetes.ImagePullPolicy != "IfNotPresent" {
 		t.Errorf("image_pull_policy = %q, want %q", cfg.Backend.Kubernetes.ImagePullPolicy, "IfNotPresent")
 	}
-	if !cfg.Backend.Kubernetes.UseImageVolumes {
+	if cfg.Backend.Kubernetes.UseImageVolumes == nil || !*cfg.Backend.Kubernetes.UseImageVolumes {
 		t.Fatal("expected use_image_volumes to be true")
 	}
 	if cfg.Backend.Kubernetes.PreflightImage != "registry.internal/platform/preflight:1.0" {
@@ -426,6 +426,40 @@ backend:
 	}
 }
 
+func TestLoadKubernetesUseImageVolumesTriState(t *testing.T) {
+	tests := []struct {
+		name       string
+		configLine string
+		wantSet    bool
+		wantValue  bool
+	}{
+		{name: "omitted"},
+		{name: "enabled", configLine: "    use_image_volumes: true\n", wantSet: true, wantValue: true},
+		{name: "disabled", configLine: "    use_image_volumes: false\n", wantSet: true},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			path := writeTestConfig(t, "backend:\n  kubernetes:\n    namespace: agents\n"+tt.configLine)
+			cfg, err := Load(path)
+			if err != nil {
+				t.Fatalf("unexpected error: %v", err)
+			}
+
+			got := cfg.Backend.Kubernetes.UseImageVolumes
+			if !tt.wantSet {
+				if got != nil {
+					t.Fatalf("UseImageVolumes = %v, want nil", *got)
+				}
+				return
+			}
+			if got == nil || *got != tt.wantValue {
+				t.Fatalf("UseImageVolumes = %v, want %t", got, tt.wantValue)
+			}
+		})
+	}
+}
+
 func TestLoadInvalidKubernetesPullPolicy(t *testing.T) {
 	path := writeTestConfig(t, `
 worker_id: "kubernetes-worker"

internal/worker/kubernetes.go

@@ -95,7 +95,7 @@ type KubernetesBackendConfig struct {
 	Kubeconfig            string
 	DefaultImage          string
 	ImagePullPolicy       string
-	UseImageVolumes       bool
+	UseImageVolumes       *bool
 	PreflightImage        string
 	SidecarImage          string
 	SetupCommand          string
@@ -136,8 +136,9 @@ func terminatedExitCode(terminated *corev1.ContainerStateTerminated) int {
 
 // KubernetesBackend executes tasks in Kubernetes Jobs.
 type KubernetesBackend struct {
-	config    KubernetesBackendConfig
-	clientset kubernetes.Interface
+	config          KubernetesBackendConfig
+	clientset       kubernetes.Interface
+	useImageVolumes bool
 }
 
 func (b *KubernetesBackend) Capabilities() backendCapabilities {
@@ -181,7 +182,7 @@ func NewKubernetesBackend(ctx context.Context, config KubernetesBackendConfig) (
 		config:    config,
 		clientset: clientset,
 	}
-	if err := backend.runStartupPreflight(ctx); err != nil {
+	if err := backend.configureImageVolumes(ctx); err != nil {
 		return nil, err
 	}
 
@@ -204,7 +205,7 @@ func (b *KubernetesBackend) ExecuteTask(ctx context.Context, params *TaskParams)
 		return executeError(newBackendFailure(metrics.TaskFailurePhaseBackend, metrics.TaskFailureReasonContainerCreate, err))
 	}
 	params.Logs.AddEnv(b.config.TaskEnv)
-	if err := validateTaskSidecars(params.Sidecars, b.config.UseImageVolumes); err != nil {
+	if err := validateTaskSidecars(params.Sidecars); err != nil {
 		return executeError(newBackendFailure(metrics.TaskFailurePhaseBackend, metrics.TaskFailureReasonSidecarPrep, err))
 	}
 
@@ -245,7 +246,7 @@ func (b *KubernetesBackend) ExecuteTask(ctx context.Context, params *TaskParams)
 
 	var initContainers []corev1.Container
 	for i, sidecar := range params.Sidecars {
-		if b.config.UseImageVolumes {
+		if b.useImageVolumes && !sidecar.ReadWrite {
 			volumeName := fmt.Sprintf("sidecar-%d-image", i)
 			volumes = append(volumes, imageVolume(volumeName, sidecar.Image, pullPolicy))
 
@@ -840,7 +841,7 @@ func workspaceVolumeName() string {
 	return "workspace"
 }
 
-func validateTaskSidecars(sidecars []types.SidecarMount, useImageVolumes bool) error {
+func validateTaskSidecars(sidecars []types.SidecarMount) error {
 	seenMountPaths := make(map[string]bool)
 	for _, sidecar := range sidecars {
 		if sidecar.Image == "" {
@@ -855,19 +856,47 @@ func validateTaskSidecars(sidecars []types.SidecarMount, useImageVolumes bool) e
 		if seenMountPaths[sidecar.MountPath] {
 			return fmt.Errorf("duplicate mount path %s for additional sidecar %s", sidecar.MountPath, sidecar.Image)
 		}
-		if useImageVolumes && sidecar.ReadWrite {
-			return fmt.Errorf("additional sidecar %s cannot request a read-write mount: kubernetes image volumes are read-only", sidecar.Image)
-		}
 		seenMountPaths[sidecar.MountPath] = true
 	}
 	return nil
 }
 
-func (b *KubernetesBackend) runStartupPreflight(ctx context.Context) error {
-	job := b.startupPreflightJob()
+func (b *KubernetesBackend) configureImageVolumes(ctx context.Context) error {
+	b.useImageVolumes = false
+	if b.config.UseImageVolumes != nil && !*b.config.UseImageVolumes {
+		if err := b.runStartupPreflight(ctx, false); err != nil {
+			return err
+		}
+		log.Infof(ctx, "Kubernetes image-volume mode resolved: forced-off (legacy copy path)")
+		return nil
+	}
+
+	if err := b.runStartupPreflight(ctx, true); err != nil {
+		if b.config.UseImageVolumes != nil {
+			return err
+		}
+		log.Warnf(ctx, "Kubernetes image-volume auto-detection failed; falling back to the legacy copy path: %v", err)
+		if legacyErr := b.runStartupPreflight(ctx, false); legacyErr != nil {
+			return legacyErr
+		}
+		log.Infof(ctx, "Kubernetes image-volume mode resolved: auto-fallback (legacy copy path)")
+		return nil
+	}
+
+	b.useImageVolumes = true
+	if b.config.UseImageVolumes != nil {
+		log.Infof(ctx, "Kubernetes image-volume mode resolved: forced-on")
+	} else {
+		log.Infof(ctx, "Kubernetes image-volume mode resolved: auto-on")
+	}
+	return nil
+}
+
+func (b *KubernetesBackend) runStartupPreflight(ctx context.Context, useImageVolumes bool) error {
+	job := b.startupPreflightJob(useImageVolumes)
 	createdJob, err := b.clientset.BatchV1().Jobs(b.config.Namespace).Create(ctx, job, metav1.CreateOptions{})
 	if err != nil {
-		return b.startupPreflightError(err)
+		return b.startupPreflightError(err, useImageVolumes)
 	}
 	defer func() {
 		if err := b.deleteJob(context.Background(), createdJob.Name); err != nil {
@@ -878,23 +907,23 @@ func (b *KubernetesBackend) runStartupPreflight(ctx context.Context) error {
 	preflightCtx, cancel := context.WithTimeout(ctx, startupPreflightTimeout)
 	defer cancel()
 
-	if err := b.waitForStartupPreflight(ctx, preflightCtx, createdJob); err != nil {
+	if err := b.waitForStartupPreflight(ctx, preflightCtx, createdJob, useImageVolumes); err != nil {
 		if err == context.Canceled && ctx.Err() != nil {
 			return ctx.Err()
 		}
-		return b.startupPreflightError(err)
+		return b.startupPreflightError(err, useImageVolumes)
 	}
 
 	return nil
 }
 
-func (b *KubernetesBackend) startupPreflightJob() *batchv1.Job {
+func (b *KubernetesBackend) startupPreflightJob(useImageVolumes bool) *batchv1.Job {
 	backoffLimit := int32(0)
 	pullPolicy := normalizePullPolicy(b.config.ImagePullPolicy)
 	podSpec := b.basePodSpec()
 	podSpec.RestartPolicy = corev1.RestartPolicyNever
 	preflightRes := b.preflightResourceRequirements()
-	if b.config.UseImageVolumes {
+	if useImageVolumes {
 		podSpec.InitContainers = nil
 		podSpec.Volumes = append(podSpec.Volumes, imageVolume(startupPreflightImageVolumeName, b.config.PreflightImage, pullPolicy))
 		podSpec.Containers = []corev1.Container{
@@ -949,8 +978,8 @@ func (b *KubernetesBackend) startupPreflightJob() *batchv1.Job {
 	return job
 }
 
-func (b *KubernetesBackend) waitForStartupPreflight(logCtx, ctx context.Context, job *batchv1.Job) error {
-	if b.config.UseImageVolumes {
+func (b *KubernetesBackend) waitForStartupPreflight(logCtx, ctx context.Context, job *batchv1.Job, useImageVolumes bool) error {
+	if useImageVolumes {
 		return b.waitForImageVolumeStartupPreflight(logCtx, ctx, job)
 	}
 	return b.waitForLegacyStartupPreflight(logCtx, ctx, job)
@@ -1077,9 +1106,9 @@ func startupPreflightFailureFromEvents(jobName string, events []corev1.Event) er
 	return nil
 }
 
-func (b *KubernetesBackend) startupPreflightError(err error) error {
-	if b.config.UseImageVolumes {
-		return fmt.Errorf("kubernetes startup preflight failed: the kubernetes backend requires creating task Jobs that mount sidecars via image volumes; verify service account/RBAC, Pod Security or admission policy, and Kubernetes/runtime image-volume support for namespace %q: %w", b.config.Namespace, err)
+func (b *KubernetesBackend) startupPreflightError(err error, useImageVolumes bool) error {
+	if useImageVolumes {
+		return fmt.Errorf("kubernetes image-volume startup preflight failed: verify service account/RBAC, Pod Security or admission policy, and Kubernetes/runtime image-volume support for namespace %q: %w", b.config.Namespace, err)
 	}
 	return fmt.Errorf("kubernetes startup preflight failed: the kubernetes backend requires creating task Jobs with a root init container for sidecar materialization; verify service account/RBAC and Pod Security or admission policy for namespace %q: %w", b.config.Namespace, err)
 }