From d75709b3bb20c0161afe981c17714ef05a9092ed Mon Sep 17 00:00:00 2001 From: "warp-agent-staging[bot]" <240773466+warp-agent-staging[bot]@users.noreply.github.com> Date: Tue, 22 Sep 2026 12:54:48 +0000 Subject: [PATCH 1/2] Enable automatic Kubernetes image volumes Make use_image_volumes tri-state so omitted configuration probes real cluster support while preserving strict explicit opt-in and opt-out behavior. Fall back to the copy path for unsupported clusters and read-write sidecars in automatic mode.\n\nREMOTE-3249 --- .github/workflows/ci.yml | 11 +- README.md | 9 +- .../oz-agent-worker/templates/configmap.yaml | 2 + charts/oz-agent-worker/values.yaml | 4 +- internal/config/config.go | 11 +- internal/config/config_test.go | 36 +++- internal/worker/kubernetes.go | 88 +++++++-- internal/worker/kubernetes_test.go | 187 ++++++++++++++++-- main.go | 2 +- main_test.go | 42 +++- 10 files changed, 345 insertions(+), 47 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index ae3875e..c3d5052 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,4 +57,13 @@ jobs: - name: Lint Helm chart run: helm lint ./charts/oz-agent-worker --set worker.workerId=ci-worker --set image.tag=ci - name: Render Helm chart - run: helm template oz-agent-worker ./charts/oz-agent-worker --namespace agents --set worker.workerId=ci-worker --set image.tag=ci >/tmp/oz-agent-worker-chart.yaml + run: | + helm template oz-agent-worker ./charts/oz-agent-worker --namespace agents --set worker.workerId=ci-worker --set image.tag=ci >/tmp/oz-agent-worker-chart.yaml + if grep -q 'use_image_volumes:' /tmp/oz-agent-worker-chart.yaml; then + echo "default chart config must omit use_image_volumes" + exit 1 + fi + helm template oz-agent-worker ./charts/oz-agent-worker --namespace agents --set worker.workerId=ci-worker --set image.tag=ci --set kubernetesBackend.useImageVolumes=true | + grep -q 'use_image_volumes: true' + helm template oz-agent-worker ./charts/oz-agent-worker --namespace agents --set worker.workerId=ci-worker --set image.tag=ci --set kubernetesBackend.useImageVolumes=false | + grep -q 'use_image_volumes: false' diff --git a/README.md b/README.md index 04b91a1..8b7deae 100644 --- a/README.md +++ b/README.md @@ -226,10 +226,11 @@ backend: ``` The custom image must have the harness binary reachable in the path that the Warp agent entrypoint scans (typically `/usr/local/bin` inside the sidecar image). `claude` must be in `PATH` when the harness process is invoked -- by default, the Kubernetes backend materializes sidecars with root init containers into `emptyDir` volumes, matching the existing behavior -- set `use_image_volumes: true` to opt into native image volumes for sidecars; in that mode, sidecar mounts are read-only and Kubernetes/runtime support for the built-in `ImageVolume` Pod volume source is required +- when `use_image_volumes` is omitted, the worker probes native image-volume support with a real startup Job. A successful probe enables image volumes; any failure is logged and falls back to root init containers that materialize sidecars into `emptyDir` volumes +- set `use_image_volumes: true` to require native image volumes and fail worker startup if the probe fails, or `use_image_volumes: false` to skip the image-volume probe and always use the legacy copy path +- image-volume sidecar mounts are read-only. In automatic mode, a task with a read-write sidecar falls back to the legacy copy path; forced-on mode rejects that task - Kubernetes `1.35+` is the recommended and tested target for `use_image_volumes: true`; Kubernetes `1.33`-`1.34` may work if `ImageVolume` is enabled and the container runtime supports image volumes -- the worker runs a short-lived startup preflight Job for the configured sidecar-loading mode and waits for either preflight success or an early controller, mount, or admission failure, so incompatible cluster/runtime policy failures surface before the worker starts accepting tasks +- the worker runs short-lived startup preflight Jobs and waits for either preflight success or an early controller, mount, or admission failure. Automatic mode reuses the image-volume preflight as its capability probe and then verifies the legacy path before falling back - `preflight_image` defaults to `busybox:1.36`; set it if your cluster only allows pulling startup-preflight images from an internal or allowlisted registry - `pod_template` accepts standard Kubernetes PodSpec YAML and is the declarative way to configure task pod scheduling, service accounts, image pull secrets, resources, and environment - when using `pod_template`, define a container named `task` if you want to customize the main task container directly; otherwise the worker appends its own `task` container to the PodSpec @@ -327,7 +328,7 @@ Recommended namespace-scoped permissions for the worker are: When upgrading an existing installation, update its Role to grant `patch` on Jobs before deploying the worker; this permission is required to expire abandoned Jobs. -The worker Deployment's `ServiceAccount` is separate from the task Job `serviceAccountName` you may set inside `backend.kubernetes.pod_template` / `kubernetesBackend.podTemplate`. The worker `Deployment` defaults to non-root. By default, task Jobs still materialize sidecars with root init containers; set `kubernetesBackend.useImageVolumes=true` to opt into native image volumes instead. Kubernetes `1.35+` is the recommended and tested target for that opt-in path, while Kubernetes `1.33`-`1.34` may work if `ImageVolume` is enabled and the container runtime supports image volumes. If your cluster restricts image sources for admission or policy reasons, set `kubernetesBackend.preflightImage` in the chart to an allowlisted image for the startup preflight Job, and configure task `imagePullSecrets` inside `podTemplate` when needed. +The worker Deployment's `ServiceAccount` is separate from the task Job `serviceAccountName` you may set inside `backend.kubernetes.pod_template` / `kubernetesBackend.podTemplate`. The worker `Deployment` defaults to non-root. The chart leaves `kubernetesBackend.useImageVolumes` unset so the worker automatically enables image volumes when its real startup Job succeeds and otherwise falls back to root init containers. Set the value to `true` to require image volumes or `false` to always use the legacy copy path. Image volumes are read-only, so automatic mode falls back per task when a sidecar requests a read-write mount. Kubernetes `1.35+` is the recommended and tested target, while Kubernetes `1.33`-`1.34` may work if `ImageVolume` is enabled and the container runtime supports image volumes. If your cluster restricts image sources for admission or policy reasons, set `kubernetesBackend.preflightImage` in the chart to an allowlisted image for the startup preflight Job, and configure task `imagePullSecrets` inside `podTemplate` when needed. ### Go Install diff --git a/charts/oz-agent-worker/templates/configmap.yaml b/charts/oz-agent-worker/templates/configmap.yaml index dc87a90..c686570 100644 --- a/charts/oz-agent-worker/templates/configmap.yaml +++ b/charts/oz-agent-worker/templates/configmap.yaml @@ -21,7 +21,9 @@ data: default_image: {{ .Values.kubernetesBackend.defaultImage | quote }} {{- end }} image_pull_policy: {{ .Values.kubernetesBackend.imagePullPolicy | quote }} + {{- if ne .Values.kubernetesBackend.useImageVolumes nil }} use_image_volumes: {{ .Values.kubernetesBackend.useImageVolumes }} + {{- end }} unschedulable_timeout: {{ .Values.kubernetesBackend.unschedulableTimeout | quote }} {{- if .Values.kubernetesBackend.preflightImage }} preflight_image: {{ .Values.kubernetesBackend.preflightImage | quote }} diff --git a/charts/oz-agent-worker/values.yaml b/charts/oz-agent-worker/values.yaml index f38cfa6..d51a37f 100644 --- a/charts/oz-agent-worker/values.yaml +++ b/charts/oz-agent-worker/values.yaml @@ -80,7 +80,9 @@ kubernetesBackend: namespace: "" defaultImage: "" imagePullPolicy: IfNotPresent - useImageVolumes: false + # null auto-detects image-volume support with the startup preflight. Set true + # to require image volumes or false to always use the legacy copy path. + useImageVolumes: null preflightImage: "" sidecarImage: "" setupCommand: "" diff --git a/internal/config/config.go b/internal/config/config.go index 8e38602..d2a7aab 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -74,11 +74,12 @@ type DirectConfig struct { // KubernetesConfig holds Kubernetes-backend-specific configuration. type KubernetesConfig struct { - Namespace string `yaml:"namespace"` - Kubeconfig string `yaml:"kubeconfig"` - DefaultImage string `yaml:"default_image" validate:"omitempty,no_whitespace"` - ImagePullPolicy string `yaml:"image_pull_policy" validate:"omitempty,oneof=Always Never IfNotPresent"` - UseImageVolumes bool `yaml:"use_image_volumes"` + Namespace string `yaml:"namespace"` + Kubeconfig string `yaml:"kubeconfig"` + DefaultImage string `yaml:"default_image" validate:"omitempty,no_whitespace"` + ImagePullPolicy string `yaml:"image_pull_policy" validate:"omitempty,oneof=Always Never IfNotPresent"` + // UseImageVolumes is nil when support should be detected by the startup preflight. + UseImageVolumes *bool `yaml:"use_image_volumes"` PreflightImage string `yaml:"preflight_image" validate:"omitempty,no_whitespace"` SidecarImage string `yaml:"sidecar_image" validate:"omitempty,no_whitespace"` SetupCommand string `yaml:"setup_command"` diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 10762e4..c9d53b3 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -402,7 +402,7 @@ backend: if cfg.Backend.Kubernetes.ImagePullPolicy != "IfNotPresent" { t.Errorf("image_pull_policy = %q, want %q", cfg.Backend.Kubernetes.ImagePullPolicy, "IfNotPresent") } - if !cfg.Backend.Kubernetes.UseImageVolumes { + if cfg.Backend.Kubernetes.UseImageVolumes == nil || !*cfg.Backend.Kubernetes.UseImageVolumes { t.Fatal("expected use_image_volumes to be true") } if cfg.Backend.Kubernetes.PreflightImage != "registry.internal/platform/preflight:1.0" { @@ -426,6 +426,40 @@ backend: } } +func TestLoadKubernetesUseImageVolumesTriState(t *testing.T) { + tests := []struct { + name string + configLine string + wantSet bool + wantValue bool + }{ + {name: "omitted"}, + {name: "enabled", configLine: " use_image_volumes: true\n", wantSet: true, wantValue: true}, + {name: "disabled", configLine: " use_image_volumes: false\n", wantSet: true}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + path := writeTestConfig(t, "backend:\n kubernetes:\n namespace: agents\n"+tt.configLine) + cfg, err := Load(path) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + got := cfg.Backend.Kubernetes.UseImageVolumes + if !tt.wantSet { + if got != nil { + t.Fatalf("UseImageVolumes = %v, want nil", *got) + } + return + } + if got == nil || *got != tt.wantValue { + t.Fatalf("UseImageVolumes = %v, want %t", got, tt.wantValue) + } + }) + } +} + func TestLoadInvalidKubernetesPullPolicy(t *testing.T) { path := writeTestConfig(t, ` worker_id: "kubernetes-worker" diff --git a/internal/worker/kubernetes.go b/internal/worker/kubernetes.go index 60373b6..6a037e1 100644 --- a/internal/worker/kubernetes.go +++ b/internal/worker/kubernetes.go @@ -95,7 +95,7 @@ type KubernetesBackendConfig struct { Kubeconfig string DefaultImage string ImagePullPolicy string - UseImageVolumes bool + UseImageVolumes *bool PreflightImage string SidecarImage string SetupCommand string @@ -136,8 +136,10 @@ func terminatedExitCode(terminated *corev1.ContainerStateTerminated) int { // KubernetesBackend executes tasks in Kubernetes Jobs. type KubernetesBackend struct { - config KubernetesBackendConfig - clientset kubernetes.Interface + config KubernetesBackendConfig + clientset kubernetes.Interface + useImageVolumes bool + imageVolumesRequired bool } func (b *KubernetesBackend) Capabilities() backendCapabilities { @@ -181,7 +183,7 @@ func NewKubernetesBackend(ctx context.Context, config KubernetesBackendConfig) ( config: config, clientset: clientset, } - if err := backend.runStartupPreflight(ctx); err != nil { + if err := backend.configureImageVolumes(ctx); err != nil { return nil, err } @@ -204,7 +206,8 @@ func (b *KubernetesBackend) ExecuteTask(ctx context.Context, params *TaskParams) return executeError(newBackendFailure(metrics.TaskFailurePhaseBackend, metrics.TaskFailureReasonContainerCreate, err)) } params.Logs.AddEnv(b.config.TaskEnv) - if err := validateTaskSidecars(params.Sidecars, b.config.UseImageVolumes); err != nil { + useImageVolumes, err := b.useImageVolumesForTask(ctx, params.Sidecars) + if err != nil { return executeError(newBackendFailure(metrics.TaskFailurePhaseBackend, metrics.TaskFailureReasonSidecarPrep, err)) } @@ -245,7 +248,7 @@ func (b *KubernetesBackend) ExecuteTask(ctx context.Context, params *TaskParams) var initContainers []corev1.Container for i, sidecar := range params.Sidecars { - if b.config.UseImageVolumes { + if useImageVolumes { volumeName := fmt.Sprintf("sidecar-%d-image", i) volumes = append(volumes, imageVolume(volumeName, sidecar.Image, pullPolicy)) @@ -863,11 +866,60 @@ func validateTaskSidecars(sidecars []types.SidecarMount, useImageVolumes bool) e return nil } -func (b *KubernetesBackend) runStartupPreflight(ctx context.Context) error { - job := b.startupPreflightJob() +func (b *KubernetesBackend) useImageVolumesForTask(ctx context.Context, sidecars []types.SidecarMount) (bool, error) { + if err := validateTaskSidecars(sidecars, b.useImageVolumes && b.imageVolumesRequired); err != nil { + return false, err + } + if !b.useImageVolumes { + return false, nil + } + for _, sidecar := range sidecars { + if sidecar.ReadWrite { + log.Infof(ctx, "Using legacy sidecar copy path because a sidecar requests a read-write mount") + return false, nil + } + } + return true, nil +} + +func (b *KubernetesBackend) configureImageVolumes(ctx context.Context) error { + b.useImageVolumes = false + b.imageVolumesRequired = false + if b.config.UseImageVolumes != nil && !*b.config.UseImageVolumes { + if err := b.runStartupPreflight(ctx, false); err != nil { + return err + } + log.Infof(ctx, "Kubernetes image-volume mode resolved: forced-off (legacy copy path)") + return nil + } + + if err := b.runStartupPreflight(ctx, true); err != nil { + if b.config.UseImageVolumes != nil { + return err + } + log.Warnf(ctx, "Kubernetes image-volume auto-detection failed; falling back to the legacy copy path: %v", err) + if legacyErr := b.runStartupPreflight(ctx, false); legacyErr != nil { + return legacyErr + } + log.Infof(ctx, "Kubernetes image-volume mode resolved: auto-fallback (legacy copy path)") + return nil + } + + b.useImageVolumes = true + b.imageVolumesRequired = b.config.UseImageVolumes != nil + if b.imageVolumesRequired { + log.Infof(ctx, "Kubernetes image-volume mode resolved: forced-on") + } else { + log.Infof(ctx, "Kubernetes image-volume mode resolved: auto-on") + } + return nil +} + +func (b *KubernetesBackend) runStartupPreflight(ctx context.Context, useImageVolumes bool) error { + job := b.startupPreflightJob(useImageVolumes) createdJob, err := b.clientset.BatchV1().Jobs(b.config.Namespace).Create(ctx, job, metav1.CreateOptions{}) if err != nil { - return b.startupPreflightError(err) + return b.startupPreflightError(err, useImageVolumes) } defer func() { if err := b.deleteJob(context.Background(), createdJob.Name); err != nil { @@ -878,23 +930,23 @@ func (b *KubernetesBackend) runStartupPreflight(ctx context.Context) error { preflightCtx, cancel := context.WithTimeout(ctx, startupPreflightTimeout) defer cancel() - if err := b.waitForStartupPreflight(ctx, preflightCtx, createdJob); err != nil { + if err := b.waitForStartupPreflight(ctx, preflightCtx, createdJob, useImageVolumes); err != nil { if err == context.Canceled && ctx.Err() != nil { return ctx.Err() } - return b.startupPreflightError(err) + return b.startupPreflightError(err, useImageVolumes) } return nil } -func (b *KubernetesBackend) startupPreflightJob() *batchv1.Job { +func (b *KubernetesBackend) startupPreflightJob(useImageVolumes bool) *batchv1.Job { backoffLimit := int32(0) pullPolicy := normalizePullPolicy(b.config.ImagePullPolicy) podSpec := b.basePodSpec() podSpec.RestartPolicy = corev1.RestartPolicyNever preflightRes := b.preflightResourceRequirements() - if b.config.UseImageVolumes { + if useImageVolumes { podSpec.InitContainers = nil podSpec.Volumes = append(podSpec.Volumes, imageVolume(startupPreflightImageVolumeName, b.config.PreflightImage, pullPolicy)) podSpec.Containers = []corev1.Container{ @@ -949,8 +1001,8 @@ func (b *KubernetesBackend) startupPreflightJob() *batchv1.Job { return job } -func (b *KubernetesBackend) waitForStartupPreflight(logCtx, ctx context.Context, job *batchv1.Job) error { - if b.config.UseImageVolumes { +func (b *KubernetesBackend) waitForStartupPreflight(logCtx, ctx context.Context, job *batchv1.Job, useImageVolumes bool) error { + if useImageVolumes { return b.waitForImageVolumeStartupPreflight(logCtx, ctx, job) } return b.waitForLegacyStartupPreflight(logCtx, ctx, job) @@ -1077,9 +1129,9 @@ func startupPreflightFailureFromEvents(jobName string, events []corev1.Event) er return nil } -func (b *KubernetesBackend) startupPreflightError(err error) error { - if b.config.UseImageVolumes { - return fmt.Errorf("kubernetes startup preflight failed: the kubernetes backend requires creating task Jobs that mount sidecars via image volumes; verify service account/RBAC, Pod Security or admission policy, and Kubernetes/runtime image-volume support for namespace %q: %w", b.config.Namespace, err) +func (b *KubernetesBackend) startupPreflightError(err error, useImageVolumes bool) error { + if useImageVolumes { + return fmt.Errorf("kubernetes image-volume startup preflight failed: verify service account/RBAC, Pod Security or admission policy, and Kubernetes/runtime image-volume support for namespace %q: %w", b.config.Namespace, err) } return fmt.Errorf("kubernetes startup preflight failed: the kubernetes backend requires creating task Jobs with a root init container for sidecar materialization; verify service account/RBAC and Pod Security or admission policy for namespace %q: %w", b.config.Namespace, err) } diff --git a/internal/worker/kubernetes_test.go b/internal/worker/kubernetes_test.go index 2b82ea5..4ca5df2 100644 --- a/internal/worker/kubernetes_test.go +++ b/internal/worker/kubernetes_test.go @@ -22,6 +22,10 @@ func durationPtr(value time.Duration) *time.Duration { return &value } +func workerBoolPtr(value bool) *bool { + return &value +} + func TestKubernetesTaskJobNameEmbedsFullRunIDAndExecSuffix(t *testing.T) { runID := "019f5de4-bfd1-762e-92ed-199c971abcba" execID := "019f5df0-aaaa-bbbb-cccc-abcdef012345" @@ -953,6 +957,28 @@ func TestValidateTaskSidecarsRejectsReadWriteMountsWhenImageVolumesEnabled(t *te } } +func TestUseImageVolumesForTask(t *testing.T) { + readWriteSidecar := []types.SidecarMount{{ + Image: "registry.internal/agent:1.0", + MountPath: "/agent", + ReadWrite: true, + }} + + autoBackend := &KubernetesBackend{useImageVolumes: true} + useImageVolumes, err := autoBackend.useImageVolumesForTask(context.Background(), readWriteSidecar) + if err != nil { + t.Fatalf("auto mode returned an unexpected error: %v", err) + } + if useImageVolumes { + t.Fatal("auto mode should use the legacy copy path for a read-write sidecar") + } + + forcedBackend := &KubernetesBackend{useImageVolumes: true, imageVolumesRequired: true} + if _, err := forcedBackend.useImageVolumesForTask(context.Background(), readWriteSidecar); err == nil { + t.Fatal("forced-on mode should reject a read-write sidecar") + } +} + func TestExecuteTaskUsesImageVolumesForSidecars(t *testing.T) { fakeClient := fake.NewSimpleClientset() jobWatch := watch.NewFake() @@ -1000,9 +1026,11 @@ func TestExecuteTaskUsesImageVolumesForSidecars(t *testing.T) { Namespace: "agents", ImagePullPolicy: string(corev1.PullAlways), SetupCommand: "true", - UseImageVolumes: true, + UseImageVolumes: workerBoolPtr(true), }, - clientset: fakeClient, + clientset: fakeClient, + useImageVolumes: true, + imageVolumesRequired: true, } result := backend.ExecuteTask(context.Background(), &TaskParams{ @@ -1821,11 +1849,144 @@ func TestRunStartupPreflightCreatesLegacyRootInitJobAndWaitsForPodCreationByDefa clientset: fakeClient, } - if err := backend.runStartupPreflight(context.Background()); err != nil { + if err := backend.runStartupPreflight(context.Background(), false); err != nil { t.Fatalf("unexpected preflight error: %v", err) } } +func newImageVolumeModeTestBackend(t *testing.T, preference *bool, imageVolumesSupported bool) (*KubernetesBackend, *[]bool) { + t.Helper() + + fakeClient := fake.NewSimpleClientset() + createdModes := make([]bool, 0, 2) + var currentJob *batchv1.Job + var currentMode bool + + fakeClient.PrependReactor("create", "jobs", func(action k8stesting.Action) (bool, runtime.Object, error) { + createAction := action.(k8stesting.CreateActionImpl) + currentJob = createAction.GetObject().(*batchv1.Job).DeepCopy() + currentJob.UID = "preflight-job-uid" + currentMode = false + for _, volume := range currentJob.Spec.Template.Spec.Volumes { + if volume.Image != nil { + currentMode = true + break + } + } + createdModes = append(createdModes, currentMode) + return true, currentJob, nil + }) + fakeClient.PrependReactor("get", "jobs", func(action k8stesting.Action) (bool, runtime.Object, error) { + job := currentJob.DeepCopy() + conditionType := batchv1.JobComplete + if currentMode && !imageVolumesSupported { + conditionType = batchv1.JobFailed + } + job.Status.Conditions = []batchv1.JobCondition{{ + Type: conditionType, + Status: corev1.ConditionTrue, + }} + return true, job, nil + }) + fakeClient.PrependReactor("list", "pods", func(action k8stesting.Action) (bool, runtime.Object, error) { + if currentMode { + return true, &corev1.PodList{}, nil + } + return true, &corev1.PodList{Items: []corev1.Pod{{ + ObjectMeta: metav1.ObjectMeta{ + Name: "preflight-pod", + Namespace: "agents", + Labels: map[string]string{"job-name": currentJob.Name}, + }, + }}}, nil + }) + fakeClient.PrependReactor("list", "events", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, &corev1.EventList{}, nil + }) + fakeClient.PrependReactor("delete", "jobs", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, nil + }) + + return &KubernetesBackend{ + config: KubernetesBackendConfig{ + WorkerID: "worker-123", + Namespace: "agents", + PreflightImage: "busybox:1.36", + UseImageVolumes: preference, + }, + clientset: fakeClient, + }, &createdModes +} + +func TestConfigureImageVolumes(t *testing.T) { + tests := []struct { + name string + preference *bool + imageVolumesSupported bool + wantModes []bool + wantUseImageVolumes bool + wantRequired bool + wantError bool + }{ + { + name: "auto probe success enables image volumes", + imageVolumesSupported: true, + wantModes: []bool{true}, + wantUseImageVolumes: true, + }, + { + name: "auto probe failure falls back to legacy preflight", + imageVolumesSupported: false, + wantModes: []bool{true, false}, + }, + { + name: "explicit true fails closed", + preference: workerBoolPtr(true), + imageVolumesSupported: false, + wantModes: []bool{true}, + wantError: true, + }, + { + name: "explicit false skips image-volume probe", + preference: workerBoolPtr(false), + imageVolumesSupported: true, + wantModes: []bool{false}, + }, + { + name: "explicit true preserves strict mode", + preference: workerBoolPtr(true), + imageVolumesSupported: true, + wantModes: []bool{true}, + wantUseImageVolumes: true, + wantRequired: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + backend, createdModes := newImageVolumeModeTestBackend(t, tt.preference, tt.imageVolumesSupported) + err := backend.configureImageVolumes(context.Background()) + if (err != nil) != tt.wantError { + t.Fatalf("configureImageVolumes() error = %v, wantError %t", err, tt.wantError) + } + if len(*createdModes) != len(tt.wantModes) { + t.Fatalf("created preflight modes = %v, want %v", *createdModes, tt.wantModes) + } + for i := range tt.wantModes { + if (*createdModes)[i] != tt.wantModes[i] { + t.Fatalf("created preflight modes = %v, want %v", *createdModes, tt.wantModes) + } + } + if backend.useImageVolumes != tt.wantUseImageVolumes { + t.Fatalf("useImageVolumes = %t, want %t", backend.useImageVolumes, tt.wantUseImageVolumes) + } + if backend.imageVolumesRequired != tt.wantRequired { + t.Fatalf("imageVolumesRequired = %t, want %t", backend.imageVolumesRequired, tt.wantRequired) + } + }) + } +} + func TestRunStartupPreflightCreatesImageVolumeJobAndWaitsForSuccess(t *testing.T) { fakeClient := fake.NewSimpleClientset() const preflightImage = "registry.internal/platform/preflight:1.0" @@ -1927,10 +2088,9 @@ func TestRunStartupPreflightCreatesImageVolumeJobAndWaitsForSuccess(t *testing.T backend := &KubernetesBackend{ config: KubernetesBackendConfig{ - WorkerID: "worker-123", - Namespace: "agents", - PreflightImage: preflightImage, - UseImageVolumes: true, + WorkerID: "worker-123", + Namespace: "agents", + PreflightImage: preflightImage, PodTemplate: &corev1.PodSpec{ ServiceAccountName: "oz-agent-worker", ImagePullSecrets: []corev1.LocalObjectReference{ @@ -1941,7 +2101,7 @@ func TestRunStartupPreflightCreatesImageVolumeJobAndWaitsForSuccess(t *testing.T clientset: fakeClient, } - if err := backend.runStartupPreflight(context.Background()); err != nil { + if err := backend.runStartupPreflight(context.Background(), true); err != nil { t.Fatalf("unexpected preflight error: %v", err) } } @@ -2003,15 +2163,14 @@ func TestRunStartupPreflightFailsOnFailedMountEvent(t *testing.T) { backend := &KubernetesBackend{ config: KubernetesBackendConfig{ - WorkerID: "worker-123", - Namespace: "agents", - PreflightImage: "registry.internal/platform/preflight:1.0", - UseImageVolumes: true, + WorkerID: "worker-123", + Namespace: "agents", + PreflightImage: "registry.internal/platform/preflight:1.0", }, clientset: fakeClient, } - err := backend.runStartupPreflight(context.Background()) + err := backend.runStartupPreflight(context.Background(), true) if err == nil { t.Fatal("expected preflight failure") } @@ -2072,7 +2231,7 @@ func TestRunStartupPreflightFailsOnFailedCreateEvent(t *testing.T) { clientset: fakeClient, } - err := backend.runStartupPreflight(context.Background()) + err := backend.runStartupPreflight(context.Background(), false) if err == nil { t.Fatal("expected preflight failure") } diff --git a/main.go b/main.go index 3a2a784..0c48933 100644 --- a/main.go +++ b/main.go @@ -202,7 +202,7 @@ func mergeConfig(fileConfig *config.FileConfig) (worker.Config, error) { kubeconfig string defaultImage string imagePullPolicy string - useImageVolumes bool + useImageVolumes *bool preflightImage string sidecarImage string codingCLISidecars map[string]string diff --git a/main_test.go b/main_test.go index 5e3beda..b277a51 100644 --- a/main_test.go +++ b/main_test.go @@ -118,7 +118,7 @@ func TestMergeConfigKubernetesFromFile(t *testing.T) { Namespace: "agents", Kubeconfig: "/tmp/kubeconfig", ImagePullPolicy: "IfNotPresent", - UseImageVolumes: true, + UseImageVolumes: boolPtr(true), PreflightImage: "registry.internal/platform/preflight:1.0", SetupCommand: "setup.sh", TeardownCommand: "teardown.sh", @@ -165,7 +165,7 @@ containers: if wc.Kubernetes.PreflightImage != "registry.internal/platform/preflight:1.0" { t.Errorf("PreflightImage = %q, want %q", wc.Kubernetes.PreflightImage, "registry.internal/platform/preflight:1.0") } - if !wc.Kubernetes.UseImageVolumes { + if wc.Kubernetes.UseImageVolumes == nil || !*wc.Kubernetes.UseImageVolumes { t.Fatal("expected UseImageVolumes to be true") } if wc.Kubernetes.TaskEnv["CLI_ONLY"] != "1" { @@ -191,6 +191,44 @@ containers: } } +func TestMergeConfigKubernetesPreservesImageVolumePreference(t *testing.T) { + tests := []struct { + name string + preference *bool + }{ + {name: "omitted"}, + {name: "enabled", preference: boolPtr(true)}, + {name: "disabled", preference: boolPtr(false)}, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + resetCLIForTest() + t.Cleanup(resetCLIForTest) + fileConfig := &config.FileConfig{ + WorkerID: "worker-123", + Backend: config.BackendConfig{ + Kubernetes: &config.KubernetesConfig{UseImageVolumes: tt.preference}, + }, + } + + wc, err := mergeConfig(fileConfig) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + if tt.preference == nil { + if wc.Kubernetes.UseImageVolumes != nil { + t.Fatalf("UseImageVolumes = %v, want nil", *wc.Kubernetes.UseImageVolumes) + } + return + } + if wc.Kubernetes.UseImageVolumes == nil || *wc.Kubernetes.UseImageVolumes != *tt.preference { + t.Fatalf("UseImageVolumes = %v, want %t", wc.Kubernetes.UseImageVolumes, *tt.preference) + } + }) + } +} + func TestMergeConfigOneShot(t *testing.T) { tests := []struct { name string From 44af877ebb7338e8a05afe2ba186b37a357aafb4 Mon Sep 17 00:00:00 2001 From: "warp-agent-staging[bot]" <240773466+warp-agent-staging[bot]@users.noreply.github.com> Date: Mon, 19 Oct 2026 16:48:15 +0000 Subject: [PATCH 2/2] Allow mixed Kubernetes sidecar volume modes --- .github/workflows/ci.yml | 11 +-- README.md | 6 +- internal/worker/kubernetes.go | 37 ++------- internal/worker/kubernetes_test.go | 124 +++++++++++++---------------- 4 files changed, 68 insertions(+), 110 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index c3d5052..ae3875e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -57,13 +57,4 @@ jobs: - name: Lint Helm chart run: helm lint ./charts/oz-agent-worker --set worker.workerId=ci-worker --set image.tag=ci - name: Render Helm chart - run: | - helm template oz-agent-worker ./charts/oz-agent-worker --namespace agents --set worker.workerId=ci-worker --set image.tag=ci >/tmp/oz-agent-worker-chart.yaml - if grep -q 'use_image_volumes:' /tmp/oz-agent-worker-chart.yaml; then - echo "default chart config must omit use_image_volumes" - exit 1 - fi - helm template oz-agent-worker ./charts/oz-agent-worker --namespace agents --set worker.workerId=ci-worker --set image.tag=ci --set kubernetesBackend.useImageVolumes=true | - grep -q 'use_image_volumes: true' - helm template oz-agent-worker ./charts/oz-agent-worker --namespace agents --set worker.workerId=ci-worker --set image.tag=ci --set kubernetesBackend.useImageVolumes=false | - grep -q 'use_image_volumes: false' + run: helm template oz-agent-worker ./charts/oz-agent-worker --namespace agents --set worker.workerId=ci-worker --set image.tag=ci >/tmp/oz-agent-worker-chart.yaml diff --git a/README.md b/README.md index 8b7deae..90510c7 100644 --- a/README.md +++ b/README.md @@ -226,9 +226,9 @@ backend: ``` The custom image must have the harness binary reachable in the path that the Warp agent entrypoint scans (typically `/usr/local/bin` inside the sidecar image). `claude` must be in `PATH` when the harness process is invoked -- when `use_image_volumes` is omitted, the worker probes native image-volume support with a real startup Job. A successful probe enables image volumes; any failure is logged and falls back to root init containers that materialize sidecars into `emptyDir` volumes +- when `use_image_volumes` is omitted, the worker probes native image-volume support with a real startup Job. A successful probe enables image volumes; any failure, including an inconclusive timeout or registry/network error, is logged and falls back to root init containers that materialize sidecars into `emptyDir` volumes - set `use_image_volumes: true` to require native image volumes and fail worker startup if the probe fails, or `use_image_volumes: false` to skip the image-volume probe and always use the legacy copy path -- image-volume sidecar mounts are read-only. In automatic mode, a task with a read-write sidecar falls back to the legacy copy path; forced-on mode rejects that task +- image-volume sidecar mounts are read-only. When image volumes are enabled, each read-only sidecar uses an image volume, while each read-write sidecar uses a root init container to copy its contents into an `emptyDir`. A task can mix both paths in automatic or forced-on mode; writable sidecars still require permission to run root init containers - Kubernetes `1.35+` is the recommended and tested target for `use_image_volumes: true`; Kubernetes `1.33`-`1.34` may work if `ImageVolume` is enabled and the container runtime supports image volumes - the worker runs short-lived startup preflight Jobs and waits for either preflight success or an early controller, mount, or admission failure. Automatic mode reuses the image-volume preflight as its capability probe and then verifies the legacy path before falling back - `preflight_image` defaults to `busybox:1.36`; set it if your cluster only allows pulling startup-preflight images from an internal or allowlisted registry @@ -328,7 +328,7 @@ Recommended namespace-scoped permissions for the worker are: When upgrading an existing installation, update its Role to grant `patch` on Jobs before deploying the worker; this permission is required to expire abandoned Jobs. -The worker Deployment's `ServiceAccount` is separate from the task Job `serviceAccountName` you may set inside `backend.kubernetes.pod_template` / `kubernetesBackend.podTemplate`. The worker `Deployment` defaults to non-root. The chart leaves `kubernetesBackend.useImageVolumes` unset so the worker automatically enables image volumes when its real startup Job succeeds and otherwise falls back to root init containers. Set the value to `true` to require image volumes or `false` to always use the legacy copy path. Image volumes are read-only, so automatic mode falls back per task when a sidecar requests a read-write mount. Kubernetes `1.35+` is the recommended and tested target, while Kubernetes `1.33`-`1.34` may work if `ImageVolume` is enabled and the container runtime supports image volumes. If your cluster restricts image sources for admission or policy reasons, set `kubernetesBackend.preflightImage` in the chart to an allowlisted image for the startup preflight Job, and configure task `imagePullSecrets` inside `podTemplate` when needed. +The worker Deployment's `ServiceAccount` is separate from the task Job `serviceAccountName` you may set inside `backend.kubernetes.pod_template` / `kubernetesBackend.podTemplate`. The worker `Deployment` defaults to non-root. The chart leaves `kubernetesBackend.useImageVolumes` unset so the worker automatically enables image volumes when its real startup Job succeeds and otherwise falls back to root init containers, including when the probe is inconclusive. Set the value to `true` to require image-volume support or `false` to always use the legacy copy path. When image volumes are enabled, read-only sidecars use image volumes and read-write sidecars still use root init containers, including in forced-on mode. Kubernetes `1.35+` is the recommended and tested target, while Kubernetes `1.33`-`1.34` may work if `ImageVolume` is enabled and the container runtime supports image volumes. If your cluster restricts image sources for admission or policy reasons, set `kubernetesBackend.preflightImage` in the chart to an allowlisted image for the startup preflight Job, and configure task `imagePullSecrets` inside `podTemplate` when needed. ### Go Install diff --git a/internal/worker/kubernetes.go b/internal/worker/kubernetes.go index 6a037e1..70b5e95 100644 --- a/internal/worker/kubernetes.go +++ b/internal/worker/kubernetes.go @@ -136,10 +136,9 @@ func terminatedExitCode(terminated *corev1.ContainerStateTerminated) int { // KubernetesBackend executes tasks in Kubernetes Jobs. type KubernetesBackend struct { - config KubernetesBackendConfig - clientset kubernetes.Interface - useImageVolumes bool - imageVolumesRequired bool + config KubernetesBackendConfig + clientset kubernetes.Interface + useImageVolumes bool } func (b *KubernetesBackend) Capabilities() backendCapabilities { @@ -206,8 +205,7 @@ func (b *KubernetesBackend) ExecuteTask(ctx context.Context, params *TaskParams) return executeError(newBackendFailure(metrics.TaskFailurePhaseBackend, metrics.TaskFailureReasonContainerCreate, err)) } params.Logs.AddEnv(b.config.TaskEnv) - useImageVolumes, err := b.useImageVolumesForTask(ctx, params.Sidecars) - if err != nil { + if err := validateTaskSidecars(params.Sidecars); err != nil { return executeError(newBackendFailure(metrics.TaskFailurePhaseBackend, metrics.TaskFailureReasonSidecarPrep, err)) } @@ -248,7 +246,7 @@ func (b *KubernetesBackend) ExecuteTask(ctx context.Context, params *TaskParams) var initContainers []corev1.Container for i, sidecar := range params.Sidecars { - if useImageVolumes { + if b.useImageVolumes && !sidecar.ReadWrite { volumeName := fmt.Sprintf("sidecar-%d-image", i) volumes = append(volumes, imageVolume(volumeName, sidecar.Image, pullPolicy)) @@ -843,7 +841,7 @@ func workspaceVolumeName() string { return "workspace" } -func validateTaskSidecars(sidecars []types.SidecarMount, useImageVolumes bool) error { +func validateTaskSidecars(sidecars []types.SidecarMount) error { seenMountPaths := make(map[string]bool) for _, sidecar := range sidecars { if sidecar.Image == "" { @@ -858,33 +856,13 @@ func validateTaskSidecars(sidecars []types.SidecarMount, useImageVolumes bool) e if seenMountPaths[sidecar.MountPath] { return fmt.Errorf("duplicate mount path %s for additional sidecar %s", sidecar.MountPath, sidecar.Image) } - if useImageVolumes && sidecar.ReadWrite { - return fmt.Errorf("additional sidecar %s cannot request a read-write mount: kubernetes image volumes are read-only", sidecar.Image) - } seenMountPaths[sidecar.MountPath] = true } return nil } -func (b *KubernetesBackend) useImageVolumesForTask(ctx context.Context, sidecars []types.SidecarMount) (bool, error) { - if err := validateTaskSidecars(sidecars, b.useImageVolumes && b.imageVolumesRequired); err != nil { - return false, err - } - if !b.useImageVolumes { - return false, nil - } - for _, sidecar := range sidecars { - if sidecar.ReadWrite { - log.Infof(ctx, "Using legacy sidecar copy path because a sidecar requests a read-write mount") - return false, nil - } - } - return true, nil -} - func (b *KubernetesBackend) configureImageVolumes(ctx context.Context) error { b.useImageVolumes = false - b.imageVolumesRequired = false if b.config.UseImageVolumes != nil && !*b.config.UseImageVolumes { if err := b.runStartupPreflight(ctx, false); err != nil { return err @@ -906,8 +884,7 @@ func (b *KubernetesBackend) configureImageVolumes(ctx context.Context) error { } b.useImageVolumes = true - b.imageVolumesRequired = b.config.UseImageVolumes != nil - if b.imageVolumesRequired { + if b.config.UseImageVolumes != nil { log.Infof(ctx, "Kubernetes image-volume mode resolved: forced-on") } else { log.Infof(ctx, "Kubernetes image-volume mode resolved: auto-on") diff --git a/internal/worker/kubernetes_test.go b/internal/worker/kubernetes_test.go index 4ca5df2..74a9765 100644 --- a/internal/worker/kubernetes_test.go +++ b/internal/worker/kubernetes_test.go @@ -928,58 +928,7 @@ func TestBuildTaskPodSpecRunnerShapeOverridesPodTemplateResources(t *testing.T) } } -func TestValidateTaskSidecarsAllowsReadWriteMountsByDefault(t *testing.T) { - err := validateTaskSidecars([]types.SidecarMount{ - { - Image: "registry.internal/agent:1.0", - MountPath: "/agent", - ReadWrite: true, - }, - }, false) - if err != nil { - t.Fatalf("expected read-write sidecar to be allowed by default, got %v", err) - } -} - -func TestValidateTaskSidecarsRejectsReadWriteMountsWhenImageVolumesEnabled(t *testing.T) { - err := validateTaskSidecars([]types.SidecarMount{ - { - Image: "registry.internal/agent:1.0", - MountPath: "/agent", - ReadWrite: true, - }, - }, true) - if err == nil { - t.Fatal("expected read-write sidecar validation failure") - } - if !strings.Contains(err.Error(), "read-write mount") { - t.Fatalf("expected read-write validation detail, got %v", err) - } -} - -func TestUseImageVolumesForTask(t *testing.T) { - readWriteSidecar := []types.SidecarMount{{ - Image: "registry.internal/agent:1.0", - MountPath: "/agent", - ReadWrite: true, - }} - - autoBackend := &KubernetesBackend{useImageVolumes: true} - useImageVolumes, err := autoBackend.useImageVolumesForTask(context.Background(), readWriteSidecar) - if err != nil { - t.Fatalf("auto mode returned an unexpected error: %v", err) - } - if useImageVolumes { - t.Fatal("auto mode should use the legacy copy path for a read-write sidecar") - } - - forcedBackend := &KubernetesBackend{useImageVolumes: true, imageVolumesRequired: true} - if _, err := forcedBackend.useImageVolumesForTask(context.Background(), readWriteSidecar); err == nil { - t.Fatal("forced-on mode should reject a read-write sidecar") - } -} - -func TestExecuteTaskUsesImageVolumesForSidecars(t *testing.T) { +func TestExecuteTaskMixesImageAndCopyVolumes(t *testing.T) { fakeClient := fake.NewSimpleClientset() jobWatch := watch.NewFake() podWatch := watch.NewFake() @@ -1028,9 +977,8 @@ func TestExecuteTaskUsesImageVolumesForSidecars(t *testing.T) { SetupCommand: "true", UseImageVolumes: workerBoolPtr(true), }, - clientset: fakeClient, - useImageVolumes: true, - imageVolumesRequired: true, + clientset: fakeClient, + useImageVolumes: true, } result := backend.ExecuteTask(context.Background(), &TaskParams{ @@ -1043,6 +991,11 @@ func TestExecuteTaskUsesImageVolumesForSidecars(t *testing.T) { Image: "registry.internal/agent:1.0", MountPath: "/agent", }, + { + Image: "registry.internal/tools:1.0", + MountPath: "/tools", + ReadWrite: true, + }, }, }) if result.Error != nil { @@ -1073,14 +1026,21 @@ func TestExecuteTaskUsesImageVolumesForSidecars(t *testing.T) { t.Fatalf("execution hash label = %q, want %q", createdJob.Labels[kubernetesExecutionHashLabel], kubernetesLabelHash("execution-1")) } - if len(createdJob.Spec.Template.Spec.InitContainers) != 1 { - t.Fatalf("expected only setup init container, got %d", len(createdJob.Spec.Template.Spec.InitContainers)) + if len(createdJob.Spec.Template.Spec.InitContainers) != 2 { + t.Fatalf("expected writable-sidecar copy and setup init containers, got %d", len(createdJob.Spec.Template.Spec.InitContainers)) } - if createdJob.Spec.Template.Spec.InitContainers[0].Name != kubernetesSetupContainerName { - t.Fatalf("expected setup init container, got %q", createdJob.Spec.Template.Spec.InitContainers[0].Name) + copyContainer := createdJob.Spec.Template.Spec.InitContainers[0] + if copyContainer.Name != "copy-sidecar-1" || copyContainer.Image != "registry.internal/tools:1.0" { + t.Fatalf("expected only the writable sidecar to be copied, got %+v", copyContainer) } - if len(createdJob.Spec.Template.Spec.Volumes) != 2 { - t.Fatalf("expected workspace and sidecar image volumes, got %d", len(createdJob.Spec.Template.Spec.Volumes)) + if len(copyContainer.VolumeMounts) != 1 || copyContainer.VolumeMounts[0].Name != "sidecar-1-data" { + t.Fatalf("expected copy output in the writable sidecar volume, got %+v", copyContainer.VolumeMounts) + } + if createdJob.Spec.Template.Spec.InitContainers[1].Name != kubernetesSetupContainerName { + t.Fatalf("expected setup after the sidecar copy, got %q", createdJob.Spec.Template.Spec.InitContainers[1].Name) + } + if len(createdJob.Spec.Template.Spec.Volumes) != 3 { + t.Fatalf("expected workspace, image, and writable data volumes, got %d", len(createdJob.Spec.Template.Spec.Volumes)) } var sidecarVolume *corev1.Volume @@ -1102,6 +1062,10 @@ func TestExecuteTaskUsesImageVolumesForSidecars(t *testing.T) { if sidecarVolume.Image.PullPolicy != corev1.PullAlways { t.Fatalf("image volume pullPolicy = %q, want %q", sidecarVolume.Image.PullPolicy, corev1.PullAlways) } + writableVolume := createdJob.Spec.Template.Spec.Volumes[2] + if writableVolume.Name != "sidecar-1-data" || writableVolume.EmptyDir == nil { + t.Fatalf("expected an emptyDir for the writable sidecar, got %+v", writableVolume) + } taskContainer := createdJob.Spec.Template.Spec.Containers[0] envMap := make(map[string]string, len(taskContainer.Env)) @@ -1141,7 +1105,7 @@ func TestExecuteTaskUsesImageVolumesForSidecars(t *testing.T) { t.Fatal("expected task sidecar mount to be read-only") } - setupContainer := createdJob.Spec.Template.Spec.InitContainers[0] + setupContainer := createdJob.Spec.Template.Spec.InitContainers[1] var setupSidecarMount *corev1.VolumeMount for i := range setupContainer.VolumeMounts { if setupContainer.VolumeMounts[i].Name == "sidecar-0-image" { @@ -1155,6 +1119,20 @@ func TestExecuteTaskUsesImageVolumesForSidecars(t *testing.T) { if !setupSidecarMount.ReadOnly { t.Fatal("expected setup sidecar mount to be read-only") } + for _, container := range []corev1.Container{taskContainer, setupContainer} { + found := false + for _, mount := range container.VolumeMounts { + if mount.MountPath == "/tools" { + found = true + if mount.Name != "sidecar-1-data" || mount.ReadOnly { + t.Fatalf("expected a writable copied-sidecar mount in %s, got %+v", container.Name, mount) + } + } + } + if !found { + t.Fatalf("missing writable sidecar mount in %s", container.Name) + } + } } func TestExecuteTaskUsesCopyInitContainersByDefault(t *testing.T) { @@ -1925,7 +1903,6 @@ func TestConfigureImageVolumes(t *testing.T) { imageVolumesSupported bool wantModes []bool wantUseImageVolumes bool - wantRequired bool wantError bool }{ { @@ -1958,7 +1935,6 @@ func TestConfigureImageVolumes(t *testing.T) { imageVolumesSupported: true, wantModes: []bool{true}, wantUseImageVolumes: true, - wantRequired: true, }, } @@ -1980,13 +1956,27 @@ func TestConfigureImageVolumes(t *testing.T) { if backend.useImageVolumes != tt.wantUseImageVolumes { t.Fatalf("useImageVolumes = %t, want %t", backend.useImageVolumes, tt.wantUseImageVolumes) } - if backend.imageVolumesRequired != tt.wantRequired { - t.Fatalf("imageVolumesRequired = %t, want %t", backend.imageVolumesRequired, tt.wantRequired) - } }) } } +func TestConfigureImageVolumesFallsBackOnInconclusiveProbe(t *testing.T) { + backend, createdModes := newImageVolumeModeTestBackend(t, nil, true) + backend.clientset.(*fake.Clientset).PrependReactor("get", "jobs", func(action k8stesting.Action) (bool, runtime.Object, error) { + return true, nil, apierrors.NewTimeoutError("temporary API timeout", 1) + }) + + if err := backend.configureImageVolumes(context.Background()); err != nil { + t.Fatalf("unexpected startup error after inconclusive image-volume probe: %v", err) + } + if backend.useImageVolumes { + t.Fatal("inconclusive probe should leave image volumes disabled") + } + if len(*createdModes) != 2 || !(*createdModes)[0] || (*createdModes)[1] { + t.Fatalf("expected image-volume probe followed by legacy preflight, got %v", *createdModes) + } +} + func TestRunStartupPreflightCreatesImageVolumeJobAndWaitsForSuccess(t *testing.T) { fakeClient := fake.NewSimpleClientset() const preflightImage = "registry.internal/platform/preflight:1.0"