From 45304d7e4c150d91880020d4541b2b6e7d2b256e Mon Sep 17 00:00:00 2001 From: Alex Rodriguez <131964409+ezekiel-alexrod@users.noreply.github.com> Date: Tue, 13 Oct 2026 12:24:57 +0200 Subject: [PATCH 1/7] ci: run the agent e2e suite on kind for each pull request The e2e suite only ran by hand, so a broken manifest, RBAC rule or DaemonSet showed up late. A new e2e job in agent-ci runs it on every pull request and on pushes to main that touch agent/. The Makefile now downloads kind at a pinned version, like the other tools, so local runs and CI use the same kind and the same node image. CI pins kubectl to the minor version of that image. No image is pushed: the suite loads it into kind. Relates-to: MK8S-450 --- .github/workflows/agent-ci.yaml | 27 +++++++++++++++++++++++++++ CONTRIBUTING.md | 8 ++++---- agent/AGENTS.md | 7 ++++--- agent/Makefile | 26 ++++++++++++++------------ agent/README.md | 7 +++---- 5 files changed, 52 insertions(+), 23 deletions(-) diff --git a/.github/workflows/agent-ci.yaml b/.github/workflows/agent-ci.yaml index ca5d852..d424781 100644 --- a/.github/workflows/agent-ci.yaml +++ b/.github/workflows/agent-ci.yaml @@ -50,3 +50,30 @@ jobs: uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - name: Build the container image run: make -C agent docker-build + + e2e: + runs-on: ubuntu-24.04 + # Without a limit, a cluster that never gets ready holds the runner for + # the 6 hour default. + timeout-minutes: 30 + env: + # Same minor version as the Kubernetes of the kind node image, which + # KIND_VERSION in agent/Makefile sets. Bump both together. + KUBECTL_VERSION: v1.37.1 + KUBECTL_SHA256: 65691ff77eb6fa44c908b77a1082c9f092c3b9733b5cefabec0d1104890e21a8 + steps: + - name: Checkout + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + - name: Set up Go + uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 + with: + go-version-file: agent/go.mod + cache-dependency-path: agent/go.sum + - name: Install kubectl + run: | + curl -fsSLo kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" + echo "${KUBECTL_SHA256} kubectl" | sha256sum --check + sudo install -m 0755 kubectl /usr/local/bin/kubectl + rm kubectl + - name: Run the e2e tests + run: make -C agent test-e2e diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 34baa17..66475c2 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -29,10 +29,10 @@ in [agent/README.md](agent/README.md#deploying). ## Development environment -The agent needs Go 1.26+. controller-gen, kustomize, envtest and golangci-lint -are downloaded into `agent/bin/` at pinned versions by the `Makefile`, so those -need no manual install. `kubectl` and `kind` do: the deploy and e2e targets -expect them on your `PATH`. +The agent needs Go 1.26+. controller-gen, kustomize, envtest, golangci-lint and +kind are downloaded into `agent/bin/` at pinned versions by the `Makefile`, so +those need no manual install. `kubectl` does: the deploy and e2e targets expect +it on your `PATH`. CI pins it in `.github/workflows/agent-ci.yaml`. One version escapes the pinning. The `logcheck` plugin of the custom golangci-lint build resolves to `latest` in `agent/.custom-gcl.yml`, so a new diff --git a/agent/AGENTS.md b/agent/AGENTS.md index 0c0010b..49c7102 100644 --- a/agent/AGENTS.md +++ b/agent/AGENTS.md @@ -57,12 +57,13 @@ and several kustomizations. make fmt vet # both run as prerequisites of test, listed here for clarity make test # unit tests plus the envtest suite make lint # custom golangci-lint build, logcheck included -make test-e2e # needs kind and an idle cluster name, see below +make test-e2e # needs kubectl and an idle cluster name, see below ``` `make test-e2e` creates the cluster `image-cache-agent-test-e2e`, builds the -image, loads it and deploys the manifests. It needs `kind` and `kubectl` on -your `PATH`; neither is downloaded by the `Makefile`. +image, loads it and deploys the manifests. The `Makefile` downloads `kind` +into `bin/`, but `kubectl` has to be on your `PATH`. CI runs it on every pull +request, in the `e2e` job of `.github/workflows/agent-ci.yaml`. A green `make lint` right after a dependency bump is not proof. Run `bin/golangci-lint cache clean` first, for the reason diff --git a/agent/Makefile b/agent/Makefile index e2d8c11..b135e33 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -63,20 +63,14 @@ vet: ## Run go vet against code. test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out -# TODO(user): To use a different vendor for e2e tests, modify the setup under 'tests/e2e'. -# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. -# kubectl kuberc is disabled by default for test isolation; enable with: +# The e2e suite runs on a kind cluster. It builds the manager image and loads it +# into kind, so no registry is involved. kubectl kuberc is disabled by default +# for test isolation; enable with: # - KUBECTL_KUBERC=true -# CertManager is installed by default; skip with: -# - CERT_MANAGER_INSTALL_SKIP=true KIND_CLUSTER ?= image-cache-agent-test-e2e .PHONY: setup-test-e2e -setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist - @command -v $(KIND) >/dev/null 2>&1 || { \ - echo "Kind is not installed. Please install Kind manually."; \ - exit 1; \ - } +setup-test-e2e: kind ## Set up a Kind cluster for e2e tests if it does not exist @case "$$($(KIND) get clusters)" in \ *"$(KIND_CLUSTER)"*) \ echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ @@ -96,7 +90,7 @@ test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expect exit $$status .PHONY: cleanup-test-e2e -cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests +cleanup-test-e2e: kind ## Tear down the Kind cluster used for e2e tests @$(KIND) delete cluster --name $(KIND_CLUSTER) .PHONY: lint @@ -194,7 +188,7 @@ $(LOCALBIN): ## Tool Binaries KUBECTL ?= kubectl -KIND ?= kind +KIND ?= $(LOCALBIN)/kind KUSTOMIZE ?= $(LOCALBIN)/kustomize CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen ENVTEST ?= $(LOCALBIN)/setup-envtest @@ -215,6 +209,9 @@ ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/') GOLANGCI_LINT_VERSION ?= v2.12.2 +# The kind release also pins the Kubernetes version of the e2e cluster, through +# its default node image. +KIND_VERSION ?= v0.33.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. $(KUSTOMIZE): $(LOCALBIN) @@ -238,6 +235,11 @@ envtest: $(ENVTEST) ## Download setup-envtest locally if necessary. $(ENVTEST): $(LOCALBIN) $(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION)) +.PHONY: kind +kind: $(KIND) ## Download kind locally if necessary. +$(KIND): $(LOCALBIN) + $(call go-install-tool,$(KIND),sigs.k8s.io/kind,$(KIND_VERSION)) + .PHONY: golangci-lint golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary. $(GOLANGCI_LINT): $(LOCALBIN) diff --git a/agent/README.md b/agent/README.md index 5730532..315e51c 100644 --- a/agent/README.md +++ b/agent/README.md @@ -17,11 +17,10 @@ All the commands below run from this directory. ## Prerequisites -- Go 1.26+ for building and testing. controller-gen, kustomize, envtest and - golangci-lint are downloaded into `bin/` at pinned versions by the +- Go 1.26+ for building and testing. controller-gen, kustomize, envtest, + golangci-lint and kind are downloaded into `bin/` at pinned versions by the `Makefile`. -- `kubectl`, and `kind` for the e2e suite. Neither is downloaded: the targets - expect them on your `PATH`. +- `kubectl`. It is not downloaded: the targets expect it on your `PATH`. - Docker, or another `CONTAINER_TOOL`, for the image. - A cluster on Kubernetes 1.25+, which is where CEL validation of custom resources landed. The CRD relies on it. From f5f2f027be9fc4cfa4cf05475fb0924e939f78ad Mon Sep 17 00:00:00 2001 From: Alex Rodriguez <131964409+ezekiel-alexrod@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:03:43 +0200 Subject: [PATCH 2/7] fix(agent): delete the e2e cluster when the suite fails test-e2e meant to tear the cluster down whatever the outcome, but the Makefile shell runs with -e, so a failing go test ended the recipe before cleanup-test-e2e. The next run then reused that cluster without a word, as the comment above the recipe warns. Relates-to: MK8S-450 --- agent/Makefile | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/agent/Makefile b/agent/Makefile index b135e33..40c8afd 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -83,9 +83,10 @@ setup-test-e2e: kind ## Set up a Kind cluster for e2e tests if it does not exist test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. # Tear the cluster down even when the tests fail, keeping their exit code: # setup-test-e2e reuses an existing cluster, so a leftover one would silently -# run the next attempt against dirty state. - KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v; \ - status=$$?; \ +# run the next attempt against dirty state. The `|| status=` keeps the shell +# alive past a failure: .SHELLFLAGS has -e. + status=0; \ + KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v || status=$$?; \ $(MAKE) cleanup-test-e2e; \ exit $$status From a6d0ea54cc8c3bd2059ef958eeb76d8401c3e7f0 Mon Sep 17 00:00:00 2001 From: Alex Rodriguez <131964409+ezekiel-alexrod@users.noreply.github.com> Date: Wed, 14 Oct 2026 12:03:43 +0200 Subject: [PATCH 3/7] test(agent): cover the sync path, TLS and node selection in e2e The e2e suite never pulled an image: it checked the DaemonSet, the garbage collection at startup and the pending label with an unreachable source. A broken pull, extraction or synced label would only show in MetalK8s. The cluster now has a control plane and a worker, and two registries run in it. One serves plain HTTP: an ImageCache selecting the worker gets synced there, with the archive and the sentinel on disk, nothing lands on the control plane, and deletion collects it. The other is signed by a throwaway CA: the agent first refuses it, then pulls once the registry-ca ConfigMap and the shipped CA patch are applied. Relates-to: MK8S-450 --- CONTRIBUTING.md | 2 +- agent/AGENTS.md | 14 +- agent/DESIGN.md | 11 +- agent/Makefile | 30 +- agent/test/e2e/e2e_test.go | 474 +++++++++++++++++++++++++++++--- agent/test/e2e/kind-config.yaml | 7 + agent/test/utils/utils.go | 37 ++- 7 files changed, 508 insertions(+), 67 deletions(-) create mode 100644 agent/test/e2e/kind-config.yaml diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66475c2..b30b7a3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,7 +74,7 @@ make -C rpm rpm EL=9 # build both RPMs into rpm/_build/ per-resource failures with `utilerrors.NewAggregate`, so one bad resource does not stall the others. - **Tests**: Go's standard `testing` for the packages, Ginkgo + Gomega with - envtest for the reconciler, and a kind-based smoke test under `test/e2e`. + envtest for the reconciler, and a kind-based e2e suite under `test/e2e`. A behaviour that can fail on a node deserves a test that reproduces it, not only a unit test of the helper it calls: a partial extraction, a missing mount, a manual deletion. diff --git a/agent/AGENTS.md b/agent/AGENTS.md index 49c7102..3514b4c 100644 --- a/agent/AGENTS.md +++ b/agent/AGENTS.md @@ -30,7 +30,7 @@ internal/controller/ the node reconciler, the filesystem watcher, the labels internal/cache/ the on-disk store: state, extraction, garbage collection internal/puller/ pulling an image and streaming its layers config/ CRD, RBAC and DaemonSet manifests (kustomize) -test/e2e/ kind-based smoke test, builds and deploys for real +test/e2e/ kind-based e2e suite, builds, deploys and pulls for real ``` ## Generated files @@ -60,10 +60,14 @@ make lint # custom golangci-lint build, logcheck included make test-e2e # needs kubectl and an idle cluster name, see below ``` -`make test-e2e` creates the cluster `image-cache-agent-test-e2e`, builds the -image, loads it and deploys the manifests. The `Makefile` downloads `kind` -into `bin/`, but `kubectl` has to be on your `PATH`. CI runs it on every pull -request, in the `e2e` job of `.github/workflows/agent-ci.yaml`. +`make test-e2e` creates the two-node cluster `image-cache-agent-test-e2e` +(`test/e2e/kind-config.yaml`), builds the image, loads it and deploys the +manifests. It deploys two registries in the cluster, one plain HTTP and one +signed by a throwaway CA, so a run pulls `registry:3.1.2` from Docker Hub +(`registryImage` in `test/e2e/e2e_test.go`, pinned by digest). The +`Makefile` downloads `kind` into `bin/`, but `kubectl` has to be on your +`PATH`. CI runs it on every pull request, in the `e2e` job of +`.github/workflows/agent-ci.yaml`. A green `make lint` right after a dependency bump is not proof. Run `bin/golangci-lint cache clean` first, for the reason diff --git a/agent/DESIGN.md b/agent/DESIGN.md index 48fd9de..73c3243 100644 --- a/agent/DESIGN.md +++ b/agent/DESIGN.md @@ -382,7 +382,10 @@ The agent never writes ImageCache resources (no status, no finalizers). - envtest: the full reconciler with a fake puller, from resource lifecycle to node labels and on-disk state: failure paths, adoption of a seeded directory, and entries the agent did not write. -- A minimal kind-based e2e smoke test, with no registry infrastructure: CRD - installed, agent running, node labelled `pending` for a resource with an - unreachable source, label cleared on deletion, and garbage collection of - root-owned directories, which needs `DAC_OVERRIDE`. +- A kind-based e2e suite on a control plane and a worker: CRD installed, + agent running on both nodes, garbage collection of root-owned directories, + which needs `DAC_OVERRIDE`, and node labels cleared on deletion. A source + nobody listens on leaves the node `pending`. Two registries run in the + cluster: a pull over plain HTTP lands on the selected node only, and a + registry signed by a private CA is refused until the shipped CA patch is + applied. diff --git a/agent/Makefile b/agent/Makefile index 40c8afd..d0700aa 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -64,29 +64,39 @@ test: manifests generate fmt vet setup-envtest ## Run tests. KUBEBUILDER_ASSETS="$(shell "$(ENVTEST)" use $(ENVTEST_K8S_VERSION) --bin-dir "$(LOCALBIN)" -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out # The e2e suite runs on a kind cluster. It builds the manager image and loads it -# into kind, so no registry is involved. kubectl kuberc is disabled by default +# into kind, so that image is never pushed. kubectl kuberc is disabled by default # for test isolation; enable with: # - KUBECTL_KUBERC=true KIND_CLUSTER ?= image-cache-agent-test-e2e +# The node count of test/e2e/kind-config.yaml. A cluster left over with another +# count would make the node assertions fail for a reason unrelated to the agent. +KIND_NODES = 2 .PHONY: setup-test-e2e setup-test-e2e: kind ## Set up a Kind cluster for e2e tests if it does not exist - @case "$$($(KIND) get clusters)" in \ - *"$(KIND_CLUSTER)"*) \ - echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ - *) \ - echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ - $(KIND) create cluster --name $(KIND_CLUSTER) ;; \ - esac + @if $(KIND) get clusters | grep -qx '$(KIND_CLUSTER)'; then \ + nodes=$$($(KIND) get nodes --name $(KIND_CLUSTER) | wc -l); \ + if [ "$$nodes" -ne $(KIND_NODES) ]; then \ + echo "Kind cluster '$(KIND_CLUSTER)' has $$nodes nodes, the e2e suite needs $(KIND_NODES)." \ + "Delete it with 'make cleanup-test-e2e'."; \ + exit 1; \ + fi; \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation."; \ + else \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) --config test/e2e/kind-config.yaml; \ + fi .PHONY: test-e2e test-e2e: setup-test-e2e manifests generate fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. # Tear the cluster down even when the tests fail, keeping their exit code: # setup-test-e2e reuses an existing cluster, so a leftover one would silently # run the next attempt against dirty state. The `|| status=` keeps the shell -# alive past a failure: .SHELLFLAGS has -e. +# alive past a failure: .SHELLFLAGS has -e. The 25m timeout replaces go +# test's 10m default, which a failing suite can reach before Ginkgo prints +# which spec failed, and stays under the 30m of the CI job. status=0; \ - KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -v -ginkgo.v || status=$$?; \ + KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./test/e2e/ -timeout 25m -v -ginkgo.v || status=$$?; \ $(MAKE) cleanup-test-e2e; \ exit $$status diff --git a/agent/test/e2e/e2e_test.go b/agent/test/e2e/e2e_test.go index d34d103..8ef3e92 100644 --- a/agent/test/e2e/e2e_test.go +++ b/agent/test/e2e/e2e_test.go @@ -20,12 +20,29 @@ limitations under the License. package e2e import ( + "bufio" + "bytes" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" "encoding/json" + "encoding/pem" "fmt" + "math/big" + "net" + "net/http" "os/exec" + "regexp" "strings" "time" + "github.com/google/go-containerregistry/pkg/crane" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -45,7 +62,39 @@ const imageCacheName = "imagecache-e2e-smoke" // nodeLabelKey is the sync-status label the agent sets on a selected node for // imageCacheName. See api/v1alpha1.ImageCache's doc comment for the contract. -const nodeLabelKey = "image-cache.scality.com/" + imageCacheName +var nodeLabelKey = labelKey(imageCacheName) + +// agentSelector selects the agent pods, one per node. +const agentSelector = "control-plane=controller-manager" + +// syncedName is the ImageCache that pulls a real image from the in-cluster +// registry, so that the whole sync path runs: pull, extraction, sentinel, +// synced label, then garbage collection once it is deleted. +const syncedName = "imagecache-e2e-synced" + +// tlsName is the ImageCache that pulls from the registry signed by a +// throwaway CA, to exercise --ca-file through the shipped patch. +const tlsName = "imagecache-e2e-tls" + +// cachePath is the agent's default --cache-path, mounted by the manifests. +const cachePath = "/var/lib/image-cache" + +// registryNamespace holds the two registries the agent pulls from, one plain +// HTTP and one TLS. Sources name them by their Service, a host name: without +// --plain-http the agent only tries HTTPS, as it would for a real registry. +const registryNamespace = "image-cache-e2e-registry" + +// registryImage is pinned by digest, as it is pulled from Docker Hub on every +// run. +const registryImage = "docker.io/library/registry:3.1.2@sha256:" + + "ddf754342cfc8acc51a56d5d0ab6af06826461864460636d8bd5c546dab2a7b8" + +// carrierFile is the archive the test image carries, and carrierBody its +// content. The agent extracts it under its base name. +const ( + carrierFile = "e2e.tar" + carrierBody = "e2e" +) // preexisting is what a node holds before the agent runs, written as root in // mode 0700: a directory imagecachectl seeded, which stays, then what goes: @@ -69,8 +118,6 @@ var seedScript = "set -e\n" + "echo '{not json' > " + preexisting[2] + "/.image-cache-agent.json\n" var _ = Describe("Manager", Ordered, func() { - var controllerPodName string - // Before running the tests, set up the environment by creating the namespace, // enforce the restricted security policy to the namespace, installing CRDs, // and deploying the controller. @@ -110,8 +157,12 @@ var _ = Describe("Manager", Ordered, func() { By("removing the preexisting directories") _, _ = utils.OnKindNode("rm -rf " + strings.Join(preexisting, " ")) - By("deleting the smoke-test ImageCache, in case an earlier step left it behind") - cmd := exec.Command("kubectl", "delete", "imagecache", imageCacheName, "--ignore-not-found") + By("deleting the e2e ImageCaches, in case an earlier step left them behind") + cmd := exec.Command("kubectl", "delete", "imagecache", imageCacheName, syncedName, tlsName, "--ignore-not-found") + _, _ = utils.Run(cmd) + + By("removing the registry") + cmd = exec.Command("kubectl", "delete", "ns", registryNamespace, "--ignore-not-found") _, _ = utils.Run(cmd) By("undeploying the controller-manager") @@ -132,8 +183,8 @@ var _ = Describe("Manager", Ordered, func() { AfterEach(func() { specReport := CurrentSpecReport() if specReport.Failed() { - By("Fetching controller manager pod logs") - cmd := exec.Command("kubectl", "logs", controllerPodName, "-n", namespace) + By("Fetching the logs of every agent pod") + cmd := exec.Command("kubectl", "logs", "-l", agentSelector, "-n", namespace, "--prefix", "--tail=-1") controllerLogs, err := utils.Run(cmd) if err == nil { _, _ = fmt.Fprintf(GinkgoWriter, "Controller logs:\n %s", controllerLogs) @@ -151,7 +202,7 @@ var _ = Describe("Manager", Ordered, func() { } By("Fetching controller manager pod description") - cmd = exec.Command("kubectl", "describe", "pod", controllerPodName, "-n", namespace) + cmd = exec.Command("kubectl", "describe", "pods", "-l", agentSelector, "-n", namespace) podDescription, err := utils.Run(cmd) if err == nil { fmt.Println("Pod description:\n", podDescription) @@ -182,30 +233,21 @@ var _ = Describe("Manager", Ordered, func() { } Eventually(verifyDaemonSetReady).Should(Succeed()) - By("getting the name of the controller-manager pod") - cmd := exec.Command("kubectl", "get", - "pods", "-l", "control-plane=controller-manager", - "-o", "go-template={{ range .items }}"+ - "{{ if not .metadata.deletionTimestamp }}"+ - "{{ .metadata.name }}"+ - "{{ \"\\n\" }}{{ end }}{{ end }}", - "-n", namespace, - ) - podOutput, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred(), "Failed to retrieve controller-manager pod information") - podNames := utils.GetNonEmptyLines(podOutput) - Expect(podNames).To(HaveLen(1), "expected 1 controller pod running") - controllerPodName = podNames[0] - Expect(controllerPodName).To(ContainSubstring("controller-manager")) + By("checking that one agent runs on each of the two nodes") + cmd := exec.Command("kubectl", "get", "pods", "-l", agentSelector, "-n", namespace, + "-o", "jsonpath={range .items[*]}{.spec.nodeName}{\"\\n\"}{end}") + out, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to list the agent pods") + Expect(utils.GetNonEmptyLines(out)).To(ConsistOf(utils.ControlPlaneNode(), utils.WorkerNode())) }) It("should collect what no resource keeps, and keep what imagecachectl seeded", func() { // They are root's, in mode 0700, and some hold a file: removing // them takes DAC_OVERRIDE. for _, gone := range preexisting[1:] { - Eventually(func() (string, error) { - return utils.OnKindNode("test -e " + gone + " && echo present || echo gone") - }).Should(Equal("gone\n"), gone) + Eventually(func() (bool, error) { + return pathExists(utils.ControlPlaneNode(), gone) + }).Should(BeFalse(), gone) } out, err := utils.OnKindNode("stat -c '%u %n' " + preexisting[0] + @@ -219,13 +261,11 @@ var _ = Describe("Manager", Ordered, func() { // exercised without any registry infrastructure by pointing source at // an address nothing listens on. It("should label the node pending for an ImageCache with an unreachable source, then clear it on deletion", func() { - By("getting the name of the (single) kind node") - cmd := exec.Command("kubectl", "get", "nodes", "-o", "jsonpath={.items[0].metadata.name}") - nodeName, err := utils.Run(cmd) - Expect(err).NotTo(HaveOccurred()) - Expect(nodeName).NotTo(BeEmpty()) + // The ImageCache selects both nodes; one is enough to check the + // contract, node selection has a spec of its own. + nodeName := utils.ControlPlaneNode() - By("creating an ImageCache selecting this node with an unreachable source") + By("creating an ImageCache selecting every node with an unreachable source") imageCacheYAML := fmt.Sprintf(`apiVersion: image-cache.scality.com/v1alpha1 kind: ImageCache metadata: @@ -235,9 +275,9 @@ spec: kubernetes.io/os: linux source: 127.0.0.1:1/nope:1 `, imageCacheName) - cmd = exec.Command("kubectl", "apply", "-f", "-") + cmd := exec.Command("kubectl", "apply", "-f", "-") cmd.Stdin = strings.NewReader(imageCacheYAML) - _, err = utils.Run(cmd) + _, err := utils.Run(cmd) Expect(err).NotTo(HaveOccurred(), "Failed to create the ImageCache") By("waiting for the node to be labelled pending") @@ -261,9 +301,375 @@ spec: } Eventually(verifyLabelGone, 2*time.Minute).Should(Succeed()) }) + + It("should pull over plain HTTP onto the selected node only, then collect it on deletion", func() { + deployRegistry("registry", "") + digest := pushCarrier("registry", nil) + + By("restarting the agent with --plain-http") + patchAgent("--type=json", + `-p=[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--plain-http"}]`) + + By("creating an ImageCache that selects the worker only") + worker, controlPlane := utils.WorkerNode(), utils.ControlPlaneNode() + applyImageCache(syncedName, worker, "registry") + syncedKey := labelKey(syncedName) + dir := cachePath + "/" + syncedName + + By("waiting for the worker to be labelled synced") + Eventually(func(g Gomega) { + labels, err := nodeLabels(worker) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(labels).To(HaveKeyWithValue(syncedKey, "synced")) + }).Should(Succeed()) + expectExtracted(worker, dir, digest) + + By("checking that the control plane, not selected, got neither label nor directory") + labels, err := nodeLabels(controlPlane) + Expect(err).NotTo(HaveOccurred()) + Expect(labels).NotTo(HaveKey(syncedKey)) + Expect(pathExists(controlPlane, dir)).To(BeFalse()) + + By("deleting the ImageCache") + cmd := exec.Command("kubectl", "delete", "imagecache", syncedName) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to delete the ImageCache") + + By("waiting for the directory and the label to go") + Eventually(func() (bool, error) { + return pathExists(worker, dir) + }).Should(BeFalse()) + Eventually(func(g Gomega) { + labels, err := nodeLabels(worker) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(labels).NotTo(HaveKey(syncedKey)) + }).Should(Succeed()) + }) + + It("should refuse a registry signed by an unknown CA, then pull once the registry CA patch is applied", func() { + ca, cert, key := newRegistryCerts("registry-tls." + registryNamespace + ".svc.cluster.local") + deployRegistry("registry-tls", tlsSecret("registry-tls", cert, key)) + digest := pushCarrier("registry-tls", ca) + + By("creating an ImageCache for the TLS registry, the agent trusting the system CAs only") + controlPlane := utils.ControlPlaneNode() + applyImageCache(tlsName, controlPlane, "registry-tls") + + By("waiting for the agent to report the unknown authority") + Eventually(func() (string, error) { + cmd := exec.Command("kubectl", "logs", "-l", agentSelector, "-n", namespace, "--tail=-1") + return utils.Run(cmd) + }).Should(ContainSubstring("certificate signed by unknown authority")) + labels, err := nodeLabels(controlPlane) + Expect(err).NotTo(HaveOccurred()) + Expect(labels).To(HaveKeyWithValue(labelKey(tlsName), "pending")) + + By("creating the registry-ca ConfigMap and applying the shipped CA patch") + cmd := exec.Command("kubectl", "create", "configmap", "registry-ca", "-n", namespace, + "--from-literal=ca.crt="+string(ca)) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create the registry-ca ConfigMap") + DeferCleanup(func() { + _, _ = utils.Run(exec.Command("kubectl", "delete", "configmap", "registry-ca", "-n", namespace)) + }) + patchAgent("--type=json", "--patch-file=config/default/manager_registry_ca_patch.yaml") + + By("waiting for the control plane to be labelled synced") + Eventually(func(g Gomega) { + labels, err := nodeLabels(controlPlane) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(labels).To(HaveKeyWithValue(labelKey(tlsName), "synced")) + }).Should(Succeed()) + expectExtracted(controlPlane, cachePath+"/"+tlsName, digest) + + By("deleting the ImageCache, so no agent-owned state outlives the run") + cmd = exec.Command("kubectl", "delete", "imagecache", tlsName) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to delete the ImageCache") + Eventually(func() (bool, error) { + return pathExists(controlPlane, cachePath+"/"+tlsName) + }).Should(BeFalse()) + Eventually(func(g Gomega) { + labels, err := nodeLabels(controlPlane) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(labels).NotTo(HaveKey(labelKey(tlsName))) + }).Should(Succeed()) + }) }) }) +// patchAgent patches the agent DaemonSet, waits for the new pods, and rolls +// the change back when the spec ends: --plain-http and --ca-file exclude each +// other, so each spec starts from the shipped manifests. +func patchAgent(args ...string) { + cmd := exec.Command("kubectl", append([]string{"patch", "daemonset", daemonSetName, "-n", namespace}, args...)...) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to patch the agent") + // Registered before the wait, so a rollout that times out is undone too. + DeferCleanup(func() { + // After the last spec, AfterAll has already removed the namespace + // when this runs, and there is nothing left to roll back. + if _, err := utils.Run(exec.Command("kubectl", "get", "ns", namespace)); err != nil { + return + } + cmd := exec.Command("kubectl", "rollout", "undo", "daemonset/"+daemonSetName, "-n", namespace) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to roll the agent back") + waitAgentRollout() + }) + waitAgentRollout() +} + +func waitAgentRollout() { + cmd := exec.Command("kubectl", "rollout", "status", "daemonset/"+daemonSetName, "-n", namespace, "--timeout=2m") + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "The agent did not roll out") +} + +// applyImageCache creates an ImageCache that selects one node and pulls the +// carrier image from the named in-cluster registry. +func applyImageCache(name, node, registry string) { + cmd := exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(fmt.Sprintf(`apiVersion: image-cache.scality.com/v1alpha1 +kind: ImageCache +metadata: + name: %s +spec: + nodeSelector: + kubernetes.io/hostname: %s + source: %s.%s.svc.cluster.local:5000/e2e/carrier:1 +`, name, node, registry, registryNamespace)) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to create the ImageCache") +} + +// expectExtracted checks the archive and the sentinel the agent wrote on node. +func expectExtracted(node, dir string, digest v1.Hash) { + By("checking the extracted archive and its sentinel on " + node) + out, err := utils.OnNode(node, "cat "+dir+"/"+carrierFile) + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(Equal(carrierBody)) + out, err = utils.OnNode(node, "cat "+dir+"/.image-cache-agent.json") + Expect(err).NotTo(HaveOccurred()) + var sentinel struct { + Digest string `json:"digest"` + Owner string `json:"owner"` + Files []string `json:"files"` + } + Expect(json.Unmarshal([]byte(out), &sentinel)).To(Succeed()) + Expect(sentinel.Owner).To(Equal("image-cache-agent")) + Expect(sentinel.Digest).To(Equal(digest.String())) + Expect(sentinel.Files).To(ConsistOf(carrierFile)) +} + +// deployRegistry deploys a single-replica registry and its Service, both +// called name, and waits for it. Given a TLS Secret manifest from tlsSecret, +// it applies it and the registry serves HTTPS with that certificate. +func deployRegistry(name, secret string) { + By("deploying the " + name + " registry in the cluster") + tlsEnv, tlsMount, tlsVolume, scheme := "", "", "", "HTTP" + if secret != "" { + tlsEnv = ` + env: + - name: REGISTRY_HTTP_TLS_CERTIFICATE + value: /certs/tls.crt + - name: REGISTRY_HTTP_TLS_KEY + value: /certs/tls.key` + tlsMount = ` + volumeMounts: + - name: certs + mountPath: /certs` + tlsVolume = ` + volumes: + - name: certs + secret: + secretName: ` + name + scheme = "HTTPS" + } + manifest := fmt.Sprintf(`apiVersion: v1 +kind: Namespace +metadata: + name: %[1]s +---%[3]s +apiVersion: apps/v1 +kind: Deployment +metadata: + name: %[2]s + namespace: %[1]s +spec: + replicas: 1 + selector: + matchLabels: + app: %[2]s + template: + metadata: + labels: + app: %[2]s + spec: + containers: + - name: registry + image: %[4]s + ports: + - containerPort: 5000%[5]s%[6]s + readinessProbe: + httpGet: + path: /v2/ + port: 5000 + scheme: %[7]s%[8]s +--- +apiVersion: v1 +kind: Service +metadata: + name: %[2]s + namespace: %[1]s +spec: + selector: + app: %[2]s + ports: + - port: 5000 +`, registryNamespace, name, secret, registryImage, tlsEnv, tlsMount, scheme, tlsVolume) + cmd := exec.Command("kubectl", "apply", "-f", "-") + cmd.Stdin = strings.NewReader(manifest) + _, err := utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to deploy the registry") + cmd = exec.Command("kubectl", "rollout", "status", "deployment/"+name, "-n", registryNamespace, "--timeout=2m") + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "The registry did not become ready") +} + +// tlsSecret returns the manifest of the Secret deployRegistry mounts for the +// registry called name, followed by the document separator it expects. +func tlsSecret(name string, cert, key []byte) string { + return fmt.Sprintf(` +apiVersion: v1 +kind: Secret +metadata: + name: %s + namespace: %s +type: kubernetes.io/tls +data: + tls.crt: %s + tls.key: %s +---`, name, registryNamespace, base64.StdEncoding.EncodeToString(cert), base64.StdEncoding.EncodeToString(key)) +} + +// pushCarrier pushes the carrier image to the named registry through a +// port-forward and returns its digest. With ca, the push trusts that CA, +// which also signed the certificate for localhost. +func pushCarrier(registry string, ca []byte) v1.Hash { + By("pushing the carrier image to " + registry + " through a port-forward") + local := portForward(registryNamespace, "svc/"+registry, 5000) + img, err := crane.Image(map[string][]byte{"images/" + carrierFile: []byte(carrierBody)}) + Expect(err).NotTo(HaveOccurred()) + opts := []crane.Option{crane.Insecure} + if ca != nil { + pool := x509.NewCertPool() + Expect(pool.AppendCertsFromPEM(ca)).To(BeTrue()) + tr := remote.DefaultTransport.(*http.Transport).Clone() + tr.TLSClientConfig = &tls.Config{RootCAs: pool, MinVersion: tls.VersionTLS12} + opts = []crane.Option{crane.WithTransport(tr)} + } + Expect(crane.Push(img, local+"/e2e/carrier:1", opts...)).To(Succeed()) + digest, err := img.Digest() + Expect(err).NotTo(HaveOccurred()) + return digest +} + +// newRegistryCerts returns a throwaway CA, and a certificate it signed for +// host and for localhost, the port-forward the test pushes through. All PEM. +func newRegistryCerts(host string) (ca, cert, key []byte) { + caKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).NotTo(HaveOccurred()) + caTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{CommonName: "image-cache e2e CA"}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + IsCA: true, + KeyUsage: x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + caDER, err := x509.CreateCertificate(rand.Reader, caTmpl, caTmpl, &caKey.PublicKey, caKey) + Expect(err).NotTo(HaveOccurred()) + + leafKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) + Expect(err).NotTo(HaveOccurred()) + leafTmpl := &x509.Certificate{ + SerialNumber: big.NewInt(2), + Subject: pkix.Name{CommonName: host}, + DNSNames: []string{host, "localhost"}, + IPAddresses: []net.IP{net.ParseIP("127.0.0.1")}, + NotBefore: time.Now().Add(-time.Hour), + NotAfter: time.Now().Add(24 * time.Hour), + KeyUsage: x509.KeyUsageDigitalSignature, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + } + leafDER, err := x509.CreateCertificate(rand.Reader, leafTmpl, caTmpl, &leafKey.PublicKey, caKey) + Expect(err).NotTo(HaveOccurred()) + keyDER, err := x509.MarshalECPrivateKey(leafKey) + Expect(err).NotTo(HaveOccurred()) + + return pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: caDER}), + pem.EncodeToMemory(&pem.Block{Type: "CERTIFICATE", Bytes: leafDER}), + pem.EncodeToMemory(&pem.Block{Type: "EC PRIVATE KEY", Bytes: keyDER}) +} + +// forwardingFrom matches the line kubectl prints once a port-forward listens. +var forwardingFrom = regexp.MustCompile(`Forwarding from 127\.0\.0\.1:(\d+)`) + +// portForward forwards a random local port to the target's port and returns +// "localhost:". The forward stops when the spec ends. +func portForward(ns, target string, port int) string { + cmd := exec.Command("kubectl", "port-forward", "-n", ns, target, fmt.Sprintf(":%d", port)) + stdout, err := cmd.StdoutPipe() + Expect(err).NotTo(HaveOccurred()) + var stderr bytes.Buffer + cmd.Stderr = &stderr + Expect(cmd.Start()).To(Succeed()) + + // One goroutine reads stdout to the end: it reports the port, then keeps + // draining so kubectl never blocks on a full pipe. Wait must only run + // once it is done, as os/exec requires. + found := make(chan string, 1) + drained := make(chan struct{}) + go func() { + defer close(drained) + lines := bufio.NewScanner(stdout) + for lines.Scan() { + if m := forwardingFrom.FindStringSubmatch(lines.Text()); m != nil { + found <- m[1] + } + } + }() + DeferCleanup(func() { + _ = cmd.Process.Kill() + <-drained + _ = cmd.Wait() + }) + + select { + case p := <-found: + return "localhost:" + p + case <-drained: + Fail("kubectl port-forward exited before listening: " + stderr.String()) + case <-time.After(30 * time.Second): + Fail("kubectl port-forward did not listen within 30s: " + stderr.String()) + } + return "" +} + +// pathExists reports whether path exists on the named kind node. +func pathExists(node, path string) (bool, error) { + out, err := utils.OnNode(node, "test -e "+path+" && echo present || echo gone") + return out == "present\n", err +} + +// labelKey returns the sync-status label the agent sets for an ImageCache. +// See api/v1alpha1.ImageCache's doc comment for the contract. +func labelKey(name string) string { + return "image-cache.scality.com/" + name +} + // nodeLabels returns the labels currently set on the named node. func nodeLabels(name string) (map[string]string, error) { cmd := exec.Command("kubectl", "get", "node", name, "-o", "jsonpath={.metadata.labels}") diff --git a/agent/test/e2e/kind-config.yaml b/agent/test/e2e/kind-config.yaml new file mode 100644 index 0000000..5a6a473 --- /dev/null +++ b/agent/test/e2e/kind-config.yaml @@ -0,0 +1,7 @@ +# Two nodes, so the e2e suite can check that an ImageCache lands on the nodes +# its nodeSelector picks and nowhere else. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker diff --git a/agent/test/utils/utils.go b/agent/test/utils/utils.go index c222ad2..1f69e69 100644 --- a/agent/test/utils/utils.go +++ b/agent/test/utils/utils.go @@ -52,28 +52,39 @@ func Run(cmd *exec.Cmd) (string, error) { return string(output), nil } -// kindNode returns the container of the kind cluster's control plane node, -// the only node the e2e cluster has. -func kindNode() string { - cluster := defaultKindCluster +// kindCluster returns the name of the e2e kind cluster. +func kindCluster() string { if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { - cluster = v + return v } - return cluster + "-control-plane" + return defaultKindCluster } -// OnKindNode runs a shell script as root on the kind node. +// ControlPlaneNode returns the name of the kind control plane node. kind names +// the node and its container the same way. +func ControlPlaneNode() string { + return kindCluster() + "-control-plane" +} + +// WorkerNode returns the name of the kind worker node, see +// test/e2e/kind-config.yaml. +func WorkerNode() string { + return kindCluster() + "-worker" +} + +// OnKindNode runs a shell script as root on the kind control plane node. func OnKindNode(script string) (string, error) { - return Run(exec.Command("docker", "exec", kindNode(), "sh", "-c", script)) + return OnNode(ControlPlaneNode(), script) +} + +// OnNode runs a shell script as root on the named kind node. +func OnNode(node, script string) (string, error) { + return Run(exec.Command("docker", "exec", node, "sh", "-c", script)) } // LoadImageToKindClusterWithName loads a local docker image to the kind cluster func LoadImageToKindClusterWithName(name string) error { - cluster := defaultKindCluster - if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { - cluster = v - } - kindOptions := []string{"load", "docker-image", name, "--name", cluster} + kindOptions := []string{"load", "docker-image", name, "--name", kindCluster()} kindBinary := defaultKindBinary if v, ok := os.LookupEnv("KIND"); ok { kindBinary = v From c53ee64e90114550870d56d73cfa12545c18a3dc Mon Sep 17 00:00:00 2001 From: Alex Rodriguez <131964409+ezekiel-alexrod@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:22:05 +0200 Subject: [PATCH 4/7] ci: let Renovate bump the e2e tools and images kind, kubectl and the registry image of the e2e only moved by hand. They get the annotations our other kubebuilder repos use. The tools the other jobs share, golangci-lint, kustomize and controller-tools, are left to their own change. kubectl now comes from the registry.k8s.io/kubectl image, pinned by tag and digest. Renovate cannot follow the sha256 of a file on dl.k8s.io, but it follows an image digest, and the binary is the same. The tag and digest sit alone in their value, as the YAML and Go regexes of the shared preset do not read a full reference. Relates-to: MK8S-450 --- .github/workflows/agent-ci.yaml | 17 +++++++++-------- agent/AGENTS.md | 2 +- agent/Makefile | 1 + agent/test/e2e/e2e_test.go | 10 ++++++---- 4 files changed, 17 insertions(+), 13 deletions(-) diff --git a/.github/workflows/agent-ci.yaml b/.github/workflows/agent-ci.yaml index d424781..10cc7f8 100644 --- a/.github/workflows/agent-ci.yaml +++ b/.github/workflows/agent-ci.yaml @@ -57,10 +57,11 @@ jobs: # the 6 hour default. timeout-minutes: 30 env: - # Same minor version as the Kubernetes of the kind node image, which - # KIND_VERSION in agent/Makefile sets. Bump both together. - KUBECTL_VERSION: v1.37.1 - KUBECTL_SHA256: 65691ff77eb6fa44c908b77a1082c9f092c3b9733b5cefabec0d1104890e21a8 + # Taken from the official image, so Renovate can bump the tag and the + # digest together. kubectl supports a cluster one minor version away, + # so it may move before KIND_VERSION in agent/Makefile. + # renovate: datasource=docker depName=registry.k8s.io/kubectl + KUBECTL_VERSION: v1.37.1@sha256:b7cab618e281b1ee7484e7b706a96e2135fbb6e072c2a573a7dab4e87d7f2385 steps: - name: Checkout uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 @@ -71,9 +72,9 @@ jobs: cache-dependency-path: agent/go.sum - name: Install kubectl run: | - curl -fsSLo kubectl "https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl" - echo "${KUBECTL_SHA256} kubectl" | sha256sum --check - sudo install -m 0755 kubectl /usr/local/bin/kubectl - rm kubectl + id=$(docker create --platform linux/amd64 "registry.k8s.io/kubectl:${KUBECTL_VERSION}") + sudo docker cp "$id":/bin/kubectl /usr/local/bin/kubectl + docker rm "$id" + kubectl version --client - name: Run the e2e tests run: make -C agent test-e2e diff --git a/agent/AGENTS.md b/agent/AGENTS.md index 3514b4c..59a58bd 100644 --- a/agent/AGENTS.md +++ b/agent/AGENTS.md @@ -64,7 +64,7 @@ make test-e2e # needs kubectl and an idle cluster name, see below (`test/e2e/kind-config.yaml`), builds the image, loads it and deploys the manifests. It deploys two registries in the cluster, one plain HTTP and one signed by a throwaway CA, so a run pulls `registry:3.1.2` from Docker Hub -(`registryImage` in `test/e2e/e2e_test.go`, pinned by digest). The +(`registryVersion` in `test/e2e/e2e_test.go`, pinned by digest). The `Makefile` downloads `kind` into `bin/`, but `kubectl` has to be on your `PATH`. CI runs it on every pull request, in the `e2e` job of `.github/workflows/agent-ci.yaml`. diff --git a/agent/Makefile b/agent/Makefile index d0700aa..13ea8a7 100644 --- a/agent/Makefile +++ b/agent/Makefile @@ -222,6 +222,7 @@ ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ GOLANGCI_LINT_VERSION ?= v2.12.2 # The kind release also pins the Kubernetes version of the e2e cluster, through # its default node image. +# renovate: datasource=github-releases depName=kubernetes-sigs/kind KIND_VERSION ?= v0.33.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. diff --git a/agent/test/e2e/e2e_test.go b/agent/test/e2e/e2e_test.go index 8ef3e92..97433dc 100644 --- a/agent/test/e2e/e2e_test.go +++ b/agent/test/e2e/e2e_test.go @@ -84,11 +84,13 @@ const cachePath = "/var/lib/image-cache" // --plain-http the agent only tries HTTPS, as it would for a real registry. const registryNamespace = "image-cache-e2e-registry" -// registryImage is pinned by digest, as it is pulled from Docker Hub on every -// run. -const registryImage = "docker.io/library/registry:3.1.2@sha256:" + - "ddf754342cfc8acc51a56d5d0ab6af06826461864460636d8bd5c546dab2a7b8" +// registryVersion pins the registry by digest, as it is pulled from Docker Hub +// on every run. Renovate reads the tag and digest alone. +// +// renovate: datasource=docker depName=docker.io/library/registry +const registryVersion = "3.1.2@sha256:ddf754342cfc8acc51a56d5d0ab6af06826461864460636d8bd5c546dab2a7b8" +const registryImage = "docker.io/library/registry:" + registryVersion // carrierFile is the archive the test image carries, and carrierBody its // content. The agent extracts it under its base name. const ( From 7c786986970b7d3422a6a9bce9e81f39fb7d2a10 Mon Sep 17 00:00:00 2001 From: Alex Rodriguez <131964409+ezekiel-alexrod@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:22:05 +0200 Subject: [PATCH 5/7] test(agent): delete every ImageCache when the e2e suite ends The cleanup listed the ImageCaches by name, so each new spec had to be added there or left one behind. They are cluster-scoped and the kind cluster is the suite's own, so --all removes only what it created. Relates-to: MK8S-450 --- agent/test/e2e/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/test/e2e/e2e_test.go b/agent/test/e2e/e2e_test.go index 97433dc..9f04374 100644 --- a/agent/test/e2e/e2e_test.go +++ b/agent/test/e2e/e2e_test.go @@ -160,7 +160,7 @@ var _ = Describe("Manager", Ordered, func() { _, _ = utils.OnKindNode("rm -rf " + strings.Join(preexisting, " ")) By("deleting the e2e ImageCaches, in case an earlier step left them behind") - cmd := exec.Command("kubectl", "delete", "imagecache", imageCacheName, syncedName, tlsName, "--ignore-not-found") + cmd := exec.Command("kubectl", "delete", "imagecache", "--all") _, _ = utils.Run(cmd) By("removing the registry") From 3804bd98d2cbbdaf8475ce3d561314e057f5c9f1 Mon Sep 17 00:00:00 2001 From: Alex Rodriguez <131964409+ezekiel-alexrod@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:22:09 +0200 Subject: [PATCH 6/7] test(agent): cover the adoption of what imagecachectl seeded in e2e The suite faked the seeded directory with a hand-written sentinel and never created a resource for it. So neither imagecachectl nor the adoption ran: a broken diff ID check would only show in MetalK8s. The spec builds imagecachectl and imports two docker archives on the worker. The agent adopts the one holding the image its resource names, with the archive untouched, and replaces the one holding another image. Relates-to: MK8S-450 --- agent/AGENTS.md | 8 +-- agent/DESIGN.md | 4 +- agent/test/e2e/e2e_test.go | 114 ++++++++++++++++++++++++++++++++++++- 3 files changed, 119 insertions(+), 7 deletions(-) diff --git a/agent/AGENTS.md b/agent/AGENTS.md index 59a58bd..58377a3 100644 --- a/agent/AGENTS.md +++ b/agent/AGENTS.md @@ -64,10 +64,10 @@ make test-e2e # needs kubectl and an idle cluster name, see below (`test/e2e/kind-config.yaml`), builds the image, loads it and deploys the manifests. It deploys two registries in the cluster, one plain HTTP and one signed by a throwaway CA, so a run pulls `registry:3.1.2` from Docker Hub -(`registryVersion` in `test/e2e/e2e_test.go`, pinned by digest). The -`Makefile` downloads `kind` into `bin/`, but `kubectl` has to be on your -`PATH`. CI runs it on every pull request, in the `e2e` job of -`.github/workflows/agent-ci.yaml`. +(`registryVersion` in `test/e2e/e2e_test.go`, pinned by digest). It also +builds `imagecachectl` and runs it on the worker. The `Makefile` downloads +`kind` into `bin/`, but `kubectl` has to be on your `PATH`. CI runs it on +every pull request, in the `e2e` job of `.github/workflows/agent-ci.yaml`. A green `make lint` right after a dependency bump is not proof. Run `bin/golangci-lint cache clean` first, for the reason diff --git a/agent/DESIGN.md b/agent/DESIGN.md index 73c3243..236fbd2 100644 --- a/agent/DESIGN.md +++ b/agent/DESIGN.md @@ -388,4 +388,6 @@ The agent never writes ImageCache resources (no status, no finalizers). nobody listens on leaves the node `pending`. Two registries run in the cluster: a pull over plain HTTP lands on the selected node only, and a registry signed by a private CA is refused until the shipped CA patch is - applied. + applied. `imagecachectl` seeds two directories on the worker: the agent + adopts the one holding the image its resource names, with no extraction, + and replaces the other. diff --git a/agent/test/e2e/e2e_test.go b/agent/test/e2e/e2e_test.go index 9f04374..a7eef39 100644 --- a/agent/test/e2e/e2e_test.go +++ b/agent/test/e2e/e2e_test.go @@ -36,13 +36,16 @@ import ( "net" "net/http" "os/exec" + "path/filepath" "regexp" "strings" "time" "github.com/google/go-containerregistry/pkg/crane" + gcrname "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/google/go-containerregistry/pkg/v1/tarball" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" @@ -76,6 +79,14 @@ const syncedName = "imagecache-e2e-synced" // throwaway CA, to exercise --ca-file through the shipped patch. const tlsName = "imagecache-e2e-tls" +// adoptedName and replacedName are the ImageCaches whose directories +// imagecachectl seeds before they exist: the first with the image the resource +// names, the second with another one. +const ( + adoptedName = "imagecache-e2e-adopted" + replacedName = "imagecache-e2e-replaced" +) + // cachePath is the agent's default --cache-path, mounted by the manifests. const cachePath = "/var/lib/image-cache" @@ -91,6 +102,7 @@ const registryNamespace = "image-cache-e2e-registry" const registryVersion = "3.1.2@sha256:ddf754342cfc8acc51a56d5d0ab6af06826461864460636d8bd5c546dab2a7b8" const registryImage = "docker.io/library/registry:" + registryVersion + // carrierFile is the archive the test image carries, and carrierBody its // content. The agent extracts it under its base name. const ( @@ -397,6 +409,61 @@ spec: g.Expect(labels).NotTo(HaveKey(labelKey(tlsName))) }).Should(Succeed()) }) + + It("should adopt what imagecachectl seeded with the same image, and replace another image", func() { + deployRegistry("registry", "") + digest := pushCarrier("registry", nil) + worker := utils.WorkerNode() + adoptedDir, replacedDir := cachePath+"/"+adoptedName, cachePath+"/"+replacedName + + bin := buildImagecachectl() + adoptedInode := seedWithImagecachectl(bin, worker, adoptedName, carrierBody) + seedWithImagecachectl(bin, worker, replacedName, "another image") + + By("restarting the agent with --plain-http") + patchAgent("--type=json", + `-p=[{"op":"add","path":"/spec/template/spec/containers/0/args/-","value":"--plain-http"}]`) + + By("creating the two ImageCaches the seeded directories stand for") + applyImageCache(adoptedName, worker, "registry") + applyImageCache(replacedName, worker, "registry") + + By("waiting for the worker to be labelled synced for both") + Eventually(func(g Gomega) { + labels, err := nodeLabels(worker) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(labels).To(HaveKeyWithValue(labelKey(adoptedName), "synced")) + g.Expect(labels).To(HaveKeyWithValue(labelKey(replacedName), "synced")) + }).Should(Succeed()) + + By("checking the same image was adopted in place, with no extraction") + // The recorder sends events in the background, so the event can + // land after the label. + Eventually(func(g Gomega) { + cmd := exec.Command("kubectl", "get", "events", "-A", "-o", "name", + "--field-selector", "reason=Adopted,involvedObject.name="+adoptedName) + out, err := utils.Run(cmd) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(out).NotTo(BeEmpty(), "no Adopted event for "+adoptedName) + }).Should(Succeed()) + out, err := utils.OnNode(worker, "stat -c %i "+adoptedDir+"/"+carrierFile+ + " && grep -c '\"owner\":\"image-cache-agent\"' "+adoptedDir+"/.image-cache-agent.json") + Expect(err).NotTo(HaveOccurred()) + Expect(out).To(Equal(adoptedInode + "1\n")) + + By("checking the other image was replaced by a pull") + expectExtracted(worker, replacedDir, digest) + + By("deleting the ImageCaches, so no agent-owned state outlives the run") + cmd := exec.Command("kubectl", "delete", "imagecache", adoptedName, replacedName) + _, err = utils.Run(cmd) + Expect(err).NotTo(HaveOccurred(), "Failed to delete the ImageCaches") + for _, dir := range []string{adoptedDir, replacedDir} { + Eventually(func() (bool, error) { + return pathExists(worker, dir) + }).Should(BeFalse()) + } + }) }) }) @@ -561,8 +628,7 @@ data: func pushCarrier(registry string, ca []byte) v1.Hash { By("pushing the carrier image to " + registry + " through a port-forward") local := portForward(registryNamespace, "svc/"+registry, 5000) - img, err := crane.Image(map[string][]byte{"images/" + carrierFile: []byte(carrierBody)}) - Expect(err).NotTo(HaveOccurred()) + img := carrierImage(carrierBody) opts := []crane.Option{crane.Insecure} if ca != nil { pool := x509.NewCertPool() @@ -577,6 +643,50 @@ func pushCarrier(registry string, ca []byte) v1.Hash { return digest } +// carrierImage returns the test image, its archive holding body. The same body +// gives the same layer, so the same diff IDs. +func carrierImage(body string) v1.Image { + img, err := crane.Image(map[string][]byte{"images/" + carrierFile: []byte(body)}) + Expect(err).NotTo(HaveOccurred()) + return img +} + +// buildImagecachectl builds imagecachectl and returns its path. Static, so it +// runs whatever libc the node image has. +func buildImagecachectl() string { + bin := filepath.Join(GinkgoT().TempDir(), "imagecachectl") + _, err := utils.Run(exec.Command("env", "CGO_ENABLED=0", "go", "build", "-o", bin, "./cmd/imagecachectl")) + Expect(err).NotTo(HaveOccurred(), "Failed to build imagecachectl") + return bin +} + +// seedWithImagecachectl copies bin to node with a docker archive of the +// carrier image holding body, and imports it there under name, as a node is +// seeded before the agent runs. It returns the inode of the extracted archive. +func seedWithImagecachectl(bin, node, name, body string) string { + By("seeding " + name + " on " + node + " with imagecachectl") + archive := filepath.Join(GinkgoT().TempDir(), name+".tar") + tag, err := gcrname.NewTag("e2e/carrier:1") + Expect(err).NotTo(HaveOccurred()) + Expect(tarball.WriteToFile(archive, tag, carrierImage(body))).To(Succeed()) + + // Not /tmp: it is a tmpfs on a kind node, and docker cp cannot write there. + dir := "/var/tmp/" + name + _, err = utils.OnNode(node, "mkdir -p "+dir) + Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { _, _ = utils.OnNode(node, "rm -rf "+dir) }) + for _, f := range []string{bin, archive} { + _, err = utils.Run(exec.Command("docker", "cp", f, node+":"+dir+"/")) + Expect(err).NotTo(HaveOccurred()) + } + _, err = utils.OnNode(node, dir+"/imagecachectl import --name "+name+" "+dir+"/"+name+".tar") + Expect(err).NotTo(HaveOccurred(), "imagecachectl import failed") + + inode, err := utils.OnNode(node, "stat -c %i "+cachePath+"/"+name+"/"+carrierFile) + Expect(err).NotTo(HaveOccurred()) + return inode +} + // newRegistryCerts returns a throwaway CA, and a certificate it signed for // host and for localhost, the port-forward the test pushes through. All PEM. func newRegistryCerts(host string) (ca, cert, key []byte) { From e646d27bc47bfef3b83ddf40ed4e512ad78311c3 Mon Sep 17 00:00:00 2001 From: Alex Rodriguez <131964409+ezekiel-alexrod@users.noreply.github.com> Date: Mon, 19 Oct 2026 13:25:16 +0200 Subject: [PATCH 7/7] test(agent): dump the events of every namespace when an e2e spec fails ImageCache is cluster-scoped, so its SyncFailed and AdoptFailed events land in the default namespace, which the dump did not read. Those are the ones that say why a sync spec failed. Relates-to: MK8S-450 --- agent/test/e2e/e2e_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/agent/test/e2e/e2e_test.go b/agent/test/e2e/e2e_test.go index a7eef39..72ff113 100644 --- a/agent/test/e2e/e2e_test.go +++ b/agent/test/e2e/e2e_test.go @@ -207,7 +207,7 @@ var _ = Describe("Manager", Ordered, func() { } By("Fetching Kubernetes events") - cmd = exec.Command("kubectl", "get", "events", "-n", namespace, "--sort-by=.lastTimestamp") + cmd = exec.Command("kubectl", "get", "events", "-A", "--sort-by=.lastTimestamp") eventsOutput, err := utils.Run(cmd) if err == nil { _, _ = fmt.Fprintf(GinkgoWriter, "Kubernetes events:\n%s", eventsOutput)