From f973d8ca5311b7cad869cde4443b8f62dd27c4fb Mon Sep 17 00:00:00 2001 From: Anthony TREUILLIER Date: Tue, 8 Sep 2026 17:27:34 +0200 Subject: [PATCH] feat: Add E2E Tests Refs: MK8S-289 Signed-off-by: Anthony TREUILLIER --- .devcontainer/Dockerfile | 14 + .github/workflows/test-e2e.yml | 15 +- CONTRIBUTING.md | 3 + Makefile | 34 ++ go.mod | 18 +- go.sum | 51 +- tests/e2e/e2e_suite_test.go | 81 +++ tests/e2e/e2e_test.go | 851 +++++++++++++++++++++++++++++++ tests/e2e/kind-config.yaml | 19 + tests/e2e/kustomization.yaml | 5 + tests/e2e/virtualip-manager.yaml | 72 +++ tests/utils/utils.go | 263 ++++++++++ 12 files changed, 1395 insertions(+), 31 deletions(-) create mode 100644 tests/e2e/e2e_suite_test.go create mode 100644 tests/e2e/e2e_test.go create mode 100644 tests/e2e/kind-config.yaml create mode 100644 tests/e2e/kustomization.yaml create mode 100644 tests/e2e/virtualip-manager.yaml create mode 100644 tests/utils/utils.go diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile index d894f0f..c346d13 100644 --- a/.devcontainer/Dockerfile +++ b/.devcontainer/Dockerfile @@ -6,3 +6,17 @@ USER vscode ARG GOLANGCI_LINT_VERSION=2.13.2 RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/HEAD/install.sh | sudo sh -s -- v${GOLANGCI_LINT_VERSION} + +# renovate: datasource=github-releases depName=kubernetes-sigs/kind +ARG KIND_VERSION=v0.32.0 + +RUN curl -sSfLO https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64 && \ + chmod +x kind-linux-amd64 && \ + sudo mv kind-linux-amd64 /usr/local/bin/kind + +# renovate: datasource=github-releases depName=kubernetes/kubernetes +ARG KUBECTL_VERSION=v1.34.8 + +RUN curl -sSfLO https://dl.k8s.io/release/${KUBECTL_VERSION}/bin/linux/amd64/kubectl && \ + chmod +x kubectl && \ + sudo mv kubectl /usr/local/bin/kubectl diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index fdde827..9bc3e8e 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -14,9 +14,22 @@ jobs: uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0 with: go-version-file: 'go.mod' + + - name: Install the latest version of kind + env: + # renovate: datasource=github-releases depName=kubernetes-sigs/kind + KIND_VERSION: v0.32.0 + run: | + curl -Lo ./kind https://kind.sigs.k8s.io/dl/${KIND_VERSION}/kind-linux-amd64 + chmod +x ./kind + sudo mv ./kind /usr/local/bin/kind + + - name: Verify kind installation + run: kind version + - name: Running Test e2e env: GIT_AUTH_TOKEN: ${{ secrets.GIT_ACCESS_TOKEN }} run: | go mod tidy - #make test-e2e + make test-e2e diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 3af7d61..dd95520 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -8,12 +8,15 @@ project conventions, and what CI expects. For the architecture, see [DESIGN.md]( - Go (the version is pinned in [`go.mod`](go.mod)). - Docker (or another `CONTAINER_TOOL`, e.g. podman) for building the image. - `golangci-lint` is installed automatically into `bin/` by `make lint`. +- [Kind](https://kind.sigs.k8s.io/) and `kubectl`, for `make test-e2e` only (install Kind manually; + override the binary with `KIND=…`). - Git for version control Everyday commands: ```sh make test # unit + integration tests +make test-e2e # end-to-end tests on a Kind cluster make lint # golangci-lint make docker-build # build the container image ``` diff --git a/Makefile b/Makefile index 33d1a53..2d7033b 100644 --- a/Makefile +++ b/Makefile @@ -44,6 +44,37 @@ vet: ## Run go vet against code. test: fmt vet ## Run tests. go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out +# The default setup assumes Kind is pre-installed and builds/loads the Manager Docker image locally. +# kubectl kuberc is disabled by default for test isolation; enable with: +# - KUBECTL_KUBERC=true +KIND_CLUSTER ?= virtualip-manager-test-e2e +# The e2e suite needs several worker nodes, so the cluster is created from a +# topology file rather than from Kind's single-node default. +KIND_CONFIG ?= tests/e2e/kind-config.yaml + +.PHONY: setup-test-e2e +setup-test-e2e: ## Set up a Kind cluster for e2e tests if it does not exist + @command -v $(KIND) >/dev/null 2>&1 || { \ + echo "Kind is not installed. Please install Kind manually."; \ + exit 1; \ + } + @case "$$($(KIND) get clusters)" in \ + *"$(KIND_CLUSTER)"*) \ + echo "Kind cluster '$(KIND_CLUSTER)' already exists. Skipping creation." ;; \ + *) \ + echo "Creating Kind cluster '$(KIND_CLUSTER)'..."; \ + $(KIND) create cluster --name $(KIND_CLUSTER) --config $(KIND_CONFIG) ;; \ + esac + +.PHONY: test-e2e +test-e2e: setup-test-e2e fmt vet ## Run the e2e tests. Expected an isolated environment using Kind. + KIND=$(KIND) KIND_CLUSTER=$(KIND_CLUSTER) go test -tags=e2e ./tests/e2e/ -v -ginkgo.v + $(MAKE) cleanup-test-e2e + +.PHONY: cleanup-test-e2e +cleanup-test-e2e: ## Tear down the Kind cluster used for e2e tests + @$(KIND) delete cluster --name $(KIND_CLUSTER) + # If you wish to build the manager image targeting other platforms you can use the --platform flag. # (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it. # More info: https://docs.docker.com/develop/develop-images/build_enhancements/ @@ -55,6 +86,9 @@ docker-build: ## Build docker image with the manager. docker-push: ## Push docker image with the manager. $(CONTAINER_TOOL) push ${IMG} +## Tool Binaries +KIND ?= kind + # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist # $1 - target path with name of binary # $2 - package url which can be installed diff --git a/go.mod b/go.mod index c6e5b37..c263ec4 100644 --- a/go.mod +++ b/go.mod @@ -12,14 +12,20 @@ require ( require ( github.com/Masterminds/semver/v3 v3.4.0 // indirect + github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/go-logr/logr v1.4.3 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect - golang.org/x/mod v0.35.0 // indirect - golang.org/x/net v0.53.0 // indirect - golang.org/x/sync v0.20.0 // indirect - golang.org/x/sys v0.43.0 // indirect - golang.org/x/text v0.36.0 // indirect - golang.org/x/tools v0.44.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect + github.com/rogpeppe/go-internal v1.14.1 // indirect + github.com/stretchr/testify v1.11.1 // indirect + golang.org/x/mod v0.37.0 // indirect + golang.org/x/net v0.57.0 // indirect + golang.org/x/sync v0.22.0 // indirect + golang.org/x/sys v0.47.0 // indirect + golang.org/x/text v0.40.0 // indirect + golang.org/x/tools v0.47.0 // indirect + google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect + gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c // indirect ) diff --git a/go.sum b/go.sum index 8714b3d..a04fdeb 100644 --- a/go.sum +++ b/go.sum @@ -1,7 +1,7 @@ github.com/Masterminds/semver/v3 v3.4.0 h1:Zog+i5UMtVoCU8oKka5P7i9q9HgrJeGzI9SA1Xbatp0= github.com/Masterminds/semver/v3 v3.4.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= -github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= +github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -20,8 +20,11 @@ github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 h1:EwtI+Al+DeppwYX2oX github.com/google/pprof v0.0.0-20260402051712-545e8a4df936/go.mod h1:MxpfABSjhmINe3F1It9d+8exIHFvUqtLIRCdOGNXqiI= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= @@ -32,16 +35,16 @@ github.com/onsi/ginkgo/v2 v2.31.0 h1:GtuJos5DFUV9EerYJo8RhYxosYNGvOdDE5haKq6Grfs github.com/onsi/ginkgo/v2 v2.31.0/go.mod h1:+aXOY+vzZ5mu2iI2HpTZUPmM//oQfsNFX6gU9kNcA44= github.com/onsi/gomega v1.41.0 h1:OwKp4pXNgVxf6sCplzYo794OFNuoL2q2SBMU5NSWOjA= github.com/onsi/gomega v1.41.0/go.mod h1:M/Uqpu/8qTjtzCLUA2zJHX9Iilrau25x1PdoSRbWh5A= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= -github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= +github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ= +github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/scality/go-errors v1.3.0 h1:InguXV1zpPyQKUCKDCA0xsNoxMR0lISW8iyOI5xHvoo= github.com/scality/go-errors v1.3.0/go.mod h1:PCgOL4vdz+9wyVrZ2/f01dBUZqLrxTbUEAEJbxLnSWY= github.com/sethvargo/go-envconfig v1.3.0 h1:gJs+Fuv8+f05omTpwWIu6KmuseFAXKrIaOZSh8RMt0U= github.com/sethvargo/go-envconfig v1.3.0/go.mod h1:JLd0KFWQYzyENqnEPWWZ49i4vzZo/6nRidxI8YvGiHw= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= +github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= github.com/tidwall/gjson v1.18.0 h1:FIDeeyB800efLX89e5a8Y0BNH+LOngJyGrIWxG2FKQY= github.com/tidwall/gjson v1.18.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= @@ -52,22 +55,22 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= -golang.org/x/mod v0.35.0 h1:Ww1D637e6Pg+Zb2KrWfHQUnH2dQRLBQyAtpr/haaJeM= -golang.org/x/mod v0.35.0/go.mod h1:+GwiRhIInF8wPm+4AoT6L0FA1QWAad3OMdTRx4tFYlU= -golang.org/x/net v0.53.0 h1:d+qAbo5L0orcWAr0a9JweQpjXF19LMXJE8Ey7hwOdUA= -golang.org/x/net v0.53.0/go.mod h1:JvMuJH7rrdiCfbeHoo3fCQU24Lf5JJwT9W3sJFulfgs= -golang.org/x/sync v0.20.0 h1:e0PTpb7pjO8GAtTs2dQ6jYa5BWYlMuX047Dco/pItO4= -golang.org/x/sync v0.20.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= -golang.org/x/sys v0.43.0 h1:Rlag2XtaFTxp19wS8MXlJwTvoh8ArU6ezoyFsMyCTNI= -golang.org/x/sys v0.43.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= -golang.org/x/text v0.36.0 h1:JfKh3XmcRPqZPKevfXVpI1wXPTqbkE5f7JA92a55Yxg= -golang.org/x/text v0.36.0/go.mod h1:NIdBknypM8iqVmPiuco0Dh6P5Jcdk8lJL0CUebqK164= -golang.org/x/tools v0.44.0 h1:UP4ajHPIcuMjT1GqzDWRlalUEoY+uzoZKnhOjbIPD2c= -golang.org/x/tools v0.44.0/go.mod h1:KA0AfVErSdxRZIsOVipbv3rQhVXTnlU6UhKxHd1seDI= -google.golang.org/protobuf v1.36.7 h1:IgrO7UwFQGJdRNXH/sQux4R1Dj1WAKcLElzeeRaXV2A= -google.golang.org/protobuf v1.36.7/go.mod h1:jduwjTPXsFjZGTmRluh+L6NjiWu7pchiJ2/5YcXBHnY= +golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= +golang.org/x/mod v0.37.0/go.mod h1:m8S8VeM9r4dzDwjrKO0a1sZP3YjeMamRRlD+fmR2Q/0= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= +golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= +golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= +golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= +golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= +golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= +golang.org/x/tools v0.47.0 h1:7Kn5x/d1svx/PzryTsqeoZN4TZwqeH5pGWjefhLi/1Q= +golang.org/x/tools v0.47.0/go.mod h1:dFHnyTvFWY212G+h7ZY4Vsp/K3U4/7W9TyVaAul8uCA= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af h1:+5/Sw3GsDNlEmu7TfklWKPdQ0Ykja5VEmq2i817+jbI= +google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/tests/e2e/e2e_suite_test.go b/tests/e2e/e2e_suite_test.go new file mode 100644 index 0000000..42748a8 --- /dev/null +++ b/tests/e2e/e2e_suite_test.go @@ -0,0 +1,81 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026 Scality. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "fmt" + "os" + "os/exec" + "testing" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/scality/virtualip-manager/tests/utils" +) + +var ( + // managerImage is the manager image to be built and loaded for testing. + managerImage = "example.com/virtualip-manager:v0.0.1" +) + +// TestE2E runs the e2e test suite to validate the solution in an isolated environment. +// The default setup requires Kind. +// +// To enable kubectl kuberc (use custom kubectl configurations), set: KUBECTL_KUBERC=true +// By default, kuberc is disabled to ensure consistent test behavior across different environments. +func TestE2E(t *testing.T) { + RegisterFailHandler(Fail) + _, _ = fmt.Fprintf(GinkgoWriter, "Starting virtualip-manager e2e test suite\n") + RunSpecs(t, "e2e suite") +} + +var _ = BeforeSuite(func() { + By("building the manager image") + cmd := exec.Command("make", "docker-build", fmt.Sprintf("IMG=%s", managerImage)) + _, err := utils.Run(cmd) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to build the manager image") + + // TODO(user): If you want to change the e2e test vendor from Kind, + // ensure the image is built and available, then remove the following block. + By("loading the manager image on Kind") + err = utils.LoadImageToKindClusterWithName(managerImage) + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to load the manager image into Kind") + + configureKubectlKubeRC() +}) + +var _ = AfterSuite(func() { +}) + +// Disable kubectl kuberc by default for test isolation. +// This prevents local kubectl configurations from affecting test behavior. +// To enable kuberc, set: KUBECTL_KUBERC=true +func configureKubectlKubeRC() { + if os.Getenv("KUBECTL_KUBERC") != "true" { + By("disabling kubectl kuberc for test isolation") + err := os.Setenv("KUBECTL_KUBERC", "false") + ExpectWithOffset(1, err).NotTo(HaveOccurred(), "Failed to disable kubectl kuberc") + _, _ = fmt.Fprintf(GinkgoWriter, + "kubectl kuberc disabled for consistent test behavior (override with KUBECTL_KUBERC=true)\n") + } else { + _, _ = fmt.Fprintf(GinkgoWriter, "kubectl kuberc enabled (KUBECTL_KUBERC=true)\n") + } +} diff --git a/tests/e2e/e2e_test.go b/tests/e2e/e2e_test.go new file mode 100644 index 0000000..0b67739 --- /dev/null +++ b/tests/e2e/e2e_test.go @@ -0,0 +1,851 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2026 Scality. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "encoding/binary" + "fmt" + "maps" + "net" + "os/exec" + "slices" + "strconv" + "strings" + "time" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + "github.com/scality/virtualip-manager/pkg/domain" + "github.com/scality/virtualip-manager/tests/utils" +) + +// namespace where the project is deployed in +const namespace = "virtualip-manager-system" + +// virtualIPConfigMapName is the ConfigMap the DaemonSet mounts as its input spec. +// The name is also hardcoded in virtualip-manager.yaml. +const virtualIPConfigMapName = "virtualip-manager-config" + +// secondaryNetworkName is the extra Docker network the suite creates so the +// DaemonSet can be exercised on more than Kind's own bridge. Kind's teardown +// does not remove it, so AfterAll does. +const secondaryNetworkName = "secondary-network" + +// managerNodeIP is the NODE_IP the DaemonSet advertises. It is hardcoded in +// virtualip-manager.yaml rather than taken from the node so the healthcheck the manager +// renders is predictable. +const managerNodeIP = "1.1.1.1" + +// generatedConfigPath is where the entrypoint renders the keepalived configuration before +// handing over to keepalived. The path is hardcoded in scripts/entrypoint.sh. +const generatedConfigPath = "/etc/keepalived/keepalived.conf" + +// The capability bits the Dockerfile grants keepalived with setcap. Nothing outside this set +// should end up in the effective set of the running process. +const ( + capSetGid = 6 + capSetUid = 7 + capNetBindService = 10 + capNetAdmin = 12 + capNetRaw = 13 +) + +//nolint:gochecknoglobals // A constant expression Go will not let us declare as a constant. +var expectedCapabilities = uint64(1<= 1; offset-- { + address := make(net.IP, net.IPv4len) + binary.BigEndian.PutUint32(address, broadcast-offset) + if !subnet.Contains(address) { + return nil, fmt.Errorf("subnet %s is too small for %d VirtualIPs", subnet, count) + } + + addresses = append(addresses, VirtualIPAddress{ + Ip: address.String(), + VrId: nextVrId(), + }) + } + + // check if the addresses are free + free, err := assertAddressesFree(kindNetwork, addresses) + if err != nil { + return nil, err + } + if !free { + return nil, fmt.Errorf("addresses are not free") + } + return addresses, nil +} + +// nextVrId hands out the next VRRP virtual_router_id. Deriving the id from the address's +// offset in its subnet collides as soon as two networks contribute an address at the same +// offset, so the ids are allocated from a single counter instead. VRRP ids are 8 bit and +// 0 is reserved. +func nextVrId() string { + lastVrId++ + Expect(lastVrId).To(BeNumerically("<=", 255), "exhausted the VRRP virtual_router_id range") + return fmt.Sprintf("%d", lastVrId) +} + +// assertAddressesFree returns false when a container already holds one of the addresses the suite +// is about to advertise. It cannot see a VIP another suite run advertises -- that address +// belongs to no container -- so it catches the common collision, not every one. +func assertAddressesFree(network string, addresses []VirtualIPAddress) (bool, error) { + output, err := utils.Run(exec.Command("docker", "network", "inspect", network, "-f", + `{{range .Containers}}{{.Name}}={{.IPv4Address}} {{end}}`)) + if err != nil { + return false, err + } + + for _, container := range strings.Fields(output) { + _, cidr, found := strings.Cut(container, "=") + if !found { + continue + } + ip, _, err := net.ParseCIDR(cidr) + if err != nil { + continue + } + if slices.ContainsFunc(addresses, func(vip VirtualIPAddress) bool { + return vip.Ip == ip.String() + }) { + return false, nil + } + } + return true, nil +} + +// nodeInterface returns the name of the interface on node whose configured subnet contains +// address. Kind names each node's container after the node, and the VIPs live in the host +// network namespace, not in any pod's -- so the container is where to look. This mirrors what +// generate-config resolves, computed independently of it. +func nodeInterface(node, address string) (string, error) { + target := net.ParseIP(address) + if target == nil { + return "", fmt.Errorf("%q is not an IP address", address) + } + + cmd := exec.Command("docker", "exec", node, "ip", "-4", "-oneline", "address", "show") + output, err := utils.Run(cmd) + if err != nil { + return "", err + } + + for _, line := range utils.GetNonEmptyLines(output) { + // "2: eth1 inet 10.0.0.11/24 brd ... scope global eth1" + fields := strings.Fields(line) + for i, field := range fields { + if field != "inet" || i+1 >= len(fields) { + continue + } + _, subnet, err := net.ParseCIDR(fields[i+1]) + if err != nil { + return "", fmt.Errorf("parsing address %q on node %s: %w", fields[i+1], node, err) + } + if subnet.Contains(target) { + return fields[1], nil + } + } + } + return "", fmt.Errorf("no interface on node %s carries the subnet of %s", node, address) +} + +// VirtualIPAddress is a simplified representation of a VirtualIPAddress. +type VirtualIPAddress struct { + Ip string + VrId string +} + +// LocalizedVirtualIPAddress is a VirtualIPAddress with a node name. +type LocalizedVirtualIPAddress struct { + VirtualIPAddress + Node string +} + +// execInManager runs a command inside the running manager pod. +func execInManager(pod string, command ...string) (string, error) { + args := append([]string{"exec", pod, "-n", namespace, "--"}, command...) + return utils.Run(exec.Command("kubectl", args...)) +} + +// generatedConfiguration returns the configuration the entrypoint rendered before it handed +// over to keepalived, which is the one keepalived is running against. +func generatedConfiguration(pod string) (string, error) { + return execInManager(pod, "cat", generatedConfigPath) +} + +// effectiveCapabilities returns the capability set PID 1 of the manager pod -- keepalived -- +// actually runs with. The image drops libcap after the setcap call, so the capabilities are +// read from procfs rather than from the file they are attached to. +func effectiveCapabilities(pod string) (uint64, error) { + status, err := execInManager(pod, "cat", "/proc/1/status") + if err != nil { + return 0, err + } + + for _, line := range utils.GetNonEmptyLines(status) { + value, found := strings.CutPrefix(strings.TrimSpace(line), "CapEff:") + if !found { + continue + } + return strconv.ParseUint(strings.TrimSpace(value), 16, 64) + } + return 0, fmt.Errorf("no CapEff line in the status of PID 1") +} + +// nodeAddressesOnInterface returns the IPv4 addresses configured on iface of node, in CIDR +// form. The VirtualIPs live in the node's network namespace, not in any pod's, so the node +// container is where to look for them. +func nodeAddressesOnInterface(node, iface string) ([]string, error) { + output, err := utils.Run(exec.Command("docker", "exec", node, + "ip", "-4", "-oneline", "address", "show", "dev", iface)) + if err != nil { + return nil, err + } + + addresses := []string{} + for _, line := range utils.GetNonEmptyLines(output) { + fields := strings.Fields(line) + for index, field := range fields { + if field == "inet" && index+1 < len(fields) { + addresses = append(addresses, fields[index+1]) + } + } + } + return addresses, nil +} + +// vrrpInstance is the parsed form of a `vrrp_instance` block of a keepalived configuration. +type vrrpInstance struct { + Name string + State string + Interface string + Priority string + VirtualRouterID string + VirtualIPAddresses []string + TrackScripts []string +} + +// keepalivedConfig holds the parts of a keepalived configuration the suite asserts on. +type keepalivedConfig struct { + Instances []vrrpInstance + // Scripts maps a vrrp_script name to its `script` argument, unquoted. + Scripts map[string]string +} + +// parseKeepalivedConfig reads the generated configuration into its meaning rather than +// comparing it to a golden file: what e2e adds over the unit tests is that the interface and +// the MASTER/BACKUP state are resolved against the real host network stack and NODE_NAME, and +// a golden file would only re-state the template's layout. Unknown directives are rejected so +// a template addition shows up here instead of being silently ignored. +func parseKeepalivedConfig(config string) (keepalivedConfig, error) { + parsed := keepalivedConfig{Scripts: map[string]string{}} + current, script, section := -1, "", "" + + for _, raw := range strings.Split(config, "\n") { + line := strings.TrimSpace(raw) + if line == "" || strings.HasPrefix(line, "#") { + continue + } + fields := strings.Fields(line) + + switch { + case line == "}": + if section != "" { + section = "" + } else { + current, script = -1, "" + } + case fields[0] == "vrrp_instance" && len(fields) == 3: + parsed.Instances = append(parsed.Instances, vrrpInstance{Name: fields[1]}) + current = len(parsed.Instances) - 1 + case fields[0] == "vrrp_script" && len(fields) == 3: + script = fields[1] + case strings.HasSuffix(line, "{"): + section = fields[0] // global_defs, virtual_ipaddress, track_script, ... + case script != "" && fields[0] == "script": + parsed.Scripts[script] = strings.Trim(strings.TrimPrefix(line, "script "), `"`) + case current < 0: + continue + case section == "virtual_ipaddress": + parsed.Instances[current].VirtualIPAddresses = append( + parsed.Instances[current].VirtualIPAddresses, fields[0]) + case section == "track_script": + parsed.Instances[current].TrackScripts = append( + parsed.Instances[current].TrackScripts, fields[0]) + case len(fields) != 2: + return parsed, fmt.Errorf("unexpected line %q in vrrp_instance %s", + line, parsed.Instances[current].Name) + default: + instance := &parsed.Instances[current] + switch fields[0] { + case "state": + instance.State = fields[1] + case "interface": + instance.Interface = fields[1] + case "priority": + instance.Priority = fields[1] + case "virtual_router_id": + instance.VirtualRouterID = fields[1] + default: + return parsed, fmt.Errorf("unknown key %q in vrrp_instance %s", fields[0], instance.Name) + } + } + } + if current >= 0 || script != "" || section != "" { + return parsed, fmt.Errorf("unterminated block in the generated configuration") + } + return parsed, nil +} + +// expectedInstances derives the vrrp_instance blocks the manager must generate for addresses +// from the point of view of node: the blocks keep the spec's order (VI_1..VI_n), the node named +// by an entry is that VIP's MASTER and every other node is a BACKUP, and the interface is the +// one that actually carries the VIP's subnet on that node. The same spec therefore renders +// differently on each node, which is what makes it worth reading the configuration of both. +func expectedInstances(node string, addresses []LocalizedVirtualIPAddress, + trackScripts []string, +) []vrrpInstance { + instances := make([]vrrpInstance, 0, len(addresses)) + for index, address := range addresses { + iface, err := nodeInterface(node, address.Ip) + Expect(err).NotTo(HaveOccurred(), "Failed to resolve the interface for %s", address.Ip) + + state, priority := "BACKUP", "80" + if address.Node == node { + state, priority = "MASTER", "130" + } + instances = append(instances, vrrpInstance{ + Name: fmt.Sprintf("VI_%d", index+1), + State: state, + Interface: iface, + Priority: priority, + VirtualRouterID: address.VrId, + VirtualIPAddresses: []string{address.Ip}, + TrackScripts: trackScripts, + }) + } + return instances +} diff --git a/tests/e2e/kind-config.yaml b/tests/e2e/kind-config.yaml new file mode 100644 index 0000000..c1e80ef --- /dev/null +++ b/tests/e2e/kind-config.yaml @@ -0,0 +1,19 @@ +# Kind cluster used by the e2e suite. Two workers are required: the per-node +# agents run as DaemonSets, and spreading the VirtualIPs over several nodes is +# part of what the suite exercises. The control-plane node keeps its NoSchedule +# taint. +kind: Cluster +apiVersion: kind.x-k8s.io/v1alpha4 +nodes: + - role: control-plane + - role: worker + labels: + # Use this label to identify this node as a virtualip endpoint + # instead of "node-role.kubernetes.io/virtualip" that make cluster creation failed + role: virtualip + - role: worker + labels: + # Use this label to identify this node as a virtualip endpoint + # instead of "node-role.kubernetes.io/virtualip" that make cluster creation failed + role: virtualip + - role: worker diff --git a/tests/e2e/kustomization.yaml b/tests/e2e/kustomization.yaml new file mode 100644 index 0000000..52dac9e --- /dev/null +++ b/tests/e2e/kustomization.yaml @@ -0,0 +1,5 @@ +# Adds namespace to all resources. +namespace: virtualip-manager-system + +resources: + - virtualip-manager.yaml diff --git a/tests/e2e/virtualip-manager.yaml b/tests/e2e/virtualip-manager.yaml new file mode 100644 index 0000000..b5d1389 --- /dev/null +++ b/tests/e2e/virtualip-manager.yaml @@ -0,0 +1,72 @@ +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + labels: + app.kubernetes.io/name: virtualip-manager + name: virtualip-manager + namespace: default +spec: + selector: + matchLabels: + app.kubernetes.io/name: virtualip-manager + template: + metadata: + labels: + app.kubernetes.io/name: virtualip-manager + spec: + containers: + - env: + # Each DaemonSet replica must identify its own node: NODE_NAME decides + # which addresses it owns (MASTER), NODE_IP is interpolated into the + # healthcheck. + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: NODE_IP + value: 1.1.1.1 + image: example.com/virtualip-manager:v0.0.1 + imagePullPolicy: IfNotPresent + name: virtualip-manager + # The image entrypoint renders the configuration and then execs keepalived; the + # argument is the spec to render, mounted from the ConfigMap. + args: + - /etc/keepalived/input/keepalived-input.yaml + resources: + limits: + memory: 64Mi + requests: + cpu: 10m + memory: 32Mi + securityContext: + allowPrivilegeEscalation: true + capabilities: + add: + - NET_ADMIN + - NET_RAW + - NET_BIND_SERVICE + - SETUID + - SETGID + drop: + - ALL + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/keepalived/input + name: virtualip-config + readOnly: true + dnsPolicy: ClusterFirstWithHostNet + hostNetwork: true + nodeSelector: + role: virtualip + restartPolicy: Always + securityContext: + runAsNonRoot: true + seccompProfile: + type: RuntimeDefault + volumes: + - configMap: + defaultMode: 420 + name: virtualip-manager-config + name: virtualip-config diff --git a/tests/utils/utils.go b/tests/utils/utils.go new file mode 100644 index 0000000..55e5537 --- /dev/null +++ b/tests/utils/utils.go @@ -0,0 +1,263 @@ +/* +Copyright 2026 Scality. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package utils + +import ( + "encoding/binary" + "fmt" + "net" + "os" + "os/exec" + "path/filepath" + "strings" + + . "github.com/onsi/ginkgo/v2" // nolint:revive,staticcheck +) + +const ( + defaultKindBinary = "kind" + defaultKindCluster = "kind" + rfc1918Private = "172.16.0.0/12" +) + +// Run executes the provided command within this context +func Run(cmd *exec.Cmd) (string, error) { + dir, err := GetProjectDir() + if err != nil { + return "", err + } + cmd.Dir = dir + + cmd.Env = append(os.Environ(), "GO111MODULE=on") + command := strings.Join(cmd.Args, " ") + _, _ = fmt.Fprintf(GinkgoWriter, "running: %q\n", command) + output, err := cmd.Output() + if err != nil { + return string(output), fmt.Errorf("%q failed with error %q: %w", command, string(output), err) + } + + return string(output), nil +} + +// LoadImageToKindClusterWithName loads a local docker image to the kind cluster +func LoadImageToKindClusterWithName(name string) error { + cluster := defaultKindCluster + if v, ok := os.LookupEnv("KIND_CLUSTER"); ok { + cluster = v + } + kindOptions := []string{"load", "docker-image", name, "--name", cluster} + kindBinary := defaultKindBinary + if v, ok := os.LookupEnv("KIND"); ok { + kindBinary = v + } + cmd := exec.Command(kindBinary, kindOptions...) + _, err := Run(cmd) + return err +} + +// GetNonEmptyLines converts given command output string into individual objects +// according to line breakers, and ignores the empty elements in it. +func GetNonEmptyLines(output string) []string { + var res []string + elements := strings.SplitSeq(output, "\n") + for element := range elements { + if element != "" { + res = append(res, element) + } + } + + return res +} + +// GetProjectDir returns the project root, i.e. the closest ancestor of the +// working directory holding a go.mod. Tests run from their own package +// directory, but the commands they shell out to (make, kustomize) expect the +// root. +func GetProjectDir() (string, error) { + wd, err := os.Getwd() + if err != nil { + return wd, fmt.Errorf("failed to get current working directory: %w", err) + } + + for dir := wd; ; { + if _, err := os.Stat(filepath.Join(dir, "go.mod")); err == nil { + return dir, nil + } + parent := filepath.Dir(dir) + if parent == dir { + return wd, fmt.Errorf("failed to find the project root (no go.mod above %q)", wd) + } + dir = parent + } +} + +// GetKindNetwork returns the Docker network a Kind node is attached to, and that network's +// IPv4 subnet. Kind lets Docker choose the subnet -- and the network's name is itself +// overridable (KIND_EXPERIMENTAL_DOCKER_NETWORK) -- so both are read back from a node +// container rather than assumed. +func GetKindNetwork(node string) (string, *net.IPNet, error) { + output, err := Run(exec.Command("docker", "inspect", node, "-f", + `{{range $net, $cfg := .NetworkSettings.Networks}}{{if $cfg.IPAddress}}{{$net}} {{end}}{{end}}`)) + if err != nil { + return "", nil, err + } + networks := strings.Fields(output) + if len(networks) != 1 { + return "", nil, fmt.Errorf("node %s is attached to %d IPv4 networks, want exactly one", node, len(networks)) + } + network := networks[0] + + output, err = Run(exec.Command("docker", "network", "inspect", network, "-f", + `{{range .IPAM.Config}}{{.Subnet}} {{end}}`)) + if err != nil { + return "", nil, err + } + // A Kind network is dual-stack, so keep the IPv4 entry: the operator manages IPv4 only. + for _, candidate := range strings.Fields(output) { + ip, subnet, err := net.ParseCIDR(candidate) + if err != nil { + return "", nil, fmt.Errorf("parsing subnet %q of network %s: %w", candidate, network, err) + } + if ip.To4() != nil { + return network, subnet, nil + } + } + return "", nil, fmt.Errorf("network %s has no IPv4 subnet", network) +} + +func CreateSecondaryNetwork(secondarySubnetName string) (*net.IPNet, error) { + secondarySubnet, err := unusedPrivateSubnet() + if err != nil { + return nil, err + } + cmd := exec.Command("docker", "network", "create", + "--subnet", secondarySubnet.String(), + secondarySubnetName) + _, err = Run(cmd) + if err != nil { + return nil, err + } + return secondarySubnet, nil +} + +// ConnectNodeToNetwork attaches a Kind node's container to an extra Docker network, giving +// the node a host interface in that network's subnet. Kind cannot declare extra networks +// itself, and the manager resolves each VirtualIP to an interface of the host network +// namespace -- without this attach, an address on a secondary network resolves to nothing. +// The address is left to Docker's IPAM, which allocates from the bottom of the subnet. +func ConnectNodeToNetwork(node, network string) error { + _, err := Run(exec.Command("docker", "network", "connect", network, node)) + return err +} + +// RemoveNetworkIfPresent makes sure a Docker network is gone, whether it was created by +// this run or left behind by a crashed one. Docker refuses to remove a network that still +// has endpoints -- exactly the state a crashed run leaves, with the Kind nodes still +// attached -- so every attached container is force-disconnected first. The endpoints are +// read back from the network rather than assumed to be the Kind nodes: a stale network may +// hold containers this suite does not know about. +func RemoveNetworkIfPresent(network string) error { + containers, err := Run(exec.Command("docker", "network", "inspect", network, "-f", + `{{range .Containers}}{{.Name}} {{end}}`)) + if err != nil { + // The only expected failure is "no such network", which is the desired state. + return nil + } + + for container := range strings.FieldsSeq(containers) { + if _, err := Run(exec.Command("docker", "network", "disconnect", + "--force", network, container)); err != nil { + return err + } + } + + _, err = Run(exec.Command("docker", "network", "rm", network)) + return err +} + +// unusedPrivateSubnet returns a /16 inside 172.16.0.0/12 that no existing Docker +// network already occupies. Kind (and Docker's default IPAM) also draw from this +// range, so the suite cannot assume a fixed subnet is free. +func unusedPrivateSubnet() (*net.IPNet, error) { + occupied, err := dockerIPv4Subnets() + if err != nil { + return nil, err + } + + _, private, err := net.ParseCIDR(rfc1918Private) + if err != nil { + return nil, err + } + + base := binary.BigEndian.Uint32(private.IP.To4()) + const prefix = 16 + ones, bits := private.Mask.Size() + count := uint32(1) << (prefix - ones) + stride := uint32(1) << (bits - prefix) + for i := uint32(0); i < count; i++ { + addr := make(net.IP, net.IPv4len) + binary.BigEndian.PutUint32(addr, base+i*stride) + candidate := &net.IPNet{IP: addr, Mask: net.CIDRMask(prefix, 32)} + if !overlapsAny(candidate, occupied) { + return candidate, nil + } + } + return nil, fmt.Errorf("no unused /%d in %s", prefix, rfc1918Private) +} + +// dockerIPv4Subnets lists the IPv4 subnets of every Docker network. Dual-stack +// networks contribute their IPv4 entry only; networks without IPAM (host, none) +// contribute nothing. +func dockerIPv4Subnets() ([]*net.IPNet, error) { + ids, err := Run(exec.Command("docker", "network", "ls", "-q")) + if err != nil { + return nil, err + } + idList := strings.Fields(ids) + if len(idList) == 0 { + return nil, nil + } + + args := append([]string{"network", "inspect", "-f", + `{{range .IPAM.Config}}{{.Subnet}} {{end}}`}, idList...) + cmd := exec.Command("docker", args...) + output, err := Run(cmd) + if err != nil { + return nil, err + } + + subnets := []*net.IPNet{} + for _, candidate := range strings.Fields(output) { + ip, subnet, err := net.ParseCIDR(candidate) + if err != nil { + return nil, fmt.Errorf("parsing Docker subnet %q: %w", candidate, err) + } + if ip.To4() != nil { + subnets = append(subnets, subnet) + } + } + return subnets, nil +} + +func overlapsAny(candidate *net.IPNet, occupied []*net.IPNet) bool { + for _, existing := range occupied { + if existing.Contains(candidate.IP) || candidate.Contains(existing.IP) { + return true + } + } + return false +}