From ab0881c2b7609c51195fd5a8e222c1a3ca5474fd Mon Sep 17 00:00:00 2001 From: Aleksandr Albekov Date: Tue, 27 Oct 2026 14:57:40 +0100 Subject: [PATCH] Run staging's e2e as its own Kustomization, on the version it was built for Staging's run of d9d24d4 tested the new e2e image against the previous version's manifests (Kratos still at 256Mi), and held that version in place. The Job was part of the stack's Kustomization: changing pins changes its spec images, Flux reconciled at once with the old source artifact and new image tags, and its 30-minute health wait on the Job kept the new revision from being applied. - deploy/staging-e2e: the Job (and a ServiceAccount allowed to read Flux Kustomizations) for a Kustomization of its own in infra, depending on the stack's, so the stack's health no longer waits for the tests. - The image carries its commit (E2E_REVISION); the Job's wait first holds until the stack's Kustomization has applied that commit and is ready, which closes the remaining window, then waits for the services. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/image.yml | 1 + deploy/{staging => staging-e2e}/e2e.yaml | 30 ++++++++++++++++++++---- deploy/staging-e2e/kustomization.yaml | 8 +++++++ deploy/staging-e2e/rbac.yaml | 22 +++++++++++++++++ deploy/staging/kustomization.yaml | 4 ++-- e2e/Dockerfile | 6 +++-- 6 files changed, 62 insertions(+), 9 deletions(-) rename deploy/{staging => staging-e2e}/e2e.yaml (56%) create mode 100644 deploy/staging-e2e/kustomization.yaml create mode 100644 deploy/staging-e2e/rbac.yaml diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index df46d55..b6bce92 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -65,6 +65,7 @@ jobs: with: context: e2e push: true + build-args: REVISION=${{ needs.ui.outputs.sha }} tags: | ghcr.io/${{ github.repository_owner }}/auth-e2e:${{ needs.ui.outputs.sha }} ghcr.io/${{ github.repository_owner }}/auth-e2e:main diff --git a/deploy/staging/e2e.yaml b/deploy/staging-e2e/e2e.yaml similarity index 56% rename from deploy/staging/e2e.yaml rename to deploy/staging-e2e/e2e.yaml index 0f54f78..b6eb7f8 100644 --- a/deploy/staging/e2e.yaml +++ b/deploy/staging-e2e/e2e.yaml @@ -1,6 +1,6 @@ -# The e2e suite (e2e/, image auth-e2e built from the same commit) against this staging stack, once per version: -# `force` recreates the Job whenever its image tag changes. infra waits for it (Flux health check), so a red run -# marks the staging deploy failed and blocks the release (infra .github/workflows/staging-gate.yml). +# The e2e suite (e2e/, image auth-e2e built from the same commit) against the staging stack, once per version: +# `force` recreates the Job whenever its image tag changes. Its own Flux Kustomization (infra apps/staging/auth) +# waits for it after the stack's is ready, and its verdict gates the release (infra .github/workflows/staging-gate.yml). apiVersion: batch/v1 kind: Job metadata: @@ -18,14 +18,34 @@ spec: runAsUser: 1000 runAsGroup: 1000 seccompProfile: { type: RuntimeDefault } + serviceAccountName: e2e initContainers: - # Start once the stack this version brought up answers, clients included. + # Test exactly this version: wait until Flux has applied the commit this image was built from (E2E_REVISION) + # to the stack's Kustomization and it is ready, then until the services answer, clients included. - name: wait image: ghcr.io/bulbashenko-com/auth-e2e:latest + env: + - { name: E2E_FLUX_KUSTOMIZATION, value: auth } + - name: NAMESPACE + valueFrom: { fieldRef: { fieldPath: metadata.namespace } } + - { name: NODE_EXTRA_CA_CERTS, value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt } command: - node - -e - | + const fs = await import("node:fs") + const sleep = (s) => new Promise((r) => setTimeout(r, s * 1000)) + const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8") + const ks = `https://kubernetes.default.svc/apis/kustomize.toolkit.fluxcd.io/v1/namespaces/${process.env.NAMESPACE}/kustomizations/${process.env.E2E_FLUX_KUSTOMIZATION}` + const want = process.env.E2E_REVISION + for (;;) { + const k = await (await fetch(ks, { headers: { authorization: `Bearer ${token}` } })).json() + const applied = k.status?.lastAppliedRevision ?? "" + const ready = (k.status?.conditions ?? []).some((c) => c.type === "Ready" && c.status === "True") + if (applied.endsWith(want) && ready) break + console.log(`waiting for ${want.slice(0, 7)} on Kustomization ${k.metadata?.name}: applied ${applied}, ready ${ready}`) + await sleep(10) + } const urls = ["http://kratos:4434/admin/health/ready", "http://hydra:4445/health/ready", "http://keto:4466/health/ready", "http://sso-ui:3000/api/health", "http://hydra:4445/admin/clients/e2e"] const ok = async (u) => { try { return (await fetch(u)).ok } catch { return false } } @@ -34,7 +54,7 @@ spec: for (const u of urls) if (!(await ok(u))) down.push(u) if (!down.length) break console.log("waiting for", down.join(" ")) - await new Promise((r) => setTimeout(r, 5000)) + await sleep(5) } resources: requests: { cpu: 10m, memory: 32Mi } diff --git a/deploy/staging-e2e/kustomization.yaml b/deploy/staging-e2e/kustomization.yaml new file mode 100644 index 0000000..6f689c4 --- /dev/null +++ b/deploy/staging-e2e/kustomization.yaml @@ -0,0 +1,8 @@ +# The e2e run against staging, deployed by bulbashenko-com/infra as a Flux Kustomization of its own (auth-e2e) that +# depends on the stack's (deploy/staging): the stack's health never waits for the tests, and the tests start on the +# version they were built for. Namespace: auth-staging (Flux targetNamespace). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rbac.yaml + - e2e.yaml diff --git a/deploy/staging-e2e/rbac.yaml b/deploy/staging-e2e/rbac.yaml new file mode 100644 index 0000000..f8a56eb --- /dev/null +++ b/deploy/staging-e2e/rbac.yaml @@ -0,0 +1,22 @@ +# The e2e Job's wait reads the stack's Flux Kustomization (which revision is applied, is it ready); nothing else. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: e2e +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: e2e +rules: + - apiGroups: [kustomize.toolkit.fluxcd.io] + resources: [kustomizations] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: e2e +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: e2e } +subjects: + - { kind: ServiceAccount, name: e2e } diff --git a/deploy/staging/kustomization.yaml b/deploy/staging/kustomization.yaml index f478d7b..237d1c9 100644 --- a/deploy/staging/kustomization.yaml +++ b/deploy/staging/kustomization.yaml @@ -2,7 +2,8 @@ # builds the root with `components: [deploy/staging]` into the namespace auth-staging (apps/staging/auth, pinned to # the same commit as production). A component, not an overlay: an overlay cannot use an ancestor directory as base. Production configs plus # override files on bulbashenko.dev (ory/, as the test stack does), its own OIDC clients (clients/), mailpit for -# Kratos' mail, the e2e stand-ins (whoami), a seeded admin account and the e2e run (e2e.yaml). Design: infra#121. +# Kratos' mail, the e2e stand-ins (whoami) and a seeded admin account. The e2e run is deploy/staging-e2e, a +# Kustomization of its own that starts after this one. Design: infra#121. apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: @@ -10,7 +11,6 @@ resources: - whoami.yaml - extras.yaml - seed-admin.yaml - - e2e.yaml - network-policy.yaml configMapGenerator: - name: ory-staging diff --git a/e2e/Dockerfile b/e2e/Dockerfile index b858840..ad691d4 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,6 +1,8 @@ -# The e2e suite as an image, for staging (deploy/staging/e2e.yaml): Playwright with Chromium, tests and config. +# The e2e suite as an image, for staging (deploy/staging-e2e/e2e.yaml): Playwright with Chromium, tests and config. # Built by .github/workflows/image.yml next to auth-ui, same tag (the commit SHA). FROM mcr.microsoft.com/playwright:v1.64.0-noble +# The commit the suite was built from; the staging Job waits until Flux has deployed that commit. +ARG REVISION=unknown WORKDIR /e2e COPY package.json package-lock.json ./ RUN npm ci --no-audit --no-fund @@ -8,6 +10,6 @@ COPY playwright.config.ts ./ COPY tests ./tests COPY run-in-cluster.sh ./ # Plain `list` output into the Job log (CI=1 would also write an HTML report into the read-only image). -ENV HOME=/tmp E2E_OUTPUT_DIR=/tmp/results +ENV HOME=/tmp E2E_OUTPUT_DIR=/tmp/results E2E_REVISION=$REVISION USER pwuser CMD ["/e2e/run-in-cluster.sh"]