diff --git a/.github/workflows/image.yml b/.github/workflows/image.yml index df46d55..b6bce92 100644 --- a/.github/workflows/image.yml +++ b/.github/workflows/image.yml @@ -65,6 +65,7 @@ jobs: with: context: e2e push: true + build-args: REVISION=${{ needs.ui.outputs.sha }} tags: | ghcr.io/${{ github.repository_owner }}/auth-e2e:${{ needs.ui.outputs.sha }} ghcr.io/${{ github.repository_owner }}/auth-e2e:main diff --git a/deploy/staging/e2e.yaml b/deploy/staging-e2e/e2e.yaml similarity index 56% rename from deploy/staging/e2e.yaml rename to deploy/staging-e2e/e2e.yaml index 0f54f78..b6eb7f8 100644 --- a/deploy/staging/e2e.yaml +++ b/deploy/staging-e2e/e2e.yaml @@ -1,6 +1,6 @@ -# The e2e suite (e2e/, image auth-e2e built from the same commit) against this staging stack, once per version: -# `force` recreates the Job whenever its image tag changes. infra waits for it (Flux health check), so a red run -# marks the staging deploy failed and blocks the release (infra .github/workflows/staging-gate.yml). +# The e2e suite (e2e/, image auth-e2e built from the same commit) against the staging stack, once per version: +# `force` recreates the Job whenever its image tag changes. Its own Flux Kustomization (infra apps/staging/auth) +# waits for it after the stack's is ready, and its verdict gates the release (infra .github/workflows/staging-gate.yml). apiVersion: batch/v1 kind: Job metadata: @@ -18,14 +18,34 @@ spec: runAsUser: 1000 runAsGroup: 1000 seccompProfile: { type: RuntimeDefault } + serviceAccountName: e2e initContainers: - # Start once the stack this version brought up answers, clients included. + # Test exactly this version: wait until Flux has applied the commit this image was built from (E2E_REVISION) + # to the stack's Kustomization and it is ready, then until the services answer, clients included. - name: wait image: ghcr.io/bulbashenko-com/auth-e2e:latest + env: + - { name: E2E_FLUX_KUSTOMIZATION, value: auth } + - name: NAMESPACE + valueFrom: { fieldRef: { fieldPath: metadata.namespace } } + - { name: NODE_EXTRA_CA_CERTS, value: /var/run/secrets/kubernetes.io/serviceaccount/ca.crt } command: - node - -e - | + const fs = await import("node:fs") + const sleep = (s) => new Promise((r) => setTimeout(r, s * 1000)) + const token = fs.readFileSync("/var/run/secrets/kubernetes.io/serviceaccount/token", "utf8") + const ks = `https://kubernetes.default.svc/apis/kustomize.toolkit.fluxcd.io/v1/namespaces/${process.env.NAMESPACE}/kustomizations/${process.env.E2E_FLUX_KUSTOMIZATION}` + const want = process.env.E2E_REVISION + for (;;) { + const k = await (await fetch(ks, { headers: { authorization: `Bearer ${token}` } })).json() + const applied = k.status?.lastAppliedRevision ?? "" + const ready = (k.status?.conditions ?? []).some((c) => c.type === "Ready" && c.status === "True") + if (applied.endsWith(want) && ready) break + console.log(`waiting for ${want.slice(0, 7)} on Kustomization ${k.metadata?.name}: applied ${applied}, ready ${ready}`) + await sleep(10) + } const urls = ["http://kratos:4434/admin/health/ready", "http://hydra:4445/health/ready", "http://keto:4466/health/ready", "http://sso-ui:3000/api/health", "http://hydra:4445/admin/clients/e2e"] const ok = async (u) => { try { return (await fetch(u)).ok } catch { return false } } @@ -34,7 +54,7 @@ spec: for (const u of urls) if (!(await ok(u))) down.push(u) if (!down.length) break console.log("waiting for", down.join(" ")) - await new Promise((r) => setTimeout(r, 5000)) + await sleep(5) } resources: requests: { cpu: 10m, memory: 32Mi } diff --git a/deploy/staging-e2e/kustomization.yaml b/deploy/staging-e2e/kustomization.yaml new file mode 100644 index 0000000..6f689c4 --- /dev/null +++ b/deploy/staging-e2e/kustomization.yaml @@ -0,0 +1,8 @@ +# The e2e run against staging, deployed by bulbashenko-com/infra as a Flux Kustomization of its own (auth-e2e) that +# depends on the stack's (deploy/staging): the stack's health never waits for the tests, and the tests start on the +# version they were built for. Namespace: auth-staging (Flux targetNamespace). +apiVersion: kustomize.config.k8s.io/v1beta1 +kind: Kustomization +resources: + - rbac.yaml + - e2e.yaml diff --git a/deploy/staging-e2e/rbac.yaml b/deploy/staging-e2e/rbac.yaml new file mode 100644 index 0000000..f8a56eb --- /dev/null +++ b/deploy/staging-e2e/rbac.yaml @@ -0,0 +1,22 @@ +# The e2e Job's wait reads the stack's Flux Kustomization (which revision is applied, is it ready); nothing else. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: e2e +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: e2e +rules: + - apiGroups: [kustomize.toolkit.fluxcd.io] + resources: [kustomizations] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: e2e +roleRef: { apiGroup: rbac.authorization.k8s.io, kind: Role, name: e2e } +subjects: + - { kind: ServiceAccount, name: e2e } diff --git a/deploy/staging/kustomization.yaml b/deploy/staging/kustomization.yaml index f478d7b..237d1c9 100644 --- a/deploy/staging/kustomization.yaml +++ b/deploy/staging/kustomization.yaml @@ -2,7 +2,8 @@ # builds the root with `components: [deploy/staging]` into the namespace auth-staging (apps/staging/auth, pinned to # the same commit as production). A component, not an overlay: an overlay cannot use an ancestor directory as base. Production configs plus # override files on bulbashenko.dev (ory/, as the test stack does), its own OIDC clients (clients/), mailpit for -# Kratos' mail, the e2e stand-ins (whoami), a seeded admin account and the e2e run (e2e.yaml). Design: infra#121. +# Kratos' mail, the e2e stand-ins (whoami) and a seeded admin account. The e2e run is deploy/staging-e2e, a +# Kustomization of its own that starts after this one. Design: infra#121. apiVersion: kustomize.config.k8s.io/v1alpha1 kind: Component resources: @@ -10,7 +11,6 @@ resources: - whoami.yaml - extras.yaml - seed-admin.yaml - - e2e.yaml - network-policy.yaml configMapGenerator: - name: ory-staging diff --git a/e2e/Dockerfile b/e2e/Dockerfile index b858840..ad691d4 100644 --- a/e2e/Dockerfile +++ b/e2e/Dockerfile @@ -1,6 +1,8 @@ -# The e2e suite as an image, for staging (deploy/staging/e2e.yaml): Playwright with Chromium, tests and config. +# The e2e suite as an image, for staging (deploy/staging-e2e/e2e.yaml): Playwright with Chromium, tests and config. # Built by .github/workflows/image.yml next to auth-ui, same tag (the commit SHA). FROM mcr.microsoft.com/playwright:v1.64.0-noble +# The commit the suite was built from; the staging Job waits until Flux has deployed that commit. +ARG REVISION=unknown WORKDIR /e2e COPY package.json package-lock.json ./ RUN npm ci --no-audit --no-fund @@ -8,6 +10,6 @@ COPY playwright.config.ts ./ COPY tests ./tests COPY run-in-cluster.sh ./ # Plain `list` output into the Job log (CI=1 would also write an HTML report into the read-only image). -ENV HOME=/tmp E2E_OUTPUT_DIR=/tmp/results +ENV HOME=/tmp E2E_OUTPUT_DIR=/tmp/results E2E_REVISION=$REVISION USER pwuser CMD ["/e2e/run-in-cluster.sh"]