From 9f682acf2c0df8d5998608bcfbe10f49a1d19ed4 Mon Sep 17 00:00:00 2001 From: Aleksandr Albekov Date: Tue, 27 Oct 2026 14:44:55 +0100 Subject: [PATCH] Keep Kratos inside its memory limit under login bursts Staging's e2e run OOM-killed Kratos (limit 256Mi) mid-run, failing every test after it. Each argon2 hash takes 64 MB (kratos.yml), and Go, not knowing the container limit, let the heap grow past it between collections. Production has the same limit; a burst of 20 logins, which the rate limiters allow, would have done the same there. - GOMEMLIMIT=400MiB makes the collector work before the limit, which rises to 512Mi. - limit-inflight: at most four password submissions reach Kratos at once (4 x 64 MB plus its baseline fit); the rest wait in Traefik. Co-Authored-By: Claude Opus 5.5 --- deploy/k8s/ory.yaml | 5 ++++- deploy/k8s/routes.yaml | 14 +++++++++++++- deploy/staging/routes.yaml | 2 +- 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/deploy/k8s/ory.yaml b/deploy/k8s/ory.yaml index c951c2a..27621dc 100644 --- a/deploy/k8s/ory.yaml +++ b/deploy/k8s/ory.yaml @@ -41,6 +41,9 @@ spec: env: - { name: SQA_OPT_OUT, value: "true" } - { name: LOG_LEVEL, value: info } + # Go does not know the container limit; without this its heap grows past it between collections. Each + # argon2 hash takes 64 MB (kratos.yml), and routes.yaml lets at most four run at once (limit-inflight). + - { name: GOMEMLIMIT, value: 400MiB } - { name: DSN, valueFrom: { secretKeyRef: { name: auth, key: KRATOS_DSN } } } - { name: SECRETS_COOKIE, valueFrom: { secretKeyRef: { name: auth, key: KRATOS_SECRETS_COOKIE } } } - { name: SECRETS_DEFAULT, valueFrom: { secretKeyRef: { name: auth, key: KRATOS_SECRETS_DEFAULT } } } @@ -57,7 +60,7 @@ spec: httpGet: { path: /admin/health/alive, port: admin } resources: requests: { cpu: 10m, memory: 48Mi } - limits: { memory: 256Mi } + limits: { memory: 512Mi } volumeMounts: - { name: config, mountPath: /etc/config/kratos } - { name: internal-ca, mountPath: /etc/internal-ca, readOnly: true } diff --git a/deploy/k8s/routes.yaml b/deploy/k8s/routes.yaml index 2591aaf..e97a90b 100644 --- a/deploy/k8s/routes.yaml +++ b/deploy/k8s/routes.yaml @@ -11,7 +11,7 @@ spec: # Form submissions that carry secrets: rate-limited (Kratos has no lockout of its own). - match: Host(`auth.bulbashenko.com`) && Method(`POST`) && PathRegexp(`^/self-service/(login|recovery|verification|settings)`) priority: 200 - middlewares: [{ name: limit-ip }, { name: limit-all }, { name: headers }] + middlewares: [{ name: limit-ip }, { name: limit-all }, { name: limit-inflight }, { name: headers }] services: [{ name: kratos, port: 4433 }] - match: Host(`auth.bulbashenko.com`) && Path(`/oauth2/token`) priority: 200 @@ -58,6 +58,18 @@ spec: sourceCriterion: requestHeaderName: CF-Connecting-IP --- +# Password hashing (argon2, 64 MB each) at most four at a time, so a burst cannot push Kratos past its memory +# limit (ory.yaml); the rest wait in Traefik instead of killing the pod. +apiVersion: traefik.io/v1alpha1 +kind: Middleware +metadata: + name: limit-inflight +spec: + inFlightReq: + amount: 4 + sourceCriterion: + requestHost: true +--- # Everyone together: 120 per minute, so rotating addresses do not help either. apiVersion: traefik.io/v1alpha1 kind: Middleware diff --git a/deploy/staging/routes.yaml b/deploy/staging/routes.yaml index 33ab463..604e327 100644 --- a/deploy/staging/routes.yaml +++ b/deploy/staging/routes.yaml @@ -10,7 +10,7 @@ spec: # Form submissions that carry secrets: rate-limited (Kratos has no lockout of its own). - match: Host(`auth.bulbashenko.dev`) && Method(`POST`) && PathRegexp(`^/self-service/(login|recovery|verification|settings)`) priority: 200 - middlewares: [{ name: staging-access }, { name: limit-ip }, { name: limit-all }, { name: headers }] + middlewares: [{ name: staging-access }, { name: limit-ip }, { name: limit-all }, { name: limit-inflight }, { name: headers }] services: [{ name: kratos, port: 4433 }] - match: Host(`auth.bulbashenko.dev`) && Path(`/oauth2/token`) priority: 200