Keep Kratos inside its memory limit under login bursts

deploy/k8s/ory.yaml

@@ -41,6 +41,9 @@ spec:
           env:
             - { name: SQA_OPT_OUT, value: "true" }
             - { name: LOG_LEVEL, value: info }
+            # Go does not know the container limit; without this its heap grows past it between collections. Each
+            # argon2 hash takes 64 MB (kratos.yml), and routes.yaml lets at most four run at once (limit-inflight).
+            - { name: GOMEMLIMIT, value: 400MiB }
             - { name: DSN, valueFrom: { secretKeyRef: { name: auth, key: KRATOS_DSN } } }
             - { name: SECRETS_COOKIE, valueFrom: { secretKeyRef: { name: auth, key: KRATOS_SECRETS_COOKIE } } }
             - { name: SECRETS_DEFAULT, valueFrom: { secretKeyRef: { name: auth, key: KRATOS_SECRETS_DEFAULT } } }
@@ -57,7 +60,7 @@ spec:
             httpGet: { path: /admin/health/alive, port: admin }
           resources:
             requests: { cpu: 10m, memory: 48Mi }
-            limits: { memory: 256Mi }
+            limits: { memory: 512Mi }
           volumeMounts:
             - { name: config, mountPath: /etc/config/kratos }
             - { name: internal-ca, mountPath: /etc/internal-ca, readOnly: true }

deploy/k8s/routes.yaml

@@ -11,7 +11,7 @@ spec:
     # Form submissions that carry secrets: rate-limited (Kratos has no lockout of its own).
     - match: Host(`auth.bulbashenko.com`) && Method(`POST`) && PathRegexp(`^/self-service/(login|recovery|verification|settings)`)
       priority: 200
-      middlewares: [{ name: limit-ip }, { name: limit-all }, { name: headers }]
+      middlewares: [{ name: limit-ip }, { name: limit-all }, { name: limit-inflight }, { name: headers }]
       services: [{ name: kratos, port: 4433 }]
     - match: Host(`auth.bulbashenko.com`) && Path(`/oauth2/token`)
       priority: 200
@@ -58,6 +58,18 @@ spec:
     sourceCriterion:
       requestHeaderName: CF-Connecting-IP
 ---
+# Password hashing (argon2, 64 MB each) at most four at a time, so a burst cannot push Kratos past its memory
+# limit (ory.yaml); the rest wait in Traefik instead of killing the pod.
+apiVersion: traefik.io/v1alpha1
+kind: Middleware
+metadata:
+  name: limit-inflight
+spec:
+  inFlightReq:
+    amount: 4
+    sourceCriterion:
+      requestHost: true
+---
 # Everyone together: 120 per minute, so rotating addresses do not help either.
 apiVersion: traefik.io/v1alpha1
 kind: Middleware

deploy/staging/routes.yaml

@@ -10,7 +10,7 @@ spec:
     # Form submissions that carry secrets: rate-limited (Kratos has no lockout of its own).
     - match: Host(`auth.bulbashenko.dev`) && Method(`POST`) && PathRegexp(`^/self-service/(login|recovery|verification|settings)`)
       priority: 200
-      middlewares: [{ name: staging-access }, { name: limit-ip }, { name: limit-all }, { name: headers }]
+      middlewares: [{ name: staging-access }, { name: limit-ip }, { name: limit-all }, { name: limit-inflight }, { name: headers }]
       services: [{ name: kratos, port: 4433 }]
     - match: Host(`auth.bulbashenko.dev`) && Path(`/oauth2/token`)
       priority: 200