DEVOPS-3406 - inject required secret keys only

packaging/helm/openwork-ee/README.md

@@ -334,9 +334,10 @@ git and in rendered manifests), use ESO mode. The chart renders an
 provider in-cluster:
 
 The chart renders `spec.data` — the oldest stable ESO shape, unchanged since
-`external-secrets.io/v1beta1` — pulling every `secret.keys.*` entry from
-`<pathPrefix>/<KEY_NAME>` in the provider. The key list is generated from
-`secret.keys`, so it can never drift from what the workloads consume:
+`external-secrets.io/v1beta1` — pulling keys from `<pathPrefix>/<KEY_NAME>` in
+the provider. Only the three boot-critical keys (`DATABASE_URL`,
+`BETTER_AUTH_SECRET`, `DEN_DB_ENCRYPTION_KEY`) are rendered by default; add
+more by name via `optionalKeys`:
 
 ```yaml
 secret:
@@ -350,19 +351,43 @@ externalSecrets:
     name: external-secrets
     kind: ClusterSecretStore
   refreshInterval: 5m
-  # Every secret.keys.* value must exist as a JSON property under this trunk,
-  # e.g. eks/openwork/prod/den/DATABASE_URL.
+  # The three boot-critical keys must exist under this trunk, e.g.
+  # eks/openwork/prod/den/DATABASE_URL.
   pathPrefix: "eks/openwork/prod/den"
+  # Additional keys to pull, by secret.keys.* name. Only listed keys are
+  # rendered, so keys you do not list need not exist in the provider.
+  optionalKeys:
+    - databaseRedisUrl
+    - emailFrom
+    - smtpHost
+    - smtpPort
+    - smtpUser
+    - smtpPass
+    - smtpSecure
 ```
 
-Every property the workloads consume must exist in your provider under
-`pathPrefix`, named like `secret.keys.*` values (`DATABASE_URL`,
-`BETTER_AUTH_SECRET`, ...) — the chart pulls each key by name and cannot
-invent missing ones. `target.deletionPolicy` defaults to `Retain`, so
-uninstalling the release keeps the materialized Secret. ESO must be installed
-on the destination cluster with a `SecretStore`/`ClusterSecretStore`; the
-chart selects `external-secrets.io/v1` or `v1beta1` from cluster capabilities
-and fails loudly at sync time if the CRDs are missing.
+The three required keys must exist in your provider under `pathPrefix` — a
+missing one fails the ExternalSecret loudly. ESO's `remoteRef` has no
+"skip-if-missing" field, so optional keys are opt-in: any `secret.keys.*` name
+you list under `optionalKeys` must exist in the provider, and keys you omit do
+not land in the Secret. Omitted keys are simply absent from the workload
+environment, so check each consumer before omitting one. Two illustrative
+cases: `DAYTONA_API_KEY` is *required* when `config.provisioner.mode` is
+`daytona` — Den API rejects startup without it, so omitting it there breaks
+boot, whereas it is safe to omit under the default `stub` provisioner; and
+omitted `SMTP_PORT`/`SMTP_SECURE` fall back to the application's own defaults
+(`587` / `false`) whenever they are absent, regardless of whether the Secret
+carries them. `optionalKeys`
+entries are `secret.keys` **property names** (camelCase, e.g. `smtpPass`); the
+provider path and the target Secret key use the corresponding **value**
+(`SMTP_PASS` by default, overridable via `secret.keys.smtpPass`). So list
+`smtpPass` here, ensure `eks/.../SMTP_PASS` (or your overridden value) exists
+in the provider, and the Secret key will be `SMTP_PASS`.
+`target.deletionPolicy` defaults to `Retain`, so uninstalling the release keeps
+the materialized Secret. ESO must be installed on the destination cluster with
+a `SecretStore`/`ClusterSecretStore`; the chart selects
+`external-secrets.io/v1` or `v1beta1` from cluster capabilities and fails
+loudly at sync time if the CRDs are missing.
 
 Set optional `DATABASE_REDIS_URL` to enable Den API Redis-backed session and query caching. Set `DAYTONA_API_KEY` when `config.provisioner.mode` is `daytona`. Set `POLAR_ACCESS_TOKEN` when Polar feature gating is enabled. Set `OPENROUTER_MANAGEMENT_API_KEY` when enabling OpenWork Models management.
 

packaging/helm/openwork-ee/templates/_helpers.tpl

@@ -89,6 +89,123 @@ app.kubernetes.io/component: {{ .component }}
 {{- include "openwork-ee.namespace" . | trimAll "\"" -}}
 {{- end -}}
 
+{{/*
+  Workload roll trigger for secret content. In inline mode the rendered
+  secret.yaml hash already changes with secret.values. In externalSecrets mode
+  secret.yaml renders empty, so hash the inputs that change the Secret ESO
+  materializes — the resolved key set, pathPrefix, the secretStoreRef, and the
+  conversion/decoding strategies (which change the decoded bytes). envFrom keys
+  and values are fixed at pod start, so any of these changing must roll the
+  workloads. Non-content ExternalSecret fields (refreshInterval, metadataPolicy,
+  hook annotations, target policies) are deliberately excluded so they do not
+  cause spurious rolls. existingSecret mode is operator-managed — no chart
+  values drive its content, so no trigger is possible there.
+*/}}
+{{- define "openwork-ee.secretChecksum" -}}
+{{- if eq .Values.secret.secretsMode "externalSecrets" -}}
+{{- $requiredKeys := list "databaseUrl" "betterAuthSecret" "denDbEncryptionKey" -}}
+{{- $optionalKeys := .Values.externalSecrets.optionalKeys | default (list) -}}
+{{- $resolvedKeys := list -}}
+{{- range $name := concat $requiredKeys $optionalKeys | uniq | sortAlpha -}}
+{{- $resolvedKeys = append $resolvedKeys (index $.Values.secret.keys $name) -}}
+{{- end -}}
+{{- $store := .Values.externalSecrets.secretStoreRef | default dict -}}
+{{- $input := dict
+    "keys" ($resolvedKeys | uniq | sortAlpha)
+    "pathPrefix" (.Values.externalSecrets.pathPrefix | toString | trim | trimSuffix "/")
+    "secretStoreName" ($store.name | default "")
+    "secretStoreKind" ($store.kind | default "")
+    "conversionStrategy" .Values.externalSecrets.conversionStrategy
+    "decodingStrategy" .Values.externalSecrets.decodingStrategy -}}
+{{- $input | toJson | sha256sum -}}
+{{- else -}}
+{{- include (print $.Template.BasePath "/secret.yaml") . | sha256sum -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+  Resolved provider key set the workloads expect in the Secret: the three
+  boot-critical keys plus opted-in optionalKeys. Drives the wait-for-secret
+  init container. Only meaningful in externalSecrets mode.
+*/}}
+{{- define "openwork-ee.expectedSecretKeys" -}}
+{{- $requiredKeys := list "databaseUrl" "betterAuthSecret" "denDbEncryptionKey" -}}
+{{- $optionalKeys := .Values.externalSecrets.optionalKeys | default (list) -}}
+{{- $resolved := list -}}
+{{- range $name := concat $requiredKeys $optionalKeys | uniq | sortAlpha -}}
+{{- $resolved = append $resolved (index $.Values.secret.keys $name) -}}
+{{- end -}}
+{{- $resolved | uniq | sortAlpha | join " " -}}
+{{- end -}}
+
+{{/*
+  Init container that blocks until the workload Secret exists and, in
+  externalSecrets mode, contains the full expected key set. envFrom imports the
+  keys present at pod start and never refreshes, so a pod that starts before
+  ESO reconciles a newly-added optional key would hold a stale env until its
+  next restart. Required keys block indefinitely (the workload cannot boot
+  without them); the optional remainder is bounded by
+  externalSecrets.optionalKeyWaitSeconds so a typo'd optional key degrades
+  (pod starts without it) rather than bricking the Deployment.
+*/}}
+{{- define "openwork-ee.waitForSecretInitContainer" -}}
+- name: wait-for-secret
+  image: "{{ .Values.migrations.kubectlImage.repository }}:{{ .Values.migrations.kubectlImage.tag }}"
+  imagePullPolicy: {{ .Values.image.pullPolicy }}
+  command:
+    - sh
+    - -c
+    - |
+        set -u
+        SECRET="{{ include "openwork-ee.secretNameRaw" . }}"
+        NS="{{ include "openwork-ee.namespaceRaw" . }}"
+        until kubectl get secret "$SECRET" -n "$NS" > /dev/null 2>&1; do
+          echo "waiting for secret $SECRET..."
+          sleep 3
+        done
+        {{- if eq .Values.secret.secretsMode "externalSecrets" }}
+        # Wait without bound for the three boot-critical keys.
+        for key in {{ include "openwork-ee.requiredSecretKeys" . }}; do
+          until kubectl get secret "$SECRET" -n "$NS" -o jsonpath="{.data.$key}" 2>/dev/null | grep -q .; do
+            echo "waiting for required key $key in secret $SECRET..."
+            sleep 3
+          done
+        done
+        # Bounded wait for the optional remainder, then proceed. Rendered
+        # directly (not via `default`) so an explicit 0 truly skips the wait —
+        # `default 60` treats numeric 0 as empty and would force 60.
+        deadline=$(( $(date +%s) + {{ .Values.externalSecrets.optionalKeyWaitSeconds }} ))
+        for key in {{ include "openwork-ee.optionalSecretKeys" . }}; do
+          while ! kubectl get secret "$SECRET" -n "$NS" -o jsonpath="{.data.$key}" 2>/dev/null | grep -q .; do
+            if [ "$(date +%s)" -ge "$deadline" ]; then
+              echo "proceeding without optional key $key (waited {{ .Values.externalSecrets.optionalKeyWaitSeconds }}s)"
+              break
+            fi
+            echo "waiting for optional key $key in secret $SECRET..."
+            sleep 3
+          done
+        done
+        {{- end }}
+{{- end -}}
+
+{{/* Required provider key names (env names) in externalSecrets mode. */}}
+{{- define "openwork-ee.requiredSecretKeys" -}}
+{{- $out := list -}}
+{{- range $name := list "databaseUrl" "betterAuthSecret" "denDbEncryptionKey" -}}
+{{- $out = append $out (index $.Values.secret.keys $name) -}}
+{{- end -}}
+{{- $out | join " " -}}
+{{- end -}}
+
+{{/* Opt-in optional provider key names (env names) in externalSecrets mode. */}}
+{{- define "openwork-ee.optionalSecretKeys" -}}
+{{- $out := list -}}
+{{- range $name := .Values.externalSecrets.optionalKeys | default (list) -}}
+{{- $out = append $out (index $.Values.secret.keys $name) -}}
+{{- end -}}
+{{- $out | join " " -}}
+{{- end -}}
+
 {{- define "openwork-ee.secretsMode.validate" -}}
 {{- if not (has .Values.secret.secretsMode (list "inline" "existingSecret" "externalSecrets")) -}}
 {{- fail "secretsMode must be one of inline, existingSecret, externalSecrets" -}}
@@ -160,6 +277,11 @@ external-secrets.io/v1beta1
 {{- if not (.Values.externalSecrets.pathPrefix | toString | trim) -}}
 {{- fail "externalSecrets.pathPrefix is required when secretsMode=externalSecrets" -}}
 {{- end -}}
+{{- range $key := .Values.externalSecrets.optionalKeys | default (list) -}}
+{{- if not (hasKey $.Values.secret.keys $key) -}}
+{{- fail (printf "externalSecrets.optionalKeys contains %q, which is not a known secret.keys.* name" $key) -}}
+{{- end -}}
+{{- end -}}
 {{- end -}}
 {{- end -}}
 

packaging/helm/openwork-ee/templates/den-api.yaml

@@ -47,7 +47,7 @@ spec:
         {{- end }}
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
-        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        checksum/secret: {{ include "openwork-ee.secretChecksum" . }}
         {{- with .Values.denApi.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -81,6 +81,11 @@ spec:
         {{- include "openwork-ee.customCa.volume" . | nindent 8 }}
         {{- end }}
       {{- end }}
+      {{- if ne .Values.secret.secretsMode "inline" }}
+      serviceAccountName: {{ include "openwork-ee.fullname" . }}-workload
+      initContainers:
+        {{- include "openwork-ee.waitForSecretInitContainer" . | nindent 8 }}
+      {{- end }}
       containers:
         - name: den-api
           image: "{{ .Values.denApi.image.repository }}:{{ default .Values.image.tag .Values.denApi.image.tag }}"

packaging/helm/openwork-ee/templates/den-web.yaml

@@ -47,7 +47,7 @@ spec:
         {{- end }}
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
-        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        checksum/secret: {{ include "openwork-ee.secretChecksum" . }}
         {{- with .Values.denWeb.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -60,6 +60,11 @@ spec:
       volumes:
         {{- include "openwork-ee.customCa.volume" . | nindent 8 }}
       {{- end }}
+      {{- if ne .Values.secret.secretsMode "inline" }}
+      serviceAccountName: {{ include "openwork-ee.fullname" . }}-workload
+      initContainers:
+        {{- include "openwork-ee.waitForSecretInitContainer" . | nindent 8 }}
+      {{- end }}
       containers:
         - name: den-web
           image: "{{ .Values.denWeb.image.repository }}:{{ default .Values.image.tag .Values.denWeb.image.tag }}"

packaging/helm/openwork-ee/templates/externalsecret.yaml

@@ -29,9 +29,18 @@ spec:
     name: {{ include "openwork-ee.secretName" . }}
     creationPolicy: {{ .Values.externalSecrets.target.creationPolicy | default "Owner" }}
     deletionPolicy: {{ .Values.externalSecrets.target.deletionPolicy | default "Retain" }}
+  {{- /*
+    Render the three boot-critical keys plus whatever optional keys the
+    operator lists in externalSecrets.optionalKeys. ESO's spec.data remoteRef
+    has no "skip if missing" field, so optionality is expressed by simply not
+    emitting a data entry for keys the operator does not have in the provider —
+    a missing rendered key would fail the whole ExternalSecret.
+  */}}
+  {{- $requiredKeys := list "databaseUrl" "betterAuthSecret" "denDbEncryptionKey" }}
+  {{- $optionalKeys := .Values.externalSecrets.optionalKeys | default (list) }}
   data:
     {{- $prefix := .Values.externalSecrets.pathPrefix | toString | trim | trimSuffix "/" }}
-    {{- range $name := keys .Values.secret.keys | sortAlpha }}
+    {{- range $name := concat $requiredKeys $optionalKeys | uniq | sortAlpha }}
     {{- $envKey := index $.Values.secret.keys $name }}
     - secretKey: {{ $envKey | quote }}
       remoteRef:

packaging/helm/openwork-ee/templates/inference.yaml

@@ -48,7 +48,7 @@ spec:
         {{- end }}
       annotations:
         checksum/config: {{ include (print $.Template.BasePath "/configmap.yaml") . | sha256sum }}
-        checksum/secret: {{ include (print $.Template.BasePath "/secret.yaml") . | sha256sum }}
+        checksum/secret: {{ include "openwork-ee.secretChecksum" . }}
         {{- with .Values.inference.podAnnotations }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
@@ -61,6 +61,11 @@ spec:
       volumes:
         {{- include "openwork-ee.customCa.volume" . | nindent 8 }}
       {{- end }}
+      {{- if ne .Values.secret.secretsMode "inline" }}
+      serviceAccountName: {{ include "openwork-ee.fullname" . }}-workload
+      initContainers:
+        {{- include "openwork-ee.waitForSecretInitContainer" . | nindent 8 }}
+      {{- end }}
       containers:
         - name: inference
           image: "{{ .Values.inference.image.repository }}:{{ default .Values.image.tag .Values.inference.image.tag }}"

packaging/helm/openwork-ee/templates/migration-job.yaml

@@ -44,22 +44,13 @@ spec:
       {{- /*
         When the Secret is materialized asynchronously (ESO or out-of-band
         creation), the Job must not fail on a missing Secret before it exists.
-        Block in an init container until it appears instead of erroring on the
-        env secretKeyRef.
+        Block in an init container until it appears (and, in externalSecrets
+        mode, holds the expected keys) instead of erroring on the env
+        secretKeyRef.
       */}}
       {{- if ne .Values.secret.secretsMode "inline" }}
       initContainers:
-        - name: wait-for-secret
-          image: "{{ .Values.migrations.kubectlImage.repository }}:{{ .Values.migrations.kubectlImage.tag }}"
-          imagePullPolicy: {{ .Values.image.pullPolicy }}
-          command:
-            - sh
-            - -c
-            - |
-              until kubectl get secret {{ include "openwork-ee.secretNameRaw" . }} -n {{ include "openwork-ee.namespaceRaw" . }} > /dev/null 2>&1; do
-                echo "waiting for secret {{ include "openwork-ee.secretNameRaw" . }}..."
-                sleep 3
-              done
+        {{- include "openwork-ee.waitForSecretInitContainer" . | nindent 8 }}
       {{- end }}
       containers:
         - name: migrate

packaging/helm/openwork-ee/templates/workload-rbac.yaml

@@ -0,0 +1,44 @@
+{{- /*
+  ServiceAccount + Role + RoleBinding for the workload Deployments
+  (den-api, den-web, inference) so their wait-for-secret init containers can
+  read the workload Secret. Rendered only outside inline mode. Not hook
+  resources — these must persist for the life of the Deployments.
+*/}}
+{{- if ne .Values.secret.secretsMode "inline" }}
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: {{ include "openwork-ee.fullname" . }}-workload
+  namespace: {{ include "openwork-ee.namespace" . }}
+  labels:
+    {{- include "openwork-ee.labels" . | nindent 4 }}
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: {{ include "openwork-ee.fullname" . }}-workload
+  namespace: {{ include "openwork-ee.namespace" . }}
+  labels:
+    {{- include "openwork-ee.labels" . | nindent 4 }}
+rules:
+  - apiGroups: [""]
+    resources: ["secrets"]
+    resourceNames: [{{ include "openwork-ee.secretName" . }}]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: {{ include "openwork-ee.fullname" . }}-workload
+  namespace: {{ include "openwork-ee.namespace" . }}
+  labels:
+    {{- include "openwork-ee.labels" . | nindent 4 }}
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: {{ include "openwork-ee.fullname" . }}-workload
+subjects:
+  - kind: ServiceAccount
+    name: {{ include "openwork-ee.fullname" . }}-workload
+    namespace: {{ include "openwork-ee.namespace" . }}
+{{- end }}