test(DEN-2724): render checksum-pinned private source on three platforms

.github/workflows/direct-source-render.yml

@@ -0,0 +1,158 @@
+name: exact direct-source snapshot Kustomize render
+
+on:
+  pull_request:
+    paths:
+      - .github/workflows/direct-source-render.yml
+      - scripts/install_pinned_kustomize.py
+      - scripts/direct_source_render_canary.py
+      - scripts/check_snapshot_endpoint_policy.py
+      - scripts/test_snapshot_endpoint_policy.py
+      - docs/direct-source-render.md
+      - fixtures/direct-source/fabrication-server.rs/**
+  workflow_dispatch:
+    inputs:
+      cluster_commit:
+        description: Exact ORESoftware/k8s-cluster catalog candidate
+        required: false
+        default: 0c71eed05f8be50ac22ebe73715ded4a871234a8
+        type: string
+      child_commit:
+        description: Exact private source commit represented by the snapshot
+        required: false
+        default: cea4a3c772012e1f2f87050bac24e911b8e2e577
+        type: string
+
+permissions:
+  contents: read
+
+concurrency:
+  group: direct-source-render-${{ github.event.pull_request.number || github.ref }}
+  cancel-in-progress: true
+
+defaults:
+  run:
+    shell: bash
+
+env:
+  K8S_CLUSTER_COMMIT: ${{ inputs.cluster_commit || '0c71eed05f8be50ac22ebe73715ded4a871234a8' }}
+  FABRICATION_SERVER_COMMIT: ${{ inputs.child_commit || 'cea4a3c772012e1f2f87050bac24e911b8e2e577' }}
+  SOURCE_SNAPSHOT: fixtures/direct-source/fabrication-server.rs/cea4a3c772012e1f2f87050bac24e911b8e2e577
+
+jobs:
+  render:
+    name: direct source snapshot render / ${{ matrix.os }}
+    runs-on: ${{ matrix.os }}
+    timeout-minutes: 20
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-24.04, macos-15, windows-2025]
+
+    steps:
+      - name: Check out independent test harness and source snapshot
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          fetch-depth: 1
+          persist-credentials: false
+          show-progress: false
+
+      - name: Check out exact cluster catalog without child worktrees
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          repository: ORESoftware/k8s-cluster
+          ref: ${{ env.K8S_CLUSTER_COMMIT }}
+          path: product/k8s-cluster
+          fetch-depth: 1
+          submodules: false
+          persist-credentials: false
+          show-progress: false
+
+      - name: Install Python 3.12
+        uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
+        with:
+          python-version: '3.12'
+
+      - name: Verify immutable cluster checkout and all test syntax
+        run: |
+          set -euo pipefail
+          [[ "$K8S_CLUSTER_COMMIT" =~ ^[0-9a-f]{40}$ ]]
+          [[ "$FABRICATION_SERVER_COMMIT" =~ ^[0-9a-f]{40}$ ]]
+          test "$(git -C product/k8s-cluster rev-parse HEAD)" = "$K8S_CLUSTER_COMMIT"
+          test -f "$SOURCE_SNAPSHOT/source.json"
+          python -m py_compile \
+            scripts/install_pinned_kustomize.py \
+            scripts/direct_source_render_canary.py \
+            scripts/check_snapshot_endpoint_policy.py \
+            scripts/test_snapshot_endpoint_policy.py \
+            "$SOURCE_SNAPSHOT/scripts/ci/check_k8s_network_contract.py" \
+            "$SOURCE_SNAPSHOT/scripts/ci/test_check_k8s_network_contract.py"
+          python scripts/test_snapshot_endpoint_policy.py
+          (
+            cd "$SOURCE_SNAPSHOT"
+            python scripts/ci/test_check_k8s_network_contract.py
+            python scripts/ci/check_k8s_network_contract.py
+          )
+          rm -rf scripts/__pycache__ "$SOURCE_SNAPSHOT/scripts/ci/__pycache__"
+
+      - name: Install checksum-pinned Kustomize v5.8.1
+        run: |
+          set -euo pipefail
+          mkdir -p artifacts
+          kustomize="$(python scripts/install_pinned_kustomize.py \
+            --destination "$RUNNER_TEMP/pinned-kustomize" \
+            --metadata "artifacts/kustomize-install-${RUNNER_OS}-${RUNNER_ARCH}.json")"
+          test -f "$kustomize"
+          echo "KUSTOMIZE_BINARY=$kustomize" >> "$GITHUB_ENV"
+
+      - name: Verify snapshot endpoint and NetworkPolicy parity independently
+        run: |
+          set -euo pipefail
+          python scripts/check_snapshot_endpoint_policy.py \
+            --deployment "$SOURCE_SNAPSHOT/k8s/deployment.yaml" \
+            --network-policy "$SOURCE_SNAPSHOT/k8s/networkpolicy.yaml" \
+            --evidence "artifacts/snapshot-endpoint-policy-${RUNNER_OS}-${RUNNER_ARCH}.json"
+
+      - name: Verify catalog provenance and render exact source snapshot twice
+        run: |
+          set -euo pipefail
+          python scripts/direct_source_render_canary.py \
+            --cluster product/k8s-cluster \
+            --snapshot "$SOURCE_SNAPSHOT" \
+            --source-metadata "$SOURCE_SNAPSHOT/source.json" \
+            --cluster-commit "$K8S_CLUSTER_COMMIT" \
+            --child-commit "$FABRICATION_SERVER_COMMIT" \
+            --kustomize "$KUSTOMIZE_BINARY" \
+            --kustomize-metadata "artifacts/kustomize-install-${RUNNER_OS}-${RUNNER_ARCH}.json" \
+            --rendered "artifacts/fabrication-render-${RUNNER_OS}-${RUNNER_ARCH}.yaml" \
+            --evidence "artifacts/direct-source-render-${RUNNER_OS}-${RUNNER_ARCH}.json"
+          rm -rf scripts/__pycache__
+
+      - name: Require pristine cluster and harness checkouts
+        run: |
+          set -euo pipefail
+          git -C product/k8s-cluster diff --exit-code
+          git -C product/k8s-cluster diff --cached --exit-code
+          cluster_status="$(git -C product/k8s-cluster status --porcelain --untracked-files=all)"
+          if [[ -n "$cluster_status" ]]; then
+            printf 'unexpected cluster drift:\n%s\n' "$cluster_status" >&2
+            exit 1
+          fi
+          harness_status="$(git status --porcelain --untracked-files=all | grep -vE '^\?\? (artifacts|product)/' || true)"
+          if [[ -n "$harness_status" ]]; then
+            printf 'unexpected harness drift:\n%s\n' "$harness_status" >&2
+            exit 1
+          fi
+
+      - name: Upload exact-source snapshot render evidence
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
+        with:
+          name: direct-source-snapshot-render-${{ runner.os }}-${{ runner.arch }}-${{ env.K8S_CLUSTER_COMMIT }}-${{ env.FABRICATION_SERVER_COMMIT }}
+          path: |
+            artifacts/kustomize-install-${{ runner.os }}-${{ runner.arch }}.json
+            artifacts/snapshot-endpoint-policy-${{ runner.os }}-${{ runner.arch }}.json
+            artifacts/direct-source-render-${{ runner.os }}-${{ runner.arch }}.json
+            artifacts/fabrication-render-${{ runner.os }}-${{ runner.arch }}.yaml
+          if-no-files-found: error
+          retention-days: 14
+          include-hidden-files: false

docs/direct-source-render.md

@@ -0,0 +1,72 @@
+# Exact direct-source snapshot Kustomize render certification
+
+This `zed-pkg-test` lane proves that the submodule-backed inventory record and the Argo CD direct-source declaration resolve to the same exact private child commit, that the declared endpoint ports are permitted by the app NetworkPolicy, and that checksum-pinned manifest bytes from that commit render deterministically without materializing the submodule in the cluster superproject.
+
+## Private-repository and billing boundaries
+
+A credential-free cross-organization checkout of `daedalus-fab/fabrication-server.rs` failed with `Repository not found`. That is the expected GitHub permission boundary: a test-repository `GITHUB_TOKEN` cannot read a private repository in another organization.
+
+The owner repository's Actions are also currently blocked before job execution by its billing/spending-limit state. Neither limitation is bypassed with a user PAT.
+
+Instead, the five deployment files were exported through the connected repository reader and committed as an immutable test fixture. Every snapshot file retains the exact Git blob SHA from the source commit, and every runner recomputes `git hash-object` before rendering. This proves source-byte provenance and renderability without claiming private-repository reachability.
+
+Private source reachability remains a separate, owner-scoped GitHub App credential contract.
+
+## Immutable inputs
+
+```text
+ORESoftware/k8s-cluster@0c71eed05f8be50ac22ebe73715ded4a871234a8
+daedalus-fab/fabrication-server.rs@cea4a3c772012e1f2f87050bac24e911b8e2e577
+Kustomize v5.8.1
+```
+
+The source commit is owner PR `daedalus-fab/fabrication-server.rs#11`. It changes OTLP egress from gRPC port 4317 to the Deployment's HTTP/protobuf endpoint port 4318 and adds a source-level parity regression.
+
+The cluster commit is stacked PR `ORESoftware/k8s-cluster#1211`. It advances the mode-160000 gitlink and both GitOps catalog pins to the exact source repair while retaining the trigger hardening from `#1209`.
+
+## Three-platform proof
+
+Ubuntu 24.04, macOS 15, and Windows Server 2025 independently:
+
+1. check out the exact cluster commit with `submodules: false`;
+2. load the five-file source snapshot and its revision/repository/blob manifest;
+3. recompute every snapshot file's Git blob SHA and require an exact match with the private source commit;
+4. verify `.gitmodules`, the mode-160000 Git index entry, catalog inventory, catalog direct source, and snapshot metadata identify the same repository and commit;
+5. prove the deployment source is not materialized beneath the superproject checkout;
+6. run unit tests proving an OTLP 4318 endpoint with only 4317 egress fails closed;
+7. extract the real snapshot's NATS and OTLP endpoint ports and require the NetworkPolicy to permit both;
+8. download the official Kustomize v5.8.1 asset selected for the runner platform;
+9. verify the release asset against a hard-coded official SHA-256 digest before safe extraction;
+10. render the checksum-pinned `k8s` snapshot twice and require byte-identical output;
+11. require exactly one `Deployment`, `Service`, `NetworkPolicy`, and `ExternalSecret` in namespace `daedalus`;
+12. reject cluster-scoped resources, a rendered Kubernetes `Secret`, or plaintext `stringData`; and
+13. upload endpoint-policy, source-blob, binary, and render digests plus the rendered YAML.
+
+The pinned release assets and digests come from the official `kubernetes-sigs/kustomize` v5.8.1 GitHub release. Archive extraction rejects path traversal and links.
+
+## Layered evidence
+
+The layers remain intentionally distinct:
+
+- static catalog validation works during a private-repository credential outage;
+- snapshot rendering proves the exact exported deployment bytes still render;
+- endpoint-policy validation proves checked-in egress permits the declared runtime URLs;
+- an authenticated GitHub App checkout must separately prove live private-repository reachability.
+
+A passing static catalog check is not treated as proof of source renderability, and a passing source snapshot does not grant repository or cluster credentials.
+
+## Isolation
+
+All repository checkouts disable persisted credentials. The canary uses no user PAT, GitHub App private key, registry credential, Kubernetes credential, Cloudflare token, DNS permission, R2 key, or database access. It runs `kustomize build` only; it never invokes `kubectl`, `argocd`, Helm installation, a registry push, or an apply operation.
+
+Passing this canary does not activate the inert ApplicationSet and does not authorize production reconciliation.
+
+## Coordination
+
+- Owner source repair: `daedalus-fab/fabrication-server.rs#11`
+- Stacked exact-pin update: `ORESoftware/k8s-cluster#1211`
+- Baseline catalog/trigger repair: `ORESoftware/k8s-cluster#1209`
+- Current-main adversarial matrix: `zed-pkg-test/security-adversarial-e2e#6`
+- Root dispatcher merge: `zed-pkg/zed-cli#242`
+- GitOps roadmap: `ORESoftware/k8s-cluster#1097`
+- Linear: `DEN-2724`, `DEN-2725`, `DEN-630`

fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/deployment.yaml

@@ -0,0 +1,101 @@
+# dd-fabrication-server — NATS-driven fabrication planner.
+#
+# Layer-2 (app-owned) manifest: namespace-scoped ONLY. The `daedalus` Namespace, its
+# ResourceQuota/LimitRange and the default-deny NetworkPolicy are Layer-1, owned by
+# ORESoftware/k8s-cluster (remote/argocd/projects/daedalus.tenant.yaml). Do NOT add a
+# Namespace/ClusterRole/CRD here — the `daedalus` AppProject enforces
+# clusterResourceWhitelist: [] and the sync will fail.
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: dd-fabrication-server
+  namespace: daedalus
+  labels:
+    app: dd-fabrication-server
+    app.kubernetes.io/name: dd-fabrication-server
+    app.kubernetes.io/part-of: daedalus
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: dd-fabrication-server
+  template:
+    metadata:
+      labels:
+        app: dd-fabrication-server
+        app.kubernetes.io/name: dd-fabrication-server
+        app.kubernetes.io/part-of: daedalus
+      annotations:
+        # Declared for discoverability; this cluster's Prometheus still scrapes
+        # from static_configs in k8s-cluster, so adding a target needs a PR to
+        # remote/argocd/observability/{prometheus,otel-collector}.configmap.yaml.
+        prometheus.io/scrape: "true"
+        prometheus.io/path: /metrics
+        prometheus.io/port: "8113"
+        prometheus.io/scheme: http
+    spec:
+      automountServiceAccountToken: false
+      securityContext:
+        runAsNonRoot: true
+        runAsUser: 10001
+        fsGroup: 10001
+        seccompProfile: { type: RuntimeDefault }
+      containers:
+        - name: fabrication-server
+          image: ghcr.io/daedalus-fab/fabrication-server.rs:main
+          imagePullPolicy: IfNotPresent
+          ports:
+            - name: http
+              containerPort: 8113
+          env:
+            - { name: HOST, value: "0.0.0.0" }
+            - { name: PORT, value: "8113" }
+            # Cluster NATS (Layer-1, messaging namespace). Subjects use the
+            # dd_nats_subject_defs defaults; override only if the contract changes.
+            - { name: NATS_URL, value: "nats://dd-nats.messaging.svc.cluster.local:4222" }
+            # 4318 (HTTP), not 4317 (gRPC): dd-telemetry exports OTLP over
+            # HTTP/protobuf (the gRPC/tonic stack is deliberately excluded) and
+            # appends /v1/traces. Aimed at 4317 every export is an HTTP POST at
+            # a gRPC listener and is dropped silently — which is why this
+            # service has never produced traces despite the NetworkPolicy
+            # already allowing the egress.
+            - { name: OTEL_EXPORTER_OTLP_ENDPOINT, value: "http://dd-otel-collector.observability.svc.cluster.local:4318" }
+            - { name: OTEL_SERVICE_NAME, value: "dd-fabrication-server" }
+            - { name: RUST_LOG, value: "info" }
+            # The DSN provisioned by k8s/externalsecret.yaml. Without this the
+            # ExternalSecret materialized a Secret that nothing read, and
+            # src/persistence.rs — which reads FABRICATION_DATABASE_URL — would
+            # start with persistence silently disabled.
+            - name: FABRICATION_DATABASE_URL
+              valueFrom:
+                secretKeyRef: { name: dd-daedalus-fabrication-secrets, key: FABRICATION_DATABASE_URL }
+            - name: POD_NAME
+              valueFrom: { fieldRef: { fieldPath: metadata.name } }
+            - name: POD_NAMESPACE
+              valueFrom: { fieldRef: { fieldPath: metadata.namespace } }
+            - name: NODE_NAME
+              valueFrom: { fieldRef: { fieldPath: spec.nodeName } }
+          readinessProbe:
+            httpGet: { path: /readyz, port: http }
+            initialDelaySeconds: 5
+            periodSeconds: 10
+          livenessProbe:
+            httpGet: { path: /healthz, port: http }
+            initialDelaySeconds: 15
+            periodSeconds: 20
+          resources:
+            requests: { cpu: 100m, memory: 128Mi }
+            limits: { cpu: 1, memory: 512Mi }
+          securityContext:
+            allowPrivilegeEscalation: false
+            readOnlyRootFilesystem: true
+            capabilities: { drop: ["ALL"] }
+          volumeMounts:
+            - { name: tmp, mountPath: /tmp }
+          lifecycle:
+            preStop:
+              # Drain: let endpoint removal propagate before shutdown begins.
+              exec: { command: ["sleep", "5"] }
+      volumes:
+        - name: tmp
+          emptyDir: {}

fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/externalsecret.yaml

@@ -0,0 +1,41 @@
+# Secrets for dd-fabrication-server, materialized by External Secrets Operator.
+#
+# Layer-2 (app-owned), namespace-scoped. The ClusterSecretStore itself is
+# Layer-1 and lives in ORESoftware/k8s-cluster.
+#
+# WHAT BELONGS HERE: only true secrets. The Supabase audience, issuer, and JWKS
+# URL are public values and are set inline in deployment.yaml — routing them
+# through ESO would imply a confidentiality they do not have and make the auth
+# configuration harder to read.
+#
+# Preferring the JWKS URL (RS256/ES256) over the legacy HS256 shared secret is
+# what keeps this Secret down to a single key: with JWKS there is no symmetric
+# secret to distribute at all. DAEDALUS_API_SUPABASE_JWT_SECRET is therefore
+# intentionally absent; add it only if the project is pinned to HS256.
+# NAME IS PER-SERVICE ON PURPOSE. All three daedalus services previously
+# declared this ExternalSecret as `dd-daedalus-secrets` with the same
+# target.name in the same namespace, while each Argo Application syncs with
+# prune+selfHeal. Three controllers claiming one Secret with three different
+# key sets fight forever: whichever syncs last wins and the other two services
+# lose their credentials. One Secret per owning service avoids that entirely.
+apiVersion: external-secrets.io/v1
+kind: ExternalSecret
+metadata:
+  name: dd-daedalus-fabrication-secrets
+  namespace: daedalus
+  labels:
+    app.kubernetes.io/part-of: daedalus
+spec:
+  refreshInterval: 15m
+  secretStoreRef:
+    name: dd-cluster-secrets
+    kind: ClusterSecretStore
+  target:
+    name: dd-daedalus-fabrication-secrets
+    creationPolicy: Owner
+  data:
+    # The daedalus Postgres DSN. Carries a password, so it is never inline.
+    - secretKey: FABRICATION_DATABASE_URL
+      remoteRef:
+        key: dd/remote-dev/daedalus-secrets
+        property: FABRICATION_DATABASE_URL

fixtures/direct-source/fabrication-server.rs/5e305b9bf40d28139bdabf9342c304f759cbfee5/k8s/kustomization.yaml

@@ -0,0 +1,13 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+
+# Namespace-scoped resources only — the `daedalus` Namespace itself is Layer-1
+# (owned by ORESoftware/k8s-cluster). See the org AGENTS.md / k8s-cluster
+# docs/app-deploy-contract.md.
+namespace: daedalus
+
+resources:
+  - deployment.yaml
+  - service.yaml
+  - networkpolicy.yaml
+  - externalsecret.yaml