diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 21161e45c..7eca868df 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -1,6 +1,6 @@ { "name": "Kubebuilder DevContainer", - "image": "golang:1.26.9@sha256:d7722066f0b60ceccb6c0643cbed1f5f9e15506ac237146c95333504d7805d89", + "image": "golang:1.27.2@sha256:5bc7f572bbaa98885a3a1fd9c0aa76b59e3e14e8628bfc316bbfd0c701e4818c", "features": { "ghcr.io/devcontainers/features/docker-in-docker:2": { "moby": false, diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 7a38a1e60..55bd156c5 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -21,7 +21,8 @@ # no required context is at stake. Locked by hack/test/ci_docs_gate_test.sh. # # The docs-only path is NOT free, and that is a deliberate trade. Where this workflow previously -# was not dispatched at all, a documentation-only PR now runs `changes`, `gitleaks` and `lint` -- +# was not dispatched at all, a documentation-only PR now runs `changes`, `gitleaks`, `lint` and +# `tool-pins-selftest` (CFC-6, the tool-pins gate's self-test, split out of lint for wall time) -- # `lint` because two existing gates (dist_ci_wiring_test.sh, repo_root_links_wiring_test.sh) # require it to be unconditional, and because it is where ci_docs_gate_test.sh runs, so leaving # it ungated is the only thing that exercises the docs-only direction at all. What such a PR no @@ -539,9 +540,67 @@ jobs: bash "${script}" echo "::endgroup::" done + # CFC-1..CFC-6 (openspec change cross-file-consistency-gates): the cross-file consistency + # gates -- the Go toolchain agrees across go.mod, the Dockerfiles and the devcontainer + # (CFC-1), every pinned tool resolves and its analysis tools run under go.mod's Go (CFC-2), + # the chart's RBAC covers config/rbac/role.yaml in both install modes (CFC-3), registered + # metrics match the docs and rules (CFC-4), and the go-patch-lag sensor keeps its verdicts + # and its workflow shape (CFC-5). Plain and --self-test are separate steps (CFC-6): the + # first reds when the tree drifts, the second when a gate can no longer fail. They run here, + # unconditionally, because lint is a required context that also runs on docs-only PRs. + # Tools: jq (runner image), yq, helm and PyYAML installed above; Go from go.mod through + # go-cache (the tool-pins gate forces GOTOOLCHAIN=go itself). + - name: Verify the Go toolchain agrees across go.mod, images and devcontainer (CFC-1) + run: bash hack/test/consistency_go_toolchain_test.sh + - name: Verify the Go toolchain gate can still fail (CFC-6) + run: bash hack/test/consistency_go_toolchain_test.sh --self-test + - name: Verify the chart RBAC covers role.yaml in both install modes (CFC-3) + run: bash hack/test/consistency_rbac_test.sh + - name: Verify the chart RBAC gate can still fail (CFC-6) + run: bash hack/test/consistency_rbac_test.sh --self-test + - name: Verify registered metrics match the docs and alert rules (CFC-4) + run: bash hack/test/consistency_metrics_test.sh + - name: Verify the metrics gate can still fail (CFC-6) + run: bash hack/test/consistency_metrics_test.sh --self-test + - name: Verify the go-patch-lag sensor verdicts and workflow shape (CFC-5) + run: bash hack/test/go_patch_lag_test.sh + - name: Verify the go-patch-lag test can still fail (CFC-6) + run: bash hack/test/go_patch_lag_test.sh --self-test + # Evidence under openspec/changes/**/evidence/ (archive included) is Markdown only: a probe + # script committed there is analysed by SonarCloud as uncovered new code although CI never + # runs it (decision 2026-10-16: inline probes in the evidence .md instead of widening + # sonar.exclusions). Plain and --self-test as separate steps, like the CFC gates above. + - name: Verify openspec evidence is Markdown only + run: bash hack/test/openspec_evidence_markdown_only_test.sh + - name: Verify the openspec evidence guard can still fail + run: bash hack/test/openspec_evidence_markdown_only_test.sh --self-test + # Network: the module proxy, golangci-lint's custom build and the vuln DB (about 30 s warm). + - name: Verify every pinned tool resolves and runs under go.mod's Go (CFC-2) + run: bash hack/test/consistency_tool_pins_test.sh - name: Lint run: task lint + # CFC-6 for CFC-2 (openspec change cross-file-consistency-gates): the tool-pins gate's + # --self-test builds each pinned analysis tool at its own and at a too-old version (about + # 7 minutes warm), so it runs in its own job, in parallel with lint, instead of doubling + # lint's wall time (operator decision 2026-10-16). Like lint it is NOT gated on the changes + # verdict: it runs on every PR, docs-only included, and is a required context + # (hack/release/verify-eligibility.sh). hack/test/ci_workflow_security_test.sh (CFC-6 rule) + # requires the self-test in this job and hack/test/ci_docs_gate_test.sh keeps the job + # unconditional. The timeout bounds a stalled module or vuln-DB download; local runs took + # 7 to 17 minutes. + tool-pins-selftest: + name: tool-pins-selftest + runs-on: ubuntu-latest + timeout-minutes: 45 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + - uses: ./.github/actions/go-cache + - name: Verify the tool-pins gate can still fail (CFC-6) + run: bash hack/test/consistency_tool_pins_test.sh --self-test + # CI-DOCSGATE-01: the REAL suite. It is deliberately NOT named `test` -- the required context # of that name is produced by the `test` job at the end of this file, which reports on every # PR. A job that legitimately skips must never carry a required context name; that is exactly diff --git a/.github/workflows/go-patch-lag.yaml b/.github/workflows/go-patch-lag.yaml new file mode 100644 index 000000000..2e151244a --- /dev/null +++ b/.github/workflows/go-patch-lag.yaml @@ -0,0 +1,36 @@ +# CFC-5 (openspec change cross-file-consistency-gates): red when go.mod's Go is behind the latest +# patch of its minor, or when that patch cannot be determined (the lookup fails closed). A Go patch +# carries stdlib security fixes; this makes the lag visible before govulncheck does. +# Pinned by hack/test/go_patch_lag_test.sh (triggers, permissions, no secrets, one plain step, +# curl's retry budget inside the step's timeout-minutes). +name: Go patch lag + +on: + schedule: + - cron: "23 6 * * 1" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: go-patch-lag-${{ github.ref }} + cancel-in-progress: false + +jobs: + go-patch-lag: + name: go-patch-lag + runs-on: ubuntu-latest + # why: above the sensor step's own timeout, so checkout and runner setup cannot end the job + # before the step does. + timeout-minutes: 6 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - name: Compare go.mod's Go with the latest patch of its minor + # why: curl's retry budget in the script (--retry-max-time + --max-time) must end inside + # this step's own timeout so a stalled go.dev fails with the script's message. + timeout-minutes: 5 + run: bash hack/ci/go-patch-lag.sh diff --git a/Makefile b/Makefile index a2d35a219..d39ba2295 100644 --- a/Makefile +++ b/Makefile @@ -181,7 +181,7 @@ ENVTEST_K8S_VERSION ?= $(shell v='$(call gomodver,k8s.io/api)'; \ [ -n "$$v" ] || { echo "Set ENVTEST_K8S_VERSION manually (k8s.io/api replace has no tag)" >&2; exit 1; }; \ printf '%s\n' "$$v" | sed -E 's/^v?[0-9]+\.([0-9]+).*/1.\1/') -GOLANGCI_LINT_VERSION ?= v2.13.1 +GOLANGCI_LINT_VERSION ?= v2.14.0 .PHONY: kustomize kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary. $(KUSTOMIZE): $(LOCALBIN) diff --git a/Taskfile.yml b/Taskfile.yml index d02328e56..6a8ee5df5 100644 --- a/Taskfile.yml +++ b/Taskfile.yml @@ -26,7 +26,7 @@ vars: GO_ARCH_LINT_GRAPH: docs/architecture-graph.svg MARKDOWNLINT_CONFIG: hack/tooling/markdownlint-cli2.yaml # renovate: datasource=github-releases depName=fe3dback/go-arch-lint - GO_ARCH_LINT_VERSION: v1.15.0 + GO_ARCH_LINT_VERSION: v1.19.0 # renovate: datasource=github-releases depName=helm/helm HELM_VERSION: v3.22.0 # renovate: datasource=github-releases depName=norwoodj/helm-docs @@ -201,7 +201,7 @@ tasks: desc: Scan for known Go vulnerabilities (govulncheck) deps: [install] cmds: - - go run golang.org/x/vuln/cmd/govulncheck@v1.1.4 ./... + - go run golang.org/x/vuln/cmd/govulncheck@v1.9.0 ./... test: desc: Run unit tests with envtest (alias for test:run) diff --git a/docs/development/setup.md b/docs/development/setup.md index d92c12af3..584f23885 100644 --- a/docs/development/setup.md +++ b/docs/development/setup.md @@ -12,7 +12,7 @@ Kubernetes cluster (typically [kind](https://kind.sigs.k8s.io/)). | Tool | Version / notes | | --- | --- | -| [Go](https://go.dev/dl/) | **1.26.9+** (see `go.mod`) | +| [Go](https://go.dev/dl/) | **1.27.2+** (see `go.mod`) | | [Docker](https://docs.docker.com/get-docker/) | For container image builds and kind | | [kubectl](https://kubernetes.io/docs/tasks/tools/) | 1.28+ recommended | | [kind](https://kind.sigs.k8s.io/docs/user/quick-start/#installation) | Local cluster for smoke / e2e | diff --git a/docs/operator-manual/metrics.md b/docs/operator-manual/metrics.md index 1d9e44170..1c205736e 100644 --- a/docs/operator-manual/metrics.md +++ b/docs/operator-manual/metrics.md @@ -100,6 +100,9 @@ All custom metrics use the `kollect_` prefix. Controller-runtime also exposes st | `kollect_reconcile_errors_total` | counter | `kind`, `error_class` | Errors by CR kind and class (`transient`, `terminal`, `forbidden`) | | `kollect_reconcile_duration_seconds` | histogram | `controller` | Reconcile latency | | `kollect_workqueue_depth` | gauge | `controller` | In-flight reconciles (approximate depth) | +| `kollect_watch_map_list_errors_total` | counter | `controller`, `watch` | Secondary watch map handlers that failed to list related objects (RBAC or API list failures) | +| `kollect_static_ref_resolution_total` | counter | `kind`, `ref_type`, `result` | Cluster-kind namespaced static-ref resolutions (`ok` / `not_found` / `forbidden`, [ADR-0208](../adr/0208-cluster-static-refs-via-namespace.md)) | +| `kollect_access_cache_total` | counter | `result` | SelfSubjectAccessReview access-cache outcomes: `hit` serves a cached decision, `miss` issues a fresh review | ### Collection @@ -118,6 +121,8 @@ All custom metrics use the `kollect_` prefix. Controller-runtime also exposes st | `kollect_export_bytes_total` | counter | `sink_type` | Payload bytes exported | | `kollect_sink_errors_total` | counter | `reason` | Export failures (`transient`, `terminal`, `forbidden`, `payload_too_large`, `spill_required`, …) | | `kollect_export_spill_warn_total` | counter | — | Payloads at/above 1 MiB spill warn threshold | +| `kollect_export_shard_warn_total` | counter | — | Inventory namespace aggregates at/above the export sharding warn row threshold (~1800 rows) | +| `kollect_namespace_fingerprint_cache_total` | counter | `controller`, `result` | Namespace content fingerprint cache outcomes: `hit` skips the snapshot + fingerprint recompute, `miss` pays for it | | `kollect_sink_connection_test_total` | counter | `type`, `result` | Git/TLS connection tests | | `kollect_cleanup_terminal_total` | counter | `kind` | Terminal sink-cleanup attempts on deleting objects (`inventory`, `cluster-inventory`). The cleanup finalizer is retained and the attempt re-checks every 5 minutes, so one wedged object increments the counter once per re-check — alert on any increase, not on the count; see [Troubleshooting](troubleshooting.md) | @@ -157,7 +162,8 @@ sum(increase(kollect_sink_errors_total[15m])) by (reason) kollect_collect_items_total ``` -Source of truth for registration: `internal/metrics/metrics.go` and `internal/metrics/metrics_catalog.go`. +Source of truth for registration: `internal/metrics/metrics.go` and `internal/metrics/aggregation*.go`. +`internal/metrics/metrics_catalog.go` documents the same names for agents; it registers nothing. ## See also diff --git a/go.mod b/go.mod index b4613df8f..a75f165e9 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/platformrelay/kollect -go 1.26.9 +go 1.27.2 require ( cloud.google.com/go/bigquery v1.85.0 diff --git a/hack/check.sh b/hack/check.sh index dd36779f5..f46365ba5 100755 --- a/hack/check.sh +++ b/hack/check.sh @@ -65,6 +65,7 @@ run_gate workflow-security-audit ./bin/zizmor --offline --no-progress --min-seve # TCE-3 mapping: required contexts whose local equivalent is a combination of the gates above. # hack/test/task_check_test.sh reads these lines; guard-sweep is the entry for the guard run. # coverage: preflight=lint:markdown,go-mod,verify,guard-sweep +# coverage: tool-pins-selftest=guard-sweep # The guard sweep: every hack/test guard, bare, plus --self-test for the scripts that parse diff --git a/hack/ci/go-patch-lag.sh b/hack/ci/go-patch-lag.sh new file mode 100755 index 000000000..f0c522c9f --- /dev/null +++ b/hack/ci/go-patch-lag.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# CFC-5 (openspec change cross-file-consistency-gates): is go.mod's Go behind the latest patch of +# its minor? +# +# Reads the `go` directive of go.mod and the go.dev release listing, and compares the directive +# with the newest STABLE release of the same minor. A newer minor existing is reported as a note, +# never as a failure. The lookup fails closed: an unreachable or unparsable listing, a minor the +# listing does not carry, or a go.mod newer than the listing is a red run, never a green one. +# +# `&include=all` is required: the default listing carries only the two newest minors, so a go.mod +# two minors behind would drop out of it and "behind" would read as "unknown". +# +# Exit codes: 0 current, 1 behind, 2 cannot determine (lookup failed or go.mod unreadable). +# +# Usage: +# hack/ci/go-patch-lag.sh [--gomod ] [--listing | --url ] [--retries ] +# --gomod go.mod to read (default: the repository's) +# --listing read the listing from a file instead of fetching it (tests) +# --url listing URL (default: https://go.dev/dl/?mode=json&include=all) +# --retries fetch retries (default 3; tests pass 0) +set -euo pipefail + +ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +GOMOD="${ROOT}/go.mod" +URL="https://go.dev/dl/?mode=json&include=all" +LISTING="" +RETRIES=3 + +while [[ $# -gt 0 ]]; do + case "$1" in + --gomod) + GOMOD="${2:?--gomod needs a path}" + shift 2 + ;; + --listing) + LISTING="${2:?--listing needs a path}" + shift 2 + ;; + --url) + URL="${2:?--url needs a URL}" + shift 2 + ;; + --retries) + RETRIES="${2:?--retries needs a number}" + [[ "${RETRIES}" =~ ^[0-9]+$ ]] || { echo "go-patch-lag: --retries needs a number" >&2; exit 2; } + shift 2 + ;; + *) + echo "go-patch-lag: unknown argument '$1'" >&2 + exit 2 + ;; + esac +done + +minor_tag="?" +undetermined() { echo "go-patch-lag: FAIL: cannot determine the latest go${minor_tag} patch: $*" >&2; exit 2; } + +command -v jq >/dev/null 2>&1 || undetermined "jq not found" + +# --- go.mod's directive ------------------------------------------------------------------------- +directive="" +if [[ -f "${GOMOD}" ]]; then + directive="$(sed -nE 's/^go[[:space:]]+([^[:space:]]+)[[:space:]]*$/\1/p' "${GOMOD}")" +fi +# Canonical numbers of at most four digits: bash arithmetic reads 08 as bad octal (an arithmetic +# error inside `if` is just "false") and wraps past 2^63 -- either way a false green. +num='(0|[1-9][0-9]{0,3})' +if [[ ! "${directive}" =~ ^${num}\.${num}\.${num}$ ]]; then + echo "go-patch-lag: FAIL: cannot read a go .. directive from ${GOMOD} (found '${directive}')" >&2 + exit 2 +fi +minor_tag="${BASH_REMATCH[1]}.${BASH_REMATCH[2]}" +patch="${BASH_REMATCH[3]}" + +# --- the listing -------------------------------------------------------------------------------- +if [[ -z "${LISTING}" ]]; then + LISTING="$(mktemp)" + trap 'rm -f "${LISTING:?}"' EXIT + # Bounded well inside the workflow step's 5-minute timeout, so a stalled go.dev ends with this + # script's own message, not a killed step: <= 200 s of retries plus one 45 s attempt. + # hack/test/go_patch_lag_test.sh fails when --retry-max-time + --max-time outgrow that timeout. + curl -fsSL --proto '=https' --tlsv1.2 --connect-timeout 10 --max-time 45 \ + --retry "${RETRIES}" --retry-all-errors --retry-max-time 200 -o "${LISTING}" "${URL}" || + undetermined "fetching ${URL} failed" +fi + +# Stable patch numbers of go.mod's minor, one per line; jq errors on anything but a JSON array. +# shellcheck disable=SC2016 # $p is a jq variable +jq_program='if type != "array" then error("not an array") else . end + | .[] | select(type == "object") | select(.stable == true) + | .version | select(type == "string") | select(startswith($p)) | ltrimstr($p) + | select(test("^(0|[1-9][0-9]{0,3})$"))' +patches="$(jq -r --arg p "go${minor_tag}." "${jq_program}" "${LISTING}" 2>/dev/null)" || + undetermined "the listing is not a JSON array of releases" +latest="$(sort -n <<<"${patches}" | tail -n 1)" +[[ -n "${latest}" ]] || undetermined "the listing has no stable go${minor_tag}.N release" + +# The newest stable release overall, for the newer-minor note only. +newest="$(jq -r '[.[] | select(type == "object") | select(.stable == true) | .version + | select(type == "string") | capture("^go(?[0-9]+)\\.(?[0-9]+)\\.(?[0-9]+)$") + | [(.a | tonumber), (.b | tonumber), (.c | tonumber)]] | max // empty + | "go\(.[0]).\(.[1]).\(.[2])"' "${LISTING}" 2>/dev/null || true)" + +if ((10#${patch} > 10#${latest})); then + undetermined "go.mod's go ${directive} is newer than go${minor_tag}.${latest}, the newest listed -- the listing is stale or go.mod names an unreleased Go" +fi + +behind() { + echo "go-patch-lag: FAIL: go.mod's go ${directive} is behind go${minor_tag}.${latest}, the latest go${minor_tag} patch -- bump go.mod (and the golang images, CFC-1)" >&2 + exit 1 +} +if ((10#${patch} < 10#${latest})); then + behind +fi +echo "go-patch-lag: ok: go.mod's go ${directive} is the latest go${minor_tag} patch" +if [[ -n "${newest}" && "${newest}" != "go${directive}" ]]; then + echo "go-patch-lag: note: a newer minor exists (${newest}); not a failure on its own" +fi diff --git a/hack/release/test-verify-eligibility.sh b/hack/release/test-verify-eligibility.sh index 9d65a3931..9dac1a921 100755 --- a/hack/release/test-verify-eligibility.sh +++ b/hack/release/test-verify-eligibility.sh @@ -10,7 +10,7 @@ trap 'rm -rf "${TMP}"' EXIT SHA="1111111111111111111111111111111111111111" MAIN="2222222222222222222222222222222222222222" -REQUIRED_NAMES='gitleaks verify audit-rbac vulncheck lint test build test-integration helm docker-build preflight kind-smoke pipeline-cli-smoke workflow-security' +REQUIRED_NAMES='gitleaks verify audit-rbac vulncheck lint test build test-integration helm docker-build preflight kind-smoke pipeline-cli-smoke workflow-security tool-pins-selftest' cat >"${TMP}/gh" <<'MOCK' #!/usr/bin/env bash diff --git a/hack/release/verify-eligibility.sh b/hack/release/verify-eligibility.sh index 7ad59638d..1c72d4093 100755 --- a/hack/release/verify-eligibility.sh +++ b/hack/release/verify-eligibility.sh @@ -18,7 +18,7 @@ fi required_checks=( gitleaks verify audit-rbac vulncheck lint test build test-integration helm - docker-build preflight kind-smoke pipeline-cli-smoke workflow-security + docker-build preflight kind-smoke pipeline-cli-smoke workflow-security tool-pins-selftest ) main_sha="$(gh api "repos/${REPO}/commits/${DEFAULT_BRANCH}" --jq .sha)" diff --git a/hack/test/ci_docs_gate_test.sh b/hack/test/ci_docs_gate_test.sh index e8dd6ea31..79c4321c7 100755 --- a/hack/test/ci_docs_gate_test.sh +++ b/hack/test/ci_docs_gate_test.sh @@ -1077,6 +1077,19 @@ assert_job_can_fail_build "${CODEQL_WORKFLOW}" analyze \ "the required context 'Analyze (Go)' must report a conclusion on every PR, including a documentation-only one" pass "preflight and Analyze (Go) run unconditionally on every PR" +# CFC-6 (openspec change cross-file-consistency-gates): the tool-pins self-test has its own +# required job, run unconditionally like lint so it reports on docs-only PRs too -- an `if:` +# (for instance on the changes verdict) would starve that required context on every docs PR. +check_tool_pins_job() { + local workflow="$1" + assert_job_can_fail_build "${workflow}" tool-pins-selftest \ + "the required context 'tool-pins-selftest' must report a conclusion on every PR, including a documentation-only one" + [[ "$(yq eval '.jobs["tool-pins-selftest"].needs' "${workflow}")" == "null" ]] || + fail "job 'tool-pins-selftest' in ${workflow} must not declare needs -- a failed or skipped dependency skips it, and a skipped required context never reports" +} +check_tool_pins_job "${CI_WORKFLOW}" +pass "tool-pins-selftest runs unconditionally on every PR" + # The bound on the documentation set comes first; the probes below are belt-and-braces. # Behaviour before text: the behavioural check is the one that cannot be evaded by restructuring. check_docs_set_behaves_like_paths_ignore "${CI_WORKFLOW}" @@ -1634,6 +1647,24 @@ mutant_rejected "${CODEQL_WORKFLOW}" "${MUTANTS}/analyze-disabled.yaml" \ assert_job_can_fail_build "${MUTANTS}/analyze-disabled.yaml" analyze \ "the required context 'Analyze (Go)' must report a conclusion on every PR, including a documentation-only one" +yq eval '.jobs["tool-pins-selftest"].if = "needs.changes.outputs.code == '"'"'true'"'"'"' "${CI_WORKFLOW}" >"${MUTANTS}/tool-pins-gated.yaml" +mutant_rejected "${CI_WORKFLOW}" "${MUTANTS}/tool-pins-gated.yaml" \ + "the tool-pins-selftest job is gated on the changes verdict" \ + 'must run unconditionally' \ + check_tool_pins_job "${MUTANTS}/tool-pins-gated.yaml" + +yq eval '.jobs["tool-pins-selftest"].needs = ["changes"]' "${CI_WORKFLOW}" >"${MUTANTS}/tool-pins-needs.yaml" +mutant_rejected "${CI_WORKFLOW}" "${MUTANTS}/tool-pins-needs.yaml" \ + "the tool-pins-selftest job depends on another job" \ + 'must not declare needs' \ + check_tool_pins_job "${MUTANTS}/tool-pins-needs.yaml" + +yq eval '.jobs["tool-pins-selftest"].continue-on-error = true' "${CI_WORKFLOW}" >"${MUTANTS}/tool-pins-soft.yaml" +mutant_rejected "${CI_WORKFLOW}" "${MUTANTS}/tool-pins-soft.yaml" \ + "the tool-pins-selftest job is soft-failed" \ + 'must not declare' \ + check_tool_pins_job "${MUTANTS}/tool-pins-soft.yaml" + # --- the classifier check-run names --- yq eval '.jobs.changes.name = "changes (ci)"' "${SMOKE_WORKFLOW}" >"${MUTANTS}/classifier-name-collision.yaml" mutant_rejected "${SMOKE_WORKFLOW}" "${MUTANTS}/classifier-name-collision.yaml" \ diff --git a/hack/test/ci_workflow_security_test.sh b/hack/test/ci_workflow_security_test.sh index fc99cf340..b0632302a 100755 --- a/hack/test/ci_workflow_security_test.sh +++ b/hack/test/ci_workflow_security_test.sh @@ -24,6 +24,10 @@ # dependency-review, and must list lint, vulncheck, workflow-security and # dependency-review. This meta-test must itself be run by ci.yaml in both of its own # modes -- a gate CI does not run guards nothing (the LAB-DEKIND defect class). +# The cross-file consistency guards (hack/test/consistency_*_test.sh and +# go_patch_lag_test.sh, openspec change cross-file-consistency-gates CFC-6) and the +# openspec evidence guard (openspec_evidence_markdown_only_test.sh) must be invoked +# that way too, so deleting their steps outright reds here as well. # CWS-7 --self-test mutates a COPY of the tree to break each of CWS-1..CWS-6 and asserts # each check reds with the MESSAGE of the assertion the mutation was built to trip # (a non-zero exit alone is not evidence), plus one unmutated no-op control. @@ -588,6 +592,31 @@ cws6_guards() { done < <(guard_modes "${script}") done <"${tmp}/scripts" + # CFC-6 (openspec change cross-file-consistency-gates): the loop above only covers guards CI + # already invokes, so a guard whose steps are deleted outright drops out silently. These guards + # SHALL run in a required job in BOTH modes (plain and --self-test), so they are named here -- + # and the modes are CFC-6's, not read from the guard's code: deleting a guard's --self-test + # handling must not delete the requirement to run it. + local cfc_guards=("${ROOT}"/hack/test/consistency_*_test.sh "${ROOT}/hack/test/go_patch_lag_test.sh" + "${ROOT}/hack/test/openspec_evidence_markdown_only_test.sh") + [[ -f "${cfc_guards[0]}" ]] || + fail "CFC-6: no hack/test/consistency_*_test.sh guard found -- the cross-file consistency gates are gone" + for script in "${cfc_guards[@]}"; do + [[ -f "${script}" ]] || fail "CFC-6: ${script#"${ROOT}"/} is missing" + for mode in bare --self-test; do + if ! awk -F'\t' -v s="${script}" -v m="${mode}" '$1 == s && $2 == m { found = 1 } END { exit !found }' "${required_rows}"; then + fail "CFC-6: guard ${script#"${ROOT}"/} is not run in its '${mode}' mode by any step of a required job -- wire it into the lint job, plain and --self-test as separate steps" + fi + done + done + # The tool-pins self-test (~7 min) has its own required job, tool-pins-selftest, so lint's wall + # time stays bounded (operator decision 2026-10-16); it must run THERE, which also requires + # tool-pins-selftest to stay in the required set (rows above are required jobs only). + local tp_script="${ROOT}/hack/test/consistency_tool_pins_test.sh" + if ! awk -F'\t' -v s="${tp_script}" '$1 == s && $2 == "--self-test" && $3 == "tool-pins-selftest" { found = 1 } END { exit !found }' "${required_rows}"; then + fail "CFC-6: hack/test/consistency_tool_pins_test.sh --self-test is not run by the required job 'tool-pins-selftest' -- the slow self-test has its own job, unconditional and required, so it neither starves nor doubles lint" + fi + # Guards CI runs INDIRECTLY through hack/docs/verify.sh (the Docs workflow's task target) # are still CI-run guards: every hack/test token in that script must also be pinned by a # required-job row, or a new guard added only to verify.sh runs solely in a non-required @@ -1056,6 +1085,45 @@ SH grep -q "zz_mode_probe_test" .github/workflows/ci.yaml || { echo "mutant did not apply"; exit 1; } ' +# CFC-6: a cross-file consistency guard deleted from CI outright (both steps) must red; the +# generic CWS-6 loop cannot see a guard CI no longer invokes at all. +mutant_rejected "CFC-6 consistency guard unwired" "CFC-6: guard hack/test/consistency_metrics_test.sh is not run" bash -c ' + set -euo pipefail + yq eval "del(.jobs.lint.steps[] | select((.run // \"\") | test(\"consistency_metrics_test\")))" .github/workflows/ci.yaml > m.yaml && mv m.yaml .github/workflows/ci.yaml + if grep -q "consistency_metrics_test" .github/workflows/ci.yaml; then echo "mutant did not apply"; exit 1; fi +' + +# The CFC-6 list names guards outside the consistency_* glob too: the openspec evidence guard +# deleted from CI outright (both steps) must red. +mutant_rejected "CFC-6 evidence guard unwired" "CFC-6: guard hack/test/openspec_evidence_markdown_only_test.sh is not run" bash -c ' + set -euo pipefail + yq eval "del(.jobs.lint.steps[] | select((.run // \"\") | test(\"openspec_evidence_markdown_only_test\")))" .github/workflows/ci.yaml > m.yaml && mv m.yaml .github/workflows/ci.yaml + if grep -q "openspec_evidence_markdown_only_test" .github/workflows/ci.yaml; then echo "mutant did not apply"; exit 1; fi +' + +# CFC-6: the required modes are CFC-6's, not the guard's: stripping a guard's --self-test handling +# and its CI step together must still red. +mutant_rejected "CFC-6 consistency guard loses its self-test" "consistency_metrics_test.sh is not run in its '--self-test' mode" bash -c ' + set -euo pipefail + yq eval "del(.jobs.lint.steps[] | select((.run // \"\") == \"bash hack/test/consistency_metrics_test.sh --self-test\"))" .github/workflows/ci.yaml > m.yaml && mv m.yaml .github/workflows/ci.yaml + sed -i "/--self-test/d" hack/test/consistency_metrics_test.sh + if grep -q -- "--self-test" hack/test/consistency_metrics_test.sh || grep -q "consistency_metrics_test.sh --self-test" .github/workflows/ci.yaml; then echo "mutant did not apply"; exit 1; fi +' + +# CFC-6: the tool-pins self-test moved back into lint (its own job deleted) must red. +mutant_rejected "CFC-6 tool-pins self-test back in lint" "is not run by the required job 'tool-pins-selftest'" bash -c ' + set -euo pipefail + yq eval "del(.jobs[\"tool-pins-selftest\"]) | .jobs.lint.steps += [{\"name\": \"tool-pins self-test\", \"run\": \"bash hack/test/consistency_tool_pins_test.sh --self-test\"}]" .github/workflows/ci.yaml > m.yaml && mv m.yaml .github/workflows/ci.yaml + if grep -q "tool-pins-selftest:" .github/workflows/ci.yaml; then echo "mutant did not apply"; exit 1; fi +' + +# CFC-6: tool-pins-selftest dropped from the required set must red (the generic CWS-6 loop fires first). +mutant_rejected "CFC-6 tool-pins-selftest not required" "consistency_tool_pins_test.sh runs in CI but its '--self-test' mode is not invoked by any step of a required job" bash -c ' + set -euo pipefail + sed -i "s/ tool-pins-selftest$//" hack/release/verify-eligibility.sh + if grep -q "tool-pins-selftest" hack/release/verify-eligibility.sh; then echo "mutant did not apply"; exit 1; fi +' + mutant_rejected "CWS-6 this gate not itself run by ci.yaml" "is not itself run by ci.yaml" bash -c ' set -euo pipefail yq eval "(.jobs.lint.steps |= map(select(.run // \"\" | test(\"ci_workflow_security_test.sh\") | not)))" .github/workflows/ci.yaml > m.yaml && mv m.yaml .github/workflows/ci.yaml diff --git a/hack/test/consistency_go_toolchain_test.sh b/hack/test/consistency_go_toolchain_test.sh new file mode 100755 index 000000000..7eb5c1335 --- /dev/null +++ b/hack/test/consistency_go_toolchain_test.sh @@ -0,0 +1,593 @@ +#!/usr/bin/env bash +# CFC-1 (openspec change cross-file-consistency-gates): the shipped binary is built with the +# tested toolchain. +# +# Tests, govulncheck and every CI Go setup run the Go that go.mod's `go` directive names +# (actions/setup-go reads it through `go-version-file: go.mod`). The release image is built by +# the `FROM golang:` line of each Dockerfile, and contributors use the golang image of +# .devcontainer/devcontainer.json. Nothing made those agree: Renovate bumps the golang image +# and, by default, never the `go` directive, so the image ran ahead and the binary that shipped +# was built by a Go no test or vulnerability scan had seen. This gate fails when they differ. +# +# What it locks: +# * every golang image (Dockerfile* FROM lines, devcontainer image) equals go.mod's `go`; +# * the images agree with each other; +# * no FROM names its image through a variable (`FROM ${B}`, `FROM $B`, `FROM ${B:-golang:x}`, +# `FROM debian:${T}`, `FROM golang:1.x@${D}`): the image is then only known at build time, +# so the gate refuses it +# rather than resolve an ARG default that `--build-arg` can replace; +# * go.mod has a `go` directive and no `toolchain` line (a toolchain line is a second source +# of truth that setup-go and the images would not follow); +# * no workflow or composite action carries a `go-version:` literal, and every +# actions/setup-go step reads `go-version-file: go.mod`; +# * renovate.json moves the `go` directive and the golang image in ONE group: the gomod +# manager is told to bump the directive (rangeStrategy bump, depType golang -- it does not +# by default), and the group rule comes AFTER the "go minor and patch dependencies" and +# "docker images" rules, because a later packageRule overrides an earlier one's groupName. +# Verified with a local Renovate lookup dry-run on 2026-10-14 (evidence/G1.md); +# * an empty scan (no golang image found) is a failure, never a pass. +# +# Usage: +# hack/test/consistency_go_toolchain_test.sh # check the repository tree +# hack/test/consistency_go_toolchain_test.sh --self-test # mutate copies, assert each check fails +# hack/test/consistency_go_toolchain_test.sh # check an arbitrary tree root +set -euo pipefail + +fail() { + echo "FAIL: $*" >&2 + exit 1 +} + +pass() { + echo "ok - $*" +} + +SELF="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/$(basename "${BASH_SOURCE[0]}")" + +MODE="check" +if [[ "${1:-}" == "--self-test" ]]; then + MODE="self-test" + shift +fi + +ROOT="${1:-}" +if [[ -z "${ROOT}" ]]; then + ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" +fi +ROOT="${ROOT%/}" + +command -v jq >/dev/null 2>&1 || fail "jq not found; install jq to run this test" + +# Every Dockerfile* under the root, relative paths, sorted. Vendored, generated and +# local-only trees are pruned; a Dockerfile with no golang FROM line is simply not a Go build. +dockerfiles() { + local root="$1" + (cd "${root}" && find . \( -name .git -o -name node_modules -o -name references -o -name bin \ + -o -name .task -o -name dist -o -name vendor -o -name worktrees \) -prune -o \ + -type f -name 'Dockerfile*' -print | sed 's|^\./||' | sort) +} + +# " " for each golang image reference: Dockerfile FROM lines and the devcontainer +# `image`. The tag is everything after `golang:` up to a digest or whitespace; an untagged or +# digest-only reference yields the tag "", which the check rejects as not comparable. +golang_images() { + local root="$1" f + while IFS= read -r f; do + [[ -n "${f}" ]] || continue + # Backslash continuations are joined first, so `FROM --platform=... \` + `golang:...` on + # the next line is one instruction. As in BuildKit's parser, a comment line is dropped before + # the continuation check (a trailing "\" does not continue a comment), and comment and blank + # lines inside a continuation are skipped (a blank one only warns, NoEmptyContinuation). + awk -v f="${f}" ' + { + if (cont != "") { + if ($0 ~ /^[[:space:]]*(#|$)/) next + line = cont " " $0 + } else { + if ($0 ~ /^[[:space:]]*#/) next + line = $0 + } + cont = "" + if (line ~ /\\[[:space:]]*$/) { sub(/\\[[:space:]]*$/, "", line); cont = line; next } + sub(/^[[:space:]]+/, "", line) + n = split(line, w, /[[:space:]]+/) + if (toupper(w[1]) != "FROM") next + for (i = 2; i <= n; i++) { + if (w[i] ~ /^--/) continue + img = w[i] + sub(/^(docker\.io\/)?(library\/)?/, "", img) + # Any "$" makes the image known only at build time (an ARG default is replaced by + # --build-arg), so it is reported, never resolved. That includes a "tag": its value + # may hold "/" and ":" (`localhost:${B}` with B=5000/golang:1.99.0 is another image). + # A golang image is no exception: `golang:1.27.2@${D}` or `golang:1.27.2-${X}` would + # pass the version parse below, which strips the digest and the tag suffix. + if (img ~ /\$/) { print f, "", w[i]; break } + if (img ~ /^golang:/) { sub(/^golang:/, "", img); sub(/@.*$/, "", img); print f, img } + else if (img ~ /^golang(@|$)/) { print f, "" } + break + } + } + ' "${root}/${f}" + done < <(dockerfiles "${root}") + + f=".devcontainer/devcontainer.json" + if [[ -f "${root}/${f}" ]]; then + # devcontainer.json is JSONC: `//` comment lines are dropped, and the key must open a line + # or follow `{` / `,`, so compact (one-line) JSON is read and a commented-out image is not. + # /* */ block comments are not handled. + { grep -vE '^[[:space:]]*//' "${root}/${f}" | + grep -oE '(^|[{,])[[:space:]]*"image"[[:space:]]*:[[:space:]]*"[^"]*"' || true; } | + sed -E 's/^[{,]?[[:space:]]*"image"[[:space:]]*:[[:space:]]*"//; s/"$//; s#^(docker\.io/)?(library/)?##' | + while IFS= read -r img; do + case "${img}" in + golang:*) img="${img#golang:}"; echo "${f} ${img%%@*}" ;; + golang | golang@*) echo "${f} " ;; + esac + done + fi +} + +# Every CI YAML that can set up Go: workflows and composite actions. +ci_files() { + local root="$1" dir + for dir in "${root}/.github/workflows" "${root}/.github/actions"; do + [[ -d "${dir}" ]] || continue + find "${dir}" -type f \( -name '*.yml' -o -name '*.yaml' \) + done | sort +} + +# ":" of every actions/setup-go step whose with: block has no +# `go-version-file: go.mod` before the next step starts. +setup_go_without_gomod() { + local root="$1" file + while IFS= read -r file; do + awk -v f="${file#"${root}"/}" ' + function close_step() { if (open && !ok) print f ":" at; open = 0 } + /^[[:space:]]*#/ { next } + /^[[:space:]]*-[[:space:]]/ { close_step() } + /["\047]?uses["\047]?:[[:space:]]*["\047]?actions\/setup-go@/ { open = 1; ok = 0; at = NR; next } + open && /^[[:space:]]*["\047]?go-version-file["\047]?:[[:space:]]*["\047]?go\.mod["\047]?[[:space:]]*(#.*)?$/ { ok = 1 } + END { close_step() } + ' "${file}" + done < <(ci_files "${root}") +} + +count_setup_go() { + local root="$1" file n=0 c + while IFS= read -r file; do + c="$(grep -Ec "^[^#]*[\"']?uses[\"']?:[[:space:]]*[\"']?actions/setup-go@" "${file}" || true)" + n=$((n + c)) + done < <(ci_files "${root}") + echo "${n}" +} + +# The whole contract for one tree. Prints one FAIL line per broken invariant, returns non-zero +# when any is broken. +check_go_toolchain() { + local root="$1" broken=0 + local gomod="${root}/go.mod" renovate="${root}/renovate.json" + local go_version="" images path tag ver distinct lowest line file n + + # --- go.mod ------------------------------------------------------------------------------- + if [[ -f "${gomod}" ]]; then + go_version="$(sed -n -E 's/^go[[:space:]]+([0-9][0-9.]*)[[:space:]]*$/\1/p' "${gomod}" | head -1)" + fi + if [[ -z "${go_version}" ]]; then + echo "FAIL: go.mod has no 'go ' directive -- there is nothing for the images to agree with" >&2 + broken=1 + fi + if [[ -f "${gomod}" ]] && grep -Eq '^[[:space:]]*toolchain[[:space:]]' "${gomod}"; then + echo "FAIL: go.mod carries a toolchain line ($(grep -E '^[[:space:]]*toolchain[[:space:]]' "${gomod}" | head -1)) -- a second Go version that setup-go and the images do not follow; keep the go directive as the one source" >&2 + broken=1 + fi + + # --- golang images ------------------------------------------------------------------------ + images="$(golang_images "${root}")" + if [[ -z "${images}" ]]; then + echo "FAIL: no golang image found in any Dockerfile* FROM line or in .devcontainer/devcontainer.json -- the scan set is empty, so the comparison would pass without comparing anything" >&2 + broken=1 + else + while read -r path tag; do + if [[ "${tag}" == " "* ]]; then + echo "FAIL: ${path}: FROM ${tag# } names its image through a variable -- whether it is a golang image, and which Go it ships, is only known at build time (--build-arg overrides an ARG default); spell the image literally" >&2 + broken=1 + continue + fi + ver="${tag%%-*}" + if [[ ! "${ver}" =~ ^[0-9]+\.[0-9]+(\.[0-9]+)?$ ]]; then + echo "FAIL: ${path}: golang:${tag} is not a literal Go version, so it cannot be compared with go.mod" >&2 + broken=1 + continue + fi + [[ -n "${go_version}" ]] || continue + [[ "${ver}" == "${go_version}" ]] && continue + lowest="$(printf '%s\n%s\n' "${ver}" "${go_version}" | sort -V | head -1)" + if [[ "${lowest}" == "${go_version}" ]]; then + echo "FAIL: ${path}: golang:${ver} is newer than go.mod's go ${go_version} -- the binary would ship built by a Go that tests and govulncheck never ran" >&2 + else + echo "FAIL: ${path}: golang:${ver} is older than go.mod's go ${go_version} -- the binary would ship built by a Go that tests and govulncheck never ran" >&2 + fi + broken=1 + done <<<"${images}" + + if [[ -f "${root}/.devcontainer/devcontainer.json" ]] && + ! grep -q '^\.devcontainer/devcontainer\.json ' <<<"${images}"; then + echo "FAIL: .devcontainer/devcontainer.json has no golang image -- CFC-1 compares it with go.mod; if the devcontainer stopped using the golang image on purpose, change this gate with it" >&2 + broken=1 + fi + + distinct="$(awk '$2 != "" { v = $2; sub(/-.*/, "", v); print v }' <<<"${images}" | sort -u -V | tr '\n' ' ')" + distinct="${distinct% }" + if [[ "${distinct}" == *" "* ]]; then + echo "FAIL: the golang images disagree with each other: ${distinct}" >&2 + broken=1 + fi + fi + + # --- CI Go setups ------------------------------------------------------------------------- + while IFS= read -r file; do + while IFS= read -r line; do + [[ -n "${line}" ]] || continue + echo "FAIL: ${file#"${root}"/}: go-version: literal ($(sed -E 's/^[[:space:]]+//' <<<"${line#*:}")) -- CI must read the Go version from go.mod via go-version-file" >&2 + broken=1 + done < <(grep -nE "^[[:space:]]*-?[[:space:]]*[\"']?go-version[\"']?:" "${file}" || true) + done < <(ci_files "${root}") + + n="$(count_setup_go "${root}")" + if [[ "${n}" -eq 0 ]]; then + echo "FAIL: no actions/setup-go step found under .github -- the CI half of this check would pass without checking anything" >&2 + broken=1 + fi + while IFS= read -r line; do + [[ -n "${line}" ]] || continue + echo "FAIL: ${line}: actions/setup-go step does not read go-version-file: go.mod" >&2 + broken=1 + done < <(setup_go_without_gomod "${root}") + + # --- Renovate ----------------------------------------------------------------------------- + if [[ ! -f "${renovate}" ]] || ! jq -e '.packageRules | type == "array"' "${renovate}" >/dev/null 2>&1; then + echo "FAIL: renovate.json is missing, unparsable or has no packageRules -- nothing moves go.mod and the golang image together" >&2 + return 1 + fi + + local group_idx gomod_idx docker_idx + group_idx="$(jq '[.packageRules | to_entries[] + | select((.value.matchDepNames // []) | (index("go") != null and index("golang") != null)) + | select(.value.groupName // "" | length > 0) | .key] | last // -1' "${renovate}")" + gomod_idx="$(jq '[.packageRules | to_entries[] + | select(.value.groupName == "go minor and patch dependencies") | .key] | last // -1' "${renovate}")" + docker_idx="$(jq '[.packageRules | to_entries[] + | select(.value.groupName == "docker images") | .key] | last // -1' "${renovate}")" + + if [[ "${group_idx}" -lt 0 ]]; then + echo "FAIL: renovate.json has no packageRule that groups depNames go and golang -- Renovate opens a lone golang image PR and go.mod stays behind" >&2 + broken=1 + else + if [[ "${gomod_idx}" -gt "${group_idx}" ]]; then + echo "FAIL: renovate.json's go/golang group rule (packageRules[${group_idx}]) comes before the \"go minor and patch dependencies\" rule (packageRules[${gomod_idx}]); the later rule wins and pulls the go directive into its own group" >&2 + broken=1 + fi + if [[ "${docker_idx}" -gt "${group_idx}" ]]; then + echo "FAIL: renovate.json's go/golang group rule (packageRules[${group_idx}]) comes before the \"docker images\" rule (packageRules[${docker_idx}]); the later rule wins and pulls the golang image into its own group" >&2 + broken=1 + fi + fi + + # Any later rule that sets a different groupName may match go or golang (a catch-all, a + # matchPackageNames or matchDatasources filter, a custom.regex dep named golang) and would win. + # Deciding which can match means re-implementing Renovate's matchers, so the group rule must + # simply be the LAST rule that sets a groupName. + if [[ "${group_idx}" -ge 0 ]]; then + local later + later="$(jq -r --argjson g "${group_idx}" '.packageRules[$g].groupName as $name + | [.packageRules | to_entries[] | select(.key > $g) + | select((.value.groupName // $name) != $name) + | "packageRules[\(.key)] (\(.value.groupName))"] | join(", ")' "${renovate}")" + if [[ -n "${later}" ]]; then + echo "FAIL: renovate.json's go/golang group rule (packageRules[${group_idx}]) is overridden by a later rule that sets another groupName: ${later} -- keep the go toolchain group the last group rule" >&2 + broken=1 + fi + fi + + if ! jq -e '[.packageRules[] + | select((.matchDepTypes // []) | index("golang") != null) + | select((.matchDepNames // ["go"]) | index("go") != null) + | select((.matchPackageNames // ["go"]) | index("go") != null) + | select(.rangeStrategy == "bump") + | select((.matchManagers // ["gomod"]) | index("gomod") != null) + | select((.matchFileNames // ["go.mod"]) | index("go.mod") != null)] | length > 0' \ + "${renovate}" >/dev/null; then + echo "FAIL: renovate.json has no packageRule that sets rangeStrategy bump for the gomod go directive (matchDepTypes golang) -- Renovate does not bump the go directive by default, so the group would carry only the image" >&2 + broken=1 + fi + + return "${broken}" +} + +if [[ "${MODE}" == "check" ]]; then + if check_go_toolchain "${ROOT}"; then + pass "go.mod, every golang image, the CI Go setups and the Renovate go group agree" + exit 0 + fi + fail "the Go toolchain is stated inconsistently (listed above) -- the binary that ships would be built by a Go the tests never ran" +fi + +# --------------------------------------------------------------------------- +# Self-test (CFC-6): copy the files the gate reads, break one invariant per copy, and assert +# the gate reds WITH THE MESSAGE of the assertion the mutant targets -- a non-zero exit alone +# is not evidence. One unmutated copy must pass. +# --------------------------------------------------------------------------- +SCRATCH="" +trap 'rm -rf ${SCRATCH} 2>/dev/null' EXIT + +make_copy() { + local dst="$1" f + mkdir -p "${dst}/.github" + cp "${ROOT}/go.mod" "${ROOT}/renovate.json" "${dst}/" + for f in "${ROOT}"/Dockerfile*; do + [[ -f "${f}" ]] && cp "${f}" "${dst}/" + done + [[ -d "${ROOT}/.devcontainer" ]] && cp -R "${ROOT}/.devcontainer" "${dst}/.devcontainer" + cp -R "${ROOT}/.github/workflows" "${dst}/.github/workflows" + [[ -d "${ROOT}/.github/actions" ]] && cp -R "${ROOT}/.github/actions" "${dst}/.github/actions" + return 0 +} + +# Called directly (never as $(new_copy)): a command substitution runs in a subshell, so its +# SCRATCH update would be lost and the EXIT trap would leave every copy behind. +COPY="" +new_copy() { + COPY="$(mktemp -d)" + SCRATCH="${SCRATCH} ${COPY}" + make_copy "${COPY}" +} + +# mutant_rejected