From 62028240c2cf8af2022b1cb17e1afabf799e6bcb Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 00:50:04 +0100 Subject: [PATCH 01/47] bsp: fix existing Jaguar U-Boot patches Pin the focused LmP v96 backport that corrects stale patch context for the handheld and Phasora images. Assisted-by: Codex --- meta-dynamicdevices-bsp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index 47542ad3..71f566e0 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit 47542ad3f8d49850df69e37a3414c1ef60c51809 +Subproject commit 71f566e04e699cd49e63cf3c8cff47a817d06956 From 7c184c7b57f348c4d1afe62e4c57519e6eab93f2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:51:00 +0100 Subject: [PATCH 02/47] ci: enforce layer adoption regression gate --- .github/workflows/layer-adoption-gate.yml | 128 ++++++++++++++++++++ ci/layer-adoption-contract.json | 5 + ci/layer-adoption-tuples.json | 20 +++ scripts/monitor-foundries-build.sh | 25 ++-- scripts/validation/capture-layer-state.sh | 80 ++++++++++++ scripts/validation/compare-layer-state.py | 73 +++++++++++ scripts/validation/detect-layer-adoption.py | 76 ++++++++++++ 7 files changed, 396 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/layer-adoption-gate.yml create mode 100644 ci/layer-adoption-contract.json create mode 100644 ci/layer-adoption-tuples.json create mode 100755 scripts/validation/capture-layer-state.sh create mode 100755 scripts/validation/compare-layer-state.py create mode 100755 scripts/validation/detect-layer-adoption.py diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml new file mode 100644 index 00000000..4813c67a --- /dev/null +++ b/.github/workflows/layer-adoption-gate.yml @@ -0,0 +1,128 @@ +name: Layer Adoption Gate + +on: + pull_request: + branches: [main, develop] + push: + branches: [main, develop] + workflow_dispatch: + inputs: + base_sha: + description: Baseline commit to compare + required: true + +concurrency: + group: layer-adoption-${${ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + detect: + name: Detect material layer change + runs-on: [self-hosted, Linux, X64] + outputs: + material: ${${ steps.detect.outputs.material }} + matrix: ${${ steps.matrix.outputs.matrix }} + base_sha: ${${ steps.base.outputs.sha }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: base + name: Resolve immutable baseline + env: + PR_BASE: ${${ github.event.pull_request.base.sha }} + PUSH_BASE: ${${ github.event.before }} + INPUT_BASE: ${${ inputs.base_sha }} + run: | + sha="${PR_BASE:-${INPUT_BASE:-${PUSH_BASE:-}}}" + if [ -z "$sha" ] || printf '%s' "$sha" | grep -Eq '^0+$'; then + sha=$(git rev-parse HEAD^) + fi + git cat-file -e "$sha^{commit}" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + - id: detect + name: Require an adoption contract + run: | + python3 scripts/validation/detect-layer-adoption.py \ + --base '${${ steps.base.outputs.sha }}' \ + --head '${${ github.sha }}' \ + --github-output "$GITHUB_OUTPUT" + - id: matrix + run: echo "matrix=$(jq -c '{include:.tuples}' ci/layer-adoption-tuples.json)" >> "$GITHUB_OUTPUT" + + regression: + name: Protect ${${ matrix.id }} + needs: detect + if: needs.detect.outputs.material == 'true' + strategy: + fail-fast: false + matrix: ${${ fromJSON(needs.detect.outputs.matrix) }} + runs-on: [self-hosted, Linux, X64] + container: + image: dynamicdevices/yocto-ci-build:latest + options: --privileged --platform linux/amd64 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: candidate + - name: Create baseline worktree + working-directory: candidate + run: git worktree add ../baseline '${${ needs.detect.outputs.base_sha }}' + - name: Install KAS when absent + run: command -v kas >/dev/null || pip3 install kas + - name: Restore Yocto caches + uses: actions/cache@v4 + with: + path: | + ~/yocto/downloads + ~/yocto/sstate-cache + key: layer-adoption-${${ matrix.machine }}-${${ hashFiles('candidate/kas/**') }} + restore-keys: | + layer-adoption-${${ matrix.machine }}- + layer-adoption- + - name: Build and capture baseline + working-directory: baseline + run: | + ../candidate/scripts/validation/capture-layer-state.sh \ + '${${ matrix.config }}' '${${ matrix.machine }}' '${${ matrix.image }}' \ + '../evidence/baseline/${${ matrix.id }}' + - name: Build and capture candidate + working-directory: candidate + run: | + scripts/validation/capture-layer-state.sh \ + '${${ matrix.config }}' '${${ matrix.machine }}' '${${ matrix.image }}' \ + '../evidence/candidate/${${ matrix.id }}' + - name: Reject unexplained contamination + run: | + python3 candidate/scripts/validation/compare-layer-state.py \ + --baseline 'evidence/baseline/${${ matrix.id }}' \ + --candidate 'evidence/candidate/${${ matrix.id }}' \ + --tuple '${${ matrix.id }}' \ + --contract candidate/ci/layer-adoption-contract.json + - name: Preserve comparison evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: layer-adoption-${${ matrix.id }} + path: evidence + retention-days: 14 + + required: + name: Layer Adoption Gate + needs: [detect, regression] + if: always() + runs-on: [self-hosted, Linux, X64] + steps: + - name: Enforce gate result + env: + DETECT: ${${ needs.detect.result }} + MATERIAL: ${${ needs.detect.outputs.material }} + REGRESSION: ${${ needs.regression.result }} + run: | + test "$DETECT" = success + if [ "$MATERIAL" = true ]; then + test "$REGRESSION" = success + else + test "$REGRESSION" = skipped + fi diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json new file mode 100644 index 00000000..6c3c21a7 --- /dev/null +++ b/ci/layer-adoption-contract.json @@ -0,0 +1,5 @@ +{ + "schema": 1, + "reason": "Adopt the isolated NXP i.MX95 partner layer without changing any pre-existing Dynamic Devices build tuple.", + "allowed_deltas": {} +} diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json new file mode 100644 index 00000000..acace193 --- /dev/null +++ b/ci/layer-adoption-tuples.json @@ -0,0 +1,20 @@ +{ + "schema": 1, + "tuples": [ + {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-inst-image", "machine": "imx8mm-jaguar-inst", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-phasora-image", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + + {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-inst-mfgtool", "machine": "imx8mm-jaguar-inst", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-phasora-mfgtool", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"} + ] +} diff --git a/scripts/monitor-foundries-build.sh b/scripts/monitor-foundries-build.sh index 42379356..2a609d54 100755 --- a/scripts/monitor-foundries-build.sh +++ b/scripts/monitor-foundries-build.sh @@ -5,7 +5,7 @@ set -e TARGET=${1:-2027} -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +POLL_INTERVAL=${FOUNDRIES_POLL_INTERVAL:-30} # Extract OAuth token from fioctl config FIOCTL_CONFIG="$HOME/.config/fioctl.yaml" @@ -43,14 +43,14 @@ show_build_info() { echo "$build_data" | jq -r '.data.build | " Build ID: \(.build_id // "N/A") Status: \(.status // "UNKNOWN") - Created: \(.created_at // "N/A") - Updated: \(.updated_at // "N/A")"' + Created: \(.created // "N/A") + Updated: \((.status_events // [] | last | .time) // "N/A")"' # Show runs if available - if echo "$build_data" | jq -e '.data.runs[]?' >/dev/null 2>&1; then + if echo "$build_data" | jq -e '.data.build.runs[]?' >/dev/null 2>&1; then echo "" echo "🏃 Build Runs:" - echo "$build_data" | jq -r '.data.runs[] | " \(.name): \(.status)"' + echo "$build_data" | jq -r '.data.build.runs[] | " \(.name): \(.status)"' else echo "" echo "🏃 Build Runs: Not started yet" @@ -64,10 +64,13 @@ echo "⏱️ Starting monitoring (Ctrl+C to stop)..." echo "" while true; do - BUILD_DATA=$(get_build_status) - - if [ $? -eq 0 ]; then - clear + if BUILD_DATA=$(get_build_status); then + # `clear` fails when there is no interactive TERM (for example when + # this monitor is run by CI or an agent). Display refresh is cosmetic + # and must never terminate authoritative build monitoring. + if [ -t 1 ] && [ -n "${TERM:-}" ]; then + clear || true + fi echo "🔍 Monitoring Foundries.io Build $TARGET - $(date)" echo "========================================" echo "" @@ -83,10 +86,10 @@ while true; do break fi - echo "🔄 Refreshing in 30 seconds..." + echo "🔄 Refreshing in ${POLL_INTERVAL} seconds..." else echo "❌ Failed to get build status" fi - sleep 30 + sleep "$POLL_INTERVAL" done diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh new file mode 100755 index 00000000..02215d4e --- /dev/null +++ b/scripts/validation/capture-layer-state.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Build one protected tuple and capture deterministic layer/package/task state. +set -euo pipefail + +if [ "$#" -ne 4 ]; then + echo "Usage: $0 KAS_CONFIG MACHINE TARGET OUTPUT_DIR" >&2 + exit 2 +fi + +config=$1 +machine=$2 +target=$3 +output_dir=$4 + +case "$output_dir" in + /*) ;; + *) output_dir="$PWD/$output_dir" ;; +esac +mkdir -p "$output_dir" + +export KAS_MACHINE="$machine" +kas checkout "$config" + +# Static layer surfaces are captured as well as BitBake's resolved view. This +# makes wildcard/dangling appends and global layer.conf policy visible even +# when they do not happen to alter the first recipe selected by BitBake. +find build/layers -type f -path '*/conf/layer.conf' -print0 \ + | sort -z \ + | xargs -0 grep -nHE \ + '(^|[[:space:]])(IMAGE_INSTALL|CORE_IMAGE_EXTRA_INSTALL|DISTRO_FEATURES|MACHINE_FEATURES|PACKAGECONFIG|PREFERRED_(VERSION|PROVIDER)|RDEPENDS)(:|[[:space:]])*([+?:.]?=)' \ + > "$output_dir/layer-conf-policy.txt" || true +find build/layers -type f -name '*.bbappend' -printf '%p\n' \ + | sed -E 's#^build/layers/[^/]+/#LAYER/#' \ + | sort -u > "$output_dir/all-bbappends.txt" + +run_bitbake() { + kas shell "$config" -c "$1" +} + +run_bitbake "bitbake-layers show-layers" \ + | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + > "$output_dir/layers.txt" +run_bitbake "bitbake-layers show-appends" \ + | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + > "$output_dir/appends.txt" +run_bitbake "bitbake-layers show-recipes" \ + | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + > "$output_dir/recipes.txt" + +run_bitbake "bitbake -g $target" +sort -u build/pn-buildlist > "$output_dir/pn-buildlist.txt" +sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ + > "$output_dir/task-depends.dot" + +# A parse-only graph is not proof that packaging, signing, recovery image size, +# or deploy layout still works. Complete the real image/recovery build for both +# baseline and candidate. +run_bitbake "bitbake $target" + +deploy_dir="build/tmp/deploy/images/$machine" +if [ ! -d "$deploy_dir" ]; then + echo "ERROR: deploy directory missing after successful build: $deploy_dir" >&2 + exit 1 +fi + +find "$deploy_dir" -maxdepth 1 -type f -printf '%f\t%s\n' \ + | sed -E 's/-[0-9]{14}(\.|-)/-TIMESTAMP\1/g' \ + | sort -u > "$output_dir/deploy-layout-and-sizes.txt" +# The expression belongs to awk; shell expansion would be a bug. +# shellcheck disable=SC2016 +find "$deploy_dir" -maxdepth 1 -type f -name '*.manifest' -print0 \ + | sort -z \ + | xargs -0 -r awk '{print $1}' \ + | sort -u > "$output_dir/packages.txt" + +# New warnings are regressions even when BitBake returns zero. +find build/tmp/log -type f -name 'console-latest.log' -print0 2>/dev/null \ + | xargs -0 -r grep -hE '(^|[[:space:]])WARNING:' \ + | sed -E "s#$PWD/##g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}[^ ]*//g" \ + | sort -u > "$output_dir/warnings.txt" || true diff --git a/scripts/validation/compare-layer-state.py b/scripts/validation/compare-layer-state.py new file mode 100755 index 00000000..13b0e77a --- /dev/null +++ b/scripts/validation/compare-layer-state.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Fail on unexplained baseline/candidate build-state differences.""" + +from __future__ import annotations + +import argparse +import difflib +import json +import re +import sys +from pathlib import Path + + +def files(root: Path) -> dict[str, list[str]]: + result: dict[str, list[str]] = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + result[str(path.relative_to(root))] = path.read_text(errors="replace").splitlines() + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--candidate", required=True, type=Path) + parser.add_argument("--tuple", required=True) + parser.add_argument("--contract", required=True, type=Path) + args = parser.parse_args() + + contract = json.loads(args.contract.read_text()) + rules = contract.get("allowed_deltas", {}).get(args.tuple, []) + compiled: list[tuple[re.Pattern[str], re.Pattern[str], str]] = [] + for rule in rules: + try: + compiled.append(( + re.compile(rule["file"]), + re.compile(rule["pattern"]), + str(rule["reason"]).strip(), + )) + except (KeyError, re.error) as exc: + print(f"ERROR: invalid allow rule for {args.tuple}: {exc}", file=sys.stderr) + return 2 + if any(not reason for _, _, reason in compiled): + print("ERROR: every allowed delta needs a reason", file=sys.stderr) + return 2 + + old, new = files(args.baseline), files(args.candidate) + unexplained: list[str] = [] + for name in sorted(set(old) | set(new)): + if old.get(name) == new.get(name): + continue + delta = list(difflib.unified_diff(old.get(name, []), new.get(name, []), lineterm="")) + changed_lines = [ + line[1:] for line in delta + if line.startswith(("+", "-")) and not line.startswith(("+++", "---")) + ] + for line in changed_lines: + if not any(file_re.search(name) and line_re.search(line) for file_re, line_re, _ in compiled): + unexplained.append(f"{name}: {line}") + + if unexplained: + print(f"ERROR: unexplained build deltas for {args.tuple}:", file=sys.stderr) + for line in unexplained[:250]: + print(f" {line}", file=sys.stderr) + if len(unexplained) > 250: + print(f" ... {len(unexplained) - 250} more", file=sys.stderr) + return 1 + print(f"PASS: {args.tuple} has no unexplained layer-adoption delta") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validation/detect-layer-adoption.py b/scripts/validation/detect-layer-adoption.py new file mode 100755 index 00000000..012ce88b --- /dev/null +++ b/scripts/validation/detect-layer-adoption.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Detect layer topology/pin changes and enforce an explicit adoption contract.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path + + +MATERIAL_PATHS = ( + re.compile(r"^\.gitmodules$"), + re.compile(r"(^|/)conf/layer\.conf$"), + re.compile(r"^kas/.*\.ya?ml$"), +) +def git(*args: str) -> str: + return subprocess.check_output(["git", *args], text=True) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base", required=True) + parser.add_argument("--head", default="HEAD") + parser.add_argument("--contract", default="ci/layer-adoption-contract.json") + parser.add_argument("--github-output") + args = parser.parse_args() + + changed = git("diff", "--name-only", f"{args.base}...{args.head}").splitlines() + material_files = [ + path for path in changed + if any(pattern.search(path) for pattern in MATERIAL_PATHS) + ] + # Treat every KAS change as material. YAML context makes line-only pin + # detection easy to evade (for example by adding a list item below an + # existing `includes:` key), and a false-positive build is safer than a + # layer adoption escaping the hard gate. + material = bool(material_files) + + if material: + if args.contract not in changed: + print( + f"ERROR: material Yocto layer change requires {args.contract} " + "to be updated in the same change", + file=sys.stderr, + ) + return 2 + try: + contract = json.loads(Path(args.contract).read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"ERROR: invalid adoption contract: {exc}", file=sys.stderr) + return 2 + if contract.get("schema") != 1 or not str(contract.get("reason", "")).strip(): + print("ERROR: contract needs schema=1 and a non-empty reason", file=sys.stderr) + return 2 + if not isinstance(contract.get("allowed_deltas"), dict): + print("ERROR: allowed_deltas must be an object", file=sys.stderr) + return 2 + + result = "true" if material else "false" + print(f"material={result}") + if material_files: + print("material candidates:") + for path in material_files: + print(f" {path}") + if args.github_output: + with open(args.github_output, "a", encoding="utf-8") as output: + output.write(f"material={result}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 82f2c51192c3c6ff9edea820baf51218a1fc614b Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:52:51 +0100 Subject: [PATCH 03/47] ci: fix layer gate expressions --- .github/workflows/layer-adoption-gate.yml | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 4813c67a..8c51d4ec 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -12,7 +12,7 @@ on: required: true concurrency: - group: layer-adoption-${${ github.event.pull_request.number || github.ref }} + group: layer-adoption-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: @@ -20,9 +20,9 @@ jobs: name: Detect material layer change runs-on: [self-hosted, Linux, X64] outputs: - material: ${${ steps.detect.outputs.material }} - matrix: ${${ steps.matrix.outputs.matrix }} - base_sha: ${${ steps.base.outputs.sha }} + material: ${{ steps.detect.outputs.material }} + matrix: ${{ steps.matrix.outputs.matrix }} + base_sha: ${{ steps.base.outputs.sha }} steps: - uses: actions/checkout@v4 with: @@ -30,9 +30,9 @@ jobs: - id: base name: Resolve immutable baseline env: - PR_BASE: ${${ github.event.pull_request.base.sha }} - PUSH_BASE: ${${ github.event.before }} - INPUT_BASE: ${${ inputs.base_sha }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + PUSH_BASE: ${{ github.event.before }} + INPUT_BASE: ${{ inputs.base_sha }} run: | sha="${PR_BASE:-${INPUT_BASE:-${PUSH_BASE:-}}}" if [ -z "$sha" ] || printf '%s' "$sha" | grep -Eq '^0+$'; then @@ -44,19 +44,19 @@ jobs: name: Require an adoption contract run: | python3 scripts/validation/detect-layer-adoption.py \ - --base '${${ steps.base.outputs.sha }}' \ - --head '${${ github.sha }}' \ + --base '${{ steps.base.outputs.sha }}' \ + --head '${{ github.sha }}' \ --github-output "$GITHUB_OUTPUT" - id: matrix run: echo "matrix=$(jq -c '{include:.tuples}' ci/layer-adoption-tuples.json)" >> "$GITHUB_OUTPUT" regression: - name: Protect ${${ matrix.id }} + name: Protect ${{ matrix.id }} needs: detect if: needs.detect.outputs.material == 'true' strategy: fail-fast: false - matrix: ${${ fromJSON(needs.detect.outputs.matrix) }} + matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} runs-on: [self-hosted, Linux, X64] container: image: dynamicdevices/yocto-ci-build:latest @@ -68,7 +68,7 @@ jobs: path: candidate - name: Create baseline worktree working-directory: candidate - run: git worktree add ../baseline '${${ needs.detect.outputs.base_sha }}' + run: git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' - name: Install KAS when absent run: command -v kas >/dev/null || pip3 install kas - name: Restore Yocto caches @@ -77,34 +77,34 @@ jobs: path: | ~/yocto/downloads ~/yocto/sstate-cache - key: layer-adoption-${${ matrix.machine }}-${${ hashFiles('candidate/kas/**') }} + key: layer-adoption-${{ matrix.machine }}-${{ hashFiles('candidate/kas/**') }} restore-keys: | - layer-adoption-${${ matrix.machine }}- + layer-adoption-${{ matrix.machine }}- layer-adoption- - name: Build and capture baseline working-directory: baseline run: | ../candidate/scripts/validation/capture-layer-state.sh \ - '${${ matrix.config }}' '${${ matrix.machine }}' '${${ matrix.image }}' \ - '../evidence/baseline/${${ matrix.id }}' + '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.image }}' \ + '../evidence/baseline/${{ matrix.id }}' - name: Build and capture candidate working-directory: candidate run: | scripts/validation/capture-layer-state.sh \ - '${${ matrix.config }}' '${${ matrix.machine }}' '${${ matrix.image }}' \ - '../evidence/candidate/${${ matrix.id }}' + '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.image }}' \ + '../evidence/candidate/${{ matrix.id }}' - name: Reject unexplained contamination run: | python3 candidate/scripts/validation/compare-layer-state.py \ - --baseline 'evidence/baseline/${${ matrix.id }}' \ - --candidate 'evidence/candidate/${${ matrix.id }}' \ - --tuple '${${ matrix.id }}' \ + --baseline 'evidence/baseline/${{ matrix.id }}' \ + --candidate 'evidence/candidate/${{ matrix.id }}' \ + --tuple '${{ matrix.id }}' \ --contract candidate/ci/layer-adoption-contract.json - name: Preserve comparison evidence if: always() uses: actions/upload-artifact@v4 with: - name: layer-adoption-${${ matrix.id }} + name: layer-adoption-${{ matrix.id }} path: evidence retention-days: 14 @@ -116,9 +116,9 @@ jobs: steps: - name: Enforce gate result env: - DETECT: ${${ needs.detect.result }} - MATERIAL: ${${ needs.detect.outputs.material }} - REGRESSION: ${${ needs.regression.result }} + DETECT: ${{ needs.detect.result }} + MATERIAL: ${{ needs.detect.outputs.material }} + REGRESSION: ${{ needs.regression.result }} run: | test "$DETECT" = success if [ "$MATERIAL" = true ]; then From 6b8e574521c8e70ba2a2151acca9bf12dfe705d8 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:53:20 +0100 Subject: [PATCH 04/47] ci: bound layer gate build concurrency --- .github/workflows/layer-adoption-gate.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 8c51d4ec..c4d2f19b 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -56,6 +56,9 @@ jobs: if: needs.detect.outputs.material == 'true' strategy: fail-fast: false + # Full Yocto baseline/candidate pairs are I/O and disk intensive. Keep + # the hard gate comprehensive without overwhelming one self-hosted host. + max-parallel: 2 matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} runs-on: [self-hosted, Linux, X64] container: From 71c01cda59edafed0896f7a0a6ff6cc06d646fe5 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:54:38 +0100 Subject: [PATCH 05/47] ci: use current Actions runtimes --- .github/workflows/layer-adoption-gate.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index c4d2f19b..28438a2d 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -24,7 +24,7 @@ jobs: matrix: ${{ steps.matrix.outputs.matrix }} base_sha: ${{ steps.base.outputs.sha }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - id: base @@ -65,7 +65,7 @@ jobs: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 path: candidate @@ -75,7 +75,7 @@ jobs: - name: Install KAS when absent run: command -v kas >/dev/null || pip3 install kas - name: Restore Yocto caches - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: | ~/yocto/downloads @@ -105,7 +105,7 @@ jobs: --contract candidate/ci/layer-adoption-contract.json - name: Preserve comparison evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: layer-adoption-${{ matrix.id }} path: evidence From c73dd4b967dbd1bfcbb06198b8c4f106a638dc03 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:57:42 +0100 Subject: [PATCH 06/47] ci: emit layer matrix without jq --- .github/workflows/layer-adoption-gate.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 28438a2d..5b165316 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -48,7 +48,17 @@ jobs: --head '${{ github.sha }}' \ --github-output "$GITHUB_OUTPUT" - id: matrix - run: echo "matrix=$(jq -c '{include:.tuples}' ci/layer-adoption-tuples.json)" >> "$GITHUB_OUTPUT" + name: Publish protected tuple matrix + run: | + python3 - <<'PY' + import json + import os + + with open("ci/layer-adoption-tuples.json", encoding="utf-8") as source: + matrix = {"include": json.load(source)["tuples"]} + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write("matrix=" + json.dumps(matrix, separators=(",", ":")) + "\n") + PY regression: name: Protect ${{ matrix.id }} From 58bb309a78024bf538fe8d4d04ce0be81c0a64cc Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 21:02:51 +0100 Subject: [PATCH 07/47] ci: route Yocto jobs to capable runner --- .github/actionlint.yaml | 3 ++ .github/workflows/kas-build-ci.yml | 62 +++++++++++------------ .github/workflows/layer-adoption-gate.yml | 2 +- 3 files changed, 35 insertions(+), 32 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..cfd084cf --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - yocto diff --git a/.github/workflows/kas-build-ci.yml b/.github/workflows/kas-build-ci.yml index e1bf06f3..517dcb41 100644 --- a/.github/workflows/kas-build-ci.yml +++ b/.github/workflows/kas-build-ci.yml @@ -46,22 +46,21 @@ jobs: # Validation job - validates Yocto layer compatibility validate: name: Validate Yocto Layers - # Pin to a Linux self-hosted runner: bare `self-hosted` also matches the - # org's macOS runners, which have no Docker and fail this container job - # in ~6s with "docker: command not found". - runs-on: [self-hosted, Linux, X64] + # Pin container work to the dedicated Yocto runner. Generic Linux labels + # also match esl-nixos, which intentionally has no Docker daemon. + runs-on: [self-hosted, Linux, X64, yocto] container: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: submodules: recursive - name: Cache KAS layers - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: | build/layers @@ -72,7 +71,7 @@ jobs: kas-layers- - name: Cache Yocto downloads - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/yocto/downloads key: yocto-downloads-${{ runner.os }}-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }} @@ -177,27 +176,28 @@ jobs: steps: - name: Generate validation summary run: | - echo "# KAS Layer Validation Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Workflow:** ${{ github.workflow }}" >> $GITHUB_STEP_SUMMARY - echo "**Trigger:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY - echo "**Commit:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - echo "**Branch:** ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Determine overall status - if [ "${{ needs.validate.result }}" = "success" ]; then - echo "## ✅ Validation Status: SUCCESS" >> $GITHUB_STEP_SUMMARY - echo "All meta-dynamicdevices layers pass comprehensive yocto-check-layer validation!" >> $GITHUB_STEP_SUMMARY - echo "Layers are ready for Yocto Project compatibility." >> $GITHUB_STEP_SUMMARY - else - echo "## ❌ Validation Status: FAILED" >> $GITHUB_STEP_SUMMARY - echo "- Layer validation: ${{ needs.validate.result }}" >> $GITHUB_STEP_SUMMARY - echo "Please fix validation issues before proceeding." >> $GITHUB_STEP_SUMMARY - fi - - echo "" >> $GITHUB_STEP_SUMMARY - echo "## Validation Coverage" >> $GITHUB_STEP_SUMMARY - echo "- **Tool:** Official yocto-check-layer" >> $GITHUB_STEP_SUMMARY - echo "- **Layers:** meta-dynamicdevices, meta-dynamicdevices-bsp, meta-dynamicdevices-distro" >> $GITHUB_STEP_SUMMARY - echo "- **Compliance:** Full Yocto Project compatibility validation" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + { + echo "# KAS Layer Validation Summary" + echo + echo "**Workflow:** ${{ github.workflow }}" + echo "**Trigger:** ${{ github.event_name }}" + echo "**Commit:** ${{ github.sha }}" + echo "**Branch:** ${{ github.ref_name }}" + echo + + if [ "${{ needs.validate.result }}" = "success" ]; then + echo "## ✅ Validation Status: SUCCESS" + echo "All meta-dynamicdevices layers pass comprehensive yocto-check-layer validation!" + echo "Layers are ready for Yocto Project compatibility." + else + echo "## ❌ Validation Status: FAILED" + echo "- Layer validation: ${{ needs.validate.result }}" + echo "Please fix validation issues before proceeding." + fi + + echo + echo "## Validation Coverage" + echo "- **Tool:** Official yocto-check-layer" + echo "- **Layers:** meta-dynamicdevices, meta-dynamicdevices-bsp, meta-dynamicdevices-distro" + echo "- **Compliance:** Full Yocto Project compatibility validation" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 5b165316..8a232115 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -70,7 +70,7 @@ jobs: # the hard gate comprehensive without overwhelming one self-hosted host. max-parallel: 2 matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} - runs-on: [self-hosted, Linux, X64] + runs-on: [self-hosted, Linux, X64, yocto] container: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 From ce0b9da24c8e2c3d8124deffb7583e5a7a841135 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 21:55:27 +0100 Subject: [PATCH 08/47] ci: strengthen layer adoption evidence --- .github/workflows/layer-adoption-gate.yml | 6 ++- scripts/validation/capture-layer-state.sh | 63 ++++++++++++++++++----- scripts/validation/compare-layer-state.py | 2 +- scripts/validation/select-bitbake-env.py | 49 ++++++++++++++++++ 4 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 scripts/validation/select-bitbake-env.py diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 8a232115..4e151c63 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -98,13 +98,15 @@ jobs: working-directory: baseline run: | ../candidate/scripts/validation/capture-layer-state.sh \ - '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.image }}' \ + '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ + '${{ matrix.image }}' \ '../evidence/baseline/${{ matrix.id }}' - name: Build and capture candidate working-directory: candidate run: | scripts/validation/capture-layer-state.sh \ - '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.image }}' \ + '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ + '${{ matrix.image }}' \ '../evidence/candidate/${{ matrix.id }}' - name: Reject unexplained contamination run: | diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 02215d4e..50661cc7 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -2,15 +2,16 @@ # Build one protected tuple and capture deterministic layer/package/task state. set -euo pipefail -if [ "$#" -ne 4 ]; then - echo "Usage: $0 KAS_CONFIG MACHINE TARGET OUTPUT_DIR" >&2 +if [ "$#" -ne 5 ]; then + echo "Usage: $0 KAS_CONFIG MACHINE DISTRO TARGET OUTPUT_DIR" >&2 exit 2 fi config=$1 machine=$2 -target=$3 -output_dir=$4 +distro=$3 +target=$4 +output_dir=$5 case "$output_dir" in /*) ;; @@ -19,43 +20,75 @@ esac mkdir -p "$output_dir" export KAS_MACHINE="$machine" +export KAS_DISTRO="$distro" +export DISTRO="$distro" kas checkout "$config" +cat > "$output_dir/metadata.json" < "$output_dir/commands.txt" + # Static layer surfaces are captured as well as BitBake's resolved view. This # makes wildcard/dangling appends and global layer.conf policy visible even # when they do not happen to alter the first recipe selected by BitBake. -find build/layers -type f -path '*/conf/layer.conf' -print0 \ +find build/layers -type f \( -path '*/conf/layer.conf' -o -name '*.bbclass' \) -print0 \ | sort -z \ | xargs -0 grep -nHE \ - '(^|[[:space:]])(IMAGE_INSTALL|CORE_IMAGE_EXTRA_INSTALL|DISTRO_FEATURES|MACHINE_FEATURES|PACKAGECONFIG|PREFERRED_(VERSION|PROVIDER)|RDEPENDS)(:|[[:space:]])*([+?:.]?=)' \ + '(^|[[:space:]])(IMAGE_INSTALL|CORE_IMAGE_|PACKAGE_INSTALL|DISTRO_FEATURES|MACHINE_FEATURES|PACKAGECONFIG|PREFERRED_(VERSION|PROVIDER)|DEFAULT_PREFERENCE|RDEPENDS|INHERIT|BBMASK|BBPATH|BBFILES|BBFILE_PRIORITY|INITRAMFS_MAXSIZE|IMAGE_FSTYPES|WKS_FILE|UBOOT_|KERNEL_|OPTEE_|SDKIMAGE_FEATURES|TOOLCHAIN_TARGET_TASK)(:|\[|[[:space:]])*([+?:.]?=)' \ > "$output_dir/layer-conf-policy.txt" || true find build/layers -type f -name '*.bbappend' -printf '%p\n' \ - | sed -E 's#^build/layers/[^/]+/#LAYER/#' \ + | sed -E 's#^build/layers/##' \ | sort -u > "$output_dir/all-bbappends.txt" run_bitbake() { kas shell "$config" -c "$1" } -run_bitbake "bitbake-layers show-layers" \ +capture_command() { + local name=$1 + shift + "$@" 2>&1 | tee "$output_dir/$name.log" +} + +capture_command show-layers run_bitbake "bitbake-layers show-layers" \ | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ > "$output_dir/layers.txt" -run_bitbake "bitbake-layers show-appends" \ +capture_command show-appends run_bitbake "bitbake-layers show-appends" \ | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ > "$output_dir/appends.txt" -run_bitbake "bitbake-layers show-recipes" \ +capture_command show-recipes run_bitbake "bitbake-layers show-recipes" \ | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ > "$output_dir/recipes.txt" -run_bitbake "bitbake -g $target" +capture_command graph run_bitbake "bitbake -g $target" sort -u build/pn-buildlist > "$output_dir/pn-buildlist.txt" sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ > "$output_dir/task-depends.dot" +sed -E "s#$PWD/##g" build/recipe-depends.dot | sort -u \ + > "$output_dir/recipe-depends.dot" + +# Capture final values and BitBake's assignment provenance for policy that a +# newly enabled layer can silently change. The full environment is retained +# temporarily only as input, avoiding volatile host variables in comparisons. +capture_command environment run_bitbake "bitbake -e $target" +python3 "$(dirname "$0")/select-bitbake-env.py" \ + "$output_dir/environment.log" > "$output_dir/selected-environment.txt" +grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" +grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" +rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, # or deploy layout still works. Complete the real image/recovery build for both # baseline and candidate. -run_bitbake "bitbake $target" +capture_command build run_bitbake "bitbake $target" deploy_dir="build/tmp/deploy/images/$machine" if [ ! -d "$deploy_dir" ]; then @@ -74,7 +107,11 @@ find "$deploy_dir" -maxdepth 1 -type f -name '*.manifest' -print0 \ | sort -u > "$output_dir/packages.txt" # New warnings are regressions even when BitBake returns zero. -find build/tmp/log -type f -name 'console-latest.log' -print0 2>/dev/null \ +find "$output_dir" -type f -name '*.log' -print0 \ | xargs -0 -r grep -hE '(^|[[:space:]])WARNING:' \ | sed -E "s#$PWD/##g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}[^ ]*//g" \ | sort -u > "$output_dir/warnings.txt" || true + +# Raw command logs are useful for diagnosis but contain progress ordering and +# timing noise. The deterministic projections above are the comparison input. +rm -f "$output_dir"/*.log diff --git a/scripts/validation/compare-layer-state.py b/scripts/validation/compare-layer-state.py index 13b0e77a..cf38a03f 100755 --- a/scripts/validation/compare-layer-state.py +++ b/scripts/validation/compare-layer-state.py @@ -14,7 +14,7 @@ def files(root: Path) -> dict[str, list[str]]: result: dict[str, list[str]] = {} for path in sorted(root.rglob("*")): - if path.is_file(): + if path.is_file() and path.name != "metadata.json": result[str(path.relative_to(root))] = path.read_text(errors="replace").splitlines() return result diff --git a/scripts/validation/select-bitbake-env.py b/scripts/validation/select-bitbake-env.py new file mode 100644 index 00000000..3a72d87d --- /dev/null +++ b/scripts/validation/select-bitbake-env.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Project BitBake -e output to stable policy values with assignment history.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + + +EXACT = { + "BBMASK", "CORE_IMAGE_BASE_INSTALL", "CORE_IMAGE_EXTRA_INSTALL", + "DISTRO", "DISTRO_FEATURES", "IMAGE_BOOT_FILES", "IMAGE_FEATURES", + "IMAGE_FSTYPES", "IMAGE_INSTALL", "IMAGE_ROOTFS_EXTRA_SPACE", + "IMAGE_ROOTFS_SIZE", "INITRAMFS_FSTYPES", "INITRAMFS_IMAGE", + "INITRAMFS_MAXSIZE", "MACHINE", "MACHINE_FEATURES", + "PACKAGE_INSTALL", "SDKIMAGE_FEATURES", "TOOLCHAIN_TARGET_TASK", + "WKS_FILE", +} +PREFIXES = ( + "FIT_", "KERNEL_", "OPTEE_", "OSTREE_", "PREFERRED_PROVIDER_", + "PREFERRED_VERSION_", "SOTA_", "UBOOT_", +) +ASSIGNMENT = re.compile(r'^([A-Za-z0-9_${}/:.+-]+)=') + + +def selected(name: str) -> bool: + return name in EXACT or name.startswith(PREFIXES) + + +def main() -> int: + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} BITBAKE_ENV", file=sys.stderr) + return 2 + comments: list[str] = [] + for line in Path(sys.argv[1]).read_text(errors="replace").splitlines(): + if line.startswith("#"): + comments.append(line) + continue + match = ASSIGNMENT.match(line) + if match and selected(match.group(1)): + print("\n".join(comments[-40:])) + print(line) + comments.clear() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 372d90c70aac023e1fff3076309093af1c015f8a Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 00:01:52 +0100 Subject: [PATCH 09/47] ci: preserve protected layer adoption tuples --- scripts/validation/detect-layer-adoption.py | 75 +++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/scripts/validation/detect-layer-adoption.py b/scripts/validation/detect-layer-adoption.py index 012ce88b..9b7fa0e0 100755 --- a/scripts/validation/detect-layer-adoption.py +++ b/scripts/validation/detect-layer-adoption.py @@ -16,19 +16,94 @@ re.compile(r"^\.gitmodules$"), re.compile(r"(^|/)conf/layer\.conf$"), re.compile(r"^kas/.*\.ya?ml$"), + re.compile(r"^ci/layer-adoption-tuples\.json$"), ) + +REQUIRED_TUPLE_FIELDS = ("id", "machine", "distro", "image", "config") + + def git(*args: str) -> str: return subprocess.check_output(["git", *args], text=True) +def parse_tuples(raw: str, source: str) -> dict[str, dict[str, str]]: + try: + document = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{source}: invalid JSON: {exc}") from exc + if document.get("schema") != 1 or not isinstance(document.get("tuples"), list): + raise ValueError(f"{source}: expected schema=1 and a tuples array") + + result: dict[str, dict[str, str]] = {} + for index, entry in enumerate(document["tuples"]): + if not isinstance(entry, dict): + raise ValueError(f"{source}: tuple {index} is not an object") + missing = [field for field in REQUIRED_TUPLE_FIELDS if not str(entry.get(field, "")).strip()] + if missing: + raise ValueError(f"{source}: tuple {index} lacks {', '.join(missing)}") + tuple_id = str(entry["id"]) + if tuple_id in result: + raise ValueError(f"{source}: duplicate tuple id {tuple_id}") + result[tuple_id] = {field: str(entry[field]) for field in REQUIRED_TUPLE_FIELDS} + if not result: + raise ValueError(f"{source}: tuple matrix must not be empty") + return result + + +def validate_tuple_matrix(base: str, path: Path) -> None: + candidate = parse_tuples(path.read_text(encoding="utf-8"), str(path)) + missing_configs = sorted( + tuple_id for tuple_id, entry in candidate.items() + if not Path(entry["config"]).is_file() + ) + if missing_configs: + raise ValueError( + f"{path}: tuple configs do not exist for: {', '.join(missing_configs)}" + ) + + baseline_result = subprocess.run( + ["git", "show", f"{base}:{path.as_posix()}"], + check=False, + capture_output=True, + text=True, + ) + if baseline_result.returncode != 0: + # Bootstrap case: the gate and its matrix are being introduced together. + return + baseline_raw = baseline_result.stdout + baseline = parse_tuples(baseline_raw, f"{base}:{path}") + removed = sorted(set(baseline) - set(candidate)) + changed = sorted( + tuple_id for tuple_id in set(baseline) & set(candidate) + if baseline[tuple_id] != candidate[tuple_id] + ) + if removed or changed: + details = [] + if removed: + details.append("removed=" + ",".join(removed)) + if changed: + details.append("redefined=" + ",".join(changed)) + raise ValueError( + "protected baseline tuples may only be extended, not removed or redefined (" + + "; ".join(details) + ")" + ) + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--base", required=True) parser.add_argument("--head", default="HEAD") parser.add_argument("--contract", default="ci/layer-adoption-contract.json") + parser.add_argument("--tuples", default="ci/layer-adoption-tuples.json") parser.add_argument("--github-output") args = parser.parse_args() + try: + validate_tuple_matrix(args.base, Path(args.tuples)) + except (OSError, ValueError) as exc: + print(f"ERROR: invalid protected tuple matrix: {exc}", file=sys.stderr) + return 2 + changed = git("diff", "--name-only", f"{args.base}...{args.head}").splitlines() material_files = [ path for path in changed From 9d6bb5ad5e7e97015eb3abb751bf4b0f3da1c544 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 00:03:57 +0100 Subject: [PATCH 10/47] ci: test protected layer adoption matrix --- .github/workflows/layer-adoption-gate.yml | 1 + .../tests/test_detect_layer_adoption.py | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 scripts/validation/tests/test_detect_layer_adoption.py diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 4e151c63..02b59dba 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -43,6 +43,7 @@ jobs: - id: detect name: Require an adoption contract run: | + python3 -m unittest discover -s scripts/validation/tests -p 'test_*.py' python3 scripts/validation/detect-layer-adoption.py \ --base '${{ steps.base.outputs.sha }}' \ --head '${{ github.sha }}' \ diff --git a/scripts/validation/tests/test_detect_layer_adoption.py b/scripts/validation/tests/test_detect_layer_adoption.py new file mode 100644 index 00000000..d264dbbc --- /dev/null +++ b/scripts/validation/tests/test_detect_layer_adoption.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Regression tests for protected layer-adoption tuple handling.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +MODULE_PATH = Path(__file__).parents[1] / "detect-layer-adoption.py" +SPEC = importlib.util.spec_from_file_location("detect_layer_adoption", MODULE_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def document(*tuples: dict[str, str]) -> str: + return json.dumps({"schema": 1, "tuples": list(tuples)}) + + +def entry(tuple_id: str, machine: str = "machine-a") -> dict[str, str]: + return { + "id": tuple_id, + "machine": machine, + "distro": "distro-a", + "image": "image-a", + "config": "kas/test.yml", + } + + +class ProtectedTupleTests(unittest.TestCase): + def validate(self, baseline: str, candidate: str) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "kas").mkdir() + (root / "kas/test.yml").touch() + matrix = root / "ci/layer-adoption-tuples.json" + matrix.parent.mkdir() + matrix.write_text(candidate, encoding="utf-8") + result = subprocess.CompletedProcess([], 0, stdout=baseline, stderr="") + with mock.patch.object(MODULE.subprocess, "run", return_value=result), mock.patch.object( + MODULE.Path, "is_file", return_value=True + ): + MODULE.validate_tuple_matrix("baseline", matrix) + + def test_extension_preserves_existing_tuple(self) -> None: + self.validate(document(entry("existing")), document(entry("existing"), entry("new"))) + + def test_removing_existing_tuple_fails(self) -> None: + with self.assertRaisesRegex(ValueError, "removed=existing"): + self.validate(document(entry("existing")), document(entry("replacement"))) + + def test_redefining_existing_tuple_fails(self) -> None: + with self.assertRaisesRegex(ValueError, "redefined=existing"): + self.validate(document(entry("existing")), document(entry("existing", "machine-b"))) + + def test_duplicate_candidate_id_fails(self) -> None: + with self.assertRaisesRegex(ValueError, "duplicate tuple id existing"): + self.validate(document(entry("existing")), document(entry("existing"), entry("existing"))) + + +if __name__ == "__main__": + unittest.main() From 06f1536da6d0e49d16b8f8aa9890bb88571fd9d3 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 01:55:44 +0100 Subject: [PATCH 11/47] ci: protect exact product feature tuples --- .github/workflows/layer-adoption-gate.yml | 4 +- ci/layer-adoption-tuples.json | 28 +++++++------- scripts/validation/capture-layer-state.sh | 38 +++++++++++++++---- scripts/validation/detect-layer-adoption.py | 11 ++++-- scripts/validation/select-bitbake-env.py | 1 + .../tests/test_detect_layer_adoption.py | 13 +++++++ 6 files changed, 68 insertions(+), 27 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 02b59dba..ff0866eb 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -100,14 +100,14 @@ jobs: run: | ../candidate/scripts/validation/capture-layer-state.sh \ '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ - '${{ matrix.image }}' \ + '${{ matrix.image }}' '${{ matrix.product_features }}' \ '../evidence/baseline/${{ matrix.id }}' - name: Build and capture candidate working-directory: candidate run: | scripts/validation/capture-layer-state.sh \ '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ - '${{ matrix.image }}' \ + '${{ matrix.image }}' '${{ matrix.product_features }}' \ '../evidence/candidate/${{ matrix.id }}' - name: Reject unexplained contamination run: | diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index acace193..bb032edd 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -1,20 +1,20 @@ { "schema": 1, "tuples": [ - {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-inst-image", "machine": "imx8mm-jaguar-inst", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-phasora-image", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv usb-gadget"}, + {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-inst-image", "machine": "imx8mm-jaguar-inst", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, + {"id": "imx8mm-jaguar-phasora-image", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "display flutter godot"}, + {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, + {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-inst-mfgtool", "machine": "imx8mm-jaguar-inst", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-phasora-mfgtool", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"} + {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-inst-mfgtool", "machine": "imx8mm-jaguar-inst", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-phasora-mfgtool", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""} ] } diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 50661cc7..a7132017 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -2,8 +2,8 @@ # Build one protected tuple and capture deterministic layer/package/task state. set -euo pipefail -if [ "$#" -ne 5 ]; then - echo "Usage: $0 KAS_CONFIG MACHINE DISTRO TARGET OUTPUT_DIR" >&2 +if [ "$#" -ne 6 ]; then + echo "Usage: $0 KAS_CONFIG MACHINE DISTRO TARGET PRODUCT_FEATURES OUTPUT_DIR" >&2 exit 2 fi @@ -11,7 +11,8 @@ config=$1 machine=$2 distro=$3 target=$4 -output_dir=$5 +product_features=$5 +output_dir=$6 case "$output_dir" in /*) ;; @@ -22,19 +23,36 @@ mkdir -p "$output_dir" export KAS_MACHINE="$machine" export KAS_DISTRO="$distro" export DISTRO="$distro" -kas checkout "$config" + +# KAS deliberately sanitises the environment before entering BitBake's build +# environment, so an exported DD_PRODUCT_FEATURES is silently lost. Inject the +# reviewed tuple value through a generated KAS overlay instead. JSON string +# quoting is also valid BitBake quoting and prevents tuple text from becoming +# local.conf syntax. +product_features_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$product_features") +overlay="$output_dir/layer-adoption-product-features.yml" +cat > "$overlay" < "$output_dir/metadata.json" < "$output_dir/commands.txt" + "bitbake $target" \ + "DD_PRODUCT_FEATURES=$product_features" > "$output_dir/commands.txt" # Static layer surfaces are captured as well as BitBake's resolved view. This # makes wildcard/dangling appends and global layer.conf policy visible even @@ -49,7 +67,7 @@ find build/layers -type f -name '*.bbappend' -printf '%p\n' \ | sort -u > "$output_dir/all-bbappends.txt" run_bitbake() { - kas shell "$config" -c "$1" + kas shell "$combined_config" -c "$1" } capture_command() { @@ -83,6 +101,7 @@ python3 "$(dirname "$0")/select-bitbake-env.py" \ "$output_dir/environment.log" > "$output_dir/selected-environment.txt" grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" +grep -Fqx "DD_PRODUCT_FEATURES=\"$product_features\"" "$output_dir/selected-environment.txt" rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, @@ -115,3 +134,6 @@ find "$output_dir" -type f -name '*.log' -print0 \ # Raw command logs are useful for diagnosis but contain progress ordering and # timing noise. The deterministic projections above are the comparison input. rm -f "$output_dir"/*.log +# The generated overlay is an input already represented in metadata.json; it +# must not become a baseline/candidate comparison artefact with differing paths. +rm -f "$overlay" diff --git a/scripts/validation/detect-layer-adoption.py b/scripts/validation/detect-layer-adoption.py index 9b7fa0e0..b0911206 100755 --- a/scripts/validation/detect-layer-adoption.py +++ b/scripts/validation/detect-layer-adoption.py @@ -19,7 +19,8 @@ re.compile(r"^ci/layer-adoption-tuples\.json$"), ) -REQUIRED_TUPLE_FIELDS = ("id", "machine", "distro", "image", "config") +TUPLE_FIELDS = ("id", "machine", "distro", "image", "config", "product_features") +NONEMPTY_TUPLE_FIELDS = ("id", "machine", "distro", "image", "config") def git(*args: str) -> str: @@ -38,13 +39,17 @@ def parse_tuples(raw: str, source: str) -> dict[str, dict[str, str]]: for index, entry in enumerate(document["tuples"]): if not isinstance(entry, dict): raise ValueError(f"{source}: tuple {index} is not an object") - missing = [field for field in REQUIRED_TUPLE_FIELDS if not str(entry.get(field, "")).strip()] + missing = [field for field in TUPLE_FIELDS if field not in entry] + missing.extend( + field for field in NONEMPTY_TUPLE_FIELDS + if field in entry and not str(entry[field]).strip() + ) if missing: raise ValueError(f"{source}: tuple {index} lacks {', '.join(missing)}") tuple_id = str(entry["id"]) if tuple_id in result: raise ValueError(f"{source}: duplicate tuple id {tuple_id}") - result[tuple_id] = {field: str(entry[field]) for field in REQUIRED_TUPLE_FIELDS} + result[tuple_id] = {field: str(entry[field]) for field in TUPLE_FIELDS} if not result: raise ValueError(f"{source}: tuple matrix must not be empty") return result diff --git a/scripts/validation/select-bitbake-env.py b/scripts/validation/select-bitbake-env.py index 3a72d87d..3ab9d22a 100644 --- a/scripts/validation/select-bitbake-env.py +++ b/scripts/validation/select-bitbake-env.py @@ -10,6 +10,7 @@ EXACT = { "BBMASK", "CORE_IMAGE_BASE_INSTALL", "CORE_IMAGE_EXTRA_INSTALL", + "DD_PRODUCT_FEATURES", "DISTRO", "DISTRO_FEATURES", "IMAGE_BOOT_FILES", "IMAGE_FEATURES", "IMAGE_FSTYPES", "IMAGE_INSTALL", "IMAGE_ROOTFS_EXTRA_SPACE", "IMAGE_ROOTFS_SIZE", "INITRAMFS_FSTYPES", "INITRAMFS_IMAGE", diff --git a/scripts/validation/tests/test_detect_layer_adoption.py b/scripts/validation/tests/test_detect_layer_adoption.py index d264dbbc..211741e7 100644 --- a/scripts/validation/tests/test_detect_layer_adoption.py +++ b/scripts/validation/tests/test_detect_layer_adoption.py @@ -30,6 +30,7 @@ def entry(tuple_id: str, machine: str = "machine-a") -> dict[str, str]: "distro": "distro-a", "image": "image-a", "config": "kas/test.yml", + "product_features": "", } @@ -63,6 +64,18 @@ def test_duplicate_candidate_id_fails(self) -> None: with self.assertRaisesRegex(ValueError, "duplicate tuple id existing"): self.validate(document(entry("existing")), document(entry("existing"), entry("existing"))) + def test_missing_product_features_fails(self) -> None: + candidate = entry("existing") + del candidate["product_features"] + with self.assertRaisesRegex(ValueError, "lacks product_features"): + self.validate(document(entry("existing")), document(candidate)) + + def test_redefining_product_features_fails(self) -> None: + candidate = entry("existing") + candidate["product_features"] = "display" + with self.assertRaisesRegex(ValueError, "redefined=existing"): + self.validate(document(entry("existing")), document(candidate)) + if __name__ == "__main__": unittest.main() From b9207b9bdb2d3495fe6dd42148cf28c3a72df96b Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 02:01:25 +0100 Subject: [PATCH 12/47] ci: exercise signing in layer adoption gate --- .github/workflows/layer-adoption-gate.yml | 4 ++ scripts/validation/capture-layer-state.sh | 45 +++++++++++++++- .../generate-layer-adoption-test-keys.sh | 53 +++++++++++++++++++ scripts/validation/select-bitbake-env.py | 7 +-- 4 files changed, 105 insertions(+), 4 deletions(-) create mode 100755 scripts/validation/generate-layer-adoption-test-keys.sh diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index ff0866eb..4bc0cac6 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -75,6 +75,8 @@ jobs: container: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 + env: + LAYER_ADOPTION_TEST_KEYS_DIR: ${{ runner.temp }}/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} steps: - uses: actions/checkout@v7 with: @@ -95,6 +97,8 @@ jobs: restore-keys: | layer-adoption-${{ matrix.machine }}- layer-adoption- + - name: Generate ephemeral signing keys + run: candidate/scripts/validation/generate-layer-adoption-test-keys.sh "$LAYER_ADOPTION_TEST_KEYS_DIR" - name: Build and capture baseline working-directory: baseline run: | diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index a7132017..8d564173 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -13,6 +13,23 @@ distro=$3 target=$4 product_features=$5 output_dir=$6 +test_keys_dir=${LAYER_ADOPTION_TEST_KEYS_DIR:-} + +if [ -z "$test_keys_dir" ] || [ ! -d "$test_keys_dir" ]; then + echo "ERROR: LAYER_ADOPTION_TEST_KEYS_DIR must name the generated test-key directory" >&2 + exit 2 +fi +test_keys_dir=$(realpath "$test_keys_dir") +for key in \ + ubootdev.key ubootdev.crt spldev.key spldev.crt \ + privkey_modsign.pem x509_modsign.crt \ + uefi/DB.key uefi/DB.crt tf-a/privkey_ec_prime256v1.pem +do + if [ ! -s "$test_keys_dir/$key" ]; then + echo "ERROR: required test signing key is missing: $key" >&2 + exit 2 + fi +done case "$output_dir" in /*) ;; @@ -37,6 +54,19 @@ header: local_conf_header: layer-adoption-product-features: | DD_PRODUCT_FEATURES = $product_features_quoted + UBOOT_SIGN_KEYDIR = "$test_keys_dir" + UEFI_SIGN_KEYDIR = "$test_keys_dir/uefi" + MODSIGN_KEY_DIR = "$test_keys_dir" + SIGNING_UBOOT_SIGN_KEY = "$test_keys_dir/ubootdev.key" + SIGNING_UBOOT_SIGN_CRT = "$test_keys_dir/ubootdev.crt" + SIGNING_UBOOT_SPL_SIGN_KEY = "$test_keys_dir/spldev.key" + SIGNING_UBOOT_SPL_SIGN_CRT = "$test_keys_dir/spldev.crt" + SIGNING_MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem" + SIGNING_MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt" + SIGNING_UEFI_SIGN_KEY = "$test_keys_dir/uefi/DB.key" + SIGNING_UEFI_SIGN_CRT = "$test_keys_dir/uefi/DB.crt" + OPTEE_TA_SIGN_KEY = "$test_keys_dir/ubootdev.key" + TF_A_SIGN_KEY_PATH = "$test_keys_dir/tf-a/privkey_ec_prime256v1.pem" EOF combined_config="${config}:${overlay}" kas checkout "$combined_config" @@ -44,6 +74,17 @@ kas checkout "$combined_config" cat > "$output_dir/metadata.json" </dev/null \ + | sha256sum | cut -d ' ' -f 1) + printf '%s\t%s\n' "$key" "$fingerprint" +done > "$output_dir/test-signing-key-fingerprints.txt" printf '%s\n' \ "kas checkout CONFIG:PRODUCT_FEATURE_OVERLAY" \ "bitbake-layers show-layers" \ @@ -98,7 +139,9 @@ sed -E "s#$PWD/##g" build/recipe-depends.dot | sort -u \ # temporarily only as input, avoiding volatile host variables in comparisons. capture_command environment run_bitbake "bitbake -e $target" python3 "$(dirname "$0")/select-bitbake-env.py" \ - "$output_dir/environment.log" > "$output_dir/selected-environment.txt" + "$output_dir/environment.log" \ + | sed -E "s#$PWD##g; s#$test_keys_dir##g" \ + > "$output_dir/selected-environment.txt" grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" grep -Fqx "DD_PRODUCT_FEATURES=\"$product_features\"" "$output_dir/selected-environment.txt" diff --git a/scripts/validation/generate-layer-adoption-test-keys.sh b/scripts/validation/generate-layer-adoption-test-keys.sh new file mode 100755 index 00000000..4f3dc91c --- /dev/null +++ b/scripts/validation/generate-layer-adoption-test-keys.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Generate one genuine but disposable signing-key set shared by a baseline and candidate build. +set -euo pipefail +umask 077 + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 OUTPUT_DIR" >&2 + exit 2 +fi + +output_dir=$(realpath -m "$1") +case "$output_dir" in + /|/home|/root|/tmp|/var|/usr) + echo "ERROR: refusing broad test-key output directory: $output_dir" >&2 + exit 2 + ;; +esac +if [ -e "$output_dir" ] && find "$output_dir" -mindepth 1 -print -quit | grep -q .; then + echo "ERROR: test-key output directory is not empty: $output_dir" >&2 + exit 2 +fi + +mkdir -p "$output_dir/uefi" "$output_dir/tf-a" + +make_rsa_certificate() { + local key=$1 + local certificate=$2 + local common_name=$3 + openssl genpkey -algorithm RSA -out "$key" -pkeyopt rsa_keygen_bits:2048 + openssl req -batch -new -x509 -sha256 -days 2 \ + -key "$key" -out "$certificate" -subj "/CN=$common_name/" + openssl pkey -in "$key" -check -noout + openssl x509 -in "$certificate" -noout +} + +make_rsa_certificate \ + "$output_dir/ubootdev.key" "$output_dir/ubootdev.crt" \ + layer-adoption-uboot +make_rsa_certificate \ + "$output_dir/spldev.key" "$output_dir/spldev.crt" \ + layer-adoption-spl +make_rsa_certificate \ + "$output_dir/privkey_modsign.pem" "$output_dir/x509_modsign.crt" \ + layer-adoption-module +make_rsa_certificate \ + "$output_dir/uefi/DB.key" "$output_dir/uefi/DB.crt" \ + layer-adoption-uefi + +openssl ecparam -name prime256v1 -genkey -noout \ + -out "$output_dir/tf-a/privkey_ec_prime256v1.pem" +openssl ec -in "$output_dir/tf-a/privkey_ec_prime256v1.pem" -check -noout + +printf 'PASS: generated ephemeral layer-adoption signing keys in %s\n' "$output_dir" diff --git a/scripts/validation/select-bitbake-env.py b/scripts/validation/select-bitbake-env.py index 3ab9d22a..c58ea9e5 100644 --- a/scripts/validation/select-bitbake-env.py +++ b/scripts/validation/select-bitbake-env.py @@ -14,13 +14,14 @@ "DISTRO", "DISTRO_FEATURES", "IMAGE_BOOT_FILES", "IMAGE_FEATURES", "IMAGE_FSTYPES", "IMAGE_INSTALL", "IMAGE_ROOTFS_EXTRA_SPACE", "IMAGE_ROOTFS_SIZE", "INITRAMFS_FSTYPES", "INITRAMFS_IMAGE", - "INITRAMFS_MAXSIZE", "MACHINE", "MACHINE_FEATURES", + "INITRAMFS_MAXSIZE", "LOCAL_DEVELOPMENT_BUILD", "MACHINE", "MACHINE_FEATURES", + "MODSIGN", "SIGN_ENABLE", "TF_A_SIGN_ENABLE", "UEFI_SIGN_ENABLE", "PACKAGE_INSTALL", "SDKIMAGE_FEATURES", "TOOLCHAIN_TARGET_TASK", "WKS_FILE", } PREFIXES = ( - "FIT_", "KERNEL_", "OPTEE_", "OSTREE_", "PREFERRED_PROVIDER_", - "PREFERRED_VERSION_", "SOTA_", "UBOOT_", + "FIT_", "KERNEL_", "MODSIGN_", "OPTEE_", "OSTREE_", "PREFERRED_PROVIDER_", + "PREFERRED_VERSION_", "SIGNING_", "SOTA_", "TF_A_", "UBOOT_", "UEFI_", ) ASSIGNMENT = re.compile(r'^([A-Za-z0-9_${}/:.+-]+)=') From 303978b4e163bae66819bf5f87346741782c4ef9 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 02:58:44 +0100 Subject: [PATCH 13/47] ci: use valid job-level key path --- .github/workflows/layer-adoption-gate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 4bc0cac6..c59754c4 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -76,7 +76,7 @@ jobs: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 env: - LAYER_ADOPTION_TEST_KEYS_DIR: ${{ runner.temp }}/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} + LAYER_ADOPTION_TEST_KEYS_DIR: /tmp/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} steps: - uses: actions/checkout@v7 with: From 4b2cfa7403a310e1dbc6b87853a8730e1d7ecbda Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 03:06:20 +0100 Subject: [PATCH 14/47] ci: bound layer gate disk usage --- .github/workflows/layer-adoption-gate.yml | 38 +++++++++++++++++++++++ scripts/validation/capture-layer-state.sh | 10 +++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index c59754c4..95b3df59 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -78,6 +78,16 @@ jobs: env: LAYER_ADOPTION_TEST_KEYS_DIR: /tmp/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} steps: + - name: Reset job-owned workspace + run: | + workspace=$(realpath -m -- "$GITHUB_WORKSPACE") + test -n "$workspace" + test "$workspace" != / + for name in candidate baseline evidence; do + target=$(realpath -m -- "$workspace/$name") + test "$(dirname "$target")" = "$workspace" + rm -rf -- "$target" + done - uses: actions/checkout@v7 with: fetch-depth: 0 @@ -97,6 +107,14 @@ jobs: restore-keys: | layer-adoption-${{ matrix.machine }}- layer-adoption- + - name: Require safe build capacity + run: | + available_kib=$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 {print $4}') + minimum_kib=$((150 * 1024 * 1024)) + if [ "$available_kib" -lt "$minimum_kib" ]; then + echo "ERROR: layer-adoption build requires at least 150 GiB free; found $((available_kib / 1024 / 1024)) GiB" >&2 + exit 1 + fi - name: Generate ephemeral signing keys run: candidate/scripts/validation/generate-layer-adoption-test-keys.sh "$LAYER_ADOPTION_TEST_KEYS_DIR" - name: Build and capture baseline @@ -106,6 +124,12 @@ jobs: '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ '${{ matrix.image }}' '${{ matrix.product_features }}' \ '../evidence/baseline/${{ matrix.id }}' + - name: Release baseline build workspace + run: | + workspace=$(realpath -m -- "$GITHUB_WORKSPACE") + target=$(realpath -m -- "$workspace/baseline/build") + test "$target" = "$workspace/baseline/build" + rm -rf -- "$target" - name: Build and capture candidate working-directory: candidate run: | @@ -127,6 +151,20 @@ jobs: name: layer-adoption-${{ matrix.id }} path: evidence retention-days: 14 + - name: Remove job-owned build trees + if: always() + run: | + workspace=$(realpath -m -- "$GITHUB_WORKSPACE") + test -n "$workspace" + test "$workspace" != / + for name in candidate/build baseline/build evidence; do + target=$(realpath -m -- "$workspace/$name") + case "$target" in + "$workspace/candidate/build"|"$workspace/baseline/build"|"$workspace/evidence") ;; + *) echo "ERROR: refusing unsafe cleanup target: $target" >&2; exit 1 ;; + esac + rm -rf -- "$target" + done required: name: Layer Adoption Gate diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 8d564173..4c325d81 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -14,12 +14,15 @@ target=$4 product_features=$5 output_dir=$6 test_keys_dir=${LAYER_ADOPTION_TEST_KEYS_DIR:-} +cache_root=${LAYER_ADOPTION_CACHE_DIR:-$HOME/yocto} if [ -z "$test_keys_dir" ] || [ ! -d "$test_keys_dir" ]; then echo "ERROR: LAYER_ADOPTION_TEST_KEYS_DIR must name the generated test-key directory" >&2 exit 2 fi test_keys_dir=$(realpath "$test_keys_dir") +mkdir -p "$cache_root/downloads" "$cache_root/sstate-cache" +cache_root=$(realpath "$cache_root") for key in \ ubootdev.key ubootdev.crt spldev.key spldev.crt \ privkey_modsign.pem x509_modsign.crt \ @@ -47,6 +50,8 @@ export DISTRO="$distro" # quoting is also valid BitBake quoting and prevents tuple text from becoming # local.conf syntax. product_features_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$product_features") +downloads_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/downloads") +sstate_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/sstate-cache") overlay="$output_dir/layer-adoption-product-features.yml" cat > "$overlay" <#g; s#$test_keys_dir##g" \ + | sed -E "s#$PWD##g; s#$test_keys_dir##g; s#$cache_root##g" \ > "$output_dir/selected-environment.txt" grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" From 5cf9737c4f39a6b9a77572833b0f69fecc1aed98 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:31:56 +0100 Subject: [PATCH 15/47] ci: consolidate layer adoption regression on ai-tools --- .github/workflows/kas-build-ci.yml | 203 ------------------ .github/workflows/layer-adoption-gate.yml | 101 +++------ .../generate-layer-adoption-test-keys.sh | 4 +- .../run-layer-adoption-regression.py | 178 +++++++++++++++ .../test_run_layer_adoption_regression.py | 46 ++++ 5 files changed, 261 insertions(+), 271 deletions(-) delete mode 100644 .github/workflows/kas-build-ci.yml create mode 100644 scripts/validation/run-layer-adoption-regression.py create mode 100644 scripts/validation/tests/test_run_layer_adoption_regression.py diff --git a/.github/workflows/kas-build-ci.yml b/.github/workflows/kas-build-ci.yml deleted file mode 100644 index 517dcb41..00000000 --- a/.github/workflows/kas-build-ci.yml +++ /dev/null @@ -1,203 +0,0 @@ -name: KAS Build CI - -on: - push: - branches: [ main, develop ] - paths: - - 'kas/**' - - 'meta-dynamicdevices-bsp/**' - - 'meta-dynamicdevices-distro/**' - - 'recipes-**' - - 'classes/**' - - 'conf/**' - - '.github/workflows/kas-build-ci.yml' - - 'scripts/kas-*.sh' - - 'scripts/validation/**' - - 'bbappends/**' - - 'custom-boot-files/**' - - pull_request: - branches: [ main, develop ] - paths: - - 'kas/**' - - 'meta-dynamicdevices-bsp/**' - - 'meta-dynamicdevices-distro/**' - - 'recipes-**' - - 'classes/**' - - 'conf/**' - - '.github/workflows/kas-build-ci.yml' - - 'scripts/kas-*.sh' - - 'scripts/validation/**' - - 'bbappends/**' - - 'custom-boot-files/**' - - workflow_dispatch: - inputs: - debug_enabled: - type: boolean - description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' - required: false - default: false - -env: - DEBIAN_FRONTEND: noninteractive - -jobs: - # Validation job - validates Yocto layer compatibility - validate: - name: Validate Yocto Layers - # Pin container work to the dedicated Yocto runner. Generic Linux labels - # also match esl-nixos, which intentionally has no Docker daemon. - runs-on: [self-hosted, Linux, X64, yocto] - container: - image: dynamicdevices/yocto-ci-build:latest - options: --privileged --platform linux/amd64 - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - submodules: recursive - - - name: Cache KAS layers - uses: actions/cache@v6 - with: - path: | - build/layers - build/cache - key: kas-layers-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }}-${{ github.sha }} - restore-keys: | - kas-layers-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }}- - kas-layers- - - - name: Cache Yocto downloads - uses: actions/cache@v6 - with: - path: ~/yocto/downloads - key: yocto-downloads-${{ runner.os }}-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }} - restore-keys: | - yocto-downloads-${{ runner.os }}- - - - name: Validate Yocto Layers - timeout-minutes: 15 - run: | - echo "🏅 Meta-DynamicDevices Layer Validation (CI)" - echo "=============================================" - echo "📋 Using official yocto-check-layer for Yocto Project compliance" - echo "" - - # Install KAS if not available - if ! command -v kas >/dev/null 2>&1; then - echo "📦 Installing KAS..." - pip3 install kas - fi - - # Create validation workspace - VALIDATION_DIR="ci-layer-validation" - rm -rf "$VALIDATION_DIR" - mkdir -p "$VALIDATION_DIR" - cd "$VALIDATION_DIR" - - echo "🔧 Setting up KAS environment for layer validation..." - - # Copy KAS configuration for validation - cp ../kas/layer-validation.yml . - - # Initialize KAS environment - echo "📋 Initializing KAS build environment..." - kas shell layer-validation.yml -c "echo 'KAS environment initialized'" - - echo "✅ KAS environment ready" - echo "" - - echo "🔍 Starting comprehensive layer validation..." - echo "" - - # Run yocto-check-layer validation - echo "1️⃣ Validating all meta-dynamicdevices layers together..." - - if kas shell layer-validation.yml -c " - # Use the yocto-check-layer script from openembedded-core - YOCTO_CHECK_LAYER='./layers/openembedded-core/scripts/yocto-check-layer' - - if [ ! -f \"\$YOCTO_CHECK_LAYER\" ]; then - echo '❌ yocto-check-layer script not found at expected location' - exit 1 - fi - - echo '✅ Found yocto-check-layer: '\$YOCTO_CHECK_LAYER - - # Clean up all potential conflicts from BitBake test data - rm -rf layers/bitbake/lib/layerindexlib/tests/testdata/ 2>/dev/null || true - find ../.. -name 'bitbake' -type d -exec rm -rf {}/lib/layerindexlib/tests/testdata/ 2>/dev/null \\; || true - - # Clean up any other build directories that might cause collection conflicts - find ../.. -maxdepth 2 -name 'build*' -type d ! -path '*/ci-layer-validation/build' -exec echo '🧹 Temporarily moving {}' \\; -exec mv {} {}.bak 2>/dev/null \\; || true - - # Run validation on all meta-dynamicdevices layers together to handle dependencies - echo '🔍 Running yocto-check-layer validation...' - python3 \"\$YOCTO_CHECK_LAYER\" \"$PWD/../meta-dynamicdevices-bsp\" \"$PWD/../meta-dynamicdevices-distro\" \"$PWD/..\" - - # Restore moved directories - find ../.. -maxdepth 2 -name 'build*.bak' -type d -exec sh -c 'mv \"\$1\" \"\${1%.bak}\"' _ {} \\; 2>/dev/null || true - "; then - echo "" - echo "✅ meta-dynamicdevices layers validation PASSED" - echo "" - echo "=============================================" - echo "✅ All layer validations PASSED" - echo "✅ All meta-dynamicdevices layers pass comprehensive yocto-check-layer validation!" - echo "✅ Layers are ready for Yocto Project compatibility." - echo "" - else - echo "" - echo "❌ meta-dynamicdevices layers validation FAILED" - echo "" - echo "=============================================" - echo "❌ Layer validation FAILED" - echo "" - echo "ℹ️ Please fix the yocto-check-layer issues above before proceeding." - echo "ℹ️ Run './scripts/validate-layers-local.sh' locally to debug issues." - echo "" - exit 1 - fi - - # Cleanup validation workspace - cd .. - rm -rf "$VALIDATION_DIR" - - # Summary job - summary: - name: Validation Summary - runs-on: [self-hosted, Linux, X64] - needs: [validate] - if: always() - - steps: - - name: Generate validation summary - run: | - { - echo "# KAS Layer Validation Summary" - echo - echo "**Workflow:** ${{ github.workflow }}" - echo "**Trigger:** ${{ github.event_name }}" - echo "**Commit:** ${{ github.sha }}" - echo "**Branch:** ${{ github.ref_name }}" - echo - - if [ "${{ needs.validate.result }}" = "success" ]; then - echo "## ✅ Validation Status: SUCCESS" - echo "All meta-dynamicdevices layers pass comprehensive yocto-check-layer validation!" - echo "Layers are ready for Yocto Project compatibility." - else - echo "## ❌ Validation Status: FAILED" - echo "- Layer validation: ${{ needs.validate.result }}" - echo "Please fix validation issues before proceeding." - fi - - echo - echo "## Validation Coverage" - echo "- **Tool:** Official yocto-check-layer" - echo "- **Layers:** meta-dynamicdevices, meta-dynamicdevices-bsp, meta-dynamicdevices-distro" - echo "- **Compliance:** Full Yocto Project compatibility validation" - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 95b3df59..4f6e219d 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -21,7 +21,6 @@ jobs: runs-on: [self-hosted, Linux, X64] outputs: material: ${{ steps.detect.outputs.material }} - matrix: ${{ steps.matrix.outputs.matrix }} base_sha: ${{ steps.base.outputs.sha }} steps: - uses: actions/checkout@v7 @@ -41,42 +40,33 @@ jobs: git cat-file -e "$sha^{commit}" echo "sha=$sha" >> "$GITHUB_OUTPUT" - id: detect - name: Require an adoption contract + name: Validate repository and require an adoption contract run: | python3 -m unittest discover -s scripts/validation/tests -p 'test_*.py' + find scripts -type f -name '*.sh' -print0 | xargs -0 -r -n1 bash -n + python3 -m json.tool ci/layer-adoption-contract.json >/dev/null + python3 -m json.tool ci/layer-adoption-tuples.json >/dev/null + # Mail-format patch payloads legitimately contain the conventional + # "-- " separator. Check repository sources without rewriting the + # third-party patches that BitBake applies. + git diff --check '${{ steps.base.outputs.sha }}...${{ github.sha }}' -- . ':(exclude)**/*.patch' python3 scripts/validation/detect-layer-adoption.py \ --base '${{ steps.base.outputs.sha }}' \ --head '${{ github.sha }}' \ --github-output "$GITHUB_OUTPUT" - - id: matrix - name: Publish protected tuple matrix - run: | - python3 - <<'PY' - import json - import os - - with open("ci/layer-adoption-tuples.json", encoding="utf-8") as source: - matrix = {"include": json.load(source)["tuples"]} - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: - output.write("matrix=" + json.dumps(matrix, separators=(",", ":")) + "\n") - PY - regression: - name: Protect ${{ matrix.id }} + name: Existing product regression needs: detect if: needs.detect.outputs.material == 'true' - strategy: - fail-fast: false - # Full Yocto baseline/candidate pairs are I/O and disk intensive. Keep - # the hard gate comprehensive without overwhelming one self-hosted host. - max-parallel: 2 - matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} - runs-on: [self-hosted, Linux, X64, yocto] + # One named ai-tools runner owns the persistent Yocto cache. Keeping the + # complete matrix inside one job makes resource use and the required gate + # obvious, while the driver still fails closed on every protected tuple. + runs-on: [self-hosted, Linux, X64, yocto, ai-tools] container: - image: dynamicdevices/yocto-ci-build:latest - options: --privileged --platform linux/amd64 + image: ghcr.io/siemens/kas/kas@sha256:d989add57fc441fe9e27bb2dd6ed98c5597b44c807928e35a72dc1cfbdda9abe + options: --privileged --platform linux/amd64 --user 0:0 -v /home/ghrunner/yocto-layer-adoption:/var/cache/layer-adoption env: - LAYER_ADOPTION_TEST_KEYS_DIR: /tmp/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} + LAYER_ADOPTION_CACHE: /var/cache/layer-adoption steps: - name: Reset job-owned workspace run: | @@ -95,18 +85,6 @@ jobs: - name: Create baseline worktree working-directory: candidate run: git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' - - name: Install KAS when absent - run: command -v kas >/dev/null || pip3 install kas - - name: Restore Yocto caches - uses: actions/cache@v6 - with: - path: | - ~/yocto/downloads - ~/yocto/sstate-cache - key: layer-adoption-${{ matrix.machine }}-${{ hashFiles('candidate/kas/**') }} - restore-keys: | - layer-adoption-${{ matrix.machine }}- - layer-adoption- - name: Require safe build capacity run: | available_kib=$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 {print $4}') @@ -115,40 +93,31 @@ jobs: echo "ERROR: layer-adoption build requires at least 150 GiB free; found $((available_kib / 1024 / 1024)) GiB" >&2 exit 1 fi - - name: Generate ephemeral signing keys - run: candidate/scripts/validation/generate-layer-adoption-test-keys.sh "$LAYER_ADOPTION_TEST_KEYS_DIR" - - name: Build and capture baseline - working-directory: baseline - run: | - ../candidate/scripts/validation/capture-layer-state.sh \ - '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ - '${{ matrix.image }}' '${{ matrix.product_features }}' \ - '../evidence/baseline/${{ matrix.id }}' - - name: Release baseline build workspace + - name: Prepare persistent test-only signing identity run: | - workspace=$(realpath -m -- "$GITHUB_WORKSPACE") - target=$(realpath -m -- "$workspace/baseline/build") - test "$target" = "$workspace/baseline/build" - rm -rf -- "$target" - - name: Build and capture candidate - working-directory: candidate - run: | - scripts/validation/capture-layer-state.sh \ - '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ - '${{ matrix.image }}' '${{ matrix.product_features }}' \ - '../evidence/candidate/${{ matrix.id }}' - - name: Reject unexplained contamination + set -euo pipefail + keys="$LAYER_ADOPTION_CACHE/test-keys" + if [ ! -s "$keys/ubootdev.key" ]; then + test "$keys" = /var/cache/layer-adoption/test-keys + rm -rf -- "$keys" + rm -rf -- "$LAYER_ADOPTION_CACHE/baselines" + temporary=$(mktemp -d "$LAYER_ADOPTION_CACHE/.test-keys.XXXXXX") + candidate/scripts/validation/generate-layer-adoption-test-keys.sh "$temporary" + mv "$temporary" "$keys" + fi + - name: Build and compare every protected tuple run: | - python3 candidate/scripts/validation/compare-layer-state.py \ - --baseline 'evidence/baseline/${{ matrix.id }}' \ - --candidate 'evidence/candidate/${{ matrix.id }}' \ - --tuple '${{ matrix.id }}' \ - --contract candidate/ci/layer-adoption-contract.json + python3 candidate/scripts/validation/run-layer-adoption-regression.py \ + --baseline baseline \ + --candidate candidate \ + --evidence evidence \ + --cache "$LAYER_ADOPTION_CACHE" \ + --test-keys "$LAYER_ADOPTION_CACHE/test-keys" - name: Preserve comparison evidence if: always() uses: actions/upload-artifact@v7 with: - name: layer-adoption-${{ matrix.id }} + name: layer-adoption-evidence path: evidence retention-days: 14 - name: Remove job-owned build trees diff --git a/scripts/validation/generate-layer-adoption-test-keys.sh b/scripts/validation/generate-layer-adoption-test-keys.sh index 4f3dc91c..6c0f0834 100755 --- a/scripts/validation/generate-layer-adoption-test-keys.sh +++ b/scripts/validation/generate-layer-adoption-test-keys.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Generate one genuine but disposable signing-key set shared by a baseline and candidate build. +# Generate one genuine, test-only signing identity shared by compared builds. set -euo pipefail umask 077 @@ -50,4 +50,4 @@ openssl ecparam -name prime256v1 -genkey -noout \ -out "$output_dir/tf-a/privkey_ec_prime256v1.pem" openssl ec -in "$output_dir/tf-a/privkey_ec_prime256v1.pem" -check -noout -printf 'PASS: generated ephemeral layer-adoption signing keys in %s\n' "$output_dir" +printf 'PASS: generated test-only layer-adoption signing keys in %s\n' "$output_dir" diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py new file mode 100644 index 00000000..9a4f7fc4 --- /dev/null +++ b/scripts/validation/run-layer-adoption-regression.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Run every protected Yocto tuple in one fail-closed regression job.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + + +MARKER = ".complete.json" +FIELDS = ("id", "machine", "distro", "image", "config", "product_features") + + +def evidence_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == MARKER: + continue + relative = path.relative_to(root).as_posix().encode() + digest.update(relative) + digest.update(b"\0") + digest.update(hashlib.sha256(path.read_bytes()).digest()) + return digest.hexdigest() + + +def valid_cached_evidence(root: Path, base_sha: str, tuple_id: str) -> bool: + try: + marker = json.loads((root / MARKER).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + return marker == { + "base_sha": base_sha, + "tuple_id": tuple_id, + "evidence_sha256": evidence_digest(root), + } + + +def load_tuples(path: Path) -> list[dict[str, str]]: + document = json.loads(path.read_text(encoding="utf-8")) + if document.get("schema") != 1 or not isinstance(document.get("tuples"), list): + raise ValueError(f"{path}: expected schema=1 and a tuples array") + tuples = [] + seen = set() + for index, raw in enumerate(document["tuples"]): + if not isinstance(raw, dict) or any(field not in raw for field in FIELDS): + raise ValueError(f"{path}: tuple {index} is incomplete") + entry = {field: str(raw[field]) for field in FIELDS} + if not entry["id"] or entry["id"] in seen: + raise ValueError(f"{path}: duplicate or empty tuple id {entry['id']!r}") + seen.add(entry["id"]) + tuples.append(entry) + if not tuples: + raise ValueError(f"{path}: no protected tuples") + return tuples + + +def remove_build_tree(repository: Path) -> None: + repository = repository.resolve() + build = (repository / "build").resolve() + if build.parent != repository or repository == Path("/"): + raise RuntimeError(f"refusing unsafe build cleanup: {build}") + shutil.rmtree(build, ignore_errors=True) + + +def capture( + script: Path, + repository: Path, + entry: dict[str, str], + output: Path, + environment: dict[str, str], +) -> None: + subprocess.run( + [ + str(script), + entry["config"], + entry["machine"], + entry["distro"], + entry["image"], + entry["product_features"], + str(output), + ], + cwd=repository, + env=environment, + check=True, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--candidate", required=True, type=Path) + parser.add_argument("--evidence", required=True, type=Path) + parser.add_argument("--cache", required=True, type=Path) + parser.add_argument("--test-keys", required=True, type=Path) + args = parser.parse_args() + + baseline = args.baseline.resolve() + candidate = args.candidate.resolve() + evidence = args.evidence.resolve() + cache = args.cache.resolve() + test_keys = args.test_keys.resolve() + capture_script = candidate / "scripts/validation/capture-layer-state.sh" + compare_script = candidate / "scripts/validation/compare-layer-state.py" + contract = candidate / "ci/layer-adoption-contract.json" + tuples = load_tuples(candidate / "ci/layer-adoption-tuples.json") + base_sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=baseline, text=True + ).strip() + + environment = os.environ.copy() + environment["LAYER_ADOPTION_TEST_KEYS_DIR"] = str(test_keys) + environment["LAYER_ADOPTION_CACHE_DIR"] = str(cache / "yocto") + shutil.rmtree(evidence, ignore_errors=True) + (evidence / "baseline").mkdir(parents=True) + (evidence / "candidate").mkdir(parents=True) + + for entry in tuples: + tuple_id = entry["id"] + print(f"::group::Protect {tuple_id}", flush=True) + cached = cache / "baselines" / base_sha / tuple_id + baseline_output = evidence / "baseline" / tuple_id + candidate_output = evidence / "candidate" / tuple_id + temporary: Path | None = None + try: + if valid_cached_evidence(cached, base_sha, tuple_id): + print(f"Reusing immutable baseline evidence for {base_sha}", flush=True) + else: + shutil.rmtree(cached, ignore_errors=True) + cached.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=f".{tuple_id}-", dir=cached.parent)) + capture(capture_script, baseline, entry, temporary, environment) + marker = { + "base_sha": base_sha, + "tuple_id": tuple_id, + "evidence_sha256": evidence_digest(temporary), + } + (temporary / MARKER).write_text( + json.dumps(marker, sort_keys=True) + "\n", encoding="utf-8" + ) + temporary.rename(cached) + temporary = None + + shutil.copytree(cached, baseline_output, ignore=shutil.ignore_patterns(MARKER)) + capture(capture_script, candidate, entry, candidate_output, environment) + subprocess.run( + [ + "python3", + str(compare_script), + "--baseline", + str(baseline_output), + "--candidate", + str(candidate_output), + "--tuple", + tuple_id, + "--contract", + str(contract), + ], + check=True, + ) + finally: + if temporary is not None: + shutil.rmtree(temporary, ignore_errors=True) + remove_build_tree(baseline) + remove_build_tree(candidate) + print("::endgroup::", flush=True) + + print(f"PASS: all {len(tuples)} protected Yocto tuples are unchanged") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py new file mode 100644 index 00000000..964b059c --- /dev/null +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Tests for immutable layer-adoption baseline evidence.""" + +from __future__ import annotations + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +MODULE_PATH = Path(__file__).parents[1] / "run-layer-adoption-regression.py" +SPEC = importlib.util.spec_from_file_location("run_layer_adoption_regression", MODULE_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class BaselineEvidenceTests(unittest.TestCase): + def test_valid_cache_is_accepted_and_tampering_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "packages.txt").write_text("package-a\n", encoding="utf-8") + marker = { + "base_sha": "abc123", + "tuple_id": "machine-image", + "evidence_sha256": MODULE.evidence_digest(root), + } + (root / MODULE.MARKER).write_text(json.dumps(marker), encoding="utf-8") + self.assertTrue(MODULE.valid_cached_evidence(root, "abc123", "machine-image")) + + (root / "packages.txt").write_text("package-b\n", encoding="utf-8") + self.assertFalse(MODULE.valid_cached_evidence(root, "abc123", "machine-image")) + + def test_marker_is_not_part_of_evidence_digest(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "state.txt").write_text("stable\n", encoding="utf-8") + before = MODULE.evidence_digest(root) + (root / MODULE.MARKER).write_text("{}\n", encoding="utf-8") + self.assertEqual(before, MODULE.evidence_digest(root)) + + +if __name__ == "__main__": + unittest.main() From 28be102167bcebc09e1fad9bb5b55b3d81fc2c17 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:33:22 +0100 Subject: [PATCH 16/47] ci: run container steps with bash --- .github/workflows/layer-adoption-gate.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 4f6e219d..64d5427d 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -15,6 +15,10 @@ concurrency: group: layer-adoption-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash + jobs: detect: name: Detect material layer change From 0ba5fa39114c8c2de97de1f18806f75b61927a30 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:35:36 +0100 Subject: [PATCH 17/47] ci: keep kas overlay inside worktree --- scripts/validation/capture-layer-state.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 4c325d81..41b00d1a 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -52,7 +52,15 @@ export DISTRO="$distro" product_features_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$product_features") downloads_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/downloads") sstate_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/sstate-cache") -overlay="$output_dir/layer-adoption-product-features.yml" +repository_root=$(git rev-parse --show-toplevel) +overlay_dir="$repository_root/.layer-adoption-overlays" +mkdir -p "$overlay_dir" +overlay=$(mktemp "$overlay_dir/product-features.XXXXXX.yml") +cleanup_overlay() { + rm -f "$overlay" + rmdir "$overlay_dir" 2>/dev/null || true +} +trap cleanup_overlay EXIT cat > "$overlay" < Date: Sat, 12 Sep 2026 13:38:44 +0100 Subject: [PATCH 18/47] ci: quote layer state normalisation safely --- scripts/validation/capture-layer-state.sh | 6 +++--- .../tests/test_capture_layer_state.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 scripts/validation/tests/test_capture_layer_state.py diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 41b00d1a..cb873fe8 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -134,13 +134,13 @@ capture_command() { } capture_command show-layers run_bitbake "bitbake-layers show-layers" \ - | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ > "$output_dir/layers.txt" capture_command show-appends run_bitbake "bitbake-layers show-appends" \ - | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ > "$output_dir/appends.txt" capture_command show-recipes run_bitbake "bitbake-layers show-recipes" \ - | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ > "$output_dir/recipes.txt" capture_command graph run_bitbake "bitbake -g $target" diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py new file mode 100644 index 00000000..44d7836b --- /dev/null +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Static regressions for the layer-state capture shell boundary.""" + +from pathlib import Path +import unittest + + +SCRIPT = Path(__file__).parents[1] / "capture-layer-state.sh" + + +class CaptureLayerStateTests(unittest.TestCase): + def test_shell_does_not_expand_sed_end_anchor_as_argument_count(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertNotIn('s#[[:space:]]+$##"', source) + self.assertEqual(source.count("-e 's#[[:space:]]+$##'"), 3) + + +if __name__ == "__main__": + unittest.main() From d58b88aa1474bed542c5013987fe983d0ff51111 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:45:16 +0100 Subject: [PATCH 19/47] ci: replay captured command failures --- scripts/validation/capture-layer-state.sh | 10 ++++- .../tests/test_capture_layer_state.py | 39 ++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index cb873fe8..961fe8dc 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -130,7 +130,15 @@ run_bitbake() { capture_command() { local name=$1 shift - "$@" 2>&1 | tee "$output_dir/$name.log" + local log="$output_dir/$name.log" + if "$@" 2>&1 | tee "$log"; then + return 0 + else + local statuses=("${PIPESTATUS[@]}") + echo "ERROR: command failed while capturing $name" >&2 + cat "$log" >&2 + return "${statuses[0]}" + fi } capture_command show-layers run_bitbake "bitbake-layers show-layers" \ diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 44d7836b..db118930 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 -"""Static regressions for the layer-state capture shell boundary.""" +"""Regressions for the layer-state capture shell boundary.""" from pathlib import Path +import re +import subprocess +import tempfile import unittest @@ -14,6 +17,40 @@ def test_shell_does_not_expand_sed_end_anchor_as_argument_count(self) -> None: self.assertNotIn('s#[[:space:]]+$##"', source) self.assertEqual(source.count("-e 's#[[:space:]]+$##'"), 3) + def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + match = re.search(r"(?ms)^capture_command\(\) \{\n.*?^\}\n", source) + self.assertIsNotNone(match) + + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run( + [ + "bash", + "-c", + match.group(0) + + """ +set -o pipefail +output_dir=$1 +capture_command failure bash -c 'printf "visible diagnostic\\n"; exit 7' \\ + | sed 's/diagnostic/output/' +""", + "capture-command-test", + directory, + ], + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 7) + self.assertEqual(result.stdout, "visible output\n") + self.assertIn("ERROR: command failed while capturing failure", result.stderr) + self.assertIn("visible diagnostic", result.stderr) + self.assertEqual( + (Path(directory) / "failure.log").read_text(encoding="utf-8"), + "visible diagnostic\n", + ) + if __name__ == "__main__": unittest.main() From 49981609161503d69e80c3ac115b8599f87675b9 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:48:57 +0100 Subject: [PATCH 20/47] ci: initialize protected layer submodules --- .github/workflows/layer-adoption-gate.yml | 10 +++++++++- .../tests/test_run_layer_adoption_regression.py | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 64d5427d..6571f490 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -88,7 +88,15 @@ jobs: path: candidate - name: Create baseline worktree working-directory: candidate - run: git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' + run: | + git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' + git -c url.https://github.com/.insteadOf=git@github.com: \ + submodule update --init --recursive \ + meta-dynamicdevices-bsp meta-dynamicdevices-distro + git -C ../baseline \ + -c url.https://github.com/.insteadOf=git@github.com: \ + submodule update --init --recursive \ + meta-dynamicdevices-bsp meta-dynamicdevices-distro - name: Require safe build capacity run: | available_kib=$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 {print $4}') diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 964b059c..762a40d4 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -11,6 +11,7 @@ MODULE_PATH = Path(__file__).parents[1] / "run-layer-adoption-regression.py" +WORKFLOW_PATH = Path(__file__).parents[3] / ".github/workflows/layer-adoption-gate.yml" SPEC = importlib.util.spec_from_file_location("run_layer_adoption_regression", MODULE_PATH) assert SPEC and SPEC.loader MODULE = importlib.util.module_from_spec(SPEC) @@ -18,6 +19,12 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_gate_initializes_product_submodules_in_both_worktrees(self) -> None: + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + self.assertEqual(workflow.count("submodule update --init --recursive"), 2) + self.assertEqual(workflow.count("meta-dynamicdevices-bsp meta-dynamicdevices-distro"), 2) + self.assertIn("git -C ../baseline", workflow) + def test_valid_cache_is_accepted_and_tampering_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 4e07c3b4d44b51c3dbbb28228c5b85b0a27fdf5a Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:52:04 +0100 Subject: [PATCH 21/47] ci: initialize protected layer worktrees --- .github/workflows/layer-adoption-gate.yml | 4 +++- .../validation/tests/test_run_layer_adoption_regression.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 6571f490..2f337eee 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -68,7 +68,9 @@ jobs: runs-on: [self-hosted, Linux, X64, yocto, ai-tools] container: image: ghcr.io/siemens/kas/kas@sha256:d989add57fc441fe9e27bb2dd6ed98c5597b44c807928e35a72dc1cfbdda9abe - options: --privileged --platform linux/amd64 --user 0:0 -v /home/ghrunner/yocto-layer-adoption:/var/cache/layer-adoption + # Match the dedicated ai-tools runner account so BitBake's root-user + # sanity check remains active and bind-mounted cache files stay writable. + options: --privileged --platform linux/amd64 --user 1002:1002 -v /home/ghrunner/yocto-layer-adoption:/var/cache/layer-adoption env: LAYER_ADOPTION_CACHE: /var/cache/layer-adoption steps: diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 762a40d4..b38de2b4 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -25,6 +25,11 @@ def test_gate_initializes_product_submodules_in_both_worktrees(self) -> None: self.assertEqual(workflow.count("meta-dynamicdevices-bsp meta-dynamicdevices-distro"), 2) self.assertIn("git -C ../baseline", workflow) + def test_gate_does_not_run_bitbake_as_root(self) -> None: + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + self.assertNotIn("--user 0:0", workflow) + self.assertIn("--user 1002:1002", workflow) + def test_valid_cache_is_accepted_and_tampering_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From aba0a4d1953e60cfa579ed3e6ac4fce8fb7f595f Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:56:07 +0100 Subject: [PATCH 22/47] ci: share kas worktree preparation --- .github/workflows/layer-adoption-gate.yml | 10 +--- .../run-layer-adoption-regression.py | 47 +++++++++++++++++ .../test_run_layer_adoption_regression.py | 50 +++++++++++++++++-- 3 files changed, 94 insertions(+), 13 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 2f337eee..42ac9d43 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -90,15 +90,7 @@ jobs: path: candidate - name: Create baseline worktree working-directory: candidate - run: | - git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' - git -c url.https://github.com/.insteadOf=git@github.com: \ - submodule update --init --recursive \ - meta-dynamicdevices-bsp meta-dynamicdevices-distro - git -C ../baseline \ - -c url.https://github.com/.insteadOf=git@github.com: \ - submodule update --init --recursive \ - meta-dynamicdevices-bsp meta-dynamicdevices-distro + run: git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' - name: Require safe build capacity run: | available_kib=$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 {print $4}') diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 9a4f7fc4..7521bd9a 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -15,6 +15,7 @@ MARKER = ".complete.json" FIELDS = ("id", "machine", "distro", "image", "config", "product_features") +PRODUCT_SUBMODULES = ("meta-dynamicdevices-bsp", "meta-dynamicdevices-distro") def evidence_digest(root: Path) -> str: @@ -68,6 +69,46 @@ def remove_build_tree(repository: Path) -> None: shutil.rmtree(build, ignore_errors=True) +def git_output(repository: Path, *args: str) -> str: + return subprocess.check_output(["git", *args], cwd=repository, text=True).strip() + + +def prepare_repository(repository: Path) -> None: + """Initialize and verify the pinned local layers used by every KAS tuple.""" + for relative in PRODUCT_SUBMODULES: + entry = git_output(repository, "ls-tree", "HEAD", "--", relative).split() + if len(entry) < 3 or entry[0] != "160000" or entry[1] != "commit": + raise RuntimeError(f"{repository}: {relative} is not a pinned git submodule") + expected = entry[2] + layer = repository / relative + layer_conf = layer / "conf/layer.conf" + if not layer_conf.is_file(): + subprocess.run( + [ + "git", + "-c", + "url.https://github.com/.insteadOf=git@github.com:", + "submodule", + "update", + "--init", + "--recursive", + "--", + relative, + ], + cwd=repository, + check=True, + ) + if not layer_conf.is_file(): + raise RuntimeError(f"{repository}: {relative}/conf/layer.conf is missing") + actual = git_output(layer, "rev-parse", "HEAD") + if actual != expected: + raise RuntimeError( + f"{repository}: {relative} is at {actual}, expected pinned {expected}" + ) + if git_output(layer, "status", "--porcelain", "--untracked-files=all"): + raise RuntimeError(f"{repository}: {relative} has uncommitted content") + + def capture( script: Path, repository: Path, @@ -113,6 +154,12 @@ def main() -> int: ["git", "rev-parse", "HEAD"], cwd=baseline, text=True ).strip() + # This preparation is intentionally owned by the shared regression driver, + # not by CI YAML. Local and hosted runs therefore build the same pinned + # submodule content through the same KAS capture path. + prepare_repository(baseline) + prepare_repository(candidate) + environment = os.environ.copy() environment["LAYER_ADOPTION_TEST_KEYS_DIR"] = str(test_keys) environment["LAYER_ADOPTION_CACHE_DIR"] = str(cache / "yocto") diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index b38de2b4..1e80ffda 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -8,6 +8,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock MODULE_PATH = Path(__file__).parents[1] / "run-layer-adoption-regression.py" @@ -19,11 +20,52 @@ class BaselineEvidenceTests(unittest.TestCase): - def test_gate_initializes_product_submodules_in_both_worktrees(self) -> None: + def test_worktree_preparation_is_owned_by_shared_driver(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") - self.assertEqual(workflow.count("submodule update --init --recursive"), 2) - self.assertEqual(workflow.count("meta-dynamicdevices-bsp meta-dynamicdevices-distro"), 2) - self.assertIn("git -C ../baseline", workflow) + self.assertNotIn("submodule update", workflow) + + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + pinned = "a" * 40 + + def git_result(command: list[str], **kwargs: object) -> str: + if "ls-tree" in command: + relative = command[-1] + return f"160000 commit {pinned}\t{relative}\n" + if "rev-parse" in command: + return pinned + "\n" + if "status" in command: + return "" + self.fail(f"unexpected git command: {command}") + + def initialize(command: list[str], **kwargs: object) -> None: + relative = command[-1] + layer_conf = repository / relative / "conf/layer.conf" + layer_conf.parent.mkdir(parents=True) + layer_conf.touch() + + with mock.patch.object( + MODULE.subprocess, "check_output", side_effect=git_result + ), mock.patch.object(MODULE.subprocess, "run", side_effect=initialize) as run: + MODULE.prepare_repository(repository) + + self.assertEqual(run.call_count, 2) + + def test_worktree_preparation_rejects_unpinned_layer(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + layer = repository / MODULE.PRODUCT_SUBMODULES[0] + (layer / "conf").mkdir(parents=True) + (layer / "conf/layer.conf").touch() + results = [ + f"160000 commit {'a' * 40}\t{layer.name}\n", + "b" * 40 + "\n", + ] + with mock.patch.object( + MODULE.subprocess, "check_output", side_effect=results + ): + with self.assertRaisesRegex(RuntimeError, "expected pinned"): + MODULE.prepare_repository(repository) def test_gate_does_not_run_bitbake_as_root(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") From c423642ffc2be96ea4a249e7f9510111aa2ae6b7 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 14:01:07 +0100 Subject: [PATCH 23/47] ci: track current bitbake dependency graph --- scripts/validation/capture-layer-state.sh | 5 +++-- scripts/validation/tests/test_capture_layer_state.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 961fe8dc..cf6cfd08 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -155,8 +155,9 @@ capture_command graph run_bitbake "bitbake -g $target" sort -u build/pn-buildlist > "$output_dir/pn-buildlist.txt" sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ > "$output_dir/task-depends.dot" -sed -E "s#$PWD/##g" build/recipe-depends.dot | sort -u \ - > "$output_dir/recipe-depends.dot" +# Current BitBake deliberately removes the obsolete recipe-depends.dot output. +# pn-buildlist plus the finer-grained task graph retain provider and dependency +# selection coverage without relying on that removed compatibility artifact. # Capture final values and BitBake's assignment provenance for policy that a # newly enabled layer can silently change. The full environment is retained diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index db118930..10def427 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -51,6 +51,12 @@ def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: "visible diagnostic\n", ) + def test_dependency_capture_uses_current_bitbake_graph_outputs(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn("build/pn-buildlist", source) + self.assertIn("build/task-depends.dot", source) + self.assertNotIn("build/recipe-depends.dot", source) + if __name__ == "__main__": unittest.main() From 67e9f6d6d94bdf4c11e277221a5ab8d1c46e0c8e Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 14:07:17 +0100 Subject: [PATCH 24/47] ci: diagnose selected environment mismatches --- scripts/validation/capture-layer-state.sh | 21 +++++++++++++++---- .../tests/test_capture_layer_state.py | 7 +++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index cf6cfd08..9d97d2bc 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -162,14 +162,27 @@ sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ # Capture final values and BitBake's assignment provenance for policy that a # newly enabled layer can silently change. The full environment is retained # temporarily only as input, avoiding volatile host variables in comparisons. -capture_command environment run_bitbake "bitbake -e $target" +# The complete environment is intentionally retained in evidence but is too +# large for routine CI output. capture_command still replays it to stderr if +# BitBake fails, so diagnostics are not lost. +capture_command environment run_bitbake "bitbake -e $target" >/dev/null python3 "$(dirname "$0")/select-bitbake-env.py" \ "$output_dir/environment.log" \ | sed -E "s#$PWD##g; s#$test_keys_dir##g; s#$cache_root##g" \ > "$output_dir/selected-environment.txt" -grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" -grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" -grep -Fqx "DD_PRODUCT_FEATURES=\"$product_features\"" "$output_dir/selected-environment.txt" +require_selected_value() { + local name=$1 + local expected=$2 + if ! grep -Fqx "$name=\"$expected\"" "$output_dir/selected-environment.txt"; then + echo "ERROR: selected BitBake environment does not contain $name=\"$expected\"" >&2 + grep -E "^${name}=" "$output_dir/selected-environment.txt" >&2 || \ + echo "ERROR: $name is absent from selected BitBake environment" >&2 + return 1 + fi +} +require_selected_value MACHINE "$machine" +require_selected_value DISTRO "$distro" +require_selected_value DD_PRODUCT_FEATURES "$product_features" rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 10def427..d302f345 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -57,6 +57,13 @@ def test_dependency_capture_uses_current_bitbake_graph_outputs(self) -> None: self.assertIn("build/task-depends.dot", source) self.assertNotIn("build/recipe-depends.dot", source) + def test_environment_assertions_emit_named_diagnostics(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn('capture_command environment run_bitbake "bitbake -e $target" >/dev/null', source) + self.assertIn("require_selected_value() {", source) + self.assertEqual(source.count("require_selected_value "), 3) + self.assertIn("ERROR: selected BitBake environment does not contain", source) + if __name__ == "__main__": unittest.main() From 62e7455c56cbd3699e70f8fac967d098191045d2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 14:13:00 +0100 Subject: [PATCH 25/47] ci: use inode-aware disk thresholds --- scripts/validation/capture-layer-state.sh | 2 +- scripts/validation/tests/test_capture_layer_state.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 9d97d2bc..6c1ae8aa 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -69,7 +69,7 @@ local_conf_header: DD_PRODUCT_FEATURES = $product_features_quoted DL_DIR = $downloads_quoted SSTATE_DIR = $sstate_quoted - BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,1G STOPTASKS,\${DL_DIR},20G,1G STOPTASKS,\${SSTATE_DIR},20G,1G HALT,\${TMPDIR},10G,1G HALT,\${DL_DIR},10G,1G HALT,\${SSTATE_DIR},10G,1G" + BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,100K STOPTASKS,\${DL_DIR},20G,100K STOPTASKS,\${SSTATE_DIR},20G,100K HALT,\${TMPDIR},10G,50K HALT,\${DL_DIR},10G,50K HALT,\${SSTATE_DIR},10G,50K" UBOOT_SIGN_KEYDIR = "$test_keys_dir" UEFI_SIGN_KEYDIR = "$test_keys_dir/uefi" MODSIGN_KEY_DIR = "$test_keys_dir" diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index d302f345..03e002d0 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -64,6 +64,15 @@ def test_environment_assertions_emit_named_diagnostics(self) -> None: self.assertEqual(source.count("require_selected_value "), 3) self.assertIn("ERROR: selected BitBake environment does not contain", source) + def test_disk_monitor_uses_inode_not_disk_units(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + disk_monitor = next( + line for line in source.splitlines() if "BB_DISKMON_DIRS =" in line + ) + self.assertNotIn(",1G", disk_monitor) + self.assertEqual(disk_monitor.count(",100K"), 3) + self.assertEqual(disk_monitor.count(",50K"), 3) + if __name__ == "__main__": unittest.main() From f7bdd2067ae282081584e40542705effb4257fde Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 14:40:02 +0100 Subject: [PATCH 26/47] ci: bind local kas changes to adoption gate --- .gitattributes | 2 + scripts/README.md | 14 +++++ scripts/kas-build-base-enhanced.sh | 2 + scripts/kas-build-base.sh | 4 ++ scripts/kas-build-mfgtools.sh | 63 ++++++++++++++++++- scripts/kas-build-profiling.sh | 2 + scripts/kas-container-image.sh | 6 ++ scripts/kas-dev-boot.sh | 2 + scripts/kas-dev-kernel.sh | 2 + scripts/kas-dev-recipe.sh | 2 + scripts/kas-shell-base.sh | 4 ++ scripts/validation/detect-layer-adoption.py | 21 +++++-- .../tests/test_detect_layer_adoption.py | 25 ++++++++ .../test_run_layer_adoption_regression.py | 21 +++++++ 14 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 scripts/kas-container-image.sh diff --git a/.gitattributes b/.gitattributes index 2288ba47..8289a470 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,4 +5,6 @@ *.bin filter=lfs diff=lfs merge=lfs -text *.itb filter=lfs diff=lfs merge=lfs -text *mfgtool* filter=lfs diff=lfs merge=lfs -text +# Build entry points are source, even when their names contain "mfgtool". +scripts/kas-build-mfgtools.sh -filter -diff -merge text eol=lf fitImage-* filter=lfs diff=lfs merge=lfs -text diff --git a/scripts/README.md b/scripts/README.md index 2818734b..ed87194e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -159,6 +159,20 @@ custom-boot-files/ # Custom boot files directory ## 🏗️ Build & Development +### Local/CI KAS parity + +Use the `scripts/kas-*.sh` entry points for local KAS work. They source +`scripts/kas-container-image.sh`, which pins the same container digest as the +Layer Adoption Gate. The gate treats changes to these wrappers, KAS YAML, +pinned layer submodules, its workflow, contract, and regression implementation +as material. Such changes must update `ci/layer-adoption-contract.json` and run +every immutable tuple in `ci/layer-adoption-tuples.json`; the tuple matrix may +be extended but existing coverage cannot be removed or redefined. + +The shared `scripts/validation/run-layer-adoption-regression.py` driver owns +repository preparation and all baseline/candidate captures in both local and +CI use. Do not duplicate KAS preparation steps in workflow YAML. + ### `kas-build-base.sh` **Purpose:** Builds the base LmP (Linux microPlatform) image using KAS configuration. diff --git a/scripts/kas-build-base-enhanced.sh b/scripts/kas-build-base-enhanced.sh index 211d1211..14cb1d7d 100755 --- a/scripts/kas-build-base-enhanced.sh +++ b/scripts/kas-build-base-enhanced.sh @@ -9,6 +9,8 @@ set -euo pipefail # Exit on error, undefined vars, pipe failures # Script configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" DEFAULT_CACHE_DIR="${HOME}/yocto" LOG_FILE="${PROJECT_ROOT}/logs/kas-build-$(date +%Y%m%d-%H%M%S).log" diff --git a/scripts/kas-build-base.sh b/scripts/kas-build-base.sh index 2c7bf21f..1b32f40e 100755 --- a/scripts/kas-build-base.sh +++ b/scripts/kas-build-base.sh @@ -1,5 +1,9 @@ #!/bin/sh +script_dir=$(CDPATH='' cd -P "$(dirname "$0")" && pwd) +# shellcheck source=kas-container-image.sh +. "$script_dir/kas-container-image.sh" + # TODO: Look at this to fix missing key issue # #conf/machine/include/lmp-factory-custom.inc:OPTEE_TA_SIGN_KEY = "${TOPDIR}/conf/factory-keys/opteedev.key" diff --git a/scripts/kas-build-mfgtools.sh b/scripts/kas-build-mfgtools.sh index 09136b4d..cfbed8bb 100755 --- a/scripts/kas-build-mfgtools.sh +++ b/scripts/kas-build-mfgtools.sh @@ -1,3 +1,60 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:180de9e62039ccce8fd0ae8ffe96091f6b4f3fdac562054fad8101055000e998 -size 1435 +#!/bin/sh + +script_dir=$(CDPATH='' cd -P "$(dirname "$0")" && pwd) +# shellcheck source=kas-container-image.sh +. "$script_dir/kas-container-image.sh" + +# Build mfgtool images for supported i.MX machines +# +# Usage: +# KAS_MACHINE=imx95-frdm-evk ./scripts/kas-build-mfgtools.sh +# KAS_MACHINE=imx93-jaguar-eink ./scripts/kas-build-mfgtools.sh +# KAS_MACHINE=imx93-11x11-lpddr4x-evk ./scripts/kas-build-mfgtools.sh +# +# The machine-specific BSP selects the correct imx-boot manufacturing target. + +# Set default machine if not specified +if [ -z "$KAS_MACHINE" ]; then + export KAS_MACHINE="imx93-11x11-lpddr4x-evk" + echo "No KAS_MACHINE specified, defaulting to $KAS_MACHINE" +else + echo "Building mfgtool for machine: $KAS_MACHINE" +fi + +KAS_CONFIG="kas/lmp-dynamicdevices-mfgtool.yml" +if [ "$KAS_MACHINE" = "imx95-frdm-evk" ]; then + # Keep local validation aligned with the Foundries manifest revisions used + # for FRDM-i.MX95 production builds. + KAS_CONFIG="kas/lmp-imx95-frdm-evk-mfgtool.yml" +fi + +# TODO: Look at this to fix missing key issue if needed +# +#conf/machine/include/lmp-factory-custom.inc:OPTEE_TA_SIGN_KEY = "${TOPDIR}/conf/factory-keys/opteedev.key" +#lmp-tools/scripts/rotate_ci_keys.sh:openssl genpkey -algorithm RSA -out factory-keys/opteedev.key \ +#lmp-tools/scripts/rotate_ci_keys.sh:openssl req -batch -new -x509 -key factory-keys/opteedev.key -out factory-keys/opteedev.crt + +if [ ! -d ~/yocto ] +then + mkdir -p ~/yocto + mkdir -p ~/yocto/downloads + mkdir -p ~/yocto/persistent + mkdir -p ~/yocto/sstate + chmod 755 ~/yocto + chmod 755 ~/yocto/downloads + chmod 755 ~/yocto/persistent + chmod 755 ~/yocto/sstate +fi + +# Pass KAS_MACHINE to kas-container to override the machine in the config file. +# Forward SSH credentials only when they exist; the standard layer URLs are HTTPS, +# so unattended builders such as ai-tools do not require an SSH agent. +set -- --runtime-args "-v ${HOME}/yocto:/var/cache -e KAS_MACHINE=$KAS_MACHINE" +if [ -n "${SSH_AUTH_SOCK:-}" ] && [ -S "${SSH_AUTH_SOCK}" ]; then + set -- --ssh-agent "$@" +fi +if [ -d "${HOME}/.ssh" ]; then + set -- --ssh-dir "${HOME}/.ssh" "$@" +fi + +kas-container "$@" build "$KAS_CONFIG" diff --git a/scripts/kas-build-profiling.sh b/scripts/kas-build-profiling.sh index 48cece16..35344070 100755 --- a/scripts/kas-build-profiling.sh +++ b/scripts/kas-build-profiling.sh @@ -7,6 +7,8 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" # Default values DEFAULT_MACHINE="imx93-jaguar-eink" diff --git a/scripts/kas-container-image.sh b/scripts/kas-container-image.sh new file mode 100644 index 00000000..e23e578a --- /dev/null +++ b/scripts/kas-container-image.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# Keep local KAS wrappers on the exact container image exercised by the +# layer-adoption gate. Update the workflow and its parity test with this pin. +KAS_CONTAINER_IMAGE="ghcr.io/siemens/kas/kas@sha256:d989add57fc441fe9e27bb2dd6ed98c5597b44c807928e35a72dc1cfbdda9abe" +export KAS_CONTAINER_IMAGE diff --git a/scripts/kas-dev-boot.sh b/scripts/kas-dev-boot.sh index 89ea7b77..a50d2c7d 100755 --- a/scripts/kas-dev-boot.sh +++ b/scripts/kas-dev-boot.sh @@ -10,6 +10,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" # Default values ACTION="" diff --git a/scripts/kas-dev-kernel.sh b/scripts/kas-dev-kernel.sh index 9c52e2b9..1ac2aa6f 100755 --- a/scripts/kas-dev-kernel.sh +++ b/scripts/kas-dev-kernel.sh @@ -10,6 +10,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" # Default values ACTION="" diff --git a/scripts/kas-dev-recipe.sh b/scripts/kas-dev-recipe.sh index c27548ee..3ef0b451 100755 --- a/scripts/kas-dev-recipe.sh +++ b/scripts/kas-dev-recipe.sh @@ -10,6 +10,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" # Default values ACTION="" diff --git a/scripts/kas-shell-base.sh b/scripts/kas-shell-base.sh index 56ff4178..b6be08a6 100755 --- a/scripts/kas-shell-base.sh +++ b/scripts/kas-shell-base.sh @@ -1,5 +1,9 @@ #!/bin/sh +script_dir=$(CDPATH='' cd -P "$(dirname "$0")" && pwd) +# shellcheck source=kas-container-image.sh +. "$script_dir/kas-container-image.sh" + # KAS Shell Base Script # Usage: ./kas-shell-base.sh [options] # -c "command" : Execute command in kas environment diff --git a/scripts/validation/detect-layer-adoption.py b/scripts/validation/detect-layer-adoption.py index b0911206..a4e34cd2 100755 --- a/scripts/validation/detect-layer-adoption.py +++ b/scripts/validation/detect-layer-adoption.py @@ -13,9 +13,20 @@ MATERIAL_PATHS = ( + re.compile(r"^\.github/workflows/layer-adoption-gate\.yml$"), + re.compile(r"^\.gitattributes$"), re.compile(r"^\.gitmodules$"), + re.compile(r"^ci/layer-adoption-contract\.json$"), re.compile(r"(^|/)conf/layer\.conf$"), re.compile(r"^kas/.*\.ya?ml$"), + re.compile(r"^meta-dynamicdevices-(?:bsp|distro)$"), + re.compile(r"^meta-partner-nxp-imx$"), + re.compile(r"^scripts/kas-.*\.sh$"), + re.compile( + r"^scripts/validation/(?:capture-layer-state\.sh|compare-layer-state\.py|" + r"detect-layer-adoption\.py|generate-layer-adoption-test-keys\.sh|" + r"run-layer-adoption-regression\.py)$" + ), re.compile(r"^ci/layer-adoption-tuples\.json$"), ) @@ -23,6 +34,11 @@ NONEMPTY_TUPLE_FIELDS = ("id", "machine", "distro", "image", "config") +def is_material_path(path: str) -> bool: + """Return whether a change can alter local or CI KAS build semantics.""" + return any(pattern.search(path) for pattern in MATERIAL_PATHS) + + def git(*args: str) -> str: return subprocess.check_output(["git", *args], text=True) @@ -110,10 +126,7 @@ def main() -> int: return 2 changed = git("diff", "--name-only", f"{args.base}...{args.head}").splitlines() - material_files = [ - path for path in changed - if any(pattern.search(path) for pattern in MATERIAL_PATHS) - ] + material_files = [path for path in changed if is_material_path(path)] # Treat every KAS change as material. YAML context makes line-only pin # detection easy to evade (for example by adding a list item below an # existing `includes:` key), and a false-positive build is safer than a diff --git a/scripts/validation/tests/test_detect_layer_adoption.py b/scripts/validation/tests/test_detect_layer_adoption.py index 211741e7..a1aecf21 100644 --- a/scripts/validation/tests/test_detect_layer_adoption.py +++ b/scripts/validation/tests/test_detect_layer_adoption.py @@ -35,6 +35,31 @@ def entry(tuple_id: str, machine: str = "machine-a") -> dict[str, str]: class ProtectedTupleTests(unittest.TestCase): + def test_local_and_ci_kas_process_changes_are_material(self) -> None: + paths = ( + ".github/workflows/layer-adoption-gate.yml", + ".gitattributes", + "ci/layer-adoption-contract.json", + "ci/layer-adoption-tuples.json", + "kas/lmp-dynamicdevices.yml", + "meta-dynamicdevices-bsp", + "meta-dynamicdevices-distro", + "meta-partner-nxp-imx", + "scripts/kas-build-base.sh", + "scripts/kas-shell-base.sh", + "scripts/validation/capture-layer-state.sh", + "scripts/validation/compare-layer-state.py", + "scripts/validation/detect-layer-adoption.py", + "scripts/validation/generate-layer-adoption-test-keys.sh", + "scripts/validation/run-layer-adoption-regression.py", + ) + for path in paths: + with self.subTest(path=path): + self.assertTrue(MODULE.is_material_path(path)) + + def test_unrelated_utility_change_is_not_material(self) -> None: + self.assertFalse(MODULE.is_material_path("scripts/analyze-boot-logs.sh")) + def validate(self, baseline: str, candidate: str) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 1e80ffda..1decdc95 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -5,6 +5,7 @@ import importlib.util import json +import re import tempfile import unittest from pathlib import Path @@ -20,6 +21,26 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_local_kas_wrappers_use_ci_container_digest(self) -> None: + root = WORKFLOW_PATH.parents[2] + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + helper_name = "kas-container-image.sh" + helper = (root / "scripts" / helper_name).read_text(encoding="utf-8") + local_pin = re.search(r'^KAS_CONTAINER_IMAGE="([^"]+)"$', helper, re.MULTILINE) + ci_pin = re.search(r"^\s+image: (\S+)$", workflow, re.MULTILINE) + self.assertIsNotNone(local_pin) + self.assertIsNotNone(ci_pin) + self.assertEqual(local_pin.group(1), ci_pin.group(1)) + + wrappers = [ + path for path in (root / "scripts").glob("kas-*.sh") + if path.name != helper_name and "kas-container" in path.read_text(encoding="utf-8") + ] + self.assertTrue(wrappers) + for wrapper in wrappers: + with self.subTest(wrapper=wrapper.name): + self.assertIn(helper_name, wrapper.read_text(encoding="utf-8")) + def test_worktree_preparation_is_owned_by_shared_driver(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") self.assertNotIn("submodule update", workflow) From b5380f59cbd0e67732d5e11fd37d75e8194f8d16 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 18:12:53 +0100 Subject: [PATCH 27/47] ci: validate kernel signing identity --- .github/workflows/layer-adoption-gate.yml | 3 +- scripts/validation/capture-layer-state.sh | 4 ++ .../generate-layer-adoption-test-keys.sh | 55 ++++++++++++++++--- .../tests/test_capture_layer_state.py | 15 ++++- .../test_run_layer_adoption_regression.py | 19 +++++++ 5 files changed, 87 insertions(+), 9 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 42ac9d43..cc5638b5 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -103,7 +103,8 @@ jobs: run: | set -euo pipefail keys="$LAYER_ADOPTION_CACHE/test-keys" - if [ ! -s "$keys/ubootdev.key" ]; then + if ! candidate/scripts/validation/generate-layer-adoption-test-keys.sh \ + --check "$keys"; then test "$keys" = /var/cache/layer-adoption/test-keys rm -rf -- "$keys" rm -rf -- "$LAYER_ADOPTION_CACHE/baselines" diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 6c1ae8aa..78ce310f 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -73,6 +73,8 @@ local_conf_header: UBOOT_SIGN_KEYDIR = "$test_keys_dir" UEFI_SIGN_KEYDIR = "$test_keys_dir/uefi" MODSIGN_KEY_DIR = "$test_keys_dir" + MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem" + MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt" SIGNING_UBOOT_SIGN_KEY = "$test_keys_dir/ubootdev.key" SIGNING_UBOOT_SIGN_CRT = "$test_keys_dir/ubootdev.crt" SIGNING_UBOOT_SPL_SIGN_KEY = "$test_keys_dir/spldev.key" @@ -183,6 +185,8 @@ require_selected_value() { require_selected_value MACHINE "$machine" require_selected_value DISTRO "$distro" require_selected_value DD_PRODUCT_FEATURES "$product_features" +require_selected_value MODSIGN_PRIVKEY "/privkey_modsign.pem" +require_selected_value MODSIGN_X509 "/x509_modsign.crt" rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, diff --git a/scripts/validation/generate-layer-adoption-test-keys.sh b/scripts/validation/generate-layer-adoption-test-keys.sh index 6c0f0834..77a38a3e 100755 --- a/scripts/validation/generate-layer-adoption-test-keys.sh +++ b/scripts/validation/generate-layer-adoption-test-keys.sh @@ -1,25 +1,66 @@ #!/usr/bin/env bash -# Generate one genuine, test-only signing identity shared by compared builds. +# Generate or validate one genuine, test-only signing identity shared by builds. set -euo pipefail umask 077 -if [ "$#" -ne 1 ]; then - echo "Usage: $0 OUTPUT_DIR" >&2 +if [ "$#" -ne 1 ] && { [ "$#" -ne 2 ] || [ "$1" != "--check" ]; }; then + echo "Usage: $0 [--check] OUTPUT_DIR" >&2 exit 2 fi -output_dir=$(realpath -m "$1") +if [ "$#" -eq 2 ]; then + check_only=true + output_dir=$(realpath -m "$2") +else + check_only=false + output_dir=$(realpath -m "$1") +fi case "$output_dir" in /|/home|/root|/tmp|/var|/usr) echo "ERROR: refusing broad test-key output directory: $output_dir" >&2 exit 2 ;; esac -if [ -e "$output_dir" ] && find "$output_dir" -mindepth 1 -print -quit | grep -q .; then +if [ "$check_only" = false ] && [ -e "$output_dir" ] && \ + find "$output_dir" -mindepth 1 -print -quit | grep -q . +then echo "ERROR: test-key output directory is not empty: $output_dir" >&2 exit 2 fi +validate_pair() { + local key=$1 + local certificate=$2 + local key_fingerprint certificate_fingerprint + openssl pkey -in "$key" -check -noout >/dev/null 2>&1 + # Never let a persistent CI identity expire during a long gate run. + openssl x509 -in "$certificate" -noout -checkend 604800 >/dev/null 2>&1 + key_fingerprint=$(openssl pkey -in "$key" -pubout -outform DER 2>/dev/null \ + | sha256sum | cut -d ' ' -f 1) + certificate_fingerprint=$(openssl x509 -in "$certificate" -pubkey -noout 2>/dev/null \ + | openssl pkey -pubin -outform DER 2>/dev/null \ + | sha256sum | cut -d ' ' -f 1) + [ "$key_fingerprint" = "$certificate_fingerprint" ] +} + +validate_keyset() { + validate_pair "$output_dir/ubootdev.key" "$output_dir/ubootdev.crt" && + validate_pair "$output_dir/spldev.key" "$output_dir/spldev.crt" && + validate_pair "$output_dir/privkey_modsign.pem" "$output_dir/x509_modsign.crt" && + validate_pair "$output_dir/uefi/DB.key" "$output_dir/uefi/DB.crt" && + openssl ec -in "$output_dir/tf-a/privkey_ec_prime256v1.pem" \ + -check -noout >/dev/null 2>&1 +} + +if [ "$check_only" = true ]; then + if validate_keyset; then + printf 'PASS: validated test-only layer-adoption signing keys in %s\n' "$output_dir" + exit 0 + fi + echo "ERROR: test-only layer-adoption signing keys are invalid or expire within seven days" >&2 + exit 1 +fi + mkdir -p "$output_dir/uefi" "$output_dir/tf-a" make_rsa_certificate() { @@ -27,7 +68,7 @@ make_rsa_certificate() { local certificate=$2 local common_name=$3 openssl genpkey -algorithm RSA -out "$key" -pkeyopt rsa_keygen_bits:2048 - openssl req -batch -new -x509 -sha256 -days 2 \ + openssl req -batch -new -x509 -sha256 -days 3650 \ -key "$key" -out "$certificate" -subj "/CN=$common_name/" openssl pkey -in "$key" -check -noout openssl x509 -in "$certificate" -noout @@ -48,6 +89,6 @@ make_rsa_certificate \ openssl ecparam -name prime256v1 -genkey -noout \ -out "$output_dir/tf-a/privkey_ec_prime256v1.pem" -openssl ec -in "$output_dir/tf-a/privkey_ec_prime256v1.pem" -check -noout +validate_keyset printf 'PASS: generated test-only layer-adoption signing keys in %s\n' "$output_dir" diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 03e002d0..1213f209 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -61,9 +61,22 @@ def test_environment_assertions_emit_named_diagnostics(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertIn('capture_command environment run_bitbake "bitbake -e $target" >/dev/null', source) self.assertIn("require_selected_value() {", source) - self.assertEqual(source.count("require_selected_value "), 3) + self.assertEqual(source.count("require_selected_value "), 5) self.assertIn("ERROR: selected BitBake environment does not contain", source) + def test_module_signing_uses_kernel_runtime_variables(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn('MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem"', source) + self.assertIn('MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt"', source) + self.assertIn( + 'require_selected_value MODSIGN_PRIVKEY "/privkey_modsign.pem"', + source, + ) + self.assertIn( + 'require_selected_value MODSIGN_X509 "/x509_modsign.crt"', + source, + ) + def test_disk_monitor_uses_inode_not_disk_units(self) -> None: source = SCRIPT.read_text(encoding="utf-8") disk_monitor = next( diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 1decdc95..6e007369 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -6,6 +6,7 @@ import importlib.util import json import re +import subprocess import tempfile import unittest from pathlib import Path @@ -21,6 +22,24 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_generated_signing_identity_is_validated_before_reuse(self) -> None: + generator = MODULE_PATH.parent / "generate-layer-adoption-test-keys.sh" + with tempfile.TemporaryDirectory() as directory: + keys = Path(directory) / "keys" + subprocess.run([str(generator), str(keys)], check=True, capture_output=True) + subprocess.run( + [str(generator), "--check", str(keys)], check=True, capture_output=True + ) + (keys / "x509_modsign.crt").write_text("invalid\n", encoding="utf-8") + invalid = subprocess.run( + [str(generator), "--check", str(keys)], + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(invalid.returncode, 0) + self.assertIn("invalid or expire within seven days", invalid.stderr) + def test_local_kas_wrappers_use_ci_container_digest(self) -> None: root = WORKFLOW_PATH.parents[2] workflow = WORKFLOW_PATH.read_text(encoding="utf-8") From b08700e580211d53c1d48ec7a19b208cc6dc3408 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 19:35:53 +0100 Subject: [PATCH 28/47] ci: force test signing paths --- scripts/validation/capture-layer-state.sh | 36 +++++++++++-------- .../tests/test_capture_layer_state.py | 32 +++++++++++++++-- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 78ce310f..151b72f8 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -70,21 +70,22 @@ local_conf_header: DL_DIR = $downloads_quoted SSTATE_DIR = $sstate_quoted BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,100K STOPTASKS,\${DL_DIR},20G,100K STOPTASKS,\${SSTATE_DIR},20G,100K HALT,\${TMPDIR},10G,50K HALT,\${DL_DIR},10G,50K HALT,\${SSTATE_DIR},10G,50K" - UBOOT_SIGN_KEYDIR = "$test_keys_dir" - UEFI_SIGN_KEYDIR = "$test_keys_dir/uefi" - MODSIGN_KEY_DIR = "$test_keys_dir" - MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem" - MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt" - SIGNING_UBOOT_SIGN_KEY = "$test_keys_dir/ubootdev.key" - SIGNING_UBOOT_SIGN_CRT = "$test_keys_dir/ubootdev.crt" - SIGNING_UBOOT_SPL_SIGN_KEY = "$test_keys_dir/spldev.key" - SIGNING_UBOOT_SPL_SIGN_CRT = "$test_keys_dir/spldev.crt" - SIGNING_MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem" - SIGNING_MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt" - SIGNING_UEFI_SIGN_KEY = "$test_keys_dir/uefi/DB.key" - SIGNING_UEFI_SIGN_CRT = "$test_keys_dir/uefi/DB.crt" - OPTEE_TA_SIGN_KEY = "$test_keys_dir/ubootdev.key" - TF_A_SIGN_KEY_PATH = "$test_keys_dir/tf-a/privkey_ec_prime256v1.pem" + UBOOT_SIGN_KEYDIR:forcevariable = "$test_keys_dir" + UBOOT_SPL_SIGN_KEYDIR:forcevariable = "$test_keys_dir" + UEFI_SIGN_KEYDIR:forcevariable = "$test_keys_dir/uefi" + MODSIGN_KEY_DIR:forcevariable = "$test_keys_dir" + MODSIGN_PRIVKEY:forcevariable = "$test_keys_dir/privkey_modsign.pem" + MODSIGN_X509:forcevariable = "$test_keys_dir/x509_modsign.crt" + SIGNING_UBOOT_SIGN_KEY:forcevariable = "$test_keys_dir/ubootdev.key" + SIGNING_UBOOT_SIGN_CRT:forcevariable = "$test_keys_dir/ubootdev.crt" + SIGNING_UBOOT_SPL_SIGN_KEY:forcevariable = "$test_keys_dir/spldev.key" + SIGNING_UBOOT_SPL_SIGN_CRT:forcevariable = "$test_keys_dir/spldev.crt" + SIGNING_MODSIGN_PRIVKEY:forcevariable = "$test_keys_dir/privkey_modsign.pem" + SIGNING_MODSIGN_X509:forcevariable = "$test_keys_dir/x509_modsign.crt" + SIGNING_UEFI_SIGN_KEY:forcevariable = "$test_keys_dir/uefi/DB.key" + SIGNING_UEFI_SIGN_CRT:forcevariable = "$test_keys_dir/uefi/DB.crt" + OPTEE_TA_SIGN_KEY:forcevariable = "$test_keys_dir/ubootdev.key" + TF_A_SIGN_KEY_PATH:forcevariable = "$test_keys_dir/tf-a/privkey_ec_prime256v1.pem" EOF combined_config="${config}:${overlay}" kas checkout "$combined_config" @@ -187,6 +188,11 @@ require_selected_value DISTRO "$distro" require_selected_value DD_PRODUCT_FEATURES "$product_features" require_selected_value MODSIGN_PRIVKEY "/privkey_modsign.pem" require_selected_value MODSIGN_X509 "/x509_modsign.crt" +require_selected_value UBOOT_SIGN_KEYDIR "" +require_selected_value UBOOT_SPL_SIGN_KEYDIR "" +require_selected_value UEFI_SIGN_KEYDIR "/uefi" +require_selected_value OPTEE_TA_SIGN_KEY "/ubootdev.key" +require_selected_value TF_A_SIGN_KEY_PATH "/tf-a/privkey_ec_prime256v1.pem" rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 1213f209..e7a98286 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -61,13 +61,19 @@ def test_environment_assertions_emit_named_diagnostics(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertIn('capture_command environment run_bitbake "bitbake -e $target" >/dev/null', source) self.assertIn("require_selected_value() {", source) - self.assertEqual(source.count("require_selected_value "), 5) + self.assertEqual(source.count("require_selected_value "), 10) self.assertIn("ERROR: selected BitBake environment does not contain", source) def test_module_signing_uses_kernel_runtime_variables(self) -> None: source = SCRIPT.read_text(encoding="utf-8") - self.assertIn('MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem"', source) - self.assertIn('MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt"', source) + self.assertIn( + 'MODSIGN_PRIVKEY:forcevariable = "$test_keys_dir/privkey_modsign.pem"', + source, + ) + self.assertIn( + 'MODSIGN_X509:forcevariable = "$test_keys_dir/x509_modsign.crt"', + source, + ) self.assertIn( 'require_selected_value MODSIGN_PRIVKEY "/privkey_modsign.pem"', source, @@ -77,6 +83,26 @@ def test_module_signing_uses_kernel_runtime_variables(self) -> None: source, ) + def test_every_signing_consumer_path_is_forced_and_asserted(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + runtime_paths = { + "UBOOT_SIGN_KEYDIR": "", + "UBOOT_SPL_SIGN_KEYDIR": "", + "UEFI_SIGN_KEYDIR": "/uefi", + "OPTEE_TA_SIGN_KEY": "/ubootdev.key", + "TF_A_SIGN_KEY_PATH": "/tf-a/privkey_ec_prime256v1.pem", + } + for name, expected in runtime_paths.items(): + with self.subTest(name=name): + self.assertIn(f"{name}:forcevariable =", source) + self.assertIn( + f'require_selected_value {name} "{expected}"', source + ) + self.assertNotRegex( + source, + r"(?m)^ (?:SIGNING_|MODSIGN_|UBOOT_|UEFI_|OPTEE_|TF_A_)[A-Z0-9_]+ =", + ) + def test_disk_monitor_uses_inode_not_disk_units(self) -> None: source = SCRIPT.read_text(encoding="utf-8") disk_monitor = next( From fc72fbd7c99b64841e3383192ec95d000d0c57ca Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 22:59:07 +0100 Subject: [PATCH 29/47] ci: canonicalise layer adoption evidence --- .../canonicalise-bitbake-layer-output.py | 68 +++++++++++++++++++ scripts/validation/capture-layer-state.sh | 44 +++++++++--- scripts/validation/compare-layer-state.py | 54 +++++++++++++-- .../test_canonicalise_bitbake_layer_output.py | 54 +++++++++++++++ .../tests/test_capture_layer_state.py | 29 +++++++- .../tests/test_compare_layer_state.py | 42 ++++++++++++ 6 files changed, 275 insertions(+), 16 deletions(-) create mode 100644 scripts/validation/canonicalise-bitbake-layer-output.py create mode 100644 scripts/validation/tests/test_canonicalise_bitbake_layer_output.py create mode 100644 scripts/validation/tests/test_compare_layer_state.py diff --git a/scripts/validation/canonicalise-bitbake-layer-output.py b/scripts/validation/canonicalise-bitbake-layer-output.py new file mode 100644 index 00000000..e59546ad --- /dev/null +++ b/scripts/validation/canonicalise-bitbake-layer-output.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Canonicalise bitbake-layers reports without discarding policy evidence.""" + +from __future__ import annotations + +import argparse +import re +import sys + + +VOLATILE_PREFIXES = ("Loaded ", "Parsing of ") + + +def canonicalise_feature_sets(line: str) -> str: + def replace(match: re.Match[str]) -> str: + words = match.group(1).split() + if len(words) > 1 and all(re.fullmatch(r"[A-Za-z0-9+_.-]+", word) for word in words): + return "'" + " ".join(sorted(words)) + "'" + return match.group(0) + + return re.sub(r"'([^']+)'", replace, line) + + +def canonicalise_blocks(lines: list[str]) -> list[str]: + result: list[str] = [] + heading = "" + for raw in lines: + line = raw.rstrip() + stripped = line.strip() + if not stripped or stripped.startswith(VOLATILE_PREFIXES): + continue + if not line[:1].isspace() and stripped.endswith(":"): + heading = stripped[:-1] + result.append(f"entry\t{heading}") + elif line[:1].isspace() and heading: + result.append(f"value\t{heading}\t{canonicalise_feature_sets(stripped)}") + else: + heading = "" + result.append(f"message\t{canonicalise_feature_sets(stripped)}") + return sorted(result) + + +def canonicalise_layers(lines: list[str]) -> list[str]: + result: list[str] = [] + for raw in lines: + fields = raw.split() + if not fields or fields[:3] == ["layer", "path", "priority"]: + continue + if len(fields) == 3 and fields[2].lstrip("-").isdigit(): + result.append("\t".join(("layer", *fields))) + else: + result.append("message\t" + canonicalise_feature_sets(raw.strip())) + return sorted(result) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=("layers", "appends", "recipes")) + args = parser.parse_args() + lines = sys.stdin.read().splitlines() + output = canonicalise_layers(lines) if args.mode == "layers" else canonicalise_blocks(lines) + if output: + print("\n".join(output)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 151b72f8..4e1d9ba3 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -117,13 +117,21 @@ printf '%s\n' \ # Static layer surfaces are captured as well as BitBake's resolved view. This # makes wildcard/dangling appends and global layer.conf policy visible even # when they do not happen to alter the first recipe selected by BitBake. -find build/layers -type f \( -path '*/conf/layer.conf' -o -name '*.bbclass' \) -print0 \ - | sort -z \ +find build/layers "$repository_root" \ + -path "$repository_root/build" -prune -o \ + -path "$repository_root/.git" -prune -o \ + -type f \( -path '*/conf/layer.conf' -o -name '*.bbclass' \) -print0 \ + | sort -zu \ | xargs -0 grep -nHE \ '(^|[[:space:]])(IMAGE_INSTALL|CORE_IMAGE_|PACKAGE_INSTALL|DISTRO_FEATURES|MACHINE_FEATURES|PACKAGECONFIG|PREFERRED_(VERSION|PROVIDER)|DEFAULT_PREFERENCE|RDEPENDS|INHERIT|BBMASK|BBPATH|BBFILES|BBFILE_PRIORITY|INITRAMFS_MAXSIZE|IMAGE_FSTYPES|WKS_FILE|UBOOT_|KERNEL_|OPTEE_|SDKIMAGE_FEATURES|TOOLCHAIN_TARGET_TASK)(:|\[|[[:space:]])*([+?:.]?=)' \ - > "$output_dir/layer-conf-policy.txt" || true -find build/layers -type f -name '*.bbappend' -printf '%p\n' \ - | sed -E 's#^build/layers/##' \ + | sed -E "s#^$repository_root/#meta-dynamicdevices/#" \ + > "$output_dir/layer-conf-policy.txt" || true +find build/layers "$repository_root" \ + -path "$repository_root/build" -prune -o \ + -path "$repository_root/.git" -prune -o \ + -type f -name '*.bbappend' -printf '%p\n' \ + | sed -E -e 's#^build/layers/##' \ + -e "s#^$repository_root/#meta-dynamicdevices/#" \ | sort -u > "$output_dir/all-bbappends.txt" run_bitbake() { @@ -144,14 +152,25 @@ capture_command() { fi } +normalise_kas_projection() { + sed -E \ + -e '/^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} - (DEBUG|INFO|WARNING|ERROR)[[:space:]]+- /d' \ + -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e "s#$PWD##g" \ + -e 's#[[:space:]]+$##' +} + capture_command show-layers run_bitbake "bitbake-layers show-layers" \ - | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ + | normalise_kas_projection \ + | python3 "$(dirname "$0")/canonicalise-bitbake-layer-output.py" layers \ > "$output_dir/layers.txt" capture_command show-appends run_bitbake "bitbake-layers show-appends" \ - | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ + | normalise_kas_projection \ + | python3 "$(dirname "$0")/canonicalise-bitbake-layer-output.py" appends \ > "$output_dir/appends.txt" capture_command show-recipes run_bitbake "bitbake-layers show-recipes" \ - | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ + | normalise_kas_projection \ + | python3 "$(dirname "$0")/canonicalise-bitbake-layer-output.py" recipes \ > "$output_dir/recipes.txt" capture_command graph run_bitbake "bitbake -g $target" @@ -171,7 +190,12 @@ sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ capture_command environment run_bitbake "bitbake -e $target" >/dev/null python3 "$(dirname "$0")/select-bitbake-env.py" \ "$output_dir/environment.log" \ - | sed -E "s#$PWD##g; s#$test_keys_dir##g; s#$cache_root##g" \ + | sed -E \ + -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e "s#$PWD##g" \ + -e "s#$test_keys_dir##g" \ + -e "s#$cache_root##g" \ + -e 's/[0-9]{14}/TIMESTAMP/g' \ > "$output_dir/selected-environment.txt" require_selected_value() { local name=$1 @@ -207,7 +231,7 @@ if [ ! -d "$deploy_dir" ]; then fi find "$deploy_dir" -maxdepth 1 -type f -printf '%f\t%s\n' \ - | sed -E 's/-[0-9]{14}(\.|-)/-TIMESTAMP\1/g' \ + | sed -E 's/-[0-9]{14}(\.|-|$)/-TIMESTAMP\1/g' \ | sort -u > "$output_dir/deploy-layout-and-sizes.txt" # The expression belongs to awk; shell expansion would be a bug. # shellcheck disable=SC2016 diff --git a/scripts/validation/compare-layer-state.py b/scripts/validation/compare-layer-state.py index cf38a03f..5f656aea 100755 --- a/scripts/validation/compare-layer-state.py +++ b/scripts/validation/compare-layer-state.py @@ -10,6 +10,43 @@ import sys from pathlib import Path +DEPLOY_SIZES = "deploy-layout-and-sizes.txt" +MAX_DEPLOY_SIZE_DRIFT_RATIO = 0.005 +MIN_DEPLOY_SIZE_DRIFT_BYTES = 4096 + + +def deploy_entries(lines: list[str]) -> dict[str, int]: + entries: dict[str, int] = {} + for line in lines: + try: + name, raw_size = line.rsplit("\t", 1) + size = int(raw_size) + except ValueError as exc: + raise ValueError(f"invalid deploy entry: {line!r}") from exc + if name in entries: + raise ValueError(f"duplicate deploy entry: {name}") + entries[name] = size + return entries + + +def deploy_deltas(old_lines: list[str], new_lines: list[str]) -> list[str]: + """Compare deploy layout exactly and sizes within reproducible-build noise.""" + old = deploy_entries(old_lines) + new = deploy_entries(new_lines) + deltas = [f"removed {name}" for name in sorted(set(old) - set(new))] + deltas.extend(f"added {name}" for name in sorted(set(new) - set(old))) + for name in sorted(set(old) & set(new)): + tolerance = max( + MIN_DEPLOY_SIZE_DRIFT_BYTES, + int(old[name] * MAX_DEPLOY_SIZE_DRIFT_RATIO), + ) + if abs(new[name] - old[name]) > tolerance: + deltas.append( + f"size {name}: {old[name]} -> {new[name]} " + f"(tolerance {tolerance})" + ) + return deltas + def files(root: Path) -> dict[str, list[str]]: result: dict[str, list[str]] = {} @@ -49,11 +86,18 @@ def main() -> int: for name in sorted(set(old) | set(new)): if old.get(name) == new.get(name): continue - delta = list(difflib.unified_diff(old.get(name, []), new.get(name, []), lineterm="")) - changed_lines = [ - line[1:] for line in delta - if line.startswith(("+", "-")) and not line.startswith(("+++", "---")) - ] + if name == DEPLOY_SIZES: + try: + changed_lines = deploy_deltas(old.get(name, []), new.get(name, [])) + except ValueError as exc: + print(f"ERROR: {name}: {exc}", file=sys.stderr) + return 2 + else: + delta = list(difflib.unified_diff(old.get(name, []), new.get(name, []), lineterm="")) + changed_lines = [ + line[1:] for line in delta + if line.startswith(("+", "-")) and not line.startswith(("+++", "---")) + ] for line in changed_lines: if not any(file_re.search(name) and line_re.search(line) for file_re, line_re, _ in compiled): unexplained.append(f"{name}: {line}") diff --git a/scripts/validation/tests/test_canonicalise_bitbake_layer_output.py b/scripts/validation/tests/test_canonicalise_bitbake_layer_output.py new file mode 100644 index 00000000..302349cb --- /dev/null +++ b/scripts/validation/tests/test_canonicalise_bitbake_layer_output.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import importlib.util +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "canonicalise-bitbake-layer-output.py" +SPEC = importlib.util.spec_from_file_location("canonicalise_output", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class CanonicaliseOutputTests(unittest.TestCase): + def test_recipe_values_keep_their_recipe_identity(self) -> None: + self.assertEqual( + MODULE.canonicalise_blocks( + ["foo:", " layer-a 1.0", "bar:", " layer-b 2.0"] + ), + [ + "entry\tbar", + "entry\tfoo", + "value\tbar\tlayer-b 2.0", + "value\tfoo\tlayer-a 1.0", + ], + ) + + def test_parse_counts_are_removed_and_feature_sets_are_sorted(self) -> None: + self.assertEqual( + MODULE.canonicalise_blocks( + [ + "Parsing of 10 .bb files complete (0 cached)", + "foo:", + " layer 1.0 (skipped: missing required distro features 'x11 opengl')", + ] + ), + [ + "entry\tfoo", + "value\tfoo\tlayer 1.0 (skipped: missing required distro features 'opengl x11')", + ], + ) + + def test_layer_spacing_is_not_evidence(self) -> None: + self.assertEqual( + MODULE.canonicalise_layers( + ["layer path priority", "meta-dd build/.. 11"] + ), + ["layer\tmeta-dd\tbuild/..\t11"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index e7a98286..f56b42f0 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -15,7 +15,34 @@ class CaptureLayerStateTests(unittest.TestCase): def test_shell_does_not_expand_sed_end_anchor_as_argument_count(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertNotIn('s#[[:space:]]+$##"', source) - self.assertEqual(source.count("-e 's#[[:space:]]+$##'"), 3) + self.assertEqual(source.count("-e 's#[[:space:]]+$##'"), 1) + self.assertEqual(source.count("| normalise_kas_projection"), 3) + + def test_kas_projection_removes_host_noise_and_sorts_at_call_site(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + match = re.search(r"(?ms)^normalise_kas_projection\(\) \{\n.*?^\}\n", source) + self.assertIsNotNone(match) + result = subprocess.run( + [ + "bash", + "-c", + match.group(0) + + """ +PWD=/workspace/candidate +printf '%s\n' \\ + '2026-09-12 20:00:00 - INFO - kas 4.7 started' \\ + '/__w/project/baseline/build/layers/meta/conf/layer.conf ' \\ + '/workspace/candidate/recipe.bb' | normalise_kas_projection +""", + ], + check=True, + capture_output=True, + text=True, + ) + self.assertEqual( + result.stdout, + "/build/layers/meta/conf/layer.conf\n/recipe.bb\n", + ) def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: source = SCRIPT.read_text(encoding="utf-8") diff --git a/scripts/validation/tests/test_compare_layer_state.py b/scripts/validation/tests/test_compare_layer_state.py new file mode 100644 index 00000000..10fa67dd --- /dev/null +++ b/scripts/validation/tests/test_compare_layer_state.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import importlib.util +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "compare-layer-state.py" +SPEC = importlib.util.spec_from_file_location("compare_layer_state", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class DeployComparisonTests(unittest.TestCase): + def test_layout_change_is_a_delta(self) -> None: + self.assertEqual( + MODULE.deploy_deltas(["old.wic\t10000"], ["new.wic\t10000"]), + ["removed old.wic", "added new.wic"], + ) + + def test_small_rebuild_size_noise_is_accepted(self) -> None: + self.assertEqual( + MODULE.deploy_deltas(["image.wic\t100000000"], ["image.wic\t100300000"]), + [], + ) + + def test_material_size_change_is_a_delta(self) -> None: + self.assertEqual( + MODULE.deploy_deltas(["image.wic\t100000000"], ["image.wic\t101000000"]), + ["size image.wic: 100000000 -> 101000000 (tolerance 500000)"], + ) + + def test_malformed_or_duplicate_entries_fail_closed(self) -> None: + with self.assertRaises(ValueError): + MODULE.deploy_entries(["missing-size"]) + with self.assertRaises(ValueError): + MODULE.deploy_entries(["image.wic\t1", "image.wic\t2"]) + + +if __name__ == "__main__": + unittest.main() From fbc6f4db157b481befdeb0efba3d2678af6acb95 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 23:05:04 +0100 Subject: [PATCH 30/47] ci: keep adoption gate on controlled runner --- .github/workflows/layer-adoption-gate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index cc5638b5..2281cb0a 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -22,7 +22,7 @@ defaults: jobs: detect: name: Detect material layer change - runs-on: [self-hosted, Linux, X64] + runs-on: [self-hosted, Linux, X64, yocto, ai-tools] outputs: material: ${{ steps.detect.outputs.material }} base_sha: ${{ steps.base.outputs.sha }} @@ -146,7 +146,7 @@ jobs: name: Layer Adoption Gate needs: [detect, regression] if: always() - runs-on: [self-hosted, Linux, X64] + runs-on: [self-hosted, Linux, X64, yocto, ai-tools] steps: - name: Enforce gate result env: From 531a6c206d75421e999d040402966ffb71d995e4 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 23:21:51 +0100 Subject: [PATCH 31/47] ci: key baseline cache by capture schema --- .../run-layer-adoption-regression.py | 25 +++++++++++++++-- .../test_run_layer_adoption_regression.py | 28 +++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 7521bd9a..88dc8876 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -16,6 +16,13 @@ MARKER = ".complete.json" FIELDS = ("id", "machine", "distro", "image", "config", "product_features") PRODUCT_SUBMODULES = ("meta-dynamicdevices-bsp", "meta-dynamicdevices-distro") +CAPTURE_SCHEMA_FILES = ( + "scripts/validation/run-layer-adoption-regression.py", + "scripts/validation/capture-layer-state.sh", + "scripts/validation/canonicalise-bitbake-layer-output.py", + "scripts/validation/select-bitbake-env.py", + "scripts/validation/generate-layer-adoption-test-keys.sh", +) def evidence_digest(root: Path) -> str: @@ -30,7 +37,18 @@ def evidence_digest(root: Path) -> str: return digest.hexdigest() -def valid_cached_evidence(root: Path, base_sha: str, tuple_id: str) -> bool: +def capture_schema_digest(repository: Path) -> str: + digest = hashlib.sha256() + for relative in CAPTURE_SCHEMA_FILES: + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(hashlib.sha256((repository / relative).read_bytes()).digest()) + return digest.hexdigest() + + +def valid_cached_evidence( + root: Path, base_sha: str, tuple_id: str, capture_schema: str +) -> bool: try: marker = json.loads((root / MARKER).read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): @@ -38,6 +56,7 @@ def valid_cached_evidence(root: Path, base_sha: str, tuple_id: str) -> bool: return marker == { "base_sha": base_sha, "tuple_id": tuple_id, + "capture_schema": capture_schema, "evidence_sha256": evidence_digest(root), } @@ -153,6 +172,7 @@ def main() -> int: base_sha = subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=baseline, text=True ).strip() + capture_schema = capture_schema_digest(candidate) # This preparation is intentionally owned by the shared regression driver, # not by CI YAML. Local and hosted runs therefore build the same pinned @@ -175,7 +195,7 @@ def main() -> int: candidate_output = evidence / "candidate" / tuple_id temporary: Path | None = None try: - if valid_cached_evidence(cached, base_sha, tuple_id): + if valid_cached_evidence(cached, base_sha, tuple_id, capture_schema): print(f"Reusing immutable baseline evidence for {base_sha}", flush=True) else: shutil.rmtree(cached, ignore_errors=True) @@ -185,6 +205,7 @@ def main() -> int: marker = { "base_sha": base_sha, "tuple_id": tuple_id, + "capture_schema": capture_schema, "evidence_sha256": evidence_digest(temporary), } (temporary / MARKER).write_text( diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 6e007369..a3ce4b84 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -119,13 +119,37 @@ def test_valid_cache_is_accepted_and_tampering_is_rejected(self) -> None: marker = { "base_sha": "abc123", "tuple_id": "machine-image", + "capture_schema": "schema-a", "evidence_sha256": MODULE.evidence_digest(root), } (root / MODULE.MARKER).write_text(json.dumps(marker), encoding="utf-8") - self.assertTrue(MODULE.valid_cached_evidence(root, "abc123", "machine-image")) + self.assertTrue( + MODULE.valid_cached_evidence( + root, "abc123", "machine-image", "schema-a" + ) + ) + self.assertFalse( + MODULE.valid_cached_evidence( + root, "abc123", "machine-image", "schema-b" + ) + ) (root / "packages.txt").write_text("package-b\n", encoding="utf-8") - self.assertFalse(MODULE.valid_cached_evidence(root, "abc123", "machine-image")) + self.assertFalse( + MODULE.valid_cached_evidence( + root, "abc123", "machine-image", "schema-a" + ) + ) + + def test_capture_schema_covers_every_evidence_producer(self) -> None: + root = MODULE_PATH.parents[2] + first = MODULE.capture_schema_digest(root) + self.assertRegex(first, r"^[0-9a-f]{64}$") + self.assertIn("scripts/validation/capture-layer-state.sh", MODULE.CAPTURE_SCHEMA_FILES) + self.assertIn( + "scripts/validation/canonicalise-bitbake-layer-output.py", + MODULE.CAPTURE_SCHEMA_FILES, + ) def test_marker_is_not_part_of_evidence_digest(self) -> None: with tempfile.TemporaryDirectory() as directory: From 9093e7ac9365d001a8c184fd6c66763bacaed543 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 23:49:48 +0100 Subject: [PATCH 32/47] ci: source warnings from cooker logs --- scripts/validation/capture-layer-state.sh | 17 ++++++++++++++--- .../tests/test_capture_layer_state.py | 6 ++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 4e1d9ba3..a8681d1b 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -155,6 +155,9 @@ capture_command() { normalise_kas_projection() { sed -E \ -e '/^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} - (DEBUG|INFO|WARNING|ERROR)[[:space:]]+- /d' \ + -e '/WARNING:/d' \ + -e '/^Summary: There were [0-9]+ WARNING messages?\.?$/d' \ + -e '/^NOTE: Starting bitbake server\.\.\.$/d' \ -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ -e "s#$PWD##g" \ -e 's#[[:space:]]+$##' @@ -241,13 +244,21 @@ find "$deploy_dir" -maxdepth 1 -type f -name '*.manifest' -print0 \ | sort -u > "$output_dir/packages.txt" # New warnings are regressions even when BitBake returns zero. -find "$output_dir" -type f -name '*.log' -print0 \ +# Cooker logs preserve one BitBake warning per line. Combined stdout/stderr +# command logs can splice concurrent parser warnings together and are not a +# deterministic warning source. +find build/tmp/log/cooker -type f -name '*.log' -print0 \ | xargs -0 -r grep -hE '(^|[[:space:]])WARNING:' \ - | sed -E "s#$PWD/##g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}[^ ]*//g" \ + | sed -E \ + -e 's/^.*WARNING:/WARNING:/' \ + -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e "s#$PWD##g" \ + -e 's/[0-9]{4}-[0-9]{2}-[0-9]{2}[^ ]*//g' \ | sort -u > "$output_dir/warnings.txt" || true # Raw command logs are useful for diagnosis but contain progress ordering and -# timing noise. The deterministic projections above are the comparison input. +# timing noise. The deterministic projections and cooker warnings above are +# the comparison input. rm -f "$output_dir"/*.log # The generated overlay is removed by the EXIT trap. Keeping it inside the # worktree satisfies KAS's same-repository rule for concatenated configs. diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index f56b42f0..8623f482 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -32,6 +32,7 @@ def test_kas_projection_removes_host_noise_and_sorts_at_call_site(self) -> None: printf '%s\n' \\ '2026-09-12 20:00:00 - INFO - kas 4.7 started' \\ '/__w/project/baseline/build/layers/meta/conf/layer.conf ' \\ + 'Parsing recipes...WARNING: deterministic warning' \\ '/workspace/candidate/recipe.bb' | normalise_kas_projection """, ], @@ -44,6 +45,11 @@ def test_kas_projection_removes_host_noise_and_sorts_at_call_site(self) -> None: "/build/layers/meta/conf/layer.conf\n/recipe.bb\n", ) + def test_warnings_come_from_cooker_logs_not_interleaved_command_logs(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn("find build/tmp/log/cooker -type f -name '*.log'", source) + self.assertNotIn("find \"$output_dir\" -type f -name '*.log'", source) + def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: source = SCRIPT.read_text(encoding="utf-8") match = re.search(r"(?ms)^capture_command\(\) \{\n.*?^\}\n", source) From 88a53cd6c603c9506463a3b9afe76f3609180052 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 00:11:31 +0100 Subject: [PATCH 33/47] ci: normalise encoded checkout provenance --- scripts/validation/capture-layer-state.sh | 2 ++ scripts/validation/tests/test_capture_layer_state.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index a8681d1b..10612119 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -159,6 +159,7 @@ normalise_kas_projection() { -e '/^Summary: There were [0-9]+ WARNING messages?\.?$/d' \ -e '/^NOTE: Starting bitbake server\.\.\.$/d' \ -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e 's/_(baseline|candidate)_build_layers_/_REPO_build_layers_/g' \ -e "s#$PWD##g" \ -e 's#[[:space:]]+$##' } @@ -195,6 +196,7 @@ python3 "$(dirname "$0")/select-bitbake-env.py" \ "$output_dir/environment.log" \ | sed -E \ -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e 's/_(baseline|candidate)_build_layers_/_REPO_build_layers_/g' \ -e "s#$PWD##g" \ -e "s#$test_keys_dir##g" \ -e "s#$cache_root##g" \ diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 8623f482..aaad7aa4 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -50,6 +50,13 @@ def test_warnings_come_from_cooker_logs_not_interleaved_command_logs(self) -> No self.assertIn("find build/tmp/log/cooker -type f -name '*.log'", source) self.assertNotIn("find \"$output_dir\" -type f -name '*.log'", source) + def test_encoded_bitbake_provenance_checkout_is_normalised(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn( + "'s/_(baseline|candidate)_build_layers_/_REPO_build_layers_/g'", + source, + ) + def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: source = SCRIPT.read_text(encoding="utf-8") match = re.search(r"(?ms)^capture_command\(\) \{\n.*?^\}\n", source) From f7c6d6e036f4ead7ae7bd83b88abc969c5eadf38 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 01:20:07 +0100 Subject: [PATCH 34/47] ci: audit immutable baseline repair Build both sides with the exact two-file Jaguar U-Boot backport while validating the old and new submodule pins, repair branch, ancestry and changed-file set. Assisted-by: Codex --- ci/layer-adoption-contract.json | 15 +++ .../run-layer-adoption-regression.py | 91 ++++++++++++++++++- .../test_run_layer_adoption_regression.py | 84 +++++++++++++++++ 3 files changed, 186 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 6c3c21a7..4f1266ca 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -1,5 +1,20 @@ { "schema": 1, "reason": "Adopt the isolated NXP i.MX95 partner layer without changing any pre-existing Dynamic Devices build tuple.", + "baseline_repairs": [ + { + "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", + "submodule": "meta-dynamicdevices-bsp", + "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", + "to": "71f566e04e699cd49e63cf3c8cff47a817d06956", + "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", + "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", + "files": [ + "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-handheld/01-customise-dtb.patch", + "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch" + ], + "reason": "The immutable baseline cannot apply these two stale patch contexts to its pinned U-Boot revision. Build both sides with the focused two-file backport while auditing the exact old-to-new submodule transition." + } + ], "allowed_deltas": {} } diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 88dc8876..44cca472 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -7,6 +7,7 @@ import hashlib import json import os +import re import shutil import subprocess import tempfile @@ -17,6 +18,7 @@ FIELDS = ("id", "machine", "distro", "image", "config", "product_features") PRODUCT_SUBMODULES = ("meta-dynamicdevices-bsp", "meta-dynamicdevices-distro") CAPTURE_SCHEMA_FILES = ( + "ci/layer-adoption-contract.json", "scripts/validation/run-layer-adoption-regression.py", "scripts/validation/capture-layer-state.sh", "scripts/validation/canonicalise-bitbake-layer-output.py", @@ -25,6 +27,13 @@ ) +def submodule_commit(repository: Path, relative: str) -> str: + entry = git_output(repository, "ls-tree", "HEAD", "--", relative).split() + if len(entry) < 3 or entry[0] != "160000" or entry[1] != "commit": + raise RuntimeError(f"{repository}: {relative} is not a pinned git submodule") + return entry[2] + + def evidence_digest(root: Path) -> str: digest = hashlib.sha256() for path in sorted(root.rglob("*")): @@ -95,10 +104,7 @@ def git_output(repository: Path, *args: str) -> str: def prepare_repository(repository: Path) -> None: """Initialize and verify the pinned local layers used by every KAS tuple.""" for relative in PRODUCT_SUBMODULES: - entry = git_output(repository, "ls-tree", "HEAD", "--", relative).split() - if len(entry) < 3 or entry[0] != "160000" or entry[1] != "commit": - raise RuntimeError(f"{repository}: {relative} is not a pinned git submodule") - expected = entry[2] + expected = submodule_commit(repository, relative) layer = repository / relative layer_conf = layer / "conf/layer.conf" if not layer_conf.is_file(): @@ -128,6 +134,82 @@ def prepare_repository(repository: Path) -> None: raise RuntimeError(f"{repository}: {relative} has uncommitted content") +def apply_baseline_repairs( + baseline: Path, candidate: Path, contract_path: Path, base_sha: str +) -> None: + """Apply an exact, audited repair to an otherwise unbuildable baseline.""" + contract = json.loads(contract_path.read_text(encoding="utf-8")) + repairs = contract.get("baseline_repairs", []) + if not isinstance(repairs, list): + raise ValueError("baseline_repairs must be an array") + for repair in repairs: + if not isinstance(repair, dict): + raise ValueError("baseline repair must be an object") + if repair.get("base_sha") != base_sha: + continue + required = {"submodule", "from", "to", "url", "ref", "files", "reason"} + if not required <= repair.keys(): + raise ValueError("baseline repair is incomplete") + relative = str(repair["submodule"]) + old = str(repair["from"]) + new = str(repair["to"]) + url = str(repair["url"]) + ref = str(repair["ref"]) + files = repair["files"] + reason = str(repair["reason"]).strip() + if relative not in PRODUCT_SUBMODULES: + raise ValueError(f"unsupported baseline repair submodule: {relative}") + if any(not re.fullmatch(r"[0-9a-f]{40}", commit) for commit in (old, new)): + raise ValueError("baseline repair pins must be full lowercase commit IDs") + if not url.startswith("https://github.com/DynamicDevices/"): + raise ValueError("baseline repair URL must use the DynamicDevices HTTPS origin") + if not ref.startswith("refs/heads/"): + raise ValueError("baseline repair ref must be an explicit branch") + if ( + not reason + or not isinstance(files, list) + or not files + or any( + not isinstance(path, str) + or Path(path).is_absolute() + or ".." in Path(path).parts + for path in files + ) + ): + raise ValueError("baseline repair needs a reason and exact file list") + if submodule_commit(baseline, relative) != old: + raise RuntimeError(f"baseline repair {relative}: unexpected source pin") + if submodule_commit(candidate, relative) != new: + raise RuntimeError(f"baseline repair {relative}: unexpected candidate pin") + + candidate_layer = candidate / relative + subprocess.run( + ["git", "merge-base", "--is-ancestor", old, new], + cwd=candidate_layer, + check=True, + ) + changed = git_output(candidate_layer, "diff", "--name-only", old, new).splitlines() + if sorted(changed) != sorted(str(path) for path in files): + raise RuntimeError( + f"baseline repair {relative}: changed files do not match contract" + ) + + baseline_layer = baseline / relative + subprocess.run(["git", "fetch", url, ref], cwd=baseline_layer, check=True) + fetched = git_output(baseline_layer, "rev-parse", "FETCH_HEAD") + if fetched != new: + raise RuntimeError(f"baseline repair {relative}: ref resolved to {fetched}") + subprocess.run( + ["git", "checkout", "--detach", new], cwd=baseline_layer, check=True + ) + if git_output(baseline_layer, "status", "--porcelain", "--untracked-files=all"): + raise RuntimeError(f"baseline repair {relative}: checkout is dirty") + print( + f"Applying audited baseline repair for {relative}: {old} -> {new}", + flush=True, + ) + + def capture( script: Path, repository: Path, @@ -179,6 +261,7 @@ def main() -> int: # submodule content through the same KAS capture path. prepare_repository(baseline) prepare_repository(candidate) + apply_baseline_repairs(baseline, candidate, contract, base_sha) environment = os.environ.copy() environment["LAYER_ADOPTION_TEST_KEYS_DIR"] = str(test_keys) diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index a3ce4b84..7de71b20 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -22,6 +22,89 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_audited_baseline_repair_is_exact_and_fail_closed(self) -> None: + old = "a" * 40 + new = "b" * 40 + changed_file = "recipes-bsp/u-boot/u-boot-fio/board/fix.patch" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + contract = root / "contract.json" + contract.write_text( + json.dumps( + { + "baseline_repairs": [ + { + "base_sha": "base", + "submodule": "meta-dynamicdevices-bsp", + "from": old, + "to": new, + "url": "https://github.com/DynamicDevices/bsp.git", + "ref": "refs/heads/focused-backport", + "files": [changed_file], + "reason": "repair an exact pre-existing patch failure", + } + ] + } + ), + encoding="utf-8", + ) + with mock.patch.object( + MODULE, "submodule_commit", side_effect=[old, new] + ), mock.patch.object( + MODULE, + "git_output", + side_effect=[changed_file + "\n", new, ""], + ), mock.patch.object(MODULE.subprocess, "run") as run: + MODULE.apply_baseline_repairs( + root / "baseline", root / "candidate", contract, "base" + ) + + self.assertEqual(run.call_count, 3) + self.assertEqual( + run.call_args_list[1].args[0], + [ + "git", + "fetch", + "https://github.com/DynamicDevices/bsp.git", + "refs/heads/focused-backport", + ], + ) + + def test_audited_baseline_repair_rejects_extra_files(self) -> None: + old = "a" * 40 + new = "b" * 40 + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + contract = root / "contract.json" + contract.write_text( + json.dumps( + { + "baseline_repairs": [ + { + "base_sha": "base", + "submodule": "meta-dynamicdevices-bsp", + "from": old, + "to": new, + "url": "https://github.com/DynamicDevices/bsp.git", + "ref": "refs/heads/focused-backport", + "files": ["expected.patch"], + "reason": "focused repair", + } + ] + } + ), + encoding="utf-8", + ) + with mock.patch.object( + MODULE, "submodule_commit", side_effect=[old, new] + ), mock.patch.object( + MODULE, "git_output", return_value="unexpected.patch\n" + ), mock.patch.object(MODULE.subprocess, "run"): + with self.assertRaisesRegex(RuntimeError, "do not match contract"): + MODULE.apply_baseline_repairs( + root / "baseline", root / "candidate", contract, "base" + ) + def test_generated_signing_identity_is_validated_before_reuse(self) -> None: generator = MODULE_PATH.parent / "generate-layer-adoption-test-keys.sh" with tempfile.TemporaryDirectory() as directory: @@ -146,6 +229,7 @@ def test_capture_schema_covers_every_evidence_producer(self) -> None: first = MODULE.capture_schema_digest(root) self.assertRegex(first, r"^[0-9a-f]{64}$") self.assertIn("scripts/validation/capture-layer-state.sh", MODULE.CAPTURE_SCHEMA_FILES) + self.assertIn("ci/layer-adoption-contract.json", MODULE.CAPTURE_SCHEMA_FILES) self.assertIn( "scripts/validation/canonicalise-bitbake-layer-output.py", MODULE.CAPTURE_SCHEMA_FILES, From 014a30118e4651935cab4d816ada52e28362ac53 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 02:13:58 +0100 Subject: [PATCH 35/47] ci: include packaging baseline repair Extend the exact LmP v96 backport contract to cover board-scripts empty-directory QA failures while preserving full baseline and candidate builds. Assisted-by: Codex --- ci/layer-adoption-contract.json | 7 ++++--- meta-dynamicdevices-bsp | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 4f1266ca..794a1578 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,14 +6,15 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "71f566e04e699cd49e63cf3c8cff47a817d06956", + "to": "a42bd66fa7e19712fe7bce8a7da892ac994423a2", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-handheld/01-customise-dtb.patch", - "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch" + "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch", + "recipes-bsp/board-scripts/board-scripts_1.0.bb" ], - "reason": "The immutable baseline cannot apply these two stale patch contexts to its pinned U-Boot revision. Build both sides with the focused two-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts and creates empty package directories for products without board scripts. Build both sides with the focused three-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index 71f566e0..a42bd66f 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit 71f566e04e699cd49e63cf3c8cff47a817d06956 +Subproject commit a42bd66fa7e19712fe7bce8a7da892ac994423a2 From d767d4c7a88fb0f21999d87811f7b5774b066ae9 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 02:47:39 +0100 Subject: [PATCH 36/47] ci: include Handheld DTS baseline repair Extend the exact LmP v96 backport contract to the undefined Handheld audio-mute pinmux while preserving complete symmetric builds. Assisted-by: Codex --- ci/layer-adoption-contract.json | 7 ++++--- meta-dynamicdevices-bsp | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 794a1578..6866a5f9 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,15 +6,16 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "a42bd66fa7e19712fe7bce8a7da892ac994423a2", + "to": "22d33e11a4c4d2a98c77ec247b20d45ffdf328d6", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-handheld/01-customise-dtb.patch", "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch", - "recipes-bsp/board-scripts/board-scripts_1.0.bb" + "recipes-bsp/board-scripts/board-scripts_1.0.bb", + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts and creates empty package directories for products without board scripts. Build both sides with the focused three-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, creates empty package directories for products without board scripts, and uses an undefined Handheld pinmux macro. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index a42bd66f..22d33e11 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit a42bd66fa7e19712fe7bce8a7da892ac994423a2 +Subproject commit 22d33e11a4c4d2a98c77ec247b20d45ffdf328d6 From 25e4e8abcb49f5b7df4e83a0667e58092a0b07bc Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 03:27:36 +0100 Subject: [PATCH 37/47] ci: include Handheld DTS baseline repair Extend the exact LmP v96 backport contract to the current EVK base and inherited regulator required by the pinned kernel. Assisted-by: Codex --- ci/layer-adoption-contract.json | 4 ++-- meta-dynamicdevices-bsp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 6866a5f9..c2b29f36 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,7 +6,7 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "22d33e11a4c4d2a98c77ec247b20d45ffdf328d6", + "to": "cb3e579f96092951b0a5c3cd69248efa58b55e13", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ @@ -15,7 +15,7 @@ "recipes-bsp/board-scripts/board-scripts_1.0.bb", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, creates empty package directories for products without board scripts, and uses an undefined Handheld pinmux macro. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, creates empty package directories for products without board scripts, and carries a Handheld DTS with an undefined pinmux macro and duplicate base-board labels. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index 22d33e11..cb3e579f 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit 22d33e11a4c4d2a98c77ec247b20d45ffdf328d6 +Subproject commit cb3e579f96092951b0a5c3cd69248efa58b55e13 From 9e66b1b92b296377a8db3921dadaca66e6294623 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 04:10:21 +0100 Subject: [PATCH 38/47] ci: retain repaired empty package baseline Extend the exact LmP v96 backport pin so products that request board-scripts can complete rootfs even when their payload is intentionally empty. Assisted-by: Codex --- ci/layer-adoption-contract.json | 4 ++-- meta-dynamicdevices-bsp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index c2b29f36..72c3d1c4 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,7 +6,7 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "cb3e579f96092951b0a5c3cd69248efa58b55e13", + "to": "768f9e8763a0797d0991fe6ed58e8e8bc85da3c2", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ @@ -15,7 +15,7 @@ "recipes-bsp/board-scripts/board-scripts_1.0.bb", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, creates empty package directories for products without board scripts, and carries a Handheld DTS with an undefined pinmux macro and duplicate base-board labels. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, and carries a Handheld DTS with an undefined pinmux macro and duplicate base-board labels. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index cb3e579f..768f9e87 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit cb3e579f96092951b0a5c3cd69248efa58b55e13 +Subproject commit 768f9e8763a0797d0991fe6ed58e8e8bc85da3c2 From cc48e390f1c47068f9350c3231e3fa511f5df7b2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 04:59:14 +0100 Subject: [PATCH 39/47] ci: include remaining DTS baseline repairs Extend the exact LmP v96 backport contract to the INST and Phasora EVK-base corrections found by the full product gate. Assisted-by: Codex --- ci/layer-adoption-contract.json | 8 +++++--- meta-dynamicdevices-bsp | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 72c3d1c4..3112e01e 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,16 +6,18 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "768f9e8763a0797d0991fe6ed58e8e8bc85da3c2", + "to": "e013655c6f78c250ad5ca4a593ff2815ed6aaaab", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-handheld/01-customise-dtb.patch", "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch", "recipes-bsp/board-scripts/board-scripts_1.0.bb", - "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts" + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts", + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-inst.dts", + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-phasora.dts" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, and carries a Handheld DTS with an undefined pinmux macro and duplicate base-board labels. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, and carries three product DTS files that no longer compile against the pinned kernel. Build both sides with the focused six-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index 768f9e87..e013655c 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit 768f9e8763a0797d0991fe6ed58e8e8bc85da3c2 +Subproject commit e013655c6f78c250ad5ca4a593ff2815ed6aaaab From 459641fc59b62ed855d2a82eb32d4cf3a2dd94ca Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 05:48:29 +0100 Subject: [PATCH 40/47] ci: make kernel warnings sstate-independent Force kernel_configcheck on both sides before collecting cooker warnings so shared sstate cannot create false warning deltas. Assisted-by: Codex --- scripts/validation/capture-layer-state.sh | 6 ++++++ scripts/validation/tests/test_capture_layer_state.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 10612119..28c101f1 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -229,6 +229,12 @@ rm "$output_dir/environment.log" # baseline and candidate. capture_command build run_bitbake "bitbake $target" +# Task warnings must not depend on whether shared sstate caused the task to run +# during this particular image build. Force the kernel's warning-producing +# configuration check on both sides before collecting cooker diagnostics. +capture_command kernel-configcheck run_bitbake \ + "bitbake -f -c kernel_configcheck virtual/kernel" >/dev/null + deploy_dir="build/tmp/deploy/images/$machine" if [ ! -d "$deploy_dir" ]; then echo "ERROR: deploy directory missing after successful build: $deploy_dir" >&2 diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index aaad7aa4..f8e60742 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -50,6 +50,13 @@ def test_warnings_come_from_cooker_logs_not_interleaved_command_logs(self) -> No self.assertIn("find build/tmp/log/cooker -type f -name '*.log'", source) self.assertNotIn("find \"$output_dir\" -type f -name '*.log'", source) + def test_kernel_warning_capture_is_independent_of_sstate_reuse(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn( + '"bitbake -f -c kernel_configcheck virtual/kernel" >/dev/null', + source, + ) + def test_encoded_bitbake_provenance_checkout_is_normalised(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertIn( From 28311cbb365d339463608c53002615b1ee48d4bc Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 07:03:49 +0100 Subject: [PATCH 41/47] ci: include Phasora SPDX baseline repair Extend the exact LmP v96 repair contract to the custom loader license text required by create-spdx. Assisted-by: Codex --- ci/layer-adoption-contract.json | 9 ++++++--- meta-dynamicdevices-bsp | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 3112e01e..93e5cf9e 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,7 +6,7 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "e013655c6f78c250ad5ca4a593ff2815ed6aaaab", + "to": "b926d4e96532fb95c83984b154d7b2f72d82471e", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ @@ -15,9 +15,12 @@ "recipes-bsp/board-scripts/board-scripts_1.0.bb", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-inst.dts", - "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-phasora.dts" + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-phasora.dts", + "recipes-bsp/upd72020x-load/upd72020x-load/LicenseRef-markusj-upd72020x-load", + "recipes-bsp/upd72020x-load/upd72020x-load/loader-upstream-license-note", + "recipes-bsp/upd72020x-load/upd72020x-load_git.bb" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, and carries three product DTS files that no longer compile against the pinned kernel. Build both sides with the focused six-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, carries three product DTS files that no longer compile against the pinned kernel, and does not expose the Phasora loader's custom license text to SPDX generation. Build both sides with the focused nine-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index e013655c..b926d4e9 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit e013655c6f78c250ad5ca4a593ff2815ed6aaaab +Subproject commit b926d4e96532fb95c83984b154d7b2f72d82471e From 843ab3bd89e2512bdde28d9218ed480eb69cc9c2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 07:10:33 +0100 Subject: [PATCH 42/47] ci: match renamed license repair surface Track Git's canonical renamed path in the exact baseline repair file set. Assisted-by: Codex --- ci/layer-adoption-contract.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 93e5cf9e..8496f62f 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -17,10 +17,9 @@ "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-inst.dts", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-phasora.dts", "recipes-bsp/upd72020x-load/upd72020x-load/LicenseRef-markusj-upd72020x-load", - "recipes-bsp/upd72020x-load/upd72020x-load/loader-upstream-license-note", "recipes-bsp/upd72020x-load/upd72020x-load_git.bb" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, carries three product DTS files that no longer compile against the pinned kernel, and does not expose the Phasora loader's custom license text to SPDX generation. Build both sides with the focused nine-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, carries three product DTS files that no longer compile against the pinned kernel, and does not expose the Phasora loader's custom license text to SPDX generation. Build both sides with the focused eight-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} From 6c4a0aa4d3fbdf34c8c79f33e209ef84d6181465 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 09:02:38 +0100 Subject: [PATCH 43/47] ci: treat removed warnings as improvements Keep the gate fail-closed for candidate-added warnings while avoiding false failures when shared sstate suppresses a pre-existing baseline warning. Assisted-by: Codex --- scripts/validation/compare-layer-state.py | 8 ++++++++ .../tests/test_compare_layer_state.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/scripts/validation/compare-layer-state.py b/scripts/validation/compare-layer-state.py index 5f656aea..a28fec01 100755 --- a/scripts/validation/compare-layer-state.py +++ b/scripts/validation/compare-layer-state.py @@ -11,6 +11,7 @@ from pathlib import Path DEPLOY_SIZES = "deploy-layout-and-sizes.txt" +WARNINGS = "warnings.txt" MAX_DEPLOY_SIZE_DRIFT_RATIO = 0.005 MIN_DEPLOY_SIZE_DRIFT_BYTES = 4096 @@ -48,6 +49,11 @@ def deploy_deltas(old_lines: list[str], new_lines: list[str]) -> list[str]: return deltas +def warning_deltas(old_lines: list[str], new_lines: list[str]) -> list[str]: + """Return candidate-added warnings; removing a baseline warning is safe.""" + return sorted(set(new_lines) - set(old_lines)) + + def files(root: Path) -> dict[str, list[str]]: result: dict[str, list[str]] = {} for path in sorted(root.rglob("*")): @@ -92,6 +98,8 @@ def main() -> int: except ValueError as exc: print(f"ERROR: {name}: {exc}", file=sys.stderr) return 2 + elif name == WARNINGS: + changed_lines = warning_deltas(old.get(name, []), new.get(name, [])) else: delta = list(difflib.unified_diff(old.get(name, []), new.get(name, []), lineterm="")) changed_lines = [ diff --git a/scripts/validation/tests/test_compare_layer_state.py b/scripts/validation/tests/test_compare_layer_state.py index 10fa67dd..0bf7a3d8 100644 --- a/scripts/validation/tests/test_compare_layer_state.py +++ b/scripts/validation/tests/test_compare_layer_state.py @@ -38,5 +38,25 @@ def test_malformed_or_duplicate_entries_fail_closed(self) -> None: MODULE.deploy_entries(["image.wic\t1", "image.wic\t2"]) +class WarningComparisonTests(unittest.TestCase): + def test_new_candidate_warning_is_a_delta(self) -> None: + self.assertEqual( + MODULE.warning_deltas( + ["WARNING: existing"], + ["WARNING: existing", "WARNING: regression"], + ), + ["WARNING: regression"], + ) + + def test_removed_baseline_warning_is_not_a_delta(self) -> None: + self.assertEqual( + MODULE.warning_deltas( + ["WARNING: fixed", "WARNING: existing"], + ["WARNING: existing"], + ), + [], + ) + + if __name__ == "__main__": unittest.main() From b47dc3765a46542d5b6ebbd936d3574b4c15a4ae Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 09:56:51 +0100 Subject: [PATCH 44/47] ci: shard layer adoption tuples Run every protected tuple as an independent fail-fast-disabled matrix shard while preserving the single required Layer Adoption Gate aggregator. Keep the shared local driver full-matrix by default. Signed-off-by: Alex Lennon Assisted-by: Codex --- .github/workflows/layer-adoption-gate.yml | 20 +++++++++++++------ .../run-layer-adoption-regression.py | 17 +++++++++++++++- .../test_run_layer_adoption_regression.py | 20 +++++++++++++++++++ 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 2281cb0a..7e875fb9 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -26,6 +26,7 @@ jobs: outputs: material: ${{ steps.detect.outputs.material }} base_sha: ${{ steps.base.outputs.sha }} + tuple_ids: ${{ steps.detect.outputs.tuple_ids }} steps: - uses: actions/checkout@v7 with: @@ -58,13 +59,19 @@ jobs: --base '${{ steps.base.outputs.sha }}' \ --head '${{ github.sha }}' \ --github-output "$GITHUB_OUTPUT" + tuple_ids=$(python3 -c 'import json; print(json.dumps([entry["id"] for entry in json.load(open("ci/layer-adoption-tuples.json"))["tuples"]], separators=(",", ":")))') + echo "tuple_ids=$tuple_ids" >> "$GITHUB_OUTPUT" regression: - name: Existing product regression + name: Existing product regression (${{ matrix.tuple_id }}) needs: detect if: needs.detect.outputs.material == 'true' - # One named ai-tools runner owns the persistent Yocto cache. Keeping the - # complete matrix inside one job makes resource use and the required gate - # obvious, while the driver still fails closed on every protected tuple. + strategy: + fail-fast: false + matrix: + tuple_id: ${{ fromJSON(needs.detect.outputs.tuple_ids) }} + # Each tuple is an independent shard so one failure cannot hide later + # product failures. The final Layer Adoption Gate remains the single + # branch-protection contract, and local driver runs still cover all tuples. runs-on: [self-hosted, Linux, X64, yocto, ai-tools] container: image: ghcr.io/siemens/kas/kas@sha256:d989add57fc441fe9e27bb2dd6ed98c5597b44c807928e35a72dc1cfbdda9abe @@ -119,12 +126,13 @@ jobs: --candidate candidate \ --evidence evidence \ --cache "$LAYER_ADOPTION_CACHE" \ - --test-keys "$LAYER_ADOPTION_CACHE/test-keys" + --test-keys "$LAYER_ADOPTION_CACHE/test-keys" \ + --tuple-id '${{ matrix.tuple_id }}' - name: Preserve comparison evidence if: always() uses: actions/upload-artifact@v7 with: - name: layer-adoption-evidence + name: layer-adoption-evidence-${{ matrix.tuple_id }} path: evidence retention-days: 14 - name: Remove job-owned build trees diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 44cca472..45889d38 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -89,6 +89,18 @@ def load_tuples(path: Path) -> list[dict[str, str]]: return tuples +def select_tuples( + tuples: list[dict[str, str]], tuple_id: str | None +) -> list[dict[str, str]]: + """Select one CI shard while keeping the local default as the full gate.""" + if tuple_id is None: + return tuples + selected = [entry for entry in tuples if entry["id"] == tuple_id] + if not selected: + raise ValueError(f"unknown protected tuple: {tuple_id}") + return selected + + def remove_build_tree(repository: Path) -> None: repository = repository.resolve() build = (repository / "build").resolve() @@ -240,6 +252,7 @@ def main() -> int: parser.add_argument("--evidence", required=True, type=Path) parser.add_argument("--cache", required=True, type=Path) parser.add_argument("--test-keys", required=True, type=Path) + parser.add_argument("--tuple-id") args = parser.parse_args() baseline = args.baseline.resolve() @@ -250,7 +263,9 @@ def main() -> int: capture_script = candidate / "scripts/validation/capture-layer-state.sh" compare_script = candidate / "scripts/validation/compare-layer-state.py" contract = candidate / "ci/layer-adoption-contract.json" - tuples = load_tuples(candidate / "ci/layer-adoption-tuples.json") + tuples = select_tuples( + load_tuples(candidate / "ci/layer-adoption-tuples.json"), args.tuple_id + ) base_sha = subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=baseline, text=True ).strip() diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 7de71b20..baa13b2c 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -22,6 +22,26 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_ci_shard_selects_exactly_one_known_tuple(self) -> None: + tuples = [ + {"id": "image-a"}, + {"id": "mfgtool-a"}, + ] + self.assertIs(MODULE.select_tuples(tuples, None), tuples) + self.assertEqual( + MODULE.select_tuples(tuples, "mfgtool-a"), + [{"id": "mfgtool-a"}], + ) + with self.assertRaisesRegex(ValueError, "unknown protected tuple"): + MODULE.select_tuples(tuples, "missing") + + def test_workflow_shards_all_tuples_without_fail_fast(self) -> None: + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + self.assertIn("fail-fast: false", workflow) + self.assertIn("fromJSON(needs.detect.outputs.tuple_ids)", workflow) + self.assertIn("--tuple-id '${{ matrix.tuple_id }}'", workflow) + self.assertIn("name: Layer Adoption Gate", workflow) + def test_audited_baseline_repair_is_exact_and_fail_closed(self) -> None: old = "a" * 40 new = "b" * 40 From e47e9da7b3bc8c75c533dcbbb5a32beb8aaecaa1 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 10:25:39 +0100 Subject: [PATCH 45/47] ci: materialize LFS-backed KAS configs Resolve protected Git LFS KAS inputs in the shared regression driver for both baseline and candidate worktrees, preserving identical local and CI preparation and failing closed if smudging does not occur. Signed-off-by: Alex Lennon Assisted-by: Codex --- .../run-layer-adoption-regression.py | 22 ++++++++++ .../test_run_layer_adoption_regression.py | 40 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 45889d38..0ce953a4 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -15,6 +15,7 @@ MARKER = ".complete.json" +LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec/v1\n" FIELDS = ("id", "machine", "distro", "image", "config", "product_features") PRODUCT_SUBMODULES = ("meta-dynamicdevices-bsp", "meta-dynamicdevices-distro") CAPTURE_SCHEMA_FILES = ( @@ -101,6 +102,24 @@ def select_tuples( return selected +def materialize_config(repository: Path, relative: str) -> None: + """Resolve a KAS config stored in Git LFS before either build starts.""" + path = repository / relative + if not path.is_file(): + raise RuntimeError(f"{repository}: protected KAS config is missing: {relative}") + if not path.read_bytes().startswith(LFS_POINTER_PREFIX): + return + subprocess.run( + ["git", "lfs", "pull", f"--include={relative}", "--exclude="], + cwd=repository, + check=True, + ) + if path.read_bytes().startswith(LFS_POINTER_PREFIX): + raise RuntimeError( + f"{repository}: Git LFS did not materialize protected KAS config: {relative}" + ) + + def remove_build_tree(repository: Path) -> None: repository = repository.resolve() build = (repository / "build").resolve() @@ -277,6 +296,9 @@ def main() -> int: prepare_repository(baseline) prepare_repository(candidate) apply_baseline_repairs(baseline, candidate, contract, base_sha) + for config in sorted({entry["config"] for entry in tuples}): + materialize_config(baseline, config) + materialize_config(candidate, config) environment = os.environ.copy() environment["LAYER_ADOPTION_TEST_KEYS_DIR"] = str(test_keys) diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index baa13b2c..04722f1f 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -35,6 +35,46 @@ def test_ci_shard_selects_exactly_one_known_tuple(self) -> None: with self.assertRaisesRegex(ValueError, "unknown protected tuple"): MODULE.select_tuples(tuples, "missing") + def test_lfs_backed_kas_config_is_materialized_by_shared_driver(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + config = repository / "kas/lmp-mfgtool.yml" + config.parent.mkdir() + config.write_bytes( + MODULE.LFS_POINTER_PREFIX + + b"oid sha256:" + b"a" * 64 + b"\nsize 42\n" + ) + + def materialize(*args: object, **kwargs: object) -> None: + config.write_text("header:\n version: 14\n", encoding="utf-8") + + with mock.patch.object( + MODULE.subprocess, "run", side_effect=materialize + ) as run: + MODULE.materialize_config(repository, "kas/lmp-mfgtool.yml") + + run.assert_called_once_with( + [ + "git", + "lfs", + "pull", + "--include=kas/lmp-mfgtool.yml", + "--exclude=", + ], + cwd=repository, + check=True, + ) + + def test_unresolved_lfs_backed_kas_config_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + config = repository / "kas/lmp-mfgtool.yml" + config.parent.mkdir() + config.write_bytes(MODULE.LFS_POINTER_PREFIX) + with mock.patch.object(MODULE.subprocess, "run"): + with self.assertRaisesRegex(RuntimeError, "did not materialize"): + MODULE.materialize_config(repository, "kas/lmp-mfgtool.yml") + def test_workflow_shards_all_tuples_without_fail_fast(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") self.assertIn("fail-fast: false", workflow) From e8c94ce3e44bbe4574d1b656d1c4741dd6b7f52e Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 12:12:16 +0100 Subject: [PATCH 46/47] ci: retire Jaguar inst and Phasora tuples Record the product-owner retirement and remove the corresponding image and mfgtool shards from the protected build matrix.\n\nAssisted-by: Codex --- ci/layer-adoption-tuples.json | 4 ---- docs/PRODUCT_TUPLE_LIFECYCLE.md | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 docs/PRODUCT_TUPLE_LIFECYCLE.md diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index bb032edd..0c72d93e 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -3,16 +3,12 @@ "tuples": [ {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv usb-gadget"}, {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-inst-image", "machine": "imx8mm-jaguar-inst", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - {"id": "imx8mm-jaguar-phasora-image", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "display flutter godot"}, {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-inst-mfgtool", "machine": "imx8mm-jaguar-inst", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-phasora-mfgtool", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""} diff --git a/docs/PRODUCT_TUPLE_LIFECYCLE.md b/docs/PRODUCT_TUPLE_LIFECYCLE.md new file mode 100644 index 00000000..ec528195 --- /dev/null +++ b/docs/PRODUCT_TUPLE_LIFECYCLE.md @@ -0,0 +1,19 @@ +# Product tuple lifecycle + +## Deprecated on 13 September 2026 + +The product owner has retired CI builds for these machine families: + +- `imx8mm-jaguar-inst` +- `imx8mm-jaguar-phasora` + +The retirement covers normal factory images and mfgtool/recovery images. For +Foundries CI it also covers the `main-jaguar-phasora`, +`main-jaguar-phasora-ext`, and `main-jaguar-inst` refs. + +These tuples are intentionally absent from `ci/layer-adoption-tuples.json` and +must not be treated as accidentally deleted regression coverage. Board source +may remain in the repository for history or possible future reactivation, but +reactivation requires an explicit product decision and restoration of both +image and recovery coverage. Every other existing product tuple remains +protected by the layer-adoption gate. From 079fe76662672aca0a13efe50c75af6b332c2088 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 12:17:44 +0100 Subject: [PATCH 47/47] ci: focus adoption gate on Jaguar screen Run the current integration phase against the screen image and recovery tuple, while documenting that other active products remain deferred rather than deprecated.\n\nAssisted-by: Codex --- ci/layer-adoption-tuples.json | 11 +---------- docs/PRODUCT_TUPLE_LIFECYCLE.md | 12 ++++++++++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index 0c72d93e..280468ab 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -1,16 +1,7 @@ { "schema": 1, "tuples": [ - {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv usb-gadget"}, - {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "display flutter godot"}, - {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - - {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""} + {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""} ] } diff --git a/docs/PRODUCT_TUPLE_LIFECYCLE.md b/docs/PRODUCT_TUPLE_LIFECYCLE.md index ec528195..cbb2a3f6 100644 --- a/docs/PRODUCT_TUPLE_LIFECYCLE.md +++ b/docs/PRODUCT_TUPLE_LIFECYCLE.md @@ -15,5 +15,13 @@ These tuples are intentionally absent from `ci/layer-adoption-tuples.json` and must not be treated as accidentally deleted regression coverage. Board source may remain in the repository for history or possible future reactivation, but reactivation requires an explicit product decision and restoration of both -image and recovery coverage. Every other existing product tuple remains -protected by the layer-adoption gate. +image and recovery coverage. + +## Temporarily deferred CI coverage + +For the current R26 screen-board integration phase, the layer-adoption workflow +runs only the `imx8mm-jaguar-screen` factory-image and mfgtool/recovery tuples. +Other still-active product families are deferred to a later CI expansion; they +are not deprecated, and this focused run must not be cited as proof that the +full historical product matrix passed. Production-wide adoption remains gated +on restoring and passing that broader coverage.