From 57d4102280d896183a3f630a7b1ab3de26e5f3a0 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Tue, 19 May 2026 13:09:41 +0100 Subject: [PATCH 01/79] fix(waydroid): lxc ThinLTO configure and python3-gbinder Cython 3 lxc meson fails with clang ThinLTO when the linker is ld.bfd; disable meson b_lto for lxc. Bump python3-gbinder to bullseye 1.1.2 (5089d76) for Cython 3 noexcept fixes required on Scarthgap/Python 3.12. Co-authored-by: Cursor --- recipes-containers/lxc/lxc_git.bbappend | 4 ++++ recipes-support/waydroid/python3-gbinder_git.bb | 7 ++++--- 2 files changed, 8 insertions(+), 3 deletions(-) create mode 100644 recipes-containers/lxc/lxc_git.bbappend diff --git a/recipes-containers/lxc/lxc_git.bbappend b/recipes-containers/lxc/lxc_git.bbappend new file mode 100644 index 00000000..985e5ecc --- /dev/null +++ b/recipes-containers/lxc/lxc_git.bbappend @@ -0,0 +1,4 @@ +# lxc meson+ninja fails at configure when clang ThinLTO is enabled but the +# default Yocto linker is ld.bfd: +# ERROR: LLVM's ThinLTO only works with gold, lld, lld-link, ld64 or mold, not ld.bfd +EXTRA_OEMESON:append = " -Db_lto=false" diff --git a/recipes-support/waydroid/python3-gbinder_git.bb b/recipes-support/waydroid/python3-gbinder_git.bb index 038e7997..27c8e833 100644 --- a/recipes-support/waydroid/python3-gbinder_git.bb +++ b/recipes-support/waydroid/python3-gbinder_git.bb @@ -7,9 +7,10 @@ LICENSE = "GPL-3.0-only" SECTION = "devel/python" LIC_FILES_CHKSUM = "file://LICENSE;md5=1ebbd3e34237af26da5dc08a4e440464" -# We're stuck @ 1.1.1 untill we are at cython3, build breaks with https://github.com/waydroid/gbinder-python/commit/4d8cb8f56da9e8159ea1b2ef76ddfa0253563db7 -PV = "1.1.1+git${SRCPV}" -SRCREV = "990c3007eeac3e015fb38aecd76dd010b4b75a1e" +# 1.1.1 + old Cython fails on Scarthgap (Python 3.12 / Cython 3.x). bullseye @ 4d8cb8f+ +# adds explicit noexcept for Cython 3; 1.1.2 is current bullseye tip. +PV = "1.1.2+git${SRCPV}" +SRCREV = "5089d76d4cd958cedda0028ffd752c25508dd382" SRC_URI = "git://github.com/waydroid/gbinder-python.git;branch=bullseye;protocol=https \ file://0001-setup.py-Migrate-away-from-deprecated-distutils.core.patch \ " From f2778cecacb140b83c109ee8633801b83634e52d Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 4 Sep 2026 11:45:54 +0100 Subject: [PATCH 02/79] feat(waydroid): provision Android images outside OSTree --- recipes-support/waydroid/waydroid.bb | 31 ++++++++++++++++++- .../waydroid/waydroid-image-provision | 18 +++++++++++ .../waydroid/waydroid-image-provision.service | 17 ++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 recipes-support/waydroid/waydroid/waydroid-image-provision create mode 100644 recipes-support/waydroid/waydroid/waydroid-image-provision.service diff --git a/recipes-support/waydroid/waydroid.bb b/recipes-support/waydroid/waydroid.bb index 759aed6c..71ce9f2d 100644 --- a/recipes-support/waydroid/waydroid.bb +++ b/recipes-support/waydroid/waydroid.bb @@ -24,6 +24,8 @@ RRECOMMENDS:${PN} += "\ SRC_URI = "git://github.com/herrie82/waydroid.git;branch=herrie/luneos;protocol=https \ file://gbinder.conf \ file://waydroid-net.sh \ + file://waydroid-image-provision \ + file://waydroid-image-provision.service \ " S = "${WORKDIR}/git" @@ -40,12 +42,22 @@ COMPATIBLE_MACHINE:pinetab2 = "(.*)" COMPATIBLE_MACHINE:mido-halium = "(.*)" COMPATIBLE_MACHINE:tissot = "(.*)" COMPATIBLE_MACHINE:imx8mm-lpddr4-evk = "(.*)" +COMPATIBLE_MACHINE:imx8mm-jaguar-screen = "(.*)" +COMPATIBLE_MACHINE:imx95-frdm-evk = "(.*)" inherit pkgconfig #inherit webos_app #inherit webos_filesystem_paths #inherit webos_systemd -inherit systemd +inherit features_check systemd + +SYSTEMD_SERVICE:${PN}:imx8mm-jaguar-screen = "waydroid-image-provision.service" +SYSTEMD_AUTO_ENABLE:${PN}:imx8mm-jaguar-screen = "enable" + +# Product configuration selects the provider-neutral `android-container` +# bundle. The distro layer expands that bundle to these implementation +# prerequisites; fail early if Waydroid is pulled into an incomplete image. +REQUIRED_DISTRO_FEATURES = "waydroid wayland opengl vulkan" WEBOS_SYSTEMD_SERVICE = "waydroid-init.service waydroid-container.service" @@ -57,6 +69,13 @@ do_install() { make install_luneos DESTDIR=${D} } +do_install:append() { + install -Dm0755 ${WORKDIR}/waydroid-image-provision \ + ${D}${libexecdir}/waydroid-image-provision + install -Dm0644 ${WORKDIR}/waydroid-image-provision.service \ + ${D}${systemd_system_unitdir}/waydroid-image-provision.service +} + # Provided by libgbinder already for Halium devices, but necessary to add for non-Halium devices. do_install:append:pinephone() { @@ -80,11 +99,21 @@ do_install:append:imx8mm-lpddr4-evk() { install -m 755 ${WORKDIR}/waydroid-net.sh ${D}/usr/lib/waydroid/data/scripts/waydroid-net.sh } +do_install:append:imx8mm-jaguar-screen() { + install -Dm644 -t "${D}${sysconfdir}" "${WORKDIR}/gbinder.conf" + install -m 755 ${WORKDIR}/waydroid-net.sh ${D}/usr/lib/waydroid/data/scripts/waydroid-net.sh +} + do_install:append:raspberrypi4-64() { install -Dm644 -t "${D}${sysconfdir}" "${WORKDIR}/gbinder.conf" install -m 755 ${WORKDIR}/waydroid-net.sh ${D}/usr/lib/waydroid/data/scripts/waydroid-net.sh } +do_install:append:imx95-frdm-evk() { + install -Dm644 -t "${D}${sysconfdir}" "${WORKDIR}/gbinder.conf" + install -m 755 ${WORKDIR}/waydroid-net.sh ${D}/usr/lib/waydroid/data/scripts/waydroid-net.sh +} + FILES:${PN} += " \ ${sysconfdir} \ ${libdir} \ diff --git a/recipes-support/waydroid/waydroid/waydroid-image-provision b/recipes-support/waydroid/waydroid/waydroid-image-provision new file mode 100644 index 00000000..7af9103d --- /dev/null +++ b/recipes-support/waydroid/waydroid/waydroid-image-provision @@ -0,0 +1,18 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-only + +set -eu + +images_dir=/var/lib/waydroid/images +config=/var/lib/waydroid/waydroid.cfg + +if [ -s "${images_dir}/system.img" ] && [ -s "${images_dir}/vendor.img" ]; then + exit 0 +fi + +# An interrupted first initialization can leave configuration claiming a +# channel revision whose image was never fully installed. Let Waydroid build +# that configuration again before its checksum-verified channel download. +rm -f "${config}" + +exec /usr/bin/waydroid init diff --git a/recipes-support/waydroid/waydroid/waydroid-image-provision.service b/recipes-support/waydroid/waydroid/waydroid-image-provision.service new file mode 100644 index 00000000..a3e69843 --- /dev/null +++ b/recipes-support/waydroid/waydroid/waydroid-image-provision.service @@ -0,0 +1,17 @@ +[Unit] +Description=Provision Waydroid Android images into persistent storage +Wants=network-online.target +After=network-online.target dbus.service +Before=waydroid-container.service +StartLimitIntervalSec=0 + +[Service] +Type=oneshot +ExecStart=/usr/libexec/waydroid-image-provision +RemainAfterExit=yes +Restart=on-failure +RestartSec=60 +TimeoutStartSec=infinity + +[Install] +WantedBy=multi-user.target From aade8ffce6a16dcb7ceb442a97effa6ed3ba90a2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 4 Sep 2026 12:16:08 +0100 Subject: [PATCH 03/79] fix(waydroid): require Etnaviv on Jaguar screen --- recipes-support/waydroid/waydroid.bb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/recipes-support/waydroid/waydroid.bb b/recipes-support/waydroid/waydroid.bb index 71ce9f2d..feb4a3c5 100644 --- a/recipes-support/waydroid/waydroid.bb +++ b/recipes-support/waydroid/waydroid.bb @@ -57,7 +57,8 @@ SYSTEMD_AUTO_ENABLE:${PN}:imx8mm-jaguar-screen = "enable" # Product configuration selects the provider-neutral `android-container` # bundle. The distro layer expands that bundle to these implementation # prerequisites; fail early if Waydroid is pulled into an incomplete image. -REQUIRED_DISTRO_FEATURES = "waydroid wayland opengl vulkan" +REQUIRED_DISTRO_FEATURES = "waydroid wayland opengl" +REQUIRED_DISTRO_FEATURES:append:imx8mm-jaguar-screen = " etnaviv" WEBOS_SYSTEMD_SERVICE = "waydroid-init.service waydroid-container.service" From 871146db2be9ca450d5e44d056350c2fe29be1ca Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 4 Sep 2026 13:16:30 +0100 Subject: [PATCH 04/79] fix(waydroid): allow Etnaviv without Vulkan on Jaguar screen --- recipes-support/waydroid/waydroid.bb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/recipes-support/waydroid/waydroid.bb b/recipes-support/waydroid/waydroid.bb index feb4a3c5..a15e9e94 100644 --- a/recipes-support/waydroid/waydroid.bb +++ b/recipes-support/waydroid/waydroid.bb @@ -57,8 +57,8 @@ SYSTEMD_AUTO_ENABLE:${PN}:imx8mm-jaguar-screen = "enable" # Product configuration selects the provider-neutral `android-container` # bundle. The distro layer expands that bundle to these implementation # prerequisites; fail early if Waydroid is pulled into an incomplete image. -REQUIRED_DISTRO_FEATURES = "waydroid wayland opengl" -REQUIRED_DISTRO_FEATURES:append:imx8mm-jaguar-screen = " etnaviv" +REQUIRED_DISTRO_FEATURES = "waydroid wayland opengl vulkan" +REQUIRED_DISTRO_FEATURES:imx8mm-jaguar-screen = "waydroid wayland opengl etnaviv" WEBOS_SYSTEMD_SERVICE = "waydroid-init.service waydroid-container.service" From e3917841cba142a948d80a3a6d6026ac0206d4d2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 6 Sep 2026 23:05:15 +0100 Subject: [PATCH 05/79] fix(waydroid): use executable LXC stop hook --- recipes-support/waydroid/waydroid.bb | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/recipes-support/waydroid/waydroid.bb b/recipes-support/waydroid/waydroid.bb index a15e9e94..79adec38 100644 --- a/recipes-support/waydroid/waydroid.bb +++ b/recipes-support/waydroid/waydroid.bb @@ -103,6 +103,16 @@ do_install:append:imx8mm-lpddr4-evk() { do_install:append:imx8mm-jaguar-screen() { install -Dm644 -t "${D}${sysconfdir}" "${WORKDIR}/gbinder.conf" install -m 755 ${WORKDIR}/waydroid-net.sh ${D}/usr/lib/waydroid/data/scripts/waydroid-net.sh + + # LXC executes hook paths. The inherited LuneOS template uses /dev/null + # as a no-op post-stop hook, which exits 126 and makes every clean Waydroid + # shutdown look like a container failure. Use an executable no-op. + config_base="${D}${libdir}/waydroid/data/configs/config_base" + if ! grep -qx 'lxc.hook.post-stop = /dev/null' "${config_base}"; then + bbfatal "unexpected Waydroid post-stop hook in ${config_base}" + fi + sed -i 's|^lxc.hook.post-stop = /dev/null$|lxc.hook.post-stop = /bin/true|' \ + "${config_base}" } do_install:append:raspberrypi4-64() { From b949eaaff18d43bb42754734c1fcf89859c6972c Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Mon, 7 Sep 2026 00:45:08 +0100 Subject: [PATCH 06/79] feat(waydroid): boot Android UI on Jaguar screen --- recipes-support/waydroid/waydroid.bb | 30 ++++++++++++++++++- .../waydroid/waydroid/90-waydroid-screen.conf | 5 ++++ .../waydroid-jaguar-container.service | 16 ++++++++++ .../waydroid/waydroid-jaguar-session.service | 26 ++++++++++++++++ .../waydroid/waydroid-jaguar-ui.service | 22 ++++++++++++++ .../waydroid/waydroid/waydroid-jaguar-wait | 23 ++++++++++++++ .../waydroid/weston-jaguar-waydroid.ini | 17 +++++++++++ 7 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 recipes-support/waydroid/waydroid/90-waydroid-screen.conf create mode 100644 recipes-support/waydroid/waydroid/waydroid-jaguar-container.service create mode 100644 recipes-support/waydroid/waydroid/waydroid-jaguar-session.service create mode 100644 recipes-support/waydroid/waydroid/waydroid-jaguar-ui.service create mode 100644 recipes-support/waydroid/waydroid/waydroid-jaguar-wait create mode 100644 recipes-support/waydroid/waydroid/weston-jaguar-waydroid.ini diff --git a/recipes-support/waydroid/waydroid.bb b/recipes-support/waydroid/waydroid.bb index 79adec38..e5bc720b 100644 --- a/recipes-support/waydroid/waydroid.bb +++ b/recipes-support/waydroid/waydroid.bb @@ -26,6 +26,12 @@ SRC_URI = "git://github.com/herrie82/waydroid.git;branch=herrie/luneos;protocol= file://waydroid-net.sh \ file://waydroid-image-provision \ file://waydroid-image-provision.service \ + file://waydroid-jaguar-wait \ + file://waydroid-jaguar-container.service \ + file://waydroid-jaguar-session.service \ + file://waydroid-jaguar-ui.service \ + file://weston-jaguar-waydroid.ini \ + file://90-waydroid-screen.conf \ " S = "${WORKDIR}/git" @@ -51,7 +57,12 @@ inherit pkgconfig #inherit webos_systemd inherit features_check systemd -SYSTEMD_SERVICE:${PN}:imx8mm-jaguar-screen = "waydroid-image-provision.service" +SYSTEMD_SERVICE:${PN}:imx8mm-jaguar-screen = " \ + waydroid-image-provision.service \ + waydroid-jaguar-container.service \ + waydroid-jaguar-session.service \ + waydroid-jaguar-ui.service \ +" SYSTEMD_AUTO_ENABLE:${PN}:imx8mm-jaguar-screen = "enable" # Product configuration selects the provider-neutral `android-container` @@ -113,6 +124,23 @@ do_install:append:imx8mm-jaguar-screen() { fi sed -i 's|^lxc.hook.post-stop = /dev/null$|lxc.hook.post-stop = /bin/true|' \ "${config_base}" + + # The display controller is card2 on this board; card0 is the boot + # framebuffer and card1 is the render-only Etnaviv node. Pinning card2 + # prevents Weston from selecting the wrong KMS device after boot. + install -Dm0644 ${WORKDIR}/weston-jaguar-waydroid.ini \ + ${D}${sysconfdir}/xdg/weston/waydroid-screen.ini + install -Dm0644 ${WORKDIR}/90-waydroid-screen.conf \ + ${D}${systemd_system_unitdir}/weston.service.d/90-waydroid-screen.conf + + install -Dm0755 ${WORKDIR}/waydroid-jaguar-wait \ + ${D}${libexecdir}/waydroid-jaguar-wait + install -Dm0644 ${WORKDIR}/waydroid-jaguar-container.service \ + ${D}${systemd_system_unitdir}/waydroid-jaguar-container.service + install -Dm0644 ${WORKDIR}/waydroid-jaguar-session.service \ + ${D}${systemd_system_unitdir}/waydroid-jaguar-session.service + install -Dm0644 ${WORKDIR}/waydroid-jaguar-ui.service \ + ${D}${systemd_system_unitdir}/waydroid-jaguar-ui.service } do_install:append:raspberrypi4-64() { diff --git a/recipes-support/waydroid/waydroid/90-waydroid-screen.conf b/recipes-support/waydroid/waydroid/90-waydroid-screen.conf new file mode 100644 index 00000000..655d290d --- /dev/null +++ b/recipes-support/waydroid/waydroid/90-waydroid-screen.conf @@ -0,0 +1,5 @@ +[Service] +WorkingDirectory=/var/rootdirs/home/weston +SupplementaryGroups=render video +ExecStart= +ExecStart=/usr/bin/weston --drm-device=card2 --config=/etc/xdg/weston/waydroid-screen.ini --modules=systemd-notify.so --log=/var/rootdirs/home/weston/weston.log diff --git a/recipes-support/waydroid/waydroid/waydroid-jaguar-container.service b/recipes-support/waydroid/waydroid/waydroid-jaguar-container.service new file mode 100644 index 00000000..edf0fbdd --- /dev/null +++ b/recipes-support/waydroid/waydroid/waydroid-jaguar-container.service @@ -0,0 +1,16 @@ +[Unit] +Description=Waydroid container on Jaguar Screen +Requires=waydroid-image-provision.service +After=waydroid-image-provision.service dbus.service +ConditionPathExists=/var/lib/waydroid/images/system.img +ConditionPathExists=/var/lib/waydroid/images/vendor.img + +[Service] +Type=simple +ExecStart=/usr/bin/waydroid container start +Restart=on-failure +RestartSec=5 +TimeoutStopSec=45 + +[Install] +WantedBy=multi-user.target diff --git a/recipes-support/waydroid/waydroid/waydroid-jaguar-session.service b/recipes-support/waydroid/waydroid/waydroid-jaguar-session.service new file mode 100644 index 00000000..7f8334c4 --- /dev/null +++ b/recipes-support/waydroid/waydroid/waydroid-jaguar-session.service @@ -0,0 +1,26 @@ +[Unit] +Description=Waydroid graphical session on Jaguar Screen +Requires=waydroid-jaguar-container.service weston.service +After=waydroid-jaguar-container.service weston.service +BindsTo=weston.service + +[Service] +Type=simple +User=weston +Group=weston +SupplementaryGroups=video render wayland +Environment=HOME=/var/rootdirs/home/weston +# Weston has the fixed system UID 63 in this image. System-service specifier +# %U expands to the manager UID (root), so keep the proven runtime path here. +Environment=XDG_RUNTIME_DIR=/run/user/63 +Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/63/bus +Environment=WAYLAND_DISPLAY=wayland-1 +# Starting the session asks the already-running ContainerManager over D-Bus to +# boot LXC. Waiting for LXC here would deadlock that request. +ExecStart=/usr/bin/waydroid session start +Restart=on-failure +RestartSec=5 +TimeoutStopSec=45 + +[Install] +WantedBy=graphical.target diff --git a/recipes-support/waydroid/waydroid/waydroid-jaguar-ui.service b/recipes-support/waydroid/waydroid/waydroid-jaguar-ui.service new file mode 100644 index 00000000..e1ce7d4d --- /dev/null +++ b/recipes-support/waydroid/waydroid/waydroid-jaguar-ui.service @@ -0,0 +1,22 @@ +[Unit] +Description=Present the Waydroid full-screen UI on Jaguar Screen +Requires=waydroid-jaguar-session.service +After=waydroid-jaguar-session.service +PartOf=waydroid-jaguar-session.service + +[Service] +Type=oneshot +User=weston +Group=weston +SupplementaryGroups=video render wayland +Environment=HOME=/var/rootdirs/home/weston +Environment=XDG_RUNTIME_DIR=/run/user/63 +Environment=DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/63/bus +Environment=WAYLAND_DISPLAY=wayland-1 +ExecStartPre=/usr/libexec/waydroid-jaguar-wait session +ExecStart=/usr/bin/waydroid show-full-ui +RemainAfterExit=yes +TimeoutStartSec=180 + +[Install] +WantedBy=graphical.target diff --git a/recipes-support/waydroid/waydroid/waydroid-jaguar-wait b/recipes-support/waydroid/waydroid/waydroid-jaguar-wait new file mode 100644 index 00000000..141c6a4c --- /dev/null +++ b/recipes-support/waydroid/waydroid/waydroid-jaguar-wait @@ -0,0 +1,23 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-only + +set -eu + +state=${1:?usage: waydroid-jaguar-wait container|session} +case "${state}" in + container) pattern='Container:[[:space:]]*RUNNING' ;; + session) pattern='Session:[[:space:]]*RUNNING' ;; + *) echo "unsupported Waydroid state: ${state}" >&2; exit 2 ;; +esac + +attempt=0 +while [ "${attempt}" -lt 120 ]; do + if /usr/bin/waydroid status 2>/dev/null | grep -Eq "${pattern}"; then + exit 0 + fi + attempt=$((attempt + 1)) + sleep 1 +done + +echo "timed out waiting for Waydroid ${state}" >&2 +exit 1 diff --git a/recipes-support/waydroid/waydroid/weston-jaguar-waydroid.ini b/recipes-support/waydroid/waydroid/weston-jaguar-waydroid.ini new file mode 100644 index 00000000..410db368 --- /dev/null +++ b/recipes-support/waydroid/waydroid/weston-jaguar-waydroid.ini @@ -0,0 +1,17 @@ +[core] +repaint-window=16 +idle-time=0 + +[shell] +background-image=/usr/share/screen-splash/active-edge-splash-1200x1920.png +background-type=scale +background-color=0xff07111f +panel-position=none + +[libinput] +touchscreen_calibrator=true + +[output] +name=DSI-1 +mode=1200x1920 +transform=rotate-90 From dc04e747ee429d04bf88edd9e13efea34e56758b Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Mon, 7 Sep 2026 01:29:43 +0100 Subject: [PATCH 07/79] fix(waydroid): use landscape splash for rotated output --- recipes-support/waydroid/waydroid/weston-jaguar-waydroid.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/recipes-support/waydroid/waydroid/weston-jaguar-waydroid.ini b/recipes-support/waydroid/waydroid/weston-jaguar-waydroid.ini index 410db368..f55a623e 100644 --- a/recipes-support/waydroid/waydroid/weston-jaguar-waydroid.ini +++ b/recipes-support/waydroid/waydroid/weston-jaguar-waydroid.ini @@ -3,7 +3,7 @@ repaint-window=16 idle-time=0 [shell] -background-image=/usr/share/screen-splash/active-edge-splash-1200x1920.png +background-image=/usr/share/screen-splash/active-edge-splash-1920x1200.png background-type=scale background-color=0xff07111f panel-position=none From 41df2bb9610fcda5397bdfa6c312ab55df5d19f1 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Mon, 7 Sep 2026 01:47:58 +0100 Subject: [PATCH 08/79] feat(demo): add Jaguar Waydroid GPU baseline --- .gitignore | 1 + demos/jaguar-waydroid-gpu-demo/README.md | 45 ++++ .../app/src/main/AndroidManifest.xml | 22 ++ .../jaguargpu/MainActivity.java | 250 ++++++++++++++++++ demos/jaguar-waydroid-gpu-demo/build.sh | 55 ++++ docs/JAGUAR-SCREEN-WAYDROID.md | 93 +++++++ docs/README.md | 10 + .../releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md | 41 +++ recipes-support/waydroid/waydroid.bb | 20 -- 9 files changed, 517 insertions(+), 20 deletions(-) create mode 100644 demos/jaguar-waydroid-gpu-demo/README.md create mode 100644 demos/jaguar-waydroid-gpu-demo/app/src/main/AndroidManifest.xml create mode 100644 demos/jaguar-waydroid-gpu-demo/app/src/main/java/com/dynamicdevices/jaguargpu/MainActivity.java create mode 100755 demos/jaguar-waydroid-gpu-demo/build.sh create mode 100644 docs/JAGUAR-SCREEN-WAYDROID.md create mode 100644 docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md diff --git a/.gitignore b/.gitignore index fe244302..a9d3a790 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ # Build outputs and layers (these are downloaded by KAS) build/ +**/out/ tmp/ tmp-glibc/ cache/ diff --git a/demos/jaguar-waydroid-gpu-demo/README.md b/demos/jaguar-waydroid-gpu-demo/README.md new file mode 100644 index 00000000..46d09378 --- /dev/null +++ b/demos/jaguar-waydroid-gpu-demo/README.md @@ -0,0 +1,45 @@ +# Jaguar Waydroid GPU demo + +An offline OpenGL ES 2.0 starfield benchmark for the Jaguar Screen Waydroid +image. It displays the renderer reported by Android, live FPS, and particle +count. Tap the screen to cycle through 2,000, 10,000, 40,000, and 80,000 +particles. + +## Build + +```sh +./build.sh +``` + +The build uses the latest Android SDK platform and build-tools installed under +`ANDROID_SDK_ROOT` or `~/Android/Sdk`. It compiles directly with the SDK +tools, so Gradle and network access are not required. The result is +`out/jaguar-gpu-demo.apk`, signed with the standard local Android debug key. + +## Install on Jaguar Screen + +Run the Waydroid commands as the same `weston` user that owns the graphical +session: + +```sh +sudo -u weston env \ + HOME=/var/rootdirs/home/weston \ + XDG_RUNTIME_DIR=/run/user/63 \ + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/63/bus \ + WAYLAND_DISPLAY=wayland-1 \ + waydroid app install /tmp/jaguar-gpu-demo.apk + +sudo -u weston env \ + HOME=/var/rootdirs/home/weston \ + XDG_RUNTIME_DIR=/run/user/63 \ + DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/63/bus \ + WAYLAND_DISPLAY=wayland-1 \ + waydroid app launch com.dynamicdevices.jaguargpu +``` + +The v1.0.0 bench baseline rendered 10,000 particles at approximately 13 fps at +the panel's full 1920x1200 logical resolution. The overlay identified +`Vivante GC600 rev 4653` and `OpenGL ES 2.0`. + +The APK uses only Android platform APIs and has no network permission or +external runtime dependency. diff --git a/demos/jaguar-waydroid-gpu-demo/app/src/main/AndroidManifest.xml b/demos/jaguar-waydroid-gpu-demo/app/src/main/AndroidManifest.xml new file mode 100644 index 00000000..6231722f --- /dev/null +++ b/demos/jaguar-waydroid-gpu-demo/app/src/main/AndroidManifest.xml @@ -0,0 +1,22 @@ + + + + + + + + + + + + + diff --git a/demos/jaguar-waydroid-gpu-demo/app/src/main/java/com/dynamicdevices/jaguargpu/MainActivity.java b/demos/jaguar-waydroid-gpu-demo/app/src/main/java/com/dynamicdevices/jaguargpu/MainActivity.java new file mode 100644 index 00000000..b1c3a579 --- /dev/null +++ b/demos/jaguar-waydroid-gpu-demo/app/src/main/java/com/dynamicdevices/jaguargpu/MainActivity.java @@ -0,0 +1,250 @@ +// SPDX-License-Identifier: GPL-3.0-only +package com.dynamicdevices.jaguargpu; + +import android.app.Activity; +import android.graphics.Color; +import android.opengl.GLES20; +import android.opengl.GLSurfaceView; +import android.os.Bundle; +import android.view.Gravity; +import android.view.View; +import android.widget.FrameLayout; +import android.widget.TextView; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.nio.FloatBuffer; +import java.util.Locale; +import java.util.Random; + +import javax.microedition.khronos.egl.EGLConfig; +import javax.microedition.khronos.opengles.GL10; + +public final class MainActivity extends Activity { + private BenchmarkRenderer renderer; + private TextView stats; + + @Override + protected void onCreate(Bundle savedInstanceState) { + super.onCreate(savedInstanceState); + getWindow().getDecorView().setSystemUiVisibility( + View.SYSTEM_UI_FLAG_FULLSCREEN + | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION + | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY); + + FrameLayout root = new FrameLayout(this); + GLSurfaceView surface = new GLSurfaceView(this); + surface.setEGLContextClientVersion(2); + surface.setPreserveEGLContextOnPause(true); + renderer = new BenchmarkRenderer(new StatsSink() { + @Override + public void update(String value) { + showStats(value); + } + }); + surface.setRenderer(renderer); + surface.setRenderMode(GLSurfaceView.RENDERMODE_CONTINUOUSLY); + root.addView(surface, new FrameLayout.LayoutParams( + FrameLayout.LayoutParams.MATCH_PARENT, + FrameLayout.LayoutParams.MATCH_PARENT)); + + stats = new TextView(this); + stats.setTextColor(Color.WHITE); + stats.setTextSize(20); + stats.setPadding(24, 16, 24, 16); + stats.setGravity(Gravity.START); + stats.setBackgroundColor(0x99030a18); + stats.setText("JAGUAR GPU DRIVE\nStarting GLES 2.0…"); + FrameLayout.LayoutParams overlay = new FrameLayout.LayoutParams( + FrameLayout.LayoutParams.WRAP_CONTENT, + FrameLayout.LayoutParams.WRAP_CONTENT, + Gravity.TOP | Gravity.START); + overlay.setMargins(24, 24, 0, 0); + root.addView(stats, overlay); + + TextView hint = new TextView(this); + hint.setTextColor(0xff79f7ff); + hint.setTextSize(18); + hint.setPadding(18, 10, 18, 10); + hint.setBackgroundColor(0x99030a18); + hint.setText("TAP TO CHANGE LOAD"); + FrameLayout.LayoutParams hintParams = new FrameLayout.LayoutParams( + FrameLayout.LayoutParams.WRAP_CONTENT, + FrameLayout.LayoutParams.WRAP_CONTENT, + Gravity.BOTTOM | Gravity.CENTER_HORIZONTAL); + hintParams.setMargins(0, 0, 0, 24); + root.addView(hint, hintParams); + + View.OnClickListener cycleLoad = new View.OnClickListener() { + @Override + public void onClick(View v) { + renderer.cycleLoad(); + } + }; + root.setOnClickListener(cycleLoad); + surface.setOnClickListener(cycleLoad); + hint.setOnClickListener(cycleLoad); + setContentView(root); + } + + private void showStats(String value) { + runOnUiThread(new Runnable() { + @Override + public void run() { + stats.setText(value); + } + }); + } + + @Override + protected void onResume() { + super.onResume(); + getWindow().getDecorView().setSystemUiVisibility( + View.SYSTEM_UI_FLAG_FULLSCREEN + | View.SYSTEM_UI_FLAG_HIDE_NAVIGATION + | View.SYSTEM_UI_FLAG_IMMERSIVE_STICKY); + } + + private static final class BenchmarkRenderer implements GLSurfaceView.Renderer { + private static final int[] LOADS = {2000, 10000, 40000, 80000}; + private static final String[] LOAD_NAMES = {"CRUISE", "FAST", "TURBO", "MAX"}; + private static final int MAX_PARTICLES = LOADS[LOADS.length - 1]; + private static final String VERTEX_SHADER = + "attribute vec4 aParticle;\n" + + "uniform float uTime;\n" + + "varying float vGlow;\n" + + "varying vec3 vColour;\n" + + "void main() {\n" + + " float z = fract(aParticle.z - uTime * aParticle.w);\n" + + " float depth = 0.08 + z;\n" + + " float twist = uTime * 0.20 + (1.0-z) * 2.4;\n" + + " mat2 r = mat2(cos(twist), -sin(twist), sin(twist), cos(twist));\n" + + " vec2 p = r * aParticle.xy / depth;\n" + + " gl_Position = vec4(p * 0.72, 0.0, 1.0);\n" + + " gl_PointSize = 1.2 + (1.0-z) * 8.0;\n" + + " vGlow = (1.0-z) * smoothstep(1.3, 0.1, length(p));\n" + + " vColour = mix(vec3(0.10,0.45,1.0), vec3(0.15,1.0,0.82), aParticle.w*18.0);\n" + + "}\n"; + private static final String FRAGMENT_SHADER = + "precision mediump float;\n" + + "varying float vGlow;\n" + + "varying vec3 vColour;\n" + + "void main() {\n" + + " vec2 q = gl_PointCoord - vec2(0.5);\n" + + " float d = length(q);\n" + + " float core = smoothstep(0.50, 0.02, d);\n" + + " float halo = smoothstep(0.50, 0.18, d);\n" + + " gl_FragColor = vec4(vColour * (core + halo*0.7) * (0.3+vGlow*1.8), core*vGlow);\n" + + "}\n"; + + private final StatsSink sink; + private final FloatBuffer particles; + private int program; + private int positionHandle; + private int timeHandle; + private int loadIndex = 1; + private long startNanos; + private long sampleNanos; + private int sampleFrames; + private String rendererName = "detecting GPU"; + private volatile boolean loadChanged; + + BenchmarkRenderer(StatsSink sink) { + this.sink = sink; + float[] data = new float[MAX_PARTICLES * 4]; + Random random = new Random(0x475055); + for (int i = 0; i < MAX_PARTICLES; i++) { + double angle = random.nextDouble() * Math.PI * 2.0; + double radius = Math.sqrt(random.nextDouble()) * 0.92; + data[i * 4] = (float) (Math.cos(angle) * radius); + data[i * 4 + 1] = (float) (Math.sin(angle) * radius); + data[i * 4 + 2] = random.nextFloat(); + data[i * 4 + 3] = 0.010f + random.nextFloat() * 0.045f; + } + particles = ByteBuffer.allocateDirect(data.length * 4) + .order(ByteOrder.nativeOrder()).asFloatBuffer(); + particles.put(data).position(0); + } + + @Override + public void onSurfaceCreated(GL10 ignored, EGLConfig config) { + program = link(VERTEX_SHADER, FRAGMENT_SHADER); + positionHandle = GLES20.glGetAttribLocation(program, "aParticle"); + timeHandle = GLES20.glGetUniformLocation(program, "uTime"); + GLES20.glEnable(GLES20.GL_BLEND); + GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE); + GLES20.glClearColor(0.005f, 0.012f, 0.045f, 1.0f); + rendererName = GLES20.glGetString(GLES20.GL_RENDERER); + startNanos = sampleNanos = System.nanoTime(); + publish(0.0); + } + + @Override + public void onSurfaceChanged(GL10 ignored, int width, int height) { + GLES20.glViewport(0, 0, width, height); + } + + @Override + public void onDrawFrame(GL10 ignored) { + long now = System.nanoTime(); + GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT); + GLES20.glUseProgram(program); + GLES20.glUniform1f(timeHandle, (now - startNanos) / 1_000_000_000.0f); + particles.position(0); + GLES20.glVertexAttribPointer(positionHandle, 4, GLES20.GL_FLOAT, false, 16, particles); + GLES20.glEnableVertexAttribArray(positionHandle); + GLES20.glDrawArrays(GLES20.GL_POINTS, 0, LOADS[loadIndex]); + + sampleFrames++; + if (loadChanged || now - sampleNanos >= 1_000_000_000L) { + double fps = sampleFrames * 1_000_000_000.0 / (now - sampleNanos); + publish(fps); + sampleFrames = 0; + sampleNanos = now; + loadChanged = false; + } + } + + void cycleLoad() { + loadIndex = (loadIndex + 1) % LOADS.length; + loadChanged = true; + } + + private void publish(double fps) { + sink.update(String.format(Locale.US, + "JAGUAR GPU DRIVE • %s\n%.1f FPS • %,d particles\n%s • OpenGL ES 2.0", + LOAD_NAMES[loadIndex], fps, LOADS[loadIndex], rendererName)); + } + + private static int link(String vertexSource, String fragmentSource) { + int vertex = compile(GLES20.GL_VERTEX_SHADER, vertexSource); + int fragment = compile(GLES20.GL_FRAGMENT_SHADER, fragmentSource); + int result = GLES20.glCreateProgram(); + GLES20.glAttachShader(result, vertex); + GLES20.glAttachShader(result, fragment); + GLES20.glLinkProgram(result); + int[] ok = new int[1]; + GLES20.glGetProgramiv(result, GLES20.GL_LINK_STATUS, ok, 0); + if (ok[0] == 0) { + throw new IllegalStateException(GLES20.glGetProgramInfoLog(result)); + } + return result; + } + + private static int compile(int type, String source) { + int shader = GLES20.glCreateShader(type); + GLES20.glShaderSource(shader, source); + GLES20.glCompileShader(shader); + int[] ok = new int[1]; + GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, ok, 0); + if (ok[0] == 0) { + throw new IllegalStateException(GLES20.glGetShaderInfoLog(shader)); + } + return shader; + } + } + + private interface StatsSink { + void update(String value); + } +} diff --git a/demos/jaguar-waydroid-gpu-demo/build.sh b/demos/jaguar-waydroid-gpu-demo/build.sh new file mode 100755 index 00000000..a855001b --- /dev/null +++ b/demos/jaguar-waydroid-gpu-demo/build.sh @@ -0,0 +1,55 @@ +#!/bin/sh +# SPDX-License-Identifier: GPL-3.0-only +set -eu + +SCRIPT_DIR="$(CDPATH= cd -- "$(dirname "$0")" && pwd)" +cd "$SCRIPT_DIR" + +SDK_ROOT="${ANDROID_SDK_ROOT:-$HOME/Android/Sdk}" +BUILD_TOOLS="${BUILD_TOOLS:-$(find "$SDK_ROOT/build-tools" -mindepth 1 -maxdepth 1 -type d | sort -V | tail -1)}" +PLATFORM="${ANDROID_PLATFORM:-$(find "$SDK_ROOT/platforms" -mindepth 1 -maxdepth 1 -type d | sort -V | tail -1)}" +ANDROID_JAR="$PLATFORM/android.jar" +OUT="$SCRIPT_DIR/out" + +mkdir -p "$OUT" +find "$OUT" -mindepth 1 -delete +mkdir -p "$OUT/classes" "$OUT/dex" + +javac -source 8 -target 8 -Xlint:-options \ + -bootclasspath "$ANDROID_JAR" \ + -d "$OUT/classes" \ + app/src/main/java/com/dynamicdevices/jaguargpu/MainActivity.java + +jar --create --file "$OUT/classes.jar" -C "$OUT/classes" . +"$BUILD_TOOLS/d8" \ + --lib "$ANDROID_JAR" \ + --min-api 29 \ + --output "$OUT/dex" \ + "$OUT/classes.jar" + +"$BUILD_TOOLS/aapt2" link \ + -I "$ANDROID_JAR" \ + --manifest app/src/main/AndroidManifest.xml \ + --min-sdk-version 29 \ + --target-sdk-version 35 \ + -o "$OUT/jaguar-gpu-demo-unsigned.apk" + +cd "$OUT/dex" +zip -q -u "$OUT/jaguar-gpu-demo-unsigned.apk" classes.dex +cd - >/dev/null + +KEYSTORE="$HOME/.android/debug.keystore" +if [ ! -f "$KEYSTORE" ]; then + mkdir -p "$(dirname "$KEYSTORE")" + keytool -genkeypair -keystore "$KEYSTORE" -storepass android -keypass android \ + -alias androiddebugkey -keyalg RSA -keysize 2048 -validity 10000 \ + -dname "CN=Android Debug,O=Android,C=US" >/dev/null 2>&1 +fi + +"$BUILD_TOOLS/zipalign" -f 4 \ + "$OUT/jaguar-gpu-demo-unsigned.apk" "$OUT/jaguar-gpu-demo-aligned.apk" +"$BUILD_TOOLS/apksigner" sign \ + --ks "$KEYSTORE" --ks-pass pass:android --key-pass pass:android \ + --out "$OUT/jaguar-gpu-demo.apk" "$OUT/jaguar-gpu-demo-aligned.apk" +"$BUILD_TOOLS/apksigner" verify --verbose "$OUT/jaguar-gpu-demo.apk" +printf 'Built %s\n' "$OUT/jaguar-gpu-demo.apk" diff --git a/docs/JAGUAR-SCREEN-WAYDROID.md b/docs/JAGUAR-SCREEN-WAYDROID.md new file mode 100644 index 00000000..3b29ceb2 --- /dev/null +++ b/docs/JAGUAR-SCREEN-WAYDROID.md @@ -0,0 +1,93 @@ +# Jaguar Screen Waydroid + +## Released baseline + +Jaguar Screen Waydroid v1.0.0 boots the physical +`imx8mm-jaguar-screen` board from a Foundries-built LmP image into a +full-screen LineageOS desktop. The first proven image was Foundries target +2887, built from manifest `88ab13ce2c5f611847566be3d1b8f9f4b4ca47cf`. + +The validated display path is: + +```text +LineageOS SurfaceFlinger + -> Waydroid minigbm / Mesa + -> etnaviv render node + -> Weston on card2 + -> DRM DSI-1 + -> ST1010B3CYOL / HX8279-D panel at 1920x1200 logical +``` + +SurfaceFlinger reported `Mesa, Vivante GC600 rev 4653, OpenGL ES 2.0 Mesa +26.0.1`. Weston used `renderD128` and the same GC600 renderer. This proves +hardware rendering on both sides of the Waydroid/Wayland boundary. + +## Boot contract + +The machine enables these services: + +1. `waydroid-image-provision.service` checks persistent + `/var/lib/waydroid/images` and runs `waydroid init` only when either + image is missing. +2. `waydroid-jaguar-container.service` owns the long-running container. +3. `waydroid-jaguar-session.service` runs as the fixed `weston` user and + binds the Android session to Weston. +4. `waydroid-jaguar-ui.service` waits for the session and opens the + full-screen Android UI. + +Weston is pinned to DRM `card2`; `card0` is the firmware framebuffer and +`card1` is the render-only etnaviv node. The physical panel scans out at +1200x1920 and Weston applies `transform=rotate-90` to expose a 1920x1200 +landscape desktop. Its handover background must therefore use +`active-edge-splash-1920x1200.png`. + +Android images are deliberately stored outside OSTree in +`/var/lib/waydroid/images`. A Foundries OS update changes the host and +services without replacing an already-provisioned Android image. Delete or +replace those images only as a deliberate image-management operation. + +## Host requirements + +The release requires: + +- Binder and Android host kernel support; +- pressure stall information for Android `lmkd`; +- etnaviv DRM and the Vivante GC600 device-tree nodes; +- Wayland, OpenGL and the Waydroid distro feature; +- persistent storage for the Android system and vendor images. + +The Screen machine accepts etnaviv/OpenGL without requiring Vulkan because the +GC600 exposes OpenGL ES 2.0. + +## Operations + +Check the complete stack: + +```sh +systemctl is-active \ + weston.service \ + waydroid-jaguar-container.service \ + waydroid-jaguar-session.service \ + waydroid-jaguar-ui.service +waydroid status +``` + +Run `waydroid app install` and `waydroid app launch` as the `weston` +session user. The exact environment is shown in +`demos/jaguar-waydroid-gpu-demo/README.md`. + +The boot firmware must leave WDOG1 disabled. Both +`CONFIG_SPL_WATCHDOG` and `CONFIG_WATCHDOG_AUTOSTART` are disabled in the +Screen U-Boot configuration so Linux can perform an immediate reset. + +## Demonstration + +`demos/jaguar-waydroid-gpu-demo` is the release demonstration and a bounded +graphics load. It is an offline GLES 2 star tunnel with live FPS, renderer +identity and tap-selectable particle counts. It avoids browser GPU policy and +network availability, making it suitable for repeatable bench and visitor +demos. + +The first physical run at 10,000 particles reported approximately 13 fps and +identified `Vivante GC600 rev 4653`. Use the lower setting for a smooth +visual introduction and increase the particle count to demonstrate scaling. diff --git a/docs/README.md b/docs/README.md index 2f6916d4..3677709e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -14,6 +14,11 @@ This folder contains formal documentation, reports, and reference materials for **Audience**: Project managers, technical leads **Contents**: Build-specific reports, testing checklists, issues tracking, next steps +### `/releases/` +**Purpose**: Tested product baselines tied to immutable source and Foundries targets +**Audience**: Engineers, release reviewers, demonstration teams +**Contents**: Release scope, evidence, known limits, and exact validation build + ### `/investigations/` **Purpose**: Technical investigation reports and research findings **Audience**: Engineers, technical team @@ -36,6 +41,11 @@ This folder contains formal documentation, reports, and reference materials for ## Engineering Documentation +- [Jaguar Screen Waydroid](JAGUAR-SCREEN-WAYDROID.md) — released Android + container, graphics, boot and operations baseline. +- [Jaguar Screen Waydroid v1.0.0](releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md) + — release evidence and known limits. + **For day-to-day engineering documentation, guides, and tutorials, see the `wiki/` folder.** The wiki contains: diff --git a/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md b/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md new file mode 100644 index 00000000..014754a4 --- /dev/null +++ b/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md @@ -0,0 +1,41 @@ +# Jaguar Screen Waydroid v1.0.0 + +Release date: 2026-09-07 + +This release establishes the first reproducible Foundries-built Jaguar Screen +image that boots directly into a GPU-accelerated LineageOS/Waydroid desktop on +the physical 1920x1200 display. + +## Included + +- seamless U-Boot to Linux display handover; +- upright Active Edge splash through the Weston handover; +- automatic persistent Android image provisioning; +- automatic Waydroid container, session and full-screen UI startup; +- Binder, pressure stall and etnaviv host support; +- Mesa GC600 acceleration in Weston and Android SurfaceFlinger; +- immediate reboot support by keeping WDOG1 disabled through SPL and U-Boot; +- offline Jaguar GPU Drive demo with live FPS and adjustable particle load. + +## Evidence + +- Board: `imx8mm-jaguar-screen-2210a09dab86563` +- First proven Foundries image: target 2887 +- Resolution: 1920x1200 logical, 1200x1920 native panel scanout +- Android renderer: `Vivante GC600 rev 4653` +- API: `OpenGL ES 2.0 Mesa 26.0.1` +- Demo baseline: approximately 13 fps at 10,000 particles + +Target 2888 is the release validation build for the final splash-orientation +and SPL watchdog fixes. The release tags identify the exact manifest and layer +commits used by the final validated target. + +## Known limits + +- Android images are provisioned separately into persistent storage; they are + not yet fetched from a product-controlled image channel. +- The GC600 exposes GLES 2.0. Chromium WebView 146 requests GLES 3 and + blocklists WebGL without development flags, so the release demo uses a + native GLES 2 application. +- Touch mapping and the visual design of the benchmark have further refinement + opportunities after this baseline. diff --git a/recipes-support/waydroid/waydroid.bb b/recipes-support/waydroid/waydroid.bb index e5bc720b..e2419773 100644 --- a/recipes-support/waydroid/waydroid.bb +++ b/recipes-support/waydroid/waydroid.bb @@ -11,7 +11,6 @@ SRCREV = "41f309f4c185a2c716723c081274eb56eb9263ff" SPV = "1.4.2" PV = "${SPV}+git${SRCPV}" - RDEPENDS:${PN} += "lxc python3-gbinder python3-pygobject libgbinder python3-pyclip python3-dbus python3-compression python3-json gobject-introspection" # these modules are directly included in android-flavored kernels @@ -161,22 +160,3 @@ FILES:${PN} += " \ ${prefix}/libexec \ /usr/palm/applications/id.waydro.container \ " - - -# Usage -# ===== -# Below is obsolete since Waydroid can now just be started from Launcher, however it's good to keep for reference -# -# mkdir -p /run/luna-session/ -# mount --bind /tmp/luna-session /run/luna-session/ -# export XDG_RUNTIME_DIR=/run/luna-session -# export XDG_SESSION_TYPE=wayland -# -- also, make sure /etc/gbinder.conf has "ApiLevel = 30" (Halium 9 needs API 28) -# -# Then: -# 0. waydroid init (just once, but needs network !) -# 1. either -# waydroid show-full-ui -# or -# waydroid session start -# waydroid app launch com.android.settings From 935b4901cab20c6f9e8081c1d414f91a25f34ed1 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Mon, 7 Sep 2026 01:49:01 +0100 Subject: [PATCH 09/79] docs(release): distinguish functional and source baselines --- docs/JAGUAR-SCREEN-WAYDROID.md | 2 ++ docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md | 7 ++++--- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/docs/JAGUAR-SCREEN-WAYDROID.md b/docs/JAGUAR-SCREEN-WAYDROID.md index 3b29ceb2..96cae43c 100644 --- a/docs/JAGUAR-SCREEN-WAYDROID.md +++ b/docs/JAGUAR-SCREEN-WAYDROID.md @@ -6,6 +6,8 @@ Jaguar Screen Waydroid v1.0.0 boots the physical `imx8mm-jaguar-screen` board from a Foundries-built LmP image into a full-screen LineageOS desktop. The first proven image was Foundries target 2887, built from manifest `88ab13ce2c5f611847566be3d1b8f9f4b4ca47cf`. +Target 2888 validates the upright Weston handover and immediate-reboot +firmware fixes. The validated display path is: diff --git a/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md b/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md index 014754a4..c6229761 100644 --- a/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md +++ b/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md @@ -26,9 +26,10 @@ the physical 1920x1200 display. - API: `OpenGL ES 2.0 Mesa 26.0.1` - Demo baseline: approximately 13 fps at 10,000 particles -Target 2888 is the release validation build for the final splash-orientation -and SPL watchdog fixes. The release tags identify the exact manifest and layer -commits used by the final validated target. +Target 2888 validates the final splash-orientation and SPL watchdog fixes. The +release manifest adds this documentation and the demo source; it does not +change the installed image. The release tags identify the exact manifest and +layer commits used by the final release target. ## Known limits From f7606956a84f1062a8c8c72203dfe05d908dfc86 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Mon, 7 Sep 2026 02:07:26 +0100 Subject: [PATCH 10/79] docs(release): reject target 2888 watchdog experiment --- docs/JAGUAR-SCREEN-WAYDROID.md | 11 ++++++----- docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md | 10 +++++----- 2 files changed, 11 insertions(+), 10 deletions(-) diff --git a/docs/JAGUAR-SCREEN-WAYDROID.md b/docs/JAGUAR-SCREEN-WAYDROID.md index 96cae43c..a006c51e 100644 --- a/docs/JAGUAR-SCREEN-WAYDROID.md +++ b/docs/JAGUAR-SCREEN-WAYDROID.md @@ -6,8 +6,8 @@ Jaguar Screen Waydroid v1.0.0 boots the physical `imx8mm-jaguar-screen` board from a Foundries-built LmP image into a full-screen LineageOS desktop. The first proven image was Foundries target 2887, built from manifest `88ab13ce2c5f611847566be3d1b8f9f4b4ca47cf`. -Target 2888 validates the upright Weston handover and immediate-reboot -firmware fixes. +Target 2888 was rejected during physical validation: its SPL watchdog change +did not remove the reboot delay and affected the proven boot presentation. The validated display path is: @@ -78,9 +78,10 @@ Run `waydroid app install` and `waydroid app launch` as the `weston` session user. The exact environment is shown in `demos/jaguar-waydroid-gpu-demo/README.md`. -The boot firmware must leave WDOG1 disabled. Both -`CONFIG_SPL_WATCHDOG` and `CONFIG_WATCHDOG_AUTOSTART` are disabled in the -Screen U-Boot configuration so Linux can perform an immediate reset. +The Screen U-Boot configuration disables `CONFIG_WATCHDOG_AUTOSTART`. Do not +change the SPL watchdog configuration as a reboot workaround: target 2888 +proved that doing so did not remove the delay and disturbed the display +baseline. Treat immediate reboot as a separate Linux restart-path requirement. ## Demonstration diff --git a/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md b/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md index c6229761..c0a64f37 100644 --- a/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md +++ b/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md @@ -14,7 +14,6 @@ the physical 1920x1200 display. - automatic Waydroid container, session and full-screen UI startup; - Binder, pressure stall and etnaviv host support; - Mesa GC600 acceleration in Weston and Android SurfaceFlinger; -- immediate reboot support by keeping WDOG1 disabled through SPL and U-Boot; - offline Jaguar GPU Drive demo with live FPS and adjustable particle load. ## Evidence @@ -26,10 +25,11 @@ the physical 1920x1200 display. - API: `OpenGL ES 2.0 Mesa 26.0.1` - Demo baseline: approximately 13 fps at 10,000 particles -Target 2888 validates the final splash-orientation and SPL watchdog fixes. The -release manifest adds this documentation and the demo source; it does not -change the installed image. The release tags identify the exact manifest and -layer commits used by the final release target. +Target 2888 was rejected after physical validation: disabling the SPL watchdog +did not remove the reboot delay and affected the proven boot presentation. A +later release candidate must restore the target 2887 boot display path and +prove immediate reboot on the board. The release tags will identify the exact +manifest and layer commits used by that final target. ## Known limits From 306e43cd04a93f42c0bd195601fa261045da39f0 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Mon, 7 Sep 2026 02:17:33 +0100 Subject: [PATCH 11/79] docs(release): record bounded reboot validation --- docs/JAGUAR-SCREEN-WAYDROID.md | 4 +++- docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md | 7 +++++-- 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/docs/JAGUAR-SCREEN-WAYDROID.md b/docs/JAGUAR-SCREEN-WAYDROID.md index a006c51e..721498a9 100644 --- a/docs/JAGUAR-SCREEN-WAYDROID.md +++ b/docs/JAGUAR-SCREEN-WAYDROID.md @@ -81,7 +81,9 @@ session user. The exact environment is shown in The Screen U-Boot configuration disables `CONFIG_WATCHDOG_AUTOSTART`. Do not change the SPL watchdog configuration as a reboot workaround: target 2888 proved that doing so did not remove the delay and disturbed the display -baseline. Treat immediate reboot as a separate Linux restart-path requirement. +baseline. The Screen-specific systemd manager drop-in sets +`RebootWatchdogSec=2s`. It bounds the final post-sync stall without changing +U-Boot, the kernel, DRM, or the splash handoff. ## Demonstration diff --git a/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md b/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md index c0a64f37..aa2ad4a0 100644 --- a/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md +++ b/docs/releases/JAGUAR-SCREEN-WAYDROID-v1.0.0.md @@ -14,6 +14,7 @@ the physical 1920x1200 display. - automatic Waydroid container, session and full-screen UI startup; - Binder, pressure stall and etnaviv host support; - Mesa GC600 acceleration in Weston and Android SurfaceFlinger; +- a two-second final reboot watchdog, applied after unmount and sync; - offline Jaguar GPU Drive demo with live FPS and adjustable particle load. ## Evidence @@ -28,8 +29,10 @@ the physical 1920x1200 display. Target 2888 was rejected after physical validation: disabling the SPL watchdog did not remove the reboot delay and affected the proven boot presentation. A later release candidate must restore the target 2887 boot display path and -prove immediate reboot on the board. The release tags will identify the exact -manifest and layer commits used by that final target. +prove the two-second final reboot watchdog on the board. A live configuration +test reduced the measured gap from `systemd-shutdown: Rebooting` to SPL from +59.5 seconds to 2.48 seconds. The release tags will identify the exact manifest +and layer commits used by the final Foundries target. ## Known limits From 62028240c2cf8af2022b1cb17e1afabf799e6bcb Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 00:50:04 +0100 Subject: [PATCH 12/79] bsp: fix existing Jaguar U-Boot patches Pin the focused LmP v96 backport that corrects stale patch context for the handheld and Phasora images. Assisted-by: Codex --- meta-dynamicdevices-bsp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index 47542ad3..71f566e0 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit 47542ad3f8d49850df69e37a3414c1ef60c51809 +Subproject commit 71f566e04e699cd49e63cf3c8cff47a817d06956 From 7c184c7b57f348c4d1afe62e4c57519e6eab93f2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:51:00 +0100 Subject: [PATCH 13/79] ci: enforce layer adoption regression gate --- .github/workflows/layer-adoption-gate.yml | 128 ++++++++++++++++++++ ci/layer-adoption-contract.json | 5 + ci/layer-adoption-tuples.json | 20 +++ scripts/monitor-foundries-build.sh | 25 ++-- scripts/validation/capture-layer-state.sh | 80 ++++++++++++ scripts/validation/compare-layer-state.py | 73 +++++++++++ scripts/validation/detect-layer-adoption.py | 76 ++++++++++++ 7 files changed, 396 insertions(+), 11 deletions(-) create mode 100644 .github/workflows/layer-adoption-gate.yml create mode 100644 ci/layer-adoption-contract.json create mode 100644 ci/layer-adoption-tuples.json create mode 100755 scripts/validation/capture-layer-state.sh create mode 100755 scripts/validation/compare-layer-state.py create mode 100755 scripts/validation/detect-layer-adoption.py diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml new file mode 100644 index 00000000..4813c67a --- /dev/null +++ b/.github/workflows/layer-adoption-gate.yml @@ -0,0 +1,128 @@ +name: Layer Adoption Gate + +on: + pull_request: + branches: [main, develop] + push: + branches: [main, develop] + workflow_dispatch: + inputs: + base_sha: + description: Baseline commit to compare + required: true + +concurrency: + group: layer-adoption-${${ github.event.pull_request.number || github.ref }} + cancel-in-progress: true + +jobs: + detect: + name: Detect material layer change + runs-on: [self-hosted, Linux, X64] + outputs: + material: ${${ steps.detect.outputs.material }} + matrix: ${${ steps.matrix.outputs.matrix }} + base_sha: ${${ steps.base.outputs.sha }} + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - id: base + name: Resolve immutable baseline + env: + PR_BASE: ${${ github.event.pull_request.base.sha }} + PUSH_BASE: ${${ github.event.before }} + INPUT_BASE: ${${ inputs.base_sha }} + run: | + sha="${PR_BASE:-${INPUT_BASE:-${PUSH_BASE:-}}}" + if [ -z "$sha" ] || printf '%s' "$sha" | grep -Eq '^0+$'; then + sha=$(git rev-parse HEAD^) + fi + git cat-file -e "$sha^{commit}" + echo "sha=$sha" >> "$GITHUB_OUTPUT" + - id: detect + name: Require an adoption contract + run: | + python3 scripts/validation/detect-layer-adoption.py \ + --base '${${ steps.base.outputs.sha }}' \ + --head '${${ github.sha }}' \ + --github-output "$GITHUB_OUTPUT" + - id: matrix + run: echo "matrix=$(jq -c '{include:.tuples}' ci/layer-adoption-tuples.json)" >> "$GITHUB_OUTPUT" + + regression: + name: Protect ${${ matrix.id }} + needs: detect + if: needs.detect.outputs.material == 'true' + strategy: + fail-fast: false + matrix: ${${ fromJSON(needs.detect.outputs.matrix) }} + runs-on: [self-hosted, Linux, X64] + container: + image: dynamicdevices/yocto-ci-build:latest + options: --privileged --platform linux/amd64 + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + path: candidate + - name: Create baseline worktree + working-directory: candidate + run: git worktree add ../baseline '${${ needs.detect.outputs.base_sha }}' + - name: Install KAS when absent + run: command -v kas >/dev/null || pip3 install kas + - name: Restore Yocto caches + uses: actions/cache@v4 + with: + path: | + ~/yocto/downloads + ~/yocto/sstate-cache + key: layer-adoption-${${ matrix.machine }}-${${ hashFiles('candidate/kas/**') }} + restore-keys: | + layer-adoption-${${ matrix.machine }}- + layer-adoption- + - name: Build and capture baseline + working-directory: baseline + run: | + ../candidate/scripts/validation/capture-layer-state.sh \ + '${${ matrix.config }}' '${${ matrix.machine }}' '${${ matrix.image }}' \ + '../evidence/baseline/${${ matrix.id }}' + - name: Build and capture candidate + working-directory: candidate + run: | + scripts/validation/capture-layer-state.sh \ + '${${ matrix.config }}' '${${ matrix.machine }}' '${${ matrix.image }}' \ + '../evidence/candidate/${${ matrix.id }}' + - name: Reject unexplained contamination + run: | + python3 candidate/scripts/validation/compare-layer-state.py \ + --baseline 'evidence/baseline/${${ matrix.id }}' \ + --candidate 'evidence/candidate/${${ matrix.id }}' \ + --tuple '${${ matrix.id }}' \ + --contract candidate/ci/layer-adoption-contract.json + - name: Preserve comparison evidence + if: always() + uses: actions/upload-artifact@v4 + with: + name: layer-adoption-${${ matrix.id }} + path: evidence + retention-days: 14 + + required: + name: Layer Adoption Gate + needs: [detect, regression] + if: always() + runs-on: [self-hosted, Linux, X64] + steps: + - name: Enforce gate result + env: + DETECT: ${${ needs.detect.result }} + MATERIAL: ${${ needs.detect.outputs.material }} + REGRESSION: ${${ needs.regression.result }} + run: | + test "$DETECT" = success + if [ "$MATERIAL" = true ]; then + test "$REGRESSION" = success + else + test "$REGRESSION" = skipped + fi diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json new file mode 100644 index 00000000..6c3c21a7 --- /dev/null +++ b/ci/layer-adoption-contract.json @@ -0,0 +1,5 @@ +{ + "schema": 1, + "reason": "Adopt the isolated NXP i.MX95 partner layer without changing any pre-existing Dynamic Devices build tuple.", + "allowed_deltas": {} +} diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json new file mode 100644 index 00000000..acace193 --- /dev/null +++ b/ci/layer-adoption-tuples.json @@ -0,0 +1,20 @@ +{ + "schema": 1, + "tuples": [ + {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-inst-image", "machine": "imx8mm-jaguar-inst", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-phasora-image", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + + {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-inst-mfgtool", "machine": "imx8mm-jaguar-inst", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-phasora-mfgtool", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, + {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"} + ] +} diff --git a/scripts/monitor-foundries-build.sh b/scripts/monitor-foundries-build.sh index 42379356..2a609d54 100755 --- a/scripts/monitor-foundries-build.sh +++ b/scripts/monitor-foundries-build.sh @@ -5,7 +5,7 @@ set -e TARGET=${1:-2027} -SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +POLL_INTERVAL=${FOUNDRIES_POLL_INTERVAL:-30} # Extract OAuth token from fioctl config FIOCTL_CONFIG="$HOME/.config/fioctl.yaml" @@ -43,14 +43,14 @@ show_build_info() { echo "$build_data" | jq -r '.data.build | " Build ID: \(.build_id // "N/A") Status: \(.status // "UNKNOWN") - Created: \(.created_at // "N/A") - Updated: \(.updated_at // "N/A")"' + Created: \(.created // "N/A") + Updated: \((.status_events // [] | last | .time) // "N/A")"' # Show runs if available - if echo "$build_data" | jq -e '.data.runs[]?' >/dev/null 2>&1; then + if echo "$build_data" | jq -e '.data.build.runs[]?' >/dev/null 2>&1; then echo "" echo "🏃 Build Runs:" - echo "$build_data" | jq -r '.data.runs[] | " \(.name): \(.status)"' + echo "$build_data" | jq -r '.data.build.runs[] | " \(.name): \(.status)"' else echo "" echo "🏃 Build Runs: Not started yet" @@ -64,10 +64,13 @@ echo "⏱️ Starting monitoring (Ctrl+C to stop)..." echo "" while true; do - BUILD_DATA=$(get_build_status) - - if [ $? -eq 0 ]; then - clear + if BUILD_DATA=$(get_build_status); then + # `clear` fails when there is no interactive TERM (for example when + # this monitor is run by CI or an agent). Display refresh is cosmetic + # and must never terminate authoritative build monitoring. + if [ -t 1 ] && [ -n "${TERM:-}" ]; then + clear || true + fi echo "🔍 Monitoring Foundries.io Build $TARGET - $(date)" echo "========================================" echo "" @@ -83,10 +86,10 @@ while true; do break fi - echo "🔄 Refreshing in 30 seconds..." + echo "🔄 Refreshing in ${POLL_INTERVAL} seconds..." else echo "❌ Failed to get build status" fi - sleep 30 + sleep "$POLL_INTERVAL" done diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh new file mode 100755 index 00000000..02215d4e --- /dev/null +++ b/scripts/validation/capture-layer-state.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash +# Build one protected tuple and capture deterministic layer/package/task state. +set -euo pipefail + +if [ "$#" -ne 4 ]; then + echo "Usage: $0 KAS_CONFIG MACHINE TARGET OUTPUT_DIR" >&2 + exit 2 +fi + +config=$1 +machine=$2 +target=$3 +output_dir=$4 + +case "$output_dir" in + /*) ;; + *) output_dir="$PWD/$output_dir" ;; +esac +mkdir -p "$output_dir" + +export KAS_MACHINE="$machine" +kas checkout "$config" + +# Static layer surfaces are captured as well as BitBake's resolved view. This +# makes wildcard/dangling appends and global layer.conf policy visible even +# when they do not happen to alter the first recipe selected by BitBake. +find build/layers -type f -path '*/conf/layer.conf' -print0 \ + | sort -z \ + | xargs -0 grep -nHE \ + '(^|[[:space:]])(IMAGE_INSTALL|CORE_IMAGE_EXTRA_INSTALL|DISTRO_FEATURES|MACHINE_FEATURES|PACKAGECONFIG|PREFERRED_(VERSION|PROVIDER)|RDEPENDS)(:|[[:space:]])*([+?:.]?=)' \ + > "$output_dir/layer-conf-policy.txt" || true +find build/layers -type f -name '*.bbappend' -printf '%p\n' \ + | sed -E 's#^build/layers/[^/]+/#LAYER/#' \ + | sort -u > "$output_dir/all-bbappends.txt" + +run_bitbake() { + kas shell "$config" -c "$1" +} + +run_bitbake "bitbake-layers show-layers" \ + | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + > "$output_dir/layers.txt" +run_bitbake "bitbake-layers show-appends" \ + | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + > "$output_dir/appends.txt" +run_bitbake "bitbake-layers show-recipes" \ + | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + > "$output_dir/recipes.txt" + +run_bitbake "bitbake -g $target" +sort -u build/pn-buildlist > "$output_dir/pn-buildlist.txt" +sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ + > "$output_dir/task-depends.dot" + +# A parse-only graph is not proof that packaging, signing, recovery image size, +# or deploy layout still works. Complete the real image/recovery build for both +# baseline and candidate. +run_bitbake "bitbake $target" + +deploy_dir="build/tmp/deploy/images/$machine" +if [ ! -d "$deploy_dir" ]; then + echo "ERROR: deploy directory missing after successful build: $deploy_dir" >&2 + exit 1 +fi + +find "$deploy_dir" -maxdepth 1 -type f -printf '%f\t%s\n' \ + | sed -E 's/-[0-9]{14}(\.|-)/-TIMESTAMP\1/g' \ + | sort -u > "$output_dir/deploy-layout-and-sizes.txt" +# The expression belongs to awk; shell expansion would be a bug. +# shellcheck disable=SC2016 +find "$deploy_dir" -maxdepth 1 -type f -name '*.manifest' -print0 \ + | sort -z \ + | xargs -0 -r awk '{print $1}' \ + | sort -u > "$output_dir/packages.txt" + +# New warnings are regressions even when BitBake returns zero. +find build/tmp/log -type f -name 'console-latest.log' -print0 2>/dev/null \ + | xargs -0 -r grep -hE '(^|[[:space:]])WARNING:' \ + | sed -E "s#$PWD/##g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}[^ ]*//g" \ + | sort -u > "$output_dir/warnings.txt" || true diff --git a/scripts/validation/compare-layer-state.py b/scripts/validation/compare-layer-state.py new file mode 100755 index 00000000..13b0e77a --- /dev/null +++ b/scripts/validation/compare-layer-state.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +"""Fail on unexplained baseline/candidate build-state differences.""" + +from __future__ import annotations + +import argparse +import difflib +import json +import re +import sys +from pathlib import Path + + +def files(root: Path) -> dict[str, list[str]]: + result: dict[str, list[str]] = {} + for path in sorted(root.rglob("*")): + if path.is_file(): + result[str(path.relative_to(root))] = path.read_text(errors="replace").splitlines() + return result + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--candidate", required=True, type=Path) + parser.add_argument("--tuple", required=True) + parser.add_argument("--contract", required=True, type=Path) + args = parser.parse_args() + + contract = json.loads(args.contract.read_text()) + rules = contract.get("allowed_deltas", {}).get(args.tuple, []) + compiled: list[tuple[re.Pattern[str], re.Pattern[str], str]] = [] + for rule in rules: + try: + compiled.append(( + re.compile(rule["file"]), + re.compile(rule["pattern"]), + str(rule["reason"]).strip(), + )) + except (KeyError, re.error) as exc: + print(f"ERROR: invalid allow rule for {args.tuple}: {exc}", file=sys.stderr) + return 2 + if any(not reason for _, _, reason in compiled): + print("ERROR: every allowed delta needs a reason", file=sys.stderr) + return 2 + + old, new = files(args.baseline), files(args.candidate) + unexplained: list[str] = [] + for name in sorted(set(old) | set(new)): + if old.get(name) == new.get(name): + continue + delta = list(difflib.unified_diff(old.get(name, []), new.get(name, []), lineterm="")) + changed_lines = [ + line[1:] for line in delta + if line.startswith(("+", "-")) and not line.startswith(("+++", "---")) + ] + for line in changed_lines: + if not any(file_re.search(name) and line_re.search(line) for file_re, line_re, _ in compiled): + unexplained.append(f"{name}: {line}") + + if unexplained: + print(f"ERROR: unexplained build deltas for {args.tuple}:", file=sys.stderr) + for line in unexplained[:250]: + print(f" {line}", file=sys.stderr) + if len(unexplained) > 250: + print(f" ... {len(unexplained) - 250} more", file=sys.stderr) + return 1 + print(f"PASS: {args.tuple} has no unexplained layer-adoption delta") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/validation/detect-layer-adoption.py b/scripts/validation/detect-layer-adoption.py new file mode 100755 index 00000000..012ce88b --- /dev/null +++ b/scripts/validation/detect-layer-adoption.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python3 +"""Detect layer topology/pin changes and enforce an explicit adoption contract.""" + +from __future__ import annotations + +import argparse +import json +import os +import re +import subprocess +import sys +from pathlib import Path + + +MATERIAL_PATHS = ( + re.compile(r"^\.gitmodules$"), + re.compile(r"(^|/)conf/layer\.conf$"), + re.compile(r"^kas/.*\.ya?ml$"), +) +def git(*args: str) -> str: + return subprocess.check_output(["git", *args], text=True) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--base", required=True) + parser.add_argument("--head", default="HEAD") + parser.add_argument("--contract", default="ci/layer-adoption-contract.json") + parser.add_argument("--github-output") + args = parser.parse_args() + + changed = git("diff", "--name-only", f"{args.base}...{args.head}").splitlines() + material_files = [ + path for path in changed + if any(pattern.search(path) for pattern in MATERIAL_PATHS) + ] + # Treat every KAS change as material. YAML context makes line-only pin + # detection easy to evade (for example by adding a list item below an + # existing `includes:` key), and a false-positive build is safer than a + # layer adoption escaping the hard gate. + material = bool(material_files) + + if material: + if args.contract not in changed: + print( + f"ERROR: material Yocto layer change requires {args.contract} " + "to be updated in the same change", + file=sys.stderr, + ) + return 2 + try: + contract = json.loads(Path(args.contract).read_text()) + except (OSError, json.JSONDecodeError) as exc: + print(f"ERROR: invalid adoption contract: {exc}", file=sys.stderr) + return 2 + if contract.get("schema") != 1 or not str(contract.get("reason", "")).strip(): + print("ERROR: contract needs schema=1 and a non-empty reason", file=sys.stderr) + return 2 + if not isinstance(contract.get("allowed_deltas"), dict): + print("ERROR: allowed_deltas must be an object", file=sys.stderr) + return 2 + + result = "true" if material else "false" + print(f"material={result}") + if material_files: + print("material candidates:") + for path in material_files: + print(f" {path}") + if args.github_output: + with open(args.github_output, "a", encoding="utf-8") as output: + output.write(f"material={result}\n") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From 82f2c51192c3c6ff9edea820baf51218a1fc614b Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:52:51 +0100 Subject: [PATCH 14/79] ci: fix layer gate expressions --- .github/workflows/layer-adoption-gate.yml | 50 +++++++++++------------ 1 file changed, 25 insertions(+), 25 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 4813c67a..8c51d4ec 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -12,7 +12,7 @@ on: required: true concurrency: - group: layer-adoption-${${ github.event.pull_request.number || github.ref }} + group: layer-adoption-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true jobs: @@ -20,9 +20,9 @@ jobs: name: Detect material layer change runs-on: [self-hosted, Linux, X64] outputs: - material: ${${ steps.detect.outputs.material }} - matrix: ${${ steps.matrix.outputs.matrix }} - base_sha: ${${ steps.base.outputs.sha }} + material: ${{ steps.detect.outputs.material }} + matrix: ${{ steps.matrix.outputs.matrix }} + base_sha: ${{ steps.base.outputs.sha }} steps: - uses: actions/checkout@v4 with: @@ -30,9 +30,9 @@ jobs: - id: base name: Resolve immutable baseline env: - PR_BASE: ${${ github.event.pull_request.base.sha }} - PUSH_BASE: ${${ github.event.before }} - INPUT_BASE: ${${ inputs.base_sha }} + PR_BASE: ${{ github.event.pull_request.base.sha }} + PUSH_BASE: ${{ github.event.before }} + INPUT_BASE: ${{ inputs.base_sha }} run: | sha="${PR_BASE:-${INPUT_BASE:-${PUSH_BASE:-}}}" if [ -z "$sha" ] || printf '%s' "$sha" | grep -Eq '^0+$'; then @@ -44,19 +44,19 @@ jobs: name: Require an adoption contract run: | python3 scripts/validation/detect-layer-adoption.py \ - --base '${${ steps.base.outputs.sha }}' \ - --head '${${ github.sha }}' \ + --base '${{ steps.base.outputs.sha }}' \ + --head '${{ github.sha }}' \ --github-output "$GITHUB_OUTPUT" - id: matrix run: echo "matrix=$(jq -c '{include:.tuples}' ci/layer-adoption-tuples.json)" >> "$GITHUB_OUTPUT" regression: - name: Protect ${${ matrix.id }} + name: Protect ${{ matrix.id }} needs: detect if: needs.detect.outputs.material == 'true' strategy: fail-fast: false - matrix: ${${ fromJSON(needs.detect.outputs.matrix) }} + matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} runs-on: [self-hosted, Linux, X64] container: image: dynamicdevices/yocto-ci-build:latest @@ -68,7 +68,7 @@ jobs: path: candidate - name: Create baseline worktree working-directory: candidate - run: git worktree add ../baseline '${${ needs.detect.outputs.base_sha }}' + run: git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' - name: Install KAS when absent run: command -v kas >/dev/null || pip3 install kas - name: Restore Yocto caches @@ -77,34 +77,34 @@ jobs: path: | ~/yocto/downloads ~/yocto/sstate-cache - key: layer-adoption-${${ matrix.machine }}-${${ hashFiles('candidate/kas/**') }} + key: layer-adoption-${{ matrix.machine }}-${{ hashFiles('candidate/kas/**') }} restore-keys: | - layer-adoption-${${ matrix.machine }}- + layer-adoption-${{ matrix.machine }}- layer-adoption- - name: Build and capture baseline working-directory: baseline run: | ../candidate/scripts/validation/capture-layer-state.sh \ - '${${ matrix.config }}' '${${ matrix.machine }}' '${${ matrix.image }}' \ - '../evidence/baseline/${${ matrix.id }}' + '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.image }}' \ + '../evidence/baseline/${{ matrix.id }}' - name: Build and capture candidate working-directory: candidate run: | scripts/validation/capture-layer-state.sh \ - '${${ matrix.config }}' '${${ matrix.machine }}' '${${ matrix.image }}' \ - '../evidence/candidate/${${ matrix.id }}' + '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.image }}' \ + '../evidence/candidate/${{ matrix.id }}' - name: Reject unexplained contamination run: | python3 candidate/scripts/validation/compare-layer-state.py \ - --baseline 'evidence/baseline/${${ matrix.id }}' \ - --candidate 'evidence/candidate/${${ matrix.id }}' \ - --tuple '${${ matrix.id }}' \ + --baseline 'evidence/baseline/${{ matrix.id }}' \ + --candidate 'evidence/candidate/${{ matrix.id }}' \ + --tuple '${{ matrix.id }}' \ --contract candidate/ci/layer-adoption-contract.json - name: Preserve comparison evidence if: always() uses: actions/upload-artifact@v4 with: - name: layer-adoption-${${ matrix.id }} + name: layer-adoption-${{ matrix.id }} path: evidence retention-days: 14 @@ -116,9 +116,9 @@ jobs: steps: - name: Enforce gate result env: - DETECT: ${${ needs.detect.result }} - MATERIAL: ${${ needs.detect.outputs.material }} - REGRESSION: ${${ needs.regression.result }} + DETECT: ${{ needs.detect.result }} + MATERIAL: ${{ needs.detect.outputs.material }} + REGRESSION: ${{ needs.regression.result }} run: | test "$DETECT" = success if [ "$MATERIAL" = true ]; then From 6b8e574521c8e70ba2a2151acca9bf12dfe705d8 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:53:20 +0100 Subject: [PATCH 15/79] ci: bound layer gate build concurrency --- .github/workflows/layer-adoption-gate.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 8c51d4ec..c4d2f19b 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -56,6 +56,9 @@ jobs: if: needs.detect.outputs.material == 'true' strategy: fail-fast: false + # Full Yocto baseline/candidate pairs are I/O and disk intensive. Keep + # the hard gate comprehensive without overwhelming one self-hosted host. + max-parallel: 2 matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} runs-on: [self-hosted, Linux, X64] container: From 71c01cda59edafed0896f7a0a6ff6cc06d646fe5 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:54:38 +0100 Subject: [PATCH 16/79] ci: use current Actions runtimes --- .github/workflows/layer-adoption-gate.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index c4d2f19b..28438a2d 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -24,7 +24,7 @@ jobs: matrix: ${{ steps.matrix.outputs.matrix }} base_sha: ${{ steps.base.outputs.sha }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 - id: base @@ -65,7 +65,7 @@ jobs: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v7 with: fetch-depth: 0 path: candidate @@ -75,7 +75,7 @@ jobs: - name: Install KAS when absent run: command -v kas >/dev/null || pip3 install kas - name: Restore Yocto caches - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: | ~/yocto/downloads @@ -105,7 +105,7 @@ jobs: --contract candidate/ci/layer-adoption-contract.json - name: Preserve comparison evidence if: always() - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: layer-adoption-${{ matrix.id }} path: evidence From c73dd4b967dbd1bfcbb06198b8c4f106a638dc03 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 20:57:42 +0100 Subject: [PATCH 17/79] ci: emit layer matrix without jq --- .github/workflows/layer-adoption-gate.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 28438a2d..5b165316 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -48,7 +48,17 @@ jobs: --head '${{ github.sha }}' \ --github-output "$GITHUB_OUTPUT" - id: matrix - run: echo "matrix=$(jq -c '{include:.tuples}' ci/layer-adoption-tuples.json)" >> "$GITHUB_OUTPUT" + name: Publish protected tuple matrix + run: | + python3 - <<'PY' + import json + import os + + with open("ci/layer-adoption-tuples.json", encoding="utf-8") as source: + matrix = {"include": json.load(source)["tuples"]} + with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + output.write("matrix=" + json.dumps(matrix, separators=(",", ":")) + "\n") + PY regression: name: Protect ${{ matrix.id }} From 58bb309a78024bf538fe8d4d04ce0be81c0a64cc Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 21:02:51 +0100 Subject: [PATCH 18/79] ci: route Yocto jobs to capable runner --- .github/actionlint.yaml | 3 ++ .github/workflows/kas-build-ci.yml | 62 +++++++++++------------ .github/workflows/layer-adoption-gate.yml | 2 +- 3 files changed, 35 insertions(+), 32 deletions(-) create mode 100644 .github/actionlint.yaml diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml new file mode 100644 index 00000000..cfd084cf --- /dev/null +++ b/.github/actionlint.yaml @@ -0,0 +1,3 @@ +self-hosted-runner: + labels: + - yocto diff --git a/.github/workflows/kas-build-ci.yml b/.github/workflows/kas-build-ci.yml index e1bf06f3..517dcb41 100644 --- a/.github/workflows/kas-build-ci.yml +++ b/.github/workflows/kas-build-ci.yml @@ -46,22 +46,21 @@ jobs: # Validation job - validates Yocto layer compatibility validate: name: Validate Yocto Layers - # Pin to a Linux self-hosted runner: bare `self-hosted` also matches the - # org's macOS runners, which have no Docker and fail this container job - # in ~6s with "docker: command not found". - runs-on: [self-hosted, Linux, X64] + # Pin container work to the dedicated Yocto runner. Generic Linux labels + # also match esl-nixos, which intentionally has no Docker daemon. + runs-on: [self-hosted, Linux, X64, yocto] container: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v7 with: submodules: recursive - name: Cache KAS layers - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: | build/layers @@ -72,7 +71,7 @@ jobs: kas-layers- - name: Cache Yocto downloads - uses: actions/cache@v4 + uses: actions/cache@v6 with: path: ~/yocto/downloads key: yocto-downloads-${{ runner.os }}-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }} @@ -177,27 +176,28 @@ jobs: steps: - name: Generate validation summary run: | - echo "# KAS Layer Validation Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "**Workflow:** ${{ github.workflow }}" >> $GITHUB_STEP_SUMMARY - echo "**Trigger:** ${{ github.event_name }}" >> $GITHUB_STEP_SUMMARY - echo "**Commit:** ${{ github.sha }}" >> $GITHUB_STEP_SUMMARY - echo "**Branch:** ${{ github.ref_name }}" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - - # Determine overall status - if [ "${{ needs.validate.result }}" = "success" ]; then - echo "## ✅ Validation Status: SUCCESS" >> $GITHUB_STEP_SUMMARY - echo "All meta-dynamicdevices layers pass comprehensive yocto-check-layer validation!" >> $GITHUB_STEP_SUMMARY - echo "Layers are ready for Yocto Project compatibility." >> $GITHUB_STEP_SUMMARY - else - echo "## ❌ Validation Status: FAILED" >> $GITHUB_STEP_SUMMARY - echo "- Layer validation: ${{ needs.validate.result }}" >> $GITHUB_STEP_SUMMARY - echo "Please fix validation issues before proceeding." >> $GITHUB_STEP_SUMMARY - fi - - echo "" >> $GITHUB_STEP_SUMMARY - echo "## Validation Coverage" >> $GITHUB_STEP_SUMMARY - echo "- **Tool:** Official yocto-check-layer" >> $GITHUB_STEP_SUMMARY - echo "- **Layers:** meta-dynamicdevices, meta-dynamicdevices-bsp, meta-dynamicdevices-distro" >> $GITHUB_STEP_SUMMARY - echo "- **Compliance:** Full Yocto Project compatibility validation" >> $GITHUB_STEP_SUMMARY \ No newline at end of file + { + echo "# KAS Layer Validation Summary" + echo + echo "**Workflow:** ${{ github.workflow }}" + echo "**Trigger:** ${{ github.event_name }}" + echo "**Commit:** ${{ github.sha }}" + echo "**Branch:** ${{ github.ref_name }}" + echo + + if [ "${{ needs.validate.result }}" = "success" ]; then + echo "## ✅ Validation Status: SUCCESS" + echo "All meta-dynamicdevices layers pass comprehensive yocto-check-layer validation!" + echo "Layers are ready for Yocto Project compatibility." + else + echo "## ❌ Validation Status: FAILED" + echo "- Layer validation: ${{ needs.validate.result }}" + echo "Please fix validation issues before proceeding." + fi + + echo + echo "## Validation Coverage" + echo "- **Tool:** Official yocto-check-layer" + echo "- **Layers:** meta-dynamicdevices, meta-dynamicdevices-bsp, meta-dynamicdevices-distro" + echo "- **Compliance:** Full Yocto Project compatibility validation" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 5b165316..8a232115 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -70,7 +70,7 @@ jobs: # the hard gate comprehensive without overwhelming one self-hosted host. max-parallel: 2 matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} - runs-on: [self-hosted, Linux, X64] + runs-on: [self-hosted, Linux, X64, yocto] container: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 From ce0b9da24c8e2c3d8124deffb7583e5a7a841135 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Fri, 11 Sep 2026 21:55:27 +0100 Subject: [PATCH 19/79] ci: strengthen layer adoption evidence --- .github/workflows/layer-adoption-gate.yml | 6 ++- scripts/validation/capture-layer-state.sh | 63 ++++++++++++++++++----- scripts/validation/compare-layer-state.py | 2 +- scripts/validation/select-bitbake-env.py | 49 ++++++++++++++++++ 4 files changed, 104 insertions(+), 16 deletions(-) create mode 100644 scripts/validation/select-bitbake-env.py diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 8a232115..4e151c63 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -98,13 +98,15 @@ jobs: working-directory: baseline run: | ../candidate/scripts/validation/capture-layer-state.sh \ - '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.image }}' \ + '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ + '${{ matrix.image }}' \ '../evidence/baseline/${{ matrix.id }}' - name: Build and capture candidate working-directory: candidate run: | scripts/validation/capture-layer-state.sh \ - '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.image }}' \ + '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ + '${{ matrix.image }}' \ '../evidence/candidate/${{ matrix.id }}' - name: Reject unexplained contamination run: | diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 02215d4e..50661cc7 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -2,15 +2,16 @@ # Build one protected tuple and capture deterministic layer/package/task state. set -euo pipefail -if [ "$#" -ne 4 ]; then - echo "Usage: $0 KAS_CONFIG MACHINE TARGET OUTPUT_DIR" >&2 +if [ "$#" -ne 5 ]; then + echo "Usage: $0 KAS_CONFIG MACHINE DISTRO TARGET OUTPUT_DIR" >&2 exit 2 fi config=$1 machine=$2 -target=$3 -output_dir=$4 +distro=$3 +target=$4 +output_dir=$5 case "$output_dir" in /*) ;; @@ -19,43 +20,75 @@ esac mkdir -p "$output_dir" export KAS_MACHINE="$machine" +export KAS_DISTRO="$distro" +export DISTRO="$distro" kas checkout "$config" +cat > "$output_dir/metadata.json" < "$output_dir/commands.txt" + # Static layer surfaces are captured as well as BitBake's resolved view. This # makes wildcard/dangling appends and global layer.conf policy visible even # when they do not happen to alter the first recipe selected by BitBake. -find build/layers -type f -path '*/conf/layer.conf' -print0 \ +find build/layers -type f \( -path '*/conf/layer.conf' -o -name '*.bbclass' \) -print0 \ | sort -z \ | xargs -0 grep -nHE \ - '(^|[[:space:]])(IMAGE_INSTALL|CORE_IMAGE_EXTRA_INSTALL|DISTRO_FEATURES|MACHINE_FEATURES|PACKAGECONFIG|PREFERRED_(VERSION|PROVIDER)|RDEPENDS)(:|[[:space:]])*([+?:.]?=)' \ + '(^|[[:space:]])(IMAGE_INSTALL|CORE_IMAGE_|PACKAGE_INSTALL|DISTRO_FEATURES|MACHINE_FEATURES|PACKAGECONFIG|PREFERRED_(VERSION|PROVIDER)|DEFAULT_PREFERENCE|RDEPENDS|INHERIT|BBMASK|BBPATH|BBFILES|BBFILE_PRIORITY|INITRAMFS_MAXSIZE|IMAGE_FSTYPES|WKS_FILE|UBOOT_|KERNEL_|OPTEE_|SDKIMAGE_FEATURES|TOOLCHAIN_TARGET_TASK)(:|\[|[[:space:]])*([+?:.]?=)' \ > "$output_dir/layer-conf-policy.txt" || true find build/layers -type f -name '*.bbappend' -printf '%p\n' \ - | sed -E 's#^build/layers/[^/]+/#LAYER/#' \ + | sed -E 's#^build/layers/##' \ | sort -u > "$output_dir/all-bbappends.txt" run_bitbake() { kas shell "$config" -c "$1" } -run_bitbake "bitbake-layers show-layers" \ +capture_command() { + local name=$1 + shift + "$@" 2>&1 | tee "$output_dir/$name.log" +} + +capture_command show-layers run_bitbake "bitbake-layers show-layers" \ | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ > "$output_dir/layers.txt" -run_bitbake "bitbake-layers show-appends" \ +capture_command show-appends run_bitbake "bitbake-layers show-appends" \ | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ > "$output_dir/appends.txt" -run_bitbake "bitbake-layers show-recipes" \ +capture_command show-recipes run_bitbake "bitbake-layers show-recipes" \ | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ > "$output_dir/recipes.txt" -run_bitbake "bitbake -g $target" +capture_command graph run_bitbake "bitbake -g $target" sort -u build/pn-buildlist > "$output_dir/pn-buildlist.txt" sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ > "$output_dir/task-depends.dot" +sed -E "s#$PWD/##g" build/recipe-depends.dot | sort -u \ + > "$output_dir/recipe-depends.dot" + +# Capture final values and BitBake's assignment provenance for policy that a +# newly enabled layer can silently change. The full environment is retained +# temporarily only as input, avoiding volatile host variables in comparisons. +capture_command environment run_bitbake "bitbake -e $target" +python3 "$(dirname "$0")/select-bitbake-env.py" \ + "$output_dir/environment.log" > "$output_dir/selected-environment.txt" +grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" +grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" +rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, # or deploy layout still works. Complete the real image/recovery build for both # baseline and candidate. -run_bitbake "bitbake $target" +capture_command build run_bitbake "bitbake $target" deploy_dir="build/tmp/deploy/images/$machine" if [ ! -d "$deploy_dir" ]; then @@ -74,7 +107,11 @@ find "$deploy_dir" -maxdepth 1 -type f -name '*.manifest' -print0 \ | sort -u > "$output_dir/packages.txt" # New warnings are regressions even when BitBake returns zero. -find build/tmp/log -type f -name 'console-latest.log' -print0 2>/dev/null \ +find "$output_dir" -type f -name '*.log' -print0 \ | xargs -0 -r grep -hE '(^|[[:space:]])WARNING:' \ | sed -E "s#$PWD/##g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}[^ ]*//g" \ | sort -u > "$output_dir/warnings.txt" || true + +# Raw command logs are useful for diagnosis but contain progress ordering and +# timing noise. The deterministic projections above are the comparison input. +rm -f "$output_dir"/*.log diff --git a/scripts/validation/compare-layer-state.py b/scripts/validation/compare-layer-state.py index 13b0e77a..cf38a03f 100755 --- a/scripts/validation/compare-layer-state.py +++ b/scripts/validation/compare-layer-state.py @@ -14,7 +14,7 @@ def files(root: Path) -> dict[str, list[str]]: result: dict[str, list[str]] = {} for path in sorted(root.rglob("*")): - if path.is_file(): + if path.is_file() and path.name != "metadata.json": result[str(path.relative_to(root))] = path.read_text(errors="replace").splitlines() return result diff --git a/scripts/validation/select-bitbake-env.py b/scripts/validation/select-bitbake-env.py new file mode 100644 index 00000000..3a72d87d --- /dev/null +++ b/scripts/validation/select-bitbake-env.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python3 +"""Project BitBake -e output to stable policy values with assignment history.""" + +from __future__ import annotations + +import re +import sys +from pathlib import Path + + +EXACT = { + "BBMASK", "CORE_IMAGE_BASE_INSTALL", "CORE_IMAGE_EXTRA_INSTALL", + "DISTRO", "DISTRO_FEATURES", "IMAGE_BOOT_FILES", "IMAGE_FEATURES", + "IMAGE_FSTYPES", "IMAGE_INSTALL", "IMAGE_ROOTFS_EXTRA_SPACE", + "IMAGE_ROOTFS_SIZE", "INITRAMFS_FSTYPES", "INITRAMFS_IMAGE", + "INITRAMFS_MAXSIZE", "MACHINE", "MACHINE_FEATURES", + "PACKAGE_INSTALL", "SDKIMAGE_FEATURES", "TOOLCHAIN_TARGET_TASK", + "WKS_FILE", +} +PREFIXES = ( + "FIT_", "KERNEL_", "OPTEE_", "OSTREE_", "PREFERRED_PROVIDER_", + "PREFERRED_VERSION_", "SOTA_", "UBOOT_", +) +ASSIGNMENT = re.compile(r'^([A-Za-z0-9_${}/:.+-]+)=') + + +def selected(name: str) -> bool: + return name in EXACT or name.startswith(PREFIXES) + + +def main() -> int: + if len(sys.argv) != 2: + print(f"Usage: {sys.argv[0]} BITBAKE_ENV", file=sys.stderr) + return 2 + comments: list[str] = [] + for line in Path(sys.argv[1]).read_text(errors="replace").splitlines(): + if line.startswith("#"): + comments.append(line) + continue + match = ASSIGNMENT.match(line) + if match and selected(match.group(1)): + print("\n".join(comments[-40:])) + print(line) + comments.clear() + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) From 372d90c70aac023e1fff3076309093af1c015f8a Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 00:01:52 +0100 Subject: [PATCH 20/79] ci: preserve protected layer adoption tuples --- scripts/validation/detect-layer-adoption.py | 75 +++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/scripts/validation/detect-layer-adoption.py b/scripts/validation/detect-layer-adoption.py index 012ce88b..9b7fa0e0 100755 --- a/scripts/validation/detect-layer-adoption.py +++ b/scripts/validation/detect-layer-adoption.py @@ -16,19 +16,94 @@ re.compile(r"^\.gitmodules$"), re.compile(r"(^|/)conf/layer\.conf$"), re.compile(r"^kas/.*\.ya?ml$"), + re.compile(r"^ci/layer-adoption-tuples\.json$"), ) + +REQUIRED_TUPLE_FIELDS = ("id", "machine", "distro", "image", "config") + + def git(*args: str) -> str: return subprocess.check_output(["git", *args], text=True) +def parse_tuples(raw: str, source: str) -> dict[str, dict[str, str]]: + try: + document = json.loads(raw) + except json.JSONDecodeError as exc: + raise ValueError(f"{source}: invalid JSON: {exc}") from exc + if document.get("schema") != 1 or not isinstance(document.get("tuples"), list): + raise ValueError(f"{source}: expected schema=1 and a tuples array") + + result: dict[str, dict[str, str]] = {} + for index, entry in enumerate(document["tuples"]): + if not isinstance(entry, dict): + raise ValueError(f"{source}: tuple {index} is not an object") + missing = [field for field in REQUIRED_TUPLE_FIELDS if not str(entry.get(field, "")).strip()] + if missing: + raise ValueError(f"{source}: tuple {index} lacks {', '.join(missing)}") + tuple_id = str(entry["id"]) + if tuple_id in result: + raise ValueError(f"{source}: duplicate tuple id {tuple_id}") + result[tuple_id] = {field: str(entry[field]) for field in REQUIRED_TUPLE_FIELDS} + if not result: + raise ValueError(f"{source}: tuple matrix must not be empty") + return result + + +def validate_tuple_matrix(base: str, path: Path) -> None: + candidate = parse_tuples(path.read_text(encoding="utf-8"), str(path)) + missing_configs = sorted( + tuple_id for tuple_id, entry in candidate.items() + if not Path(entry["config"]).is_file() + ) + if missing_configs: + raise ValueError( + f"{path}: tuple configs do not exist for: {', '.join(missing_configs)}" + ) + + baseline_result = subprocess.run( + ["git", "show", f"{base}:{path.as_posix()}"], + check=False, + capture_output=True, + text=True, + ) + if baseline_result.returncode != 0: + # Bootstrap case: the gate and its matrix are being introduced together. + return + baseline_raw = baseline_result.stdout + baseline = parse_tuples(baseline_raw, f"{base}:{path}") + removed = sorted(set(baseline) - set(candidate)) + changed = sorted( + tuple_id for tuple_id in set(baseline) & set(candidate) + if baseline[tuple_id] != candidate[tuple_id] + ) + if removed or changed: + details = [] + if removed: + details.append("removed=" + ",".join(removed)) + if changed: + details.append("redefined=" + ",".join(changed)) + raise ValueError( + "protected baseline tuples may only be extended, not removed or redefined (" + + "; ".join(details) + ")" + ) + + def main() -> int: parser = argparse.ArgumentParser() parser.add_argument("--base", required=True) parser.add_argument("--head", default="HEAD") parser.add_argument("--contract", default="ci/layer-adoption-contract.json") + parser.add_argument("--tuples", default="ci/layer-adoption-tuples.json") parser.add_argument("--github-output") args = parser.parse_args() + try: + validate_tuple_matrix(args.base, Path(args.tuples)) + except (OSError, ValueError) as exc: + print(f"ERROR: invalid protected tuple matrix: {exc}", file=sys.stderr) + return 2 + changed = git("diff", "--name-only", f"{args.base}...{args.head}").splitlines() material_files = [ path for path in changed From 9d6bb5ad5e7e97015eb3abb751bf4b0f3da1c544 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 00:03:57 +0100 Subject: [PATCH 21/79] ci: test protected layer adoption matrix --- .github/workflows/layer-adoption-gate.yml | 1 + .../tests/test_detect_layer_adoption.py | 68 +++++++++++++++++++ 2 files changed, 69 insertions(+) create mode 100644 scripts/validation/tests/test_detect_layer_adoption.py diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 4e151c63..02b59dba 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -43,6 +43,7 @@ jobs: - id: detect name: Require an adoption contract run: | + python3 -m unittest discover -s scripts/validation/tests -p 'test_*.py' python3 scripts/validation/detect-layer-adoption.py \ --base '${{ steps.base.outputs.sha }}' \ --head '${{ github.sha }}' \ diff --git a/scripts/validation/tests/test_detect_layer_adoption.py b/scripts/validation/tests/test_detect_layer_adoption.py new file mode 100644 index 00000000..d264dbbc --- /dev/null +++ b/scripts/validation/tests/test_detect_layer_adoption.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Regression tests for protected layer-adoption tuple handling.""" + +from __future__ import annotations + +import importlib.util +import json +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest import mock + + +MODULE_PATH = Path(__file__).parents[1] / "detect-layer-adoption.py" +SPEC = importlib.util.spec_from_file_location("detect_layer_adoption", MODULE_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +def document(*tuples: dict[str, str]) -> str: + return json.dumps({"schema": 1, "tuples": list(tuples)}) + + +def entry(tuple_id: str, machine: str = "machine-a") -> dict[str, str]: + return { + "id": tuple_id, + "machine": machine, + "distro": "distro-a", + "image": "image-a", + "config": "kas/test.yml", + } + + +class ProtectedTupleTests(unittest.TestCase): + def validate(self, baseline: str, candidate: str) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "kas").mkdir() + (root / "kas/test.yml").touch() + matrix = root / "ci/layer-adoption-tuples.json" + matrix.parent.mkdir() + matrix.write_text(candidate, encoding="utf-8") + result = subprocess.CompletedProcess([], 0, stdout=baseline, stderr="") + with mock.patch.object(MODULE.subprocess, "run", return_value=result), mock.patch.object( + MODULE.Path, "is_file", return_value=True + ): + MODULE.validate_tuple_matrix("baseline", matrix) + + def test_extension_preserves_existing_tuple(self) -> None: + self.validate(document(entry("existing")), document(entry("existing"), entry("new"))) + + def test_removing_existing_tuple_fails(self) -> None: + with self.assertRaisesRegex(ValueError, "removed=existing"): + self.validate(document(entry("existing")), document(entry("replacement"))) + + def test_redefining_existing_tuple_fails(self) -> None: + with self.assertRaisesRegex(ValueError, "redefined=existing"): + self.validate(document(entry("existing")), document(entry("existing", "machine-b"))) + + def test_duplicate_candidate_id_fails(self) -> None: + with self.assertRaisesRegex(ValueError, "duplicate tuple id existing"): + self.validate(document(entry("existing")), document(entry("existing"), entry("existing"))) + + +if __name__ == "__main__": + unittest.main() From 06f1536da6d0e49d16b8f8aa9890bb88571fd9d3 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 01:55:44 +0100 Subject: [PATCH 22/79] ci: protect exact product feature tuples --- .github/workflows/layer-adoption-gate.yml | 4 +- ci/layer-adoption-tuples.json | 28 +++++++------- scripts/validation/capture-layer-state.sh | 38 +++++++++++++++---- scripts/validation/detect-layer-adoption.py | 11 ++++-- scripts/validation/select-bitbake-env.py | 1 + .../tests/test_detect_layer_adoption.py | 13 +++++++ 6 files changed, 68 insertions(+), 27 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 02b59dba..ff0866eb 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -100,14 +100,14 @@ jobs: run: | ../candidate/scripts/validation/capture-layer-state.sh \ '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ - '${{ matrix.image }}' \ + '${{ matrix.image }}' '${{ matrix.product_features }}' \ '../evidence/baseline/${{ matrix.id }}' - name: Build and capture candidate working-directory: candidate run: | scripts/validation/capture-layer-state.sh \ '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ - '${{ matrix.image }}' \ + '${{ matrix.image }}' '${{ matrix.product_features }}' \ '../evidence/candidate/${{ matrix.id }}' - name: Reject unexplained contamination run: | diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index acace193..bb032edd 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -1,20 +1,20 @@ { "schema": 1, "tuples": [ - {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-inst-image", "machine": "imx8mm-jaguar-inst", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-phasora-image", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, - {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml"}, + {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv usb-gadget"}, + {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-inst-image", "machine": "imx8mm-jaguar-inst", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, + {"id": "imx8mm-jaguar-phasora-image", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "display flutter godot"}, + {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, + {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-inst-mfgtool", "machine": "imx8mm-jaguar-inst", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-phasora-mfgtool", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"}, - {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml"} + {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-inst-mfgtool", "machine": "imx8mm-jaguar-inst", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-phasora-mfgtool", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, + {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""} ] } diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 50661cc7..a7132017 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -2,8 +2,8 @@ # Build one protected tuple and capture deterministic layer/package/task state. set -euo pipefail -if [ "$#" -ne 5 ]; then - echo "Usage: $0 KAS_CONFIG MACHINE DISTRO TARGET OUTPUT_DIR" >&2 +if [ "$#" -ne 6 ]; then + echo "Usage: $0 KAS_CONFIG MACHINE DISTRO TARGET PRODUCT_FEATURES OUTPUT_DIR" >&2 exit 2 fi @@ -11,7 +11,8 @@ config=$1 machine=$2 distro=$3 target=$4 -output_dir=$5 +product_features=$5 +output_dir=$6 case "$output_dir" in /*) ;; @@ -22,19 +23,36 @@ mkdir -p "$output_dir" export KAS_MACHINE="$machine" export KAS_DISTRO="$distro" export DISTRO="$distro" -kas checkout "$config" + +# KAS deliberately sanitises the environment before entering BitBake's build +# environment, so an exported DD_PRODUCT_FEATURES is silently lost. Inject the +# reviewed tuple value through a generated KAS overlay instead. JSON string +# quoting is also valid BitBake quoting and prevents tuple text from becoming +# local.conf syntax. +product_features_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$product_features") +overlay="$output_dir/layer-adoption-product-features.yml" +cat > "$overlay" < "$output_dir/metadata.json" < "$output_dir/commands.txt" + "bitbake $target" \ + "DD_PRODUCT_FEATURES=$product_features" > "$output_dir/commands.txt" # Static layer surfaces are captured as well as BitBake's resolved view. This # makes wildcard/dangling appends and global layer.conf policy visible even @@ -49,7 +67,7 @@ find build/layers -type f -name '*.bbappend' -printf '%p\n' \ | sort -u > "$output_dir/all-bbappends.txt" run_bitbake() { - kas shell "$config" -c "$1" + kas shell "$combined_config" -c "$1" } capture_command() { @@ -83,6 +101,7 @@ python3 "$(dirname "$0")/select-bitbake-env.py" \ "$output_dir/environment.log" > "$output_dir/selected-environment.txt" grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" +grep -Fqx "DD_PRODUCT_FEATURES=\"$product_features\"" "$output_dir/selected-environment.txt" rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, @@ -115,3 +134,6 @@ find "$output_dir" -type f -name '*.log' -print0 \ # Raw command logs are useful for diagnosis but contain progress ordering and # timing noise. The deterministic projections above are the comparison input. rm -f "$output_dir"/*.log +# The generated overlay is an input already represented in metadata.json; it +# must not become a baseline/candidate comparison artefact with differing paths. +rm -f "$overlay" diff --git a/scripts/validation/detect-layer-adoption.py b/scripts/validation/detect-layer-adoption.py index 9b7fa0e0..b0911206 100755 --- a/scripts/validation/detect-layer-adoption.py +++ b/scripts/validation/detect-layer-adoption.py @@ -19,7 +19,8 @@ re.compile(r"^ci/layer-adoption-tuples\.json$"), ) -REQUIRED_TUPLE_FIELDS = ("id", "machine", "distro", "image", "config") +TUPLE_FIELDS = ("id", "machine", "distro", "image", "config", "product_features") +NONEMPTY_TUPLE_FIELDS = ("id", "machine", "distro", "image", "config") def git(*args: str) -> str: @@ -38,13 +39,17 @@ def parse_tuples(raw: str, source: str) -> dict[str, dict[str, str]]: for index, entry in enumerate(document["tuples"]): if not isinstance(entry, dict): raise ValueError(f"{source}: tuple {index} is not an object") - missing = [field for field in REQUIRED_TUPLE_FIELDS if not str(entry.get(field, "")).strip()] + missing = [field for field in TUPLE_FIELDS if field not in entry] + missing.extend( + field for field in NONEMPTY_TUPLE_FIELDS + if field in entry and not str(entry[field]).strip() + ) if missing: raise ValueError(f"{source}: tuple {index} lacks {', '.join(missing)}") tuple_id = str(entry["id"]) if tuple_id in result: raise ValueError(f"{source}: duplicate tuple id {tuple_id}") - result[tuple_id] = {field: str(entry[field]) for field in REQUIRED_TUPLE_FIELDS} + result[tuple_id] = {field: str(entry[field]) for field in TUPLE_FIELDS} if not result: raise ValueError(f"{source}: tuple matrix must not be empty") return result diff --git a/scripts/validation/select-bitbake-env.py b/scripts/validation/select-bitbake-env.py index 3a72d87d..3ab9d22a 100644 --- a/scripts/validation/select-bitbake-env.py +++ b/scripts/validation/select-bitbake-env.py @@ -10,6 +10,7 @@ EXACT = { "BBMASK", "CORE_IMAGE_BASE_INSTALL", "CORE_IMAGE_EXTRA_INSTALL", + "DD_PRODUCT_FEATURES", "DISTRO", "DISTRO_FEATURES", "IMAGE_BOOT_FILES", "IMAGE_FEATURES", "IMAGE_FSTYPES", "IMAGE_INSTALL", "IMAGE_ROOTFS_EXTRA_SPACE", "IMAGE_ROOTFS_SIZE", "INITRAMFS_FSTYPES", "INITRAMFS_IMAGE", diff --git a/scripts/validation/tests/test_detect_layer_adoption.py b/scripts/validation/tests/test_detect_layer_adoption.py index d264dbbc..211741e7 100644 --- a/scripts/validation/tests/test_detect_layer_adoption.py +++ b/scripts/validation/tests/test_detect_layer_adoption.py @@ -30,6 +30,7 @@ def entry(tuple_id: str, machine: str = "machine-a") -> dict[str, str]: "distro": "distro-a", "image": "image-a", "config": "kas/test.yml", + "product_features": "", } @@ -63,6 +64,18 @@ def test_duplicate_candidate_id_fails(self) -> None: with self.assertRaisesRegex(ValueError, "duplicate tuple id existing"): self.validate(document(entry("existing")), document(entry("existing"), entry("existing"))) + def test_missing_product_features_fails(self) -> None: + candidate = entry("existing") + del candidate["product_features"] + with self.assertRaisesRegex(ValueError, "lacks product_features"): + self.validate(document(entry("existing")), document(candidate)) + + def test_redefining_product_features_fails(self) -> None: + candidate = entry("existing") + candidate["product_features"] = "display" + with self.assertRaisesRegex(ValueError, "redefined=existing"): + self.validate(document(entry("existing")), document(candidate)) + if __name__ == "__main__": unittest.main() From b9207b9bdb2d3495fe6dd42148cf28c3a72df96b Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 02:01:25 +0100 Subject: [PATCH 23/79] ci: exercise signing in layer adoption gate --- .github/workflows/layer-adoption-gate.yml | 4 ++ scripts/validation/capture-layer-state.sh | 45 +++++++++++++++- .../generate-layer-adoption-test-keys.sh | 53 +++++++++++++++++++ scripts/validation/select-bitbake-env.py | 7 +-- 4 files changed, 105 insertions(+), 4 deletions(-) create mode 100755 scripts/validation/generate-layer-adoption-test-keys.sh diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index ff0866eb..4bc0cac6 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -75,6 +75,8 @@ jobs: container: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 + env: + LAYER_ADOPTION_TEST_KEYS_DIR: ${{ runner.temp }}/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} steps: - uses: actions/checkout@v7 with: @@ -95,6 +97,8 @@ jobs: restore-keys: | layer-adoption-${{ matrix.machine }}- layer-adoption- + - name: Generate ephemeral signing keys + run: candidate/scripts/validation/generate-layer-adoption-test-keys.sh "$LAYER_ADOPTION_TEST_KEYS_DIR" - name: Build and capture baseline working-directory: baseline run: | diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index a7132017..8d564173 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -13,6 +13,23 @@ distro=$3 target=$4 product_features=$5 output_dir=$6 +test_keys_dir=${LAYER_ADOPTION_TEST_KEYS_DIR:-} + +if [ -z "$test_keys_dir" ] || [ ! -d "$test_keys_dir" ]; then + echo "ERROR: LAYER_ADOPTION_TEST_KEYS_DIR must name the generated test-key directory" >&2 + exit 2 +fi +test_keys_dir=$(realpath "$test_keys_dir") +for key in \ + ubootdev.key ubootdev.crt spldev.key spldev.crt \ + privkey_modsign.pem x509_modsign.crt \ + uefi/DB.key uefi/DB.crt tf-a/privkey_ec_prime256v1.pem +do + if [ ! -s "$test_keys_dir/$key" ]; then + echo "ERROR: required test signing key is missing: $key" >&2 + exit 2 + fi +done case "$output_dir" in /*) ;; @@ -37,6 +54,19 @@ header: local_conf_header: layer-adoption-product-features: | DD_PRODUCT_FEATURES = $product_features_quoted + UBOOT_SIGN_KEYDIR = "$test_keys_dir" + UEFI_SIGN_KEYDIR = "$test_keys_dir/uefi" + MODSIGN_KEY_DIR = "$test_keys_dir" + SIGNING_UBOOT_SIGN_KEY = "$test_keys_dir/ubootdev.key" + SIGNING_UBOOT_SIGN_CRT = "$test_keys_dir/ubootdev.crt" + SIGNING_UBOOT_SPL_SIGN_KEY = "$test_keys_dir/spldev.key" + SIGNING_UBOOT_SPL_SIGN_CRT = "$test_keys_dir/spldev.crt" + SIGNING_MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem" + SIGNING_MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt" + SIGNING_UEFI_SIGN_KEY = "$test_keys_dir/uefi/DB.key" + SIGNING_UEFI_SIGN_CRT = "$test_keys_dir/uefi/DB.crt" + OPTEE_TA_SIGN_KEY = "$test_keys_dir/ubootdev.key" + TF_A_SIGN_KEY_PATH = "$test_keys_dir/tf-a/privkey_ec_prime256v1.pem" EOF combined_config="${config}:${overlay}" kas checkout "$combined_config" @@ -44,6 +74,17 @@ kas checkout "$combined_config" cat > "$output_dir/metadata.json" </dev/null \ + | sha256sum | cut -d ' ' -f 1) + printf '%s\t%s\n' "$key" "$fingerprint" +done > "$output_dir/test-signing-key-fingerprints.txt" printf '%s\n' \ "kas checkout CONFIG:PRODUCT_FEATURE_OVERLAY" \ "bitbake-layers show-layers" \ @@ -98,7 +139,9 @@ sed -E "s#$PWD/##g" build/recipe-depends.dot | sort -u \ # temporarily only as input, avoiding volatile host variables in comparisons. capture_command environment run_bitbake "bitbake -e $target" python3 "$(dirname "$0")/select-bitbake-env.py" \ - "$output_dir/environment.log" > "$output_dir/selected-environment.txt" + "$output_dir/environment.log" \ + | sed -E "s#$PWD##g; s#$test_keys_dir##g" \ + > "$output_dir/selected-environment.txt" grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" grep -Fqx "DD_PRODUCT_FEATURES=\"$product_features\"" "$output_dir/selected-environment.txt" diff --git a/scripts/validation/generate-layer-adoption-test-keys.sh b/scripts/validation/generate-layer-adoption-test-keys.sh new file mode 100755 index 00000000..4f3dc91c --- /dev/null +++ b/scripts/validation/generate-layer-adoption-test-keys.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Generate one genuine but disposable signing-key set shared by a baseline and candidate build. +set -euo pipefail +umask 077 + +if [ "$#" -ne 1 ]; then + echo "Usage: $0 OUTPUT_DIR" >&2 + exit 2 +fi + +output_dir=$(realpath -m "$1") +case "$output_dir" in + /|/home|/root|/tmp|/var|/usr) + echo "ERROR: refusing broad test-key output directory: $output_dir" >&2 + exit 2 + ;; +esac +if [ -e "$output_dir" ] && find "$output_dir" -mindepth 1 -print -quit | grep -q .; then + echo "ERROR: test-key output directory is not empty: $output_dir" >&2 + exit 2 +fi + +mkdir -p "$output_dir/uefi" "$output_dir/tf-a" + +make_rsa_certificate() { + local key=$1 + local certificate=$2 + local common_name=$3 + openssl genpkey -algorithm RSA -out "$key" -pkeyopt rsa_keygen_bits:2048 + openssl req -batch -new -x509 -sha256 -days 2 \ + -key "$key" -out "$certificate" -subj "/CN=$common_name/" + openssl pkey -in "$key" -check -noout + openssl x509 -in "$certificate" -noout +} + +make_rsa_certificate \ + "$output_dir/ubootdev.key" "$output_dir/ubootdev.crt" \ + layer-adoption-uboot +make_rsa_certificate \ + "$output_dir/spldev.key" "$output_dir/spldev.crt" \ + layer-adoption-spl +make_rsa_certificate \ + "$output_dir/privkey_modsign.pem" "$output_dir/x509_modsign.crt" \ + layer-adoption-module +make_rsa_certificate \ + "$output_dir/uefi/DB.key" "$output_dir/uefi/DB.crt" \ + layer-adoption-uefi + +openssl ecparam -name prime256v1 -genkey -noout \ + -out "$output_dir/tf-a/privkey_ec_prime256v1.pem" +openssl ec -in "$output_dir/tf-a/privkey_ec_prime256v1.pem" -check -noout + +printf 'PASS: generated ephemeral layer-adoption signing keys in %s\n' "$output_dir" diff --git a/scripts/validation/select-bitbake-env.py b/scripts/validation/select-bitbake-env.py index 3ab9d22a..c58ea9e5 100644 --- a/scripts/validation/select-bitbake-env.py +++ b/scripts/validation/select-bitbake-env.py @@ -14,13 +14,14 @@ "DISTRO", "DISTRO_FEATURES", "IMAGE_BOOT_FILES", "IMAGE_FEATURES", "IMAGE_FSTYPES", "IMAGE_INSTALL", "IMAGE_ROOTFS_EXTRA_SPACE", "IMAGE_ROOTFS_SIZE", "INITRAMFS_FSTYPES", "INITRAMFS_IMAGE", - "INITRAMFS_MAXSIZE", "MACHINE", "MACHINE_FEATURES", + "INITRAMFS_MAXSIZE", "LOCAL_DEVELOPMENT_BUILD", "MACHINE", "MACHINE_FEATURES", + "MODSIGN", "SIGN_ENABLE", "TF_A_SIGN_ENABLE", "UEFI_SIGN_ENABLE", "PACKAGE_INSTALL", "SDKIMAGE_FEATURES", "TOOLCHAIN_TARGET_TASK", "WKS_FILE", } PREFIXES = ( - "FIT_", "KERNEL_", "OPTEE_", "OSTREE_", "PREFERRED_PROVIDER_", - "PREFERRED_VERSION_", "SOTA_", "UBOOT_", + "FIT_", "KERNEL_", "MODSIGN_", "OPTEE_", "OSTREE_", "PREFERRED_PROVIDER_", + "PREFERRED_VERSION_", "SIGNING_", "SOTA_", "TF_A_", "UBOOT_", "UEFI_", ) ASSIGNMENT = re.compile(r'^([A-Za-z0-9_${}/:.+-]+)=') From 303978b4e163bae66819bf5f87346741782c4ef9 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 02:58:44 +0100 Subject: [PATCH 24/79] ci: use valid job-level key path --- .github/workflows/layer-adoption-gate.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 4bc0cac6..c59754c4 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -76,7 +76,7 @@ jobs: image: dynamicdevices/yocto-ci-build:latest options: --privileged --platform linux/amd64 env: - LAYER_ADOPTION_TEST_KEYS_DIR: ${{ runner.temp }}/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} + LAYER_ADOPTION_TEST_KEYS_DIR: /tmp/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} steps: - uses: actions/checkout@v7 with: From 4b2cfa7403a310e1dbc6b87853a8730e1d7ecbda Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 03:06:20 +0100 Subject: [PATCH 25/79] ci: bound layer gate disk usage --- .github/workflows/layer-adoption-gate.yml | 38 +++++++++++++++++++++++ scripts/validation/capture-layer-state.sh | 10 +++++- 2 files changed, 47 insertions(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index c59754c4..95b3df59 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -78,6 +78,16 @@ jobs: env: LAYER_ADOPTION_TEST_KEYS_DIR: /tmp/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} steps: + - name: Reset job-owned workspace + run: | + workspace=$(realpath -m -- "$GITHUB_WORKSPACE") + test -n "$workspace" + test "$workspace" != / + for name in candidate baseline evidence; do + target=$(realpath -m -- "$workspace/$name") + test "$(dirname "$target")" = "$workspace" + rm -rf -- "$target" + done - uses: actions/checkout@v7 with: fetch-depth: 0 @@ -97,6 +107,14 @@ jobs: restore-keys: | layer-adoption-${{ matrix.machine }}- layer-adoption- + - name: Require safe build capacity + run: | + available_kib=$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 {print $4}') + minimum_kib=$((150 * 1024 * 1024)) + if [ "$available_kib" -lt "$minimum_kib" ]; then + echo "ERROR: layer-adoption build requires at least 150 GiB free; found $((available_kib / 1024 / 1024)) GiB" >&2 + exit 1 + fi - name: Generate ephemeral signing keys run: candidate/scripts/validation/generate-layer-adoption-test-keys.sh "$LAYER_ADOPTION_TEST_KEYS_DIR" - name: Build and capture baseline @@ -106,6 +124,12 @@ jobs: '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ '${{ matrix.image }}' '${{ matrix.product_features }}' \ '../evidence/baseline/${{ matrix.id }}' + - name: Release baseline build workspace + run: | + workspace=$(realpath -m -- "$GITHUB_WORKSPACE") + target=$(realpath -m -- "$workspace/baseline/build") + test "$target" = "$workspace/baseline/build" + rm -rf -- "$target" - name: Build and capture candidate working-directory: candidate run: | @@ -127,6 +151,20 @@ jobs: name: layer-adoption-${{ matrix.id }} path: evidence retention-days: 14 + - name: Remove job-owned build trees + if: always() + run: | + workspace=$(realpath -m -- "$GITHUB_WORKSPACE") + test -n "$workspace" + test "$workspace" != / + for name in candidate/build baseline/build evidence; do + target=$(realpath -m -- "$workspace/$name") + case "$target" in + "$workspace/candidate/build"|"$workspace/baseline/build"|"$workspace/evidence") ;; + *) echo "ERROR: refusing unsafe cleanup target: $target" >&2; exit 1 ;; + esac + rm -rf -- "$target" + done required: name: Layer Adoption Gate diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 8d564173..4c325d81 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -14,12 +14,15 @@ target=$4 product_features=$5 output_dir=$6 test_keys_dir=${LAYER_ADOPTION_TEST_KEYS_DIR:-} +cache_root=${LAYER_ADOPTION_CACHE_DIR:-$HOME/yocto} if [ -z "$test_keys_dir" ] || [ ! -d "$test_keys_dir" ]; then echo "ERROR: LAYER_ADOPTION_TEST_KEYS_DIR must name the generated test-key directory" >&2 exit 2 fi test_keys_dir=$(realpath "$test_keys_dir") +mkdir -p "$cache_root/downloads" "$cache_root/sstate-cache" +cache_root=$(realpath "$cache_root") for key in \ ubootdev.key ubootdev.crt spldev.key spldev.crt \ privkey_modsign.pem x509_modsign.crt \ @@ -47,6 +50,8 @@ export DISTRO="$distro" # quoting is also valid BitBake quoting and prevents tuple text from becoming # local.conf syntax. product_features_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$product_features") +downloads_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/downloads") +sstate_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/sstate-cache") overlay="$output_dir/layer-adoption-product-features.yml" cat > "$overlay" <#g; s#$test_keys_dir##g" \ + | sed -E "s#$PWD##g; s#$test_keys_dir##g; s#$cache_root##g" \ > "$output_dir/selected-environment.txt" grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" From 5cf9737c4f39a6b9a77572833b0f69fecc1aed98 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:31:56 +0100 Subject: [PATCH 26/79] ci: consolidate layer adoption regression on ai-tools --- .github/workflows/kas-build-ci.yml | 203 ------------------ .github/workflows/layer-adoption-gate.yml | 101 +++------ .../generate-layer-adoption-test-keys.sh | 4 +- .../run-layer-adoption-regression.py | 178 +++++++++++++++ .../test_run_layer_adoption_regression.py | 46 ++++ 5 files changed, 261 insertions(+), 271 deletions(-) delete mode 100644 .github/workflows/kas-build-ci.yml create mode 100644 scripts/validation/run-layer-adoption-regression.py create mode 100644 scripts/validation/tests/test_run_layer_adoption_regression.py diff --git a/.github/workflows/kas-build-ci.yml b/.github/workflows/kas-build-ci.yml deleted file mode 100644 index 517dcb41..00000000 --- a/.github/workflows/kas-build-ci.yml +++ /dev/null @@ -1,203 +0,0 @@ -name: KAS Build CI - -on: - push: - branches: [ main, develop ] - paths: - - 'kas/**' - - 'meta-dynamicdevices-bsp/**' - - 'meta-dynamicdevices-distro/**' - - 'recipes-**' - - 'classes/**' - - 'conf/**' - - '.github/workflows/kas-build-ci.yml' - - 'scripts/kas-*.sh' - - 'scripts/validation/**' - - 'bbappends/**' - - 'custom-boot-files/**' - - pull_request: - branches: [ main, develop ] - paths: - - 'kas/**' - - 'meta-dynamicdevices-bsp/**' - - 'meta-dynamicdevices-distro/**' - - 'recipes-**' - - 'classes/**' - - 'conf/**' - - '.github/workflows/kas-build-ci.yml' - - 'scripts/kas-*.sh' - - 'scripts/validation/**' - - 'bbappends/**' - - 'custom-boot-files/**' - - workflow_dispatch: - inputs: - debug_enabled: - type: boolean - description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)' - required: false - default: false - -env: - DEBIAN_FRONTEND: noninteractive - -jobs: - # Validation job - validates Yocto layer compatibility - validate: - name: Validate Yocto Layers - # Pin container work to the dedicated Yocto runner. Generic Linux labels - # also match esl-nixos, which intentionally has no Docker daemon. - runs-on: [self-hosted, Linux, X64, yocto] - container: - image: dynamicdevices/yocto-ci-build:latest - options: --privileged --platform linux/amd64 - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - with: - submodules: recursive - - - name: Cache KAS layers - uses: actions/cache@v6 - with: - path: | - build/layers - build/cache - key: kas-layers-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }}-${{ github.sha }} - restore-keys: | - kas-layers-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }}- - kas-layers- - - - name: Cache Yocto downloads - uses: actions/cache@v6 - with: - path: ~/yocto/downloads - key: yocto-downloads-${{ runner.os }}-${{ hashFiles('kas/lmp-dynamicdevices-base.yml') }} - restore-keys: | - yocto-downloads-${{ runner.os }}- - - - name: Validate Yocto Layers - timeout-minutes: 15 - run: | - echo "🏅 Meta-DynamicDevices Layer Validation (CI)" - echo "=============================================" - echo "📋 Using official yocto-check-layer for Yocto Project compliance" - echo "" - - # Install KAS if not available - if ! command -v kas >/dev/null 2>&1; then - echo "📦 Installing KAS..." - pip3 install kas - fi - - # Create validation workspace - VALIDATION_DIR="ci-layer-validation" - rm -rf "$VALIDATION_DIR" - mkdir -p "$VALIDATION_DIR" - cd "$VALIDATION_DIR" - - echo "🔧 Setting up KAS environment for layer validation..." - - # Copy KAS configuration for validation - cp ../kas/layer-validation.yml . - - # Initialize KAS environment - echo "📋 Initializing KAS build environment..." - kas shell layer-validation.yml -c "echo 'KAS environment initialized'" - - echo "✅ KAS environment ready" - echo "" - - echo "🔍 Starting comprehensive layer validation..." - echo "" - - # Run yocto-check-layer validation - echo "1️⃣ Validating all meta-dynamicdevices layers together..." - - if kas shell layer-validation.yml -c " - # Use the yocto-check-layer script from openembedded-core - YOCTO_CHECK_LAYER='./layers/openembedded-core/scripts/yocto-check-layer' - - if [ ! -f \"\$YOCTO_CHECK_LAYER\" ]; then - echo '❌ yocto-check-layer script not found at expected location' - exit 1 - fi - - echo '✅ Found yocto-check-layer: '\$YOCTO_CHECK_LAYER - - # Clean up all potential conflicts from BitBake test data - rm -rf layers/bitbake/lib/layerindexlib/tests/testdata/ 2>/dev/null || true - find ../.. -name 'bitbake' -type d -exec rm -rf {}/lib/layerindexlib/tests/testdata/ 2>/dev/null \\; || true - - # Clean up any other build directories that might cause collection conflicts - find ../.. -maxdepth 2 -name 'build*' -type d ! -path '*/ci-layer-validation/build' -exec echo '🧹 Temporarily moving {}' \\; -exec mv {} {}.bak 2>/dev/null \\; || true - - # Run validation on all meta-dynamicdevices layers together to handle dependencies - echo '🔍 Running yocto-check-layer validation...' - python3 \"\$YOCTO_CHECK_LAYER\" \"$PWD/../meta-dynamicdevices-bsp\" \"$PWD/../meta-dynamicdevices-distro\" \"$PWD/..\" - - # Restore moved directories - find ../.. -maxdepth 2 -name 'build*.bak' -type d -exec sh -c 'mv \"\$1\" \"\${1%.bak}\"' _ {} \\; 2>/dev/null || true - "; then - echo "" - echo "✅ meta-dynamicdevices layers validation PASSED" - echo "" - echo "=============================================" - echo "✅ All layer validations PASSED" - echo "✅ All meta-dynamicdevices layers pass comprehensive yocto-check-layer validation!" - echo "✅ Layers are ready for Yocto Project compatibility." - echo "" - else - echo "" - echo "❌ meta-dynamicdevices layers validation FAILED" - echo "" - echo "=============================================" - echo "❌ Layer validation FAILED" - echo "" - echo "ℹ️ Please fix the yocto-check-layer issues above before proceeding." - echo "ℹ️ Run './scripts/validate-layers-local.sh' locally to debug issues." - echo "" - exit 1 - fi - - # Cleanup validation workspace - cd .. - rm -rf "$VALIDATION_DIR" - - # Summary job - summary: - name: Validation Summary - runs-on: [self-hosted, Linux, X64] - needs: [validate] - if: always() - - steps: - - name: Generate validation summary - run: | - { - echo "# KAS Layer Validation Summary" - echo - echo "**Workflow:** ${{ github.workflow }}" - echo "**Trigger:** ${{ github.event_name }}" - echo "**Commit:** ${{ github.sha }}" - echo "**Branch:** ${{ github.ref_name }}" - echo - - if [ "${{ needs.validate.result }}" = "success" ]; then - echo "## ✅ Validation Status: SUCCESS" - echo "All meta-dynamicdevices layers pass comprehensive yocto-check-layer validation!" - echo "Layers are ready for Yocto Project compatibility." - else - echo "## ❌ Validation Status: FAILED" - echo "- Layer validation: ${{ needs.validate.result }}" - echo "Please fix validation issues before proceeding." - fi - - echo - echo "## Validation Coverage" - echo "- **Tool:** Official yocto-check-layer" - echo "- **Layers:** meta-dynamicdevices, meta-dynamicdevices-bsp, meta-dynamicdevices-distro" - echo "- **Compliance:** Full Yocto Project compatibility validation" - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 95b3df59..4f6e219d 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -21,7 +21,6 @@ jobs: runs-on: [self-hosted, Linux, X64] outputs: material: ${{ steps.detect.outputs.material }} - matrix: ${{ steps.matrix.outputs.matrix }} base_sha: ${{ steps.base.outputs.sha }} steps: - uses: actions/checkout@v7 @@ -41,42 +40,33 @@ jobs: git cat-file -e "$sha^{commit}" echo "sha=$sha" >> "$GITHUB_OUTPUT" - id: detect - name: Require an adoption contract + name: Validate repository and require an adoption contract run: | python3 -m unittest discover -s scripts/validation/tests -p 'test_*.py' + find scripts -type f -name '*.sh' -print0 | xargs -0 -r -n1 bash -n + python3 -m json.tool ci/layer-adoption-contract.json >/dev/null + python3 -m json.tool ci/layer-adoption-tuples.json >/dev/null + # Mail-format patch payloads legitimately contain the conventional + # "-- " separator. Check repository sources without rewriting the + # third-party patches that BitBake applies. + git diff --check '${{ steps.base.outputs.sha }}...${{ github.sha }}' -- . ':(exclude)**/*.patch' python3 scripts/validation/detect-layer-adoption.py \ --base '${{ steps.base.outputs.sha }}' \ --head '${{ github.sha }}' \ --github-output "$GITHUB_OUTPUT" - - id: matrix - name: Publish protected tuple matrix - run: | - python3 - <<'PY' - import json - import os - - with open("ci/layer-adoption-tuples.json", encoding="utf-8") as source: - matrix = {"include": json.load(source)["tuples"]} - with open(os.environ["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: - output.write("matrix=" + json.dumps(matrix, separators=(",", ":")) + "\n") - PY - regression: - name: Protect ${{ matrix.id }} + name: Existing product regression needs: detect if: needs.detect.outputs.material == 'true' - strategy: - fail-fast: false - # Full Yocto baseline/candidate pairs are I/O and disk intensive. Keep - # the hard gate comprehensive without overwhelming one self-hosted host. - max-parallel: 2 - matrix: ${{ fromJSON(needs.detect.outputs.matrix) }} - runs-on: [self-hosted, Linux, X64, yocto] + # One named ai-tools runner owns the persistent Yocto cache. Keeping the + # complete matrix inside one job makes resource use and the required gate + # obvious, while the driver still fails closed on every protected tuple. + runs-on: [self-hosted, Linux, X64, yocto, ai-tools] container: - image: dynamicdevices/yocto-ci-build:latest - options: --privileged --platform linux/amd64 + image: ghcr.io/siemens/kas/kas@sha256:d989add57fc441fe9e27bb2dd6ed98c5597b44c807928e35a72dc1cfbdda9abe + options: --privileged --platform linux/amd64 --user 0:0 -v /home/ghrunner/yocto-layer-adoption:/var/cache/layer-adoption env: - LAYER_ADOPTION_TEST_KEYS_DIR: /tmp/layer-adoption-keys-${{ github.run_id }}-${{ github.run_attempt }}-${{ matrix.id }} + LAYER_ADOPTION_CACHE: /var/cache/layer-adoption steps: - name: Reset job-owned workspace run: | @@ -95,18 +85,6 @@ jobs: - name: Create baseline worktree working-directory: candidate run: git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' - - name: Install KAS when absent - run: command -v kas >/dev/null || pip3 install kas - - name: Restore Yocto caches - uses: actions/cache@v6 - with: - path: | - ~/yocto/downloads - ~/yocto/sstate-cache - key: layer-adoption-${{ matrix.machine }}-${{ hashFiles('candidate/kas/**') }} - restore-keys: | - layer-adoption-${{ matrix.machine }}- - layer-adoption- - name: Require safe build capacity run: | available_kib=$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 {print $4}') @@ -115,40 +93,31 @@ jobs: echo "ERROR: layer-adoption build requires at least 150 GiB free; found $((available_kib / 1024 / 1024)) GiB" >&2 exit 1 fi - - name: Generate ephemeral signing keys - run: candidate/scripts/validation/generate-layer-adoption-test-keys.sh "$LAYER_ADOPTION_TEST_KEYS_DIR" - - name: Build and capture baseline - working-directory: baseline - run: | - ../candidate/scripts/validation/capture-layer-state.sh \ - '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ - '${{ matrix.image }}' '${{ matrix.product_features }}' \ - '../evidence/baseline/${{ matrix.id }}' - - name: Release baseline build workspace + - name: Prepare persistent test-only signing identity run: | - workspace=$(realpath -m -- "$GITHUB_WORKSPACE") - target=$(realpath -m -- "$workspace/baseline/build") - test "$target" = "$workspace/baseline/build" - rm -rf -- "$target" - - name: Build and capture candidate - working-directory: candidate - run: | - scripts/validation/capture-layer-state.sh \ - '${{ matrix.config }}' '${{ matrix.machine }}' '${{ matrix.distro }}' \ - '${{ matrix.image }}' '${{ matrix.product_features }}' \ - '../evidence/candidate/${{ matrix.id }}' - - name: Reject unexplained contamination + set -euo pipefail + keys="$LAYER_ADOPTION_CACHE/test-keys" + if [ ! -s "$keys/ubootdev.key" ]; then + test "$keys" = /var/cache/layer-adoption/test-keys + rm -rf -- "$keys" + rm -rf -- "$LAYER_ADOPTION_CACHE/baselines" + temporary=$(mktemp -d "$LAYER_ADOPTION_CACHE/.test-keys.XXXXXX") + candidate/scripts/validation/generate-layer-adoption-test-keys.sh "$temporary" + mv "$temporary" "$keys" + fi + - name: Build and compare every protected tuple run: | - python3 candidate/scripts/validation/compare-layer-state.py \ - --baseline 'evidence/baseline/${{ matrix.id }}' \ - --candidate 'evidence/candidate/${{ matrix.id }}' \ - --tuple '${{ matrix.id }}' \ - --contract candidate/ci/layer-adoption-contract.json + python3 candidate/scripts/validation/run-layer-adoption-regression.py \ + --baseline baseline \ + --candidate candidate \ + --evidence evidence \ + --cache "$LAYER_ADOPTION_CACHE" \ + --test-keys "$LAYER_ADOPTION_CACHE/test-keys" - name: Preserve comparison evidence if: always() uses: actions/upload-artifact@v7 with: - name: layer-adoption-${{ matrix.id }} + name: layer-adoption-evidence path: evidence retention-days: 14 - name: Remove job-owned build trees diff --git a/scripts/validation/generate-layer-adoption-test-keys.sh b/scripts/validation/generate-layer-adoption-test-keys.sh index 4f3dc91c..6c0f0834 100755 --- a/scripts/validation/generate-layer-adoption-test-keys.sh +++ b/scripts/validation/generate-layer-adoption-test-keys.sh @@ -1,5 +1,5 @@ #!/usr/bin/env bash -# Generate one genuine but disposable signing-key set shared by a baseline and candidate build. +# Generate one genuine, test-only signing identity shared by compared builds. set -euo pipefail umask 077 @@ -50,4 +50,4 @@ openssl ecparam -name prime256v1 -genkey -noout \ -out "$output_dir/tf-a/privkey_ec_prime256v1.pem" openssl ec -in "$output_dir/tf-a/privkey_ec_prime256v1.pem" -check -noout -printf 'PASS: generated ephemeral layer-adoption signing keys in %s\n' "$output_dir" +printf 'PASS: generated test-only layer-adoption signing keys in %s\n' "$output_dir" diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py new file mode 100644 index 00000000..9a4f7fc4 --- /dev/null +++ b/scripts/validation/run-layer-adoption-regression.py @@ -0,0 +1,178 @@ +#!/usr/bin/env python3 +"""Run every protected Yocto tuple in one fail-closed regression job.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + + +MARKER = ".complete.json" +FIELDS = ("id", "machine", "distro", "image", "config", "product_features") + + +def evidence_digest(root: Path) -> str: + digest = hashlib.sha256() + for path in sorted(root.rglob("*")): + if not path.is_file() or path.name == MARKER: + continue + relative = path.relative_to(root).as_posix().encode() + digest.update(relative) + digest.update(b"\0") + digest.update(hashlib.sha256(path.read_bytes()).digest()) + return digest.hexdigest() + + +def valid_cached_evidence(root: Path, base_sha: str, tuple_id: str) -> bool: + try: + marker = json.loads((root / MARKER).read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return False + return marker == { + "base_sha": base_sha, + "tuple_id": tuple_id, + "evidence_sha256": evidence_digest(root), + } + + +def load_tuples(path: Path) -> list[dict[str, str]]: + document = json.loads(path.read_text(encoding="utf-8")) + if document.get("schema") != 1 or not isinstance(document.get("tuples"), list): + raise ValueError(f"{path}: expected schema=1 and a tuples array") + tuples = [] + seen = set() + for index, raw in enumerate(document["tuples"]): + if not isinstance(raw, dict) or any(field not in raw for field in FIELDS): + raise ValueError(f"{path}: tuple {index} is incomplete") + entry = {field: str(raw[field]) for field in FIELDS} + if not entry["id"] or entry["id"] in seen: + raise ValueError(f"{path}: duplicate or empty tuple id {entry['id']!r}") + seen.add(entry["id"]) + tuples.append(entry) + if not tuples: + raise ValueError(f"{path}: no protected tuples") + return tuples + + +def remove_build_tree(repository: Path) -> None: + repository = repository.resolve() + build = (repository / "build").resolve() + if build.parent != repository or repository == Path("/"): + raise RuntimeError(f"refusing unsafe build cleanup: {build}") + shutil.rmtree(build, ignore_errors=True) + + +def capture( + script: Path, + repository: Path, + entry: dict[str, str], + output: Path, + environment: dict[str, str], +) -> None: + subprocess.run( + [ + str(script), + entry["config"], + entry["machine"], + entry["distro"], + entry["image"], + entry["product_features"], + str(output), + ], + cwd=repository, + env=environment, + check=True, + ) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("--baseline", required=True, type=Path) + parser.add_argument("--candidate", required=True, type=Path) + parser.add_argument("--evidence", required=True, type=Path) + parser.add_argument("--cache", required=True, type=Path) + parser.add_argument("--test-keys", required=True, type=Path) + args = parser.parse_args() + + baseline = args.baseline.resolve() + candidate = args.candidate.resolve() + evidence = args.evidence.resolve() + cache = args.cache.resolve() + test_keys = args.test_keys.resolve() + capture_script = candidate / "scripts/validation/capture-layer-state.sh" + compare_script = candidate / "scripts/validation/compare-layer-state.py" + contract = candidate / "ci/layer-adoption-contract.json" + tuples = load_tuples(candidate / "ci/layer-adoption-tuples.json") + base_sha = subprocess.check_output( + ["git", "rev-parse", "HEAD"], cwd=baseline, text=True + ).strip() + + environment = os.environ.copy() + environment["LAYER_ADOPTION_TEST_KEYS_DIR"] = str(test_keys) + environment["LAYER_ADOPTION_CACHE_DIR"] = str(cache / "yocto") + shutil.rmtree(evidence, ignore_errors=True) + (evidence / "baseline").mkdir(parents=True) + (evidence / "candidate").mkdir(parents=True) + + for entry in tuples: + tuple_id = entry["id"] + print(f"::group::Protect {tuple_id}", flush=True) + cached = cache / "baselines" / base_sha / tuple_id + baseline_output = evidence / "baseline" / tuple_id + candidate_output = evidence / "candidate" / tuple_id + temporary: Path | None = None + try: + if valid_cached_evidence(cached, base_sha, tuple_id): + print(f"Reusing immutable baseline evidence for {base_sha}", flush=True) + else: + shutil.rmtree(cached, ignore_errors=True) + cached.parent.mkdir(parents=True, exist_ok=True) + temporary = Path(tempfile.mkdtemp(prefix=f".{tuple_id}-", dir=cached.parent)) + capture(capture_script, baseline, entry, temporary, environment) + marker = { + "base_sha": base_sha, + "tuple_id": tuple_id, + "evidence_sha256": evidence_digest(temporary), + } + (temporary / MARKER).write_text( + json.dumps(marker, sort_keys=True) + "\n", encoding="utf-8" + ) + temporary.rename(cached) + temporary = None + + shutil.copytree(cached, baseline_output, ignore=shutil.ignore_patterns(MARKER)) + capture(capture_script, candidate, entry, candidate_output, environment) + subprocess.run( + [ + "python3", + str(compare_script), + "--baseline", + str(baseline_output), + "--candidate", + str(candidate_output), + "--tuple", + tuple_id, + "--contract", + str(contract), + ], + check=True, + ) + finally: + if temporary is not None: + shutil.rmtree(temporary, ignore_errors=True) + remove_build_tree(baseline) + remove_build_tree(candidate) + print("::endgroup::", flush=True) + + print(f"PASS: all {len(tuples)} protected Yocto tuples are unchanged") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py new file mode 100644 index 00000000..964b059c --- /dev/null +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -0,0 +1,46 @@ +#!/usr/bin/env python3 +"""Tests for immutable layer-adoption baseline evidence.""" + +from __future__ import annotations + +import importlib.util +import json +import tempfile +import unittest +from pathlib import Path + + +MODULE_PATH = Path(__file__).parents[1] / "run-layer-adoption-regression.py" +SPEC = importlib.util.spec_from_file_location("run_layer_adoption_regression", MODULE_PATH) +assert SPEC and SPEC.loader +MODULE = importlib.util.module_from_spec(SPEC) +SPEC.loader.exec_module(MODULE) + + +class BaselineEvidenceTests(unittest.TestCase): + def test_valid_cache_is_accepted_and_tampering_is_rejected(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "packages.txt").write_text("package-a\n", encoding="utf-8") + marker = { + "base_sha": "abc123", + "tuple_id": "machine-image", + "evidence_sha256": MODULE.evidence_digest(root), + } + (root / MODULE.MARKER).write_text(json.dumps(marker), encoding="utf-8") + self.assertTrue(MODULE.valid_cached_evidence(root, "abc123", "machine-image")) + + (root / "packages.txt").write_text("package-b\n", encoding="utf-8") + self.assertFalse(MODULE.valid_cached_evidence(root, "abc123", "machine-image")) + + def test_marker_is_not_part_of_evidence_digest(self) -> None: + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + (root / "state.txt").write_text("stable\n", encoding="utf-8") + before = MODULE.evidence_digest(root) + (root / MODULE.MARKER).write_text("{}\n", encoding="utf-8") + self.assertEqual(before, MODULE.evidence_digest(root)) + + +if __name__ == "__main__": + unittest.main() From 28be102167bcebc09e1fad9bb5b55b3d81fc2c17 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:33:22 +0100 Subject: [PATCH 27/79] ci: run container steps with bash --- .github/workflows/layer-adoption-gate.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 4f6e219d..64d5427d 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -15,6 +15,10 @@ concurrency: group: layer-adoption-${{ github.event.pull_request.number || github.ref }} cancel-in-progress: true +defaults: + run: + shell: bash + jobs: detect: name: Detect material layer change From 0ba5fa39114c8c2de97de1f18806f75b61927a30 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:35:36 +0100 Subject: [PATCH 28/79] ci: keep kas overlay inside worktree --- scripts/validation/capture-layer-state.sh | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 4c325d81..41b00d1a 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -52,7 +52,15 @@ export DISTRO="$distro" product_features_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$product_features") downloads_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/downloads") sstate_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/sstate-cache") -overlay="$output_dir/layer-adoption-product-features.yml" +repository_root=$(git rev-parse --show-toplevel) +overlay_dir="$repository_root/.layer-adoption-overlays" +mkdir -p "$overlay_dir" +overlay=$(mktemp "$overlay_dir/product-features.XXXXXX.yml") +cleanup_overlay() { + rm -f "$overlay" + rmdir "$overlay_dir" 2>/dev/null || true +} +trap cleanup_overlay EXIT cat > "$overlay" < Date: Sat, 12 Sep 2026 13:38:44 +0100 Subject: [PATCH 29/79] ci: quote layer state normalisation safely --- scripts/validation/capture-layer-state.sh | 6 +++--- .../tests/test_capture_layer_state.py | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 scripts/validation/tests/test_capture_layer_state.py diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 41b00d1a..cb873fe8 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -134,13 +134,13 @@ capture_command() { } capture_command show-layers run_bitbake "bitbake-layers show-layers" \ - | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ > "$output_dir/layers.txt" capture_command show-appends run_bitbake "bitbake-layers show-appends" \ - | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ > "$output_dir/appends.txt" capture_command show-recipes run_bitbake "bitbake-layers show-recipes" \ - | sed -E "s#$PWD/##g; s#[[:space:]]+$##" \ + | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ > "$output_dir/recipes.txt" capture_command graph run_bitbake "bitbake -g $target" diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py new file mode 100644 index 00000000..44d7836b --- /dev/null +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -0,0 +1,19 @@ +#!/usr/bin/env python3 +"""Static regressions for the layer-state capture shell boundary.""" + +from pathlib import Path +import unittest + + +SCRIPT = Path(__file__).parents[1] / "capture-layer-state.sh" + + +class CaptureLayerStateTests(unittest.TestCase): + def test_shell_does_not_expand_sed_end_anchor_as_argument_count(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertNotIn('s#[[:space:]]+$##"', source) + self.assertEqual(source.count("-e 's#[[:space:]]+$##'"), 3) + + +if __name__ == "__main__": + unittest.main() From d58b88aa1474bed542c5013987fe983d0ff51111 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:45:16 +0100 Subject: [PATCH 30/79] ci: replay captured command failures --- scripts/validation/capture-layer-state.sh | 10 ++++- .../tests/test_capture_layer_state.py | 39 ++++++++++++++++++- 2 files changed, 47 insertions(+), 2 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index cb873fe8..961fe8dc 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -130,7 +130,15 @@ run_bitbake() { capture_command() { local name=$1 shift - "$@" 2>&1 | tee "$output_dir/$name.log" + local log="$output_dir/$name.log" + if "$@" 2>&1 | tee "$log"; then + return 0 + else + local statuses=("${PIPESTATUS[@]}") + echo "ERROR: command failed while capturing $name" >&2 + cat "$log" >&2 + return "${statuses[0]}" + fi } capture_command show-layers run_bitbake "bitbake-layers show-layers" \ diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 44d7836b..db118930 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -1,7 +1,10 @@ #!/usr/bin/env python3 -"""Static regressions for the layer-state capture shell boundary.""" +"""Regressions for the layer-state capture shell boundary.""" from pathlib import Path +import re +import subprocess +import tempfile import unittest @@ -14,6 +17,40 @@ def test_shell_does_not_expand_sed_end_anchor_as_argument_count(self) -> None: self.assertNotIn('s#[[:space:]]+$##"', source) self.assertEqual(source.count("-e 's#[[:space:]]+$##'"), 3) + def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + match = re.search(r"(?ms)^capture_command\(\) \{\n.*?^\}\n", source) + self.assertIsNotNone(match) + + with tempfile.TemporaryDirectory() as directory: + result = subprocess.run( + [ + "bash", + "-c", + match.group(0) + + """ +set -o pipefail +output_dir=$1 +capture_command failure bash -c 'printf "visible diagnostic\\n"; exit 7' \\ + | sed 's/diagnostic/output/' +""", + "capture-command-test", + directory, + ], + check=False, + capture_output=True, + text=True, + ) + + self.assertEqual(result.returncode, 7) + self.assertEqual(result.stdout, "visible output\n") + self.assertIn("ERROR: command failed while capturing failure", result.stderr) + self.assertIn("visible diagnostic", result.stderr) + self.assertEqual( + (Path(directory) / "failure.log").read_text(encoding="utf-8"), + "visible diagnostic\n", + ) + if __name__ == "__main__": unittest.main() From 49981609161503d69e80c3ac115b8599f87675b9 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:48:57 +0100 Subject: [PATCH 31/79] ci: initialize protected layer submodules --- .github/workflows/layer-adoption-gate.yml | 10 +++++++++- .../tests/test_run_layer_adoption_regression.py | 7 +++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 64d5427d..6571f490 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -88,7 +88,15 @@ jobs: path: candidate - name: Create baseline worktree working-directory: candidate - run: git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' + run: | + git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' + git -c url.https://github.com/.insteadOf=git@github.com: \ + submodule update --init --recursive \ + meta-dynamicdevices-bsp meta-dynamicdevices-distro + git -C ../baseline \ + -c url.https://github.com/.insteadOf=git@github.com: \ + submodule update --init --recursive \ + meta-dynamicdevices-bsp meta-dynamicdevices-distro - name: Require safe build capacity run: | available_kib=$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 {print $4}') diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 964b059c..762a40d4 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -11,6 +11,7 @@ MODULE_PATH = Path(__file__).parents[1] / "run-layer-adoption-regression.py" +WORKFLOW_PATH = Path(__file__).parents[3] / ".github/workflows/layer-adoption-gate.yml" SPEC = importlib.util.spec_from_file_location("run_layer_adoption_regression", MODULE_PATH) assert SPEC and SPEC.loader MODULE = importlib.util.module_from_spec(SPEC) @@ -18,6 +19,12 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_gate_initializes_product_submodules_in_both_worktrees(self) -> None: + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + self.assertEqual(workflow.count("submodule update --init --recursive"), 2) + self.assertEqual(workflow.count("meta-dynamicdevices-bsp meta-dynamicdevices-distro"), 2) + self.assertIn("git -C ../baseline", workflow) + def test_valid_cache_is_accepted_and_tampering_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From 4e07c3b4d44b51c3dbbb28228c5b85b0a27fdf5a Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:52:04 +0100 Subject: [PATCH 32/79] ci: initialize protected layer worktrees --- .github/workflows/layer-adoption-gate.yml | 4 +++- .../validation/tests/test_run_layer_adoption_regression.py | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 6571f490..2f337eee 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -68,7 +68,9 @@ jobs: runs-on: [self-hosted, Linux, X64, yocto, ai-tools] container: image: ghcr.io/siemens/kas/kas@sha256:d989add57fc441fe9e27bb2dd6ed98c5597b44c807928e35a72dc1cfbdda9abe - options: --privileged --platform linux/amd64 --user 0:0 -v /home/ghrunner/yocto-layer-adoption:/var/cache/layer-adoption + # Match the dedicated ai-tools runner account so BitBake's root-user + # sanity check remains active and bind-mounted cache files stay writable. + options: --privileged --platform linux/amd64 --user 1002:1002 -v /home/ghrunner/yocto-layer-adoption:/var/cache/layer-adoption env: LAYER_ADOPTION_CACHE: /var/cache/layer-adoption steps: diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 762a40d4..b38de2b4 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -25,6 +25,11 @@ def test_gate_initializes_product_submodules_in_both_worktrees(self) -> None: self.assertEqual(workflow.count("meta-dynamicdevices-bsp meta-dynamicdevices-distro"), 2) self.assertIn("git -C ../baseline", workflow) + def test_gate_does_not_run_bitbake_as_root(self) -> None: + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + self.assertNotIn("--user 0:0", workflow) + self.assertIn("--user 1002:1002", workflow) + def test_valid_cache_is_accepted_and_tampering_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) From aba0a4d1953e60cfa579ed3e6ac4fce8fb7f595f Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 13:56:07 +0100 Subject: [PATCH 33/79] ci: share kas worktree preparation --- .github/workflows/layer-adoption-gate.yml | 10 +--- .../run-layer-adoption-regression.py | 47 +++++++++++++++++ .../test_run_layer_adoption_regression.py | 50 +++++++++++++++++-- 3 files changed, 94 insertions(+), 13 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 2f337eee..42ac9d43 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -90,15 +90,7 @@ jobs: path: candidate - name: Create baseline worktree working-directory: candidate - run: | - git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' - git -c url.https://github.com/.insteadOf=git@github.com: \ - submodule update --init --recursive \ - meta-dynamicdevices-bsp meta-dynamicdevices-distro - git -C ../baseline \ - -c url.https://github.com/.insteadOf=git@github.com: \ - submodule update --init --recursive \ - meta-dynamicdevices-bsp meta-dynamicdevices-distro + run: git worktree add ../baseline '${{ needs.detect.outputs.base_sha }}' - name: Require safe build capacity run: | available_kib=$(df -Pk "$GITHUB_WORKSPACE" | awk 'NR == 2 {print $4}') diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 9a4f7fc4..7521bd9a 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -15,6 +15,7 @@ MARKER = ".complete.json" FIELDS = ("id", "machine", "distro", "image", "config", "product_features") +PRODUCT_SUBMODULES = ("meta-dynamicdevices-bsp", "meta-dynamicdevices-distro") def evidence_digest(root: Path) -> str: @@ -68,6 +69,46 @@ def remove_build_tree(repository: Path) -> None: shutil.rmtree(build, ignore_errors=True) +def git_output(repository: Path, *args: str) -> str: + return subprocess.check_output(["git", *args], cwd=repository, text=True).strip() + + +def prepare_repository(repository: Path) -> None: + """Initialize and verify the pinned local layers used by every KAS tuple.""" + for relative in PRODUCT_SUBMODULES: + entry = git_output(repository, "ls-tree", "HEAD", "--", relative).split() + if len(entry) < 3 or entry[0] != "160000" or entry[1] != "commit": + raise RuntimeError(f"{repository}: {relative} is not a pinned git submodule") + expected = entry[2] + layer = repository / relative + layer_conf = layer / "conf/layer.conf" + if not layer_conf.is_file(): + subprocess.run( + [ + "git", + "-c", + "url.https://github.com/.insteadOf=git@github.com:", + "submodule", + "update", + "--init", + "--recursive", + "--", + relative, + ], + cwd=repository, + check=True, + ) + if not layer_conf.is_file(): + raise RuntimeError(f"{repository}: {relative}/conf/layer.conf is missing") + actual = git_output(layer, "rev-parse", "HEAD") + if actual != expected: + raise RuntimeError( + f"{repository}: {relative} is at {actual}, expected pinned {expected}" + ) + if git_output(layer, "status", "--porcelain", "--untracked-files=all"): + raise RuntimeError(f"{repository}: {relative} has uncommitted content") + + def capture( script: Path, repository: Path, @@ -113,6 +154,12 @@ def main() -> int: ["git", "rev-parse", "HEAD"], cwd=baseline, text=True ).strip() + # This preparation is intentionally owned by the shared regression driver, + # not by CI YAML. Local and hosted runs therefore build the same pinned + # submodule content through the same KAS capture path. + prepare_repository(baseline) + prepare_repository(candidate) + environment = os.environ.copy() environment["LAYER_ADOPTION_TEST_KEYS_DIR"] = str(test_keys) environment["LAYER_ADOPTION_CACHE_DIR"] = str(cache / "yocto") diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index b38de2b4..1e80ffda 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -8,6 +8,7 @@ import tempfile import unittest from pathlib import Path +from unittest import mock MODULE_PATH = Path(__file__).parents[1] / "run-layer-adoption-regression.py" @@ -19,11 +20,52 @@ class BaselineEvidenceTests(unittest.TestCase): - def test_gate_initializes_product_submodules_in_both_worktrees(self) -> None: + def test_worktree_preparation_is_owned_by_shared_driver(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") - self.assertEqual(workflow.count("submodule update --init --recursive"), 2) - self.assertEqual(workflow.count("meta-dynamicdevices-bsp meta-dynamicdevices-distro"), 2) - self.assertIn("git -C ../baseline", workflow) + self.assertNotIn("submodule update", workflow) + + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + pinned = "a" * 40 + + def git_result(command: list[str], **kwargs: object) -> str: + if "ls-tree" in command: + relative = command[-1] + return f"160000 commit {pinned}\t{relative}\n" + if "rev-parse" in command: + return pinned + "\n" + if "status" in command: + return "" + self.fail(f"unexpected git command: {command}") + + def initialize(command: list[str], **kwargs: object) -> None: + relative = command[-1] + layer_conf = repository / relative / "conf/layer.conf" + layer_conf.parent.mkdir(parents=True) + layer_conf.touch() + + with mock.patch.object( + MODULE.subprocess, "check_output", side_effect=git_result + ), mock.patch.object(MODULE.subprocess, "run", side_effect=initialize) as run: + MODULE.prepare_repository(repository) + + self.assertEqual(run.call_count, 2) + + def test_worktree_preparation_rejects_unpinned_layer(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + layer = repository / MODULE.PRODUCT_SUBMODULES[0] + (layer / "conf").mkdir(parents=True) + (layer / "conf/layer.conf").touch() + results = [ + f"160000 commit {'a' * 40}\t{layer.name}\n", + "b" * 40 + "\n", + ] + with mock.patch.object( + MODULE.subprocess, "check_output", side_effect=results + ): + with self.assertRaisesRegex(RuntimeError, "expected pinned"): + MODULE.prepare_repository(repository) def test_gate_does_not_run_bitbake_as_root(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") From c423642ffc2be96ea4a249e7f9510111aa2ae6b7 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 14:01:07 +0100 Subject: [PATCH 34/79] ci: track current bitbake dependency graph --- scripts/validation/capture-layer-state.sh | 5 +++-- scripts/validation/tests/test_capture_layer_state.py | 6 ++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 961fe8dc..cf6cfd08 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -155,8 +155,9 @@ capture_command graph run_bitbake "bitbake -g $target" sort -u build/pn-buildlist > "$output_dir/pn-buildlist.txt" sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ > "$output_dir/task-depends.dot" -sed -E "s#$PWD/##g" build/recipe-depends.dot | sort -u \ - > "$output_dir/recipe-depends.dot" +# Current BitBake deliberately removes the obsolete recipe-depends.dot output. +# pn-buildlist plus the finer-grained task graph retain provider and dependency +# selection coverage without relying on that removed compatibility artifact. # Capture final values and BitBake's assignment provenance for policy that a # newly enabled layer can silently change. The full environment is retained diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index db118930..10def427 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -51,6 +51,12 @@ def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: "visible diagnostic\n", ) + def test_dependency_capture_uses_current_bitbake_graph_outputs(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn("build/pn-buildlist", source) + self.assertIn("build/task-depends.dot", source) + self.assertNotIn("build/recipe-depends.dot", source) + if __name__ == "__main__": unittest.main() From 67e9f6d6d94bdf4c11e277221a5ab8d1c46e0c8e Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 14:07:17 +0100 Subject: [PATCH 35/79] ci: diagnose selected environment mismatches --- scripts/validation/capture-layer-state.sh | 21 +++++++++++++++---- .../tests/test_capture_layer_state.py | 7 +++++++ 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index cf6cfd08..9d97d2bc 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -162,14 +162,27 @@ sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ # Capture final values and BitBake's assignment provenance for policy that a # newly enabled layer can silently change. The full environment is retained # temporarily only as input, avoiding volatile host variables in comparisons. -capture_command environment run_bitbake "bitbake -e $target" +# The complete environment is intentionally retained in evidence but is too +# large for routine CI output. capture_command still replays it to stderr if +# BitBake fails, so diagnostics are not lost. +capture_command environment run_bitbake "bitbake -e $target" >/dev/null python3 "$(dirname "$0")/select-bitbake-env.py" \ "$output_dir/environment.log" \ | sed -E "s#$PWD##g; s#$test_keys_dir##g; s#$cache_root##g" \ > "$output_dir/selected-environment.txt" -grep -Fqx "MACHINE=\"$machine\"" "$output_dir/selected-environment.txt" -grep -Fqx "DISTRO=\"$distro\"" "$output_dir/selected-environment.txt" -grep -Fqx "DD_PRODUCT_FEATURES=\"$product_features\"" "$output_dir/selected-environment.txt" +require_selected_value() { + local name=$1 + local expected=$2 + if ! grep -Fqx "$name=\"$expected\"" "$output_dir/selected-environment.txt"; then + echo "ERROR: selected BitBake environment does not contain $name=\"$expected\"" >&2 + grep -E "^${name}=" "$output_dir/selected-environment.txt" >&2 || \ + echo "ERROR: $name is absent from selected BitBake environment" >&2 + return 1 + fi +} +require_selected_value MACHINE "$machine" +require_selected_value DISTRO "$distro" +require_selected_value DD_PRODUCT_FEATURES "$product_features" rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 10def427..d302f345 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -57,6 +57,13 @@ def test_dependency_capture_uses_current_bitbake_graph_outputs(self) -> None: self.assertIn("build/task-depends.dot", source) self.assertNotIn("build/recipe-depends.dot", source) + def test_environment_assertions_emit_named_diagnostics(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn('capture_command environment run_bitbake "bitbake -e $target" >/dev/null', source) + self.assertIn("require_selected_value() {", source) + self.assertEqual(source.count("require_selected_value "), 3) + self.assertIn("ERROR: selected BitBake environment does not contain", source) + if __name__ == "__main__": unittest.main() From 62e7455c56cbd3699e70f8fac967d098191045d2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 14:13:00 +0100 Subject: [PATCH 36/79] ci: use inode-aware disk thresholds --- scripts/validation/capture-layer-state.sh | 2 +- scripts/validation/tests/test_capture_layer_state.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 9d97d2bc..6c1ae8aa 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -69,7 +69,7 @@ local_conf_header: DD_PRODUCT_FEATURES = $product_features_quoted DL_DIR = $downloads_quoted SSTATE_DIR = $sstate_quoted - BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,1G STOPTASKS,\${DL_DIR},20G,1G STOPTASKS,\${SSTATE_DIR},20G,1G HALT,\${TMPDIR},10G,1G HALT,\${DL_DIR},10G,1G HALT,\${SSTATE_DIR},10G,1G" + BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,100K STOPTASKS,\${DL_DIR},20G,100K STOPTASKS,\${SSTATE_DIR},20G,100K HALT,\${TMPDIR},10G,50K HALT,\${DL_DIR},10G,50K HALT,\${SSTATE_DIR},10G,50K" UBOOT_SIGN_KEYDIR = "$test_keys_dir" UEFI_SIGN_KEYDIR = "$test_keys_dir/uefi" MODSIGN_KEY_DIR = "$test_keys_dir" diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index d302f345..03e002d0 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -64,6 +64,15 @@ def test_environment_assertions_emit_named_diagnostics(self) -> None: self.assertEqual(source.count("require_selected_value "), 3) self.assertIn("ERROR: selected BitBake environment does not contain", source) + def test_disk_monitor_uses_inode_not_disk_units(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + disk_monitor = next( + line for line in source.splitlines() if "BB_DISKMON_DIRS =" in line + ) + self.assertNotIn(",1G", disk_monitor) + self.assertEqual(disk_monitor.count(",100K"), 3) + self.assertEqual(disk_monitor.count(",50K"), 3) + if __name__ == "__main__": unittest.main() From f7bdd2067ae282081584e40542705effb4257fde Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 14:40:02 +0100 Subject: [PATCH 37/79] ci: bind local kas changes to adoption gate --- .gitattributes | 2 + scripts/README.md | 14 +++++ scripts/kas-build-base-enhanced.sh | 2 + scripts/kas-build-base.sh | 4 ++ scripts/kas-build-mfgtools.sh | 63 ++++++++++++++++++- scripts/kas-build-profiling.sh | 2 + scripts/kas-container-image.sh | 6 ++ scripts/kas-dev-boot.sh | 2 + scripts/kas-dev-kernel.sh | 2 + scripts/kas-dev-recipe.sh | 2 + scripts/kas-shell-base.sh | 4 ++ scripts/validation/detect-layer-adoption.py | 21 +++++-- .../tests/test_detect_layer_adoption.py | 25 ++++++++ .../test_run_layer_adoption_regression.py | 21 +++++++ 14 files changed, 163 insertions(+), 7 deletions(-) create mode 100644 scripts/kas-container-image.sh diff --git a/.gitattributes b/.gitattributes index 2288ba47..8289a470 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,4 +5,6 @@ *.bin filter=lfs diff=lfs merge=lfs -text *.itb filter=lfs diff=lfs merge=lfs -text *mfgtool* filter=lfs diff=lfs merge=lfs -text +# Build entry points are source, even when their names contain "mfgtool". +scripts/kas-build-mfgtools.sh -filter -diff -merge text eol=lf fitImage-* filter=lfs diff=lfs merge=lfs -text diff --git a/scripts/README.md b/scripts/README.md index 2818734b..ed87194e 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -159,6 +159,20 @@ custom-boot-files/ # Custom boot files directory ## 🏗️ Build & Development +### Local/CI KAS parity + +Use the `scripts/kas-*.sh` entry points for local KAS work. They source +`scripts/kas-container-image.sh`, which pins the same container digest as the +Layer Adoption Gate. The gate treats changes to these wrappers, KAS YAML, +pinned layer submodules, its workflow, contract, and regression implementation +as material. Such changes must update `ci/layer-adoption-contract.json` and run +every immutable tuple in `ci/layer-adoption-tuples.json`; the tuple matrix may +be extended but existing coverage cannot be removed or redefined. + +The shared `scripts/validation/run-layer-adoption-regression.py` driver owns +repository preparation and all baseline/candidate captures in both local and +CI use. Do not duplicate KAS preparation steps in workflow YAML. + ### `kas-build-base.sh` **Purpose:** Builds the base LmP (Linux microPlatform) image using KAS configuration. diff --git a/scripts/kas-build-base-enhanced.sh b/scripts/kas-build-base-enhanced.sh index 211d1211..14cb1d7d 100755 --- a/scripts/kas-build-base-enhanced.sh +++ b/scripts/kas-build-base-enhanced.sh @@ -9,6 +9,8 @@ set -euo pipefail # Exit on error, undefined vars, pipe failures # Script configuration SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" DEFAULT_CACHE_DIR="${HOME}/yocto" LOG_FILE="${PROJECT_ROOT}/logs/kas-build-$(date +%Y%m%d-%H%M%S).log" diff --git a/scripts/kas-build-base.sh b/scripts/kas-build-base.sh index 2c7bf21f..1b32f40e 100755 --- a/scripts/kas-build-base.sh +++ b/scripts/kas-build-base.sh @@ -1,5 +1,9 @@ #!/bin/sh +script_dir=$(CDPATH='' cd -P "$(dirname "$0")" && pwd) +# shellcheck source=kas-container-image.sh +. "$script_dir/kas-container-image.sh" + # TODO: Look at this to fix missing key issue # #conf/machine/include/lmp-factory-custom.inc:OPTEE_TA_SIGN_KEY = "${TOPDIR}/conf/factory-keys/opteedev.key" diff --git a/scripts/kas-build-mfgtools.sh b/scripts/kas-build-mfgtools.sh index 09136b4d..cfbed8bb 100755 --- a/scripts/kas-build-mfgtools.sh +++ b/scripts/kas-build-mfgtools.sh @@ -1,3 +1,60 @@ -version https://git-lfs.github.com/spec/v1 -oid sha256:180de9e62039ccce8fd0ae8ffe96091f6b4f3fdac562054fad8101055000e998 -size 1435 +#!/bin/sh + +script_dir=$(CDPATH='' cd -P "$(dirname "$0")" && pwd) +# shellcheck source=kas-container-image.sh +. "$script_dir/kas-container-image.sh" + +# Build mfgtool images for supported i.MX machines +# +# Usage: +# KAS_MACHINE=imx95-frdm-evk ./scripts/kas-build-mfgtools.sh +# KAS_MACHINE=imx93-jaguar-eink ./scripts/kas-build-mfgtools.sh +# KAS_MACHINE=imx93-11x11-lpddr4x-evk ./scripts/kas-build-mfgtools.sh +# +# The machine-specific BSP selects the correct imx-boot manufacturing target. + +# Set default machine if not specified +if [ -z "$KAS_MACHINE" ]; then + export KAS_MACHINE="imx93-11x11-lpddr4x-evk" + echo "No KAS_MACHINE specified, defaulting to $KAS_MACHINE" +else + echo "Building mfgtool for machine: $KAS_MACHINE" +fi + +KAS_CONFIG="kas/lmp-dynamicdevices-mfgtool.yml" +if [ "$KAS_MACHINE" = "imx95-frdm-evk" ]; then + # Keep local validation aligned with the Foundries manifest revisions used + # for FRDM-i.MX95 production builds. + KAS_CONFIG="kas/lmp-imx95-frdm-evk-mfgtool.yml" +fi + +# TODO: Look at this to fix missing key issue if needed +# +#conf/machine/include/lmp-factory-custom.inc:OPTEE_TA_SIGN_KEY = "${TOPDIR}/conf/factory-keys/opteedev.key" +#lmp-tools/scripts/rotate_ci_keys.sh:openssl genpkey -algorithm RSA -out factory-keys/opteedev.key \ +#lmp-tools/scripts/rotate_ci_keys.sh:openssl req -batch -new -x509 -key factory-keys/opteedev.key -out factory-keys/opteedev.crt + +if [ ! -d ~/yocto ] +then + mkdir -p ~/yocto + mkdir -p ~/yocto/downloads + mkdir -p ~/yocto/persistent + mkdir -p ~/yocto/sstate + chmod 755 ~/yocto + chmod 755 ~/yocto/downloads + chmod 755 ~/yocto/persistent + chmod 755 ~/yocto/sstate +fi + +# Pass KAS_MACHINE to kas-container to override the machine in the config file. +# Forward SSH credentials only when they exist; the standard layer URLs are HTTPS, +# so unattended builders such as ai-tools do not require an SSH agent. +set -- --runtime-args "-v ${HOME}/yocto:/var/cache -e KAS_MACHINE=$KAS_MACHINE" +if [ -n "${SSH_AUTH_SOCK:-}" ] && [ -S "${SSH_AUTH_SOCK}" ]; then + set -- --ssh-agent "$@" +fi +if [ -d "${HOME}/.ssh" ]; then + set -- --ssh-dir "${HOME}/.ssh" "$@" +fi + +kas-container "$@" build "$KAS_CONFIG" diff --git a/scripts/kas-build-profiling.sh b/scripts/kas-build-profiling.sh index 48cece16..35344070 100755 --- a/scripts/kas-build-profiling.sh +++ b/scripts/kas-build-profiling.sh @@ -7,6 +7,8 @@ set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "$SCRIPT_DIR")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" # Default values DEFAULT_MACHINE="imx93-jaguar-eink" diff --git a/scripts/kas-container-image.sh b/scripts/kas-container-image.sh new file mode 100644 index 00000000..e23e578a --- /dev/null +++ b/scripts/kas-container-image.sh @@ -0,0 +1,6 @@ +#!/bin/sh + +# Keep local KAS wrappers on the exact container image exercised by the +# layer-adoption gate. Update the workflow and its parity test with this pin. +KAS_CONTAINER_IMAGE="ghcr.io/siemens/kas/kas@sha256:d989add57fc441fe9e27bb2dd6ed98c5597b44c807928e35a72dc1cfbdda9abe" +export KAS_CONTAINER_IMAGE diff --git a/scripts/kas-dev-boot.sh b/scripts/kas-dev-boot.sh index 89ea7b77..a50d2c7d 100755 --- a/scripts/kas-dev-boot.sh +++ b/scripts/kas-dev-boot.sh @@ -10,6 +10,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" # Default values ACTION="" diff --git a/scripts/kas-dev-kernel.sh b/scripts/kas-dev-kernel.sh index 9c52e2b9..1ac2aa6f 100755 --- a/scripts/kas-dev-kernel.sh +++ b/scripts/kas-dev-kernel.sh @@ -10,6 +10,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" # Default values ACTION="" diff --git a/scripts/kas-dev-recipe.sh b/scripts/kas-dev-recipe.sh index c27548ee..3ef0b451 100755 --- a/scripts/kas-dev-recipe.sh +++ b/scripts/kas-dev-recipe.sh @@ -10,6 +10,8 @@ set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PROJECT_ROOT="$(dirname "${SCRIPT_DIR}")" +# shellcheck source=kas-container-image.sh +. "$SCRIPT_DIR/kas-container-image.sh" # Default values ACTION="" diff --git a/scripts/kas-shell-base.sh b/scripts/kas-shell-base.sh index 56ff4178..b6be08a6 100755 --- a/scripts/kas-shell-base.sh +++ b/scripts/kas-shell-base.sh @@ -1,5 +1,9 @@ #!/bin/sh +script_dir=$(CDPATH='' cd -P "$(dirname "$0")" && pwd) +# shellcheck source=kas-container-image.sh +. "$script_dir/kas-container-image.sh" + # KAS Shell Base Script # Usage: ./kas-shell-base.sh [options] # -c "command" : Execute command in kas environment diff --git a/scripts/validation/detect-layer-adoption.py b/scripts/validation/detect-layer-adoption.py index b0911206..a4e34cd2 100755 --- a/scripts/validation/detect-layer-adoption.py +++ b/scripts/validation/detect-layer-adoption.py @@ -13,9 +13,20 @@ MATERIAL_PATHS = ( + re.compile(r"^\.github/workflows/layer-adoption-gate\.yml$"), + re.compile(r"^\.gitattributes$"), re.compile(r"^\.gitmodules$"), + re.compile(r"^ci/layer-adoption-contract\.json$"), re.compile(r"(^|/)conf/layer\.conf$"), re.compile(r"^kas/.*\.ya?ml$"), + re.compile(r"^meta-dynamicdevices-(?:bsp|distro)$"), + re.compile(r"^meta-partner-nxp-imx$"), + re.compile(r"^scripts/kas-.*\.sh$"), + re.compile( + r"^scripts/validation/(?:capture-layer-state\.sh|compare-layer-state\.py|" + r"detect-layer-adoption\.py|generate-layer-adoption-test-keys\.sh|" + r"run-layer-adoption-regression\.py)$" + ), re.compile(r"^ci/layer-adoption-tuples\.json$"), ) @@ -23,6 +34,11 @@ NONEMPTY_TUPLE_FIELDS = ("id", "machine", "distro", "image", "config") +def is_material_path(path: str) -> bool: + """Return whether a change can alter local or CI KAS build semantics.""" + return any(pattern.search(path) for pattern in MATERIAL_PATHS) + + def git(*args: str) -> str: return subprocess.check_output(["git", *args], text=True) @@ -110,10 +126,7 @@ def main() -> int: return 2 changed = git("diff", "--name-only", f"{args.base}...{args.head}").splitlines() - material_files = [ - path for path in changed - if any(pattern.search(path) for pattern in MATERIAL_PATHS) - ] + material_files = [path for path in changed if is_material_path(path)] # Treat every KAS change as material. YAML context makes line-only pin # detection easy to evade (for example by adding a list item below an # existing `includes:` key), and a false-positive build is safer than a diff --git a/scripts/validation/tests/test_detect_layer_adoption.py b/scripts/validation/tests/test_detect_layer_adoption.py index 211741e7..a1aecf21 100644 --- a/scripts/validation/tests/test_detect_layer_adoption.py +++ b/scripts/validation/tests/test_detect_layer_adoption.py @@ -35,6 +35,31 @@ def entry(tuple_id: str, machine: str = "machine-a") -> dict[str, str]: class ProtectedTupleTests(unittest.TestCase): + def test_local_and_ci_kas_process_changes_are_material(self) -> None: + paths = ( + ".github/workflows/layer-adoption-gate.yml", + ".gitattributes", + "ci/layer-adoption-contract.json", + "ci/layer-adoption-tuples.json", + "kas/lmp-dynamicdevices.yml", + "meta-dynamicdevices-bsp", + "meta-dynamicdevices-distro", + "meta-partner-nxp-imx", + "scripts/kas-build-base.sh", + "scripts/kas-shell-base.sh", + "scripts/validation/capture-layer-state.sh", + "scripts/validation/compare-layer-state.py", + "scripts/validation/detect-layer-adoption.py", + "scripts/validation/generate-layer-adoption-test-keys.sh", + "scripts/validation/run-layer-adoption-regression.py", + ) + for path in paths: + with self.subTest(path=path): + self.assertTrue(MODULE.is_material_path(path)) + + def test_unrelated_utility_change_is_not_material(self) -> None: + self.assertFalse(MODULE.is_material_path("scripts/analyze-boot-logs.sh")) + def validate(self, baseline: str, candidate: str) -> None: with tempfile.TemporaryDirectory() as directory: root = Path(directory) diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 1e80ffda..1decdc95 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -5,6 +5,7 @@ import importlib.util import json +import re import tempfile import unittest from pathlib import Path @@ -20,6 +21,26 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_local_kas_wrappers_use_ci_container_digest(self) -> None: + root = WORKFLOW_PATH.parents[2] + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + helper_name = "kas-container-image.sh" + helper = (root / "scripts" / helper_name).read_text(encoding="utf-8") + local_pin = re.search(r'^KAS_CONTAINER_IMAGE="([^"]+)"$', helper, re.MULTILINE) + ci_pin = re.search(r"^\s+image: (\S+)$", workflow, re.MULTILINE) + self.assertIsNotNone(local_pin) + self.assertIsNotNone(ci_pin) + self.assertEqual(local_pin.group(1), ci_pin.group(1)) + + wrappers = [ + path for path in (root / "scripts").glob("kas-*.sh") + if path.name != helper_name and "kas-container" in path.read_text(encoding="utf-8") + ] + self.assertTrue(wrappers) + for wrapper in wrappers: + with self.subTest(wrapper=wrapper.name): + self.assertIn(helper_name, wrapper.read_text(encoding="utf-8")) + def test_worktree_preparation_is_owned_by_shared_driver(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") self.assertNotIn("submodule update", workflow) From b5380f59cbd0e67732d5e11fd37d75e8194f8d16 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 18:12:53 +0100 Subject: [PATCH 38/79] ci: validate kernel signing identity --- .github/workflows/layer-adoption-gate.yml | 3 +- scripts/validation/capture-layer-state.sh | 4 ++ .../generate-layer-adoption-test-keys.sh | 55 ++++++++++++++++--- .../tests/test_capture_layer_state.py | 15 ++++- .../test_run_layer_adoption_regression.py | 19 +++++++ 5 files changed, 87 insertions(+), 9 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 42ac9d43..cc5638b5 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -103,7 +103,8 @@ jobs: run: | set -euo pipefail keys="$LAYER_ADOPTION_CACHE/test-keys" - if [ ! -s "$keys/ubootdev.key" ]; then + if ! candidate/scripts/validation/generate-layer-adoption-test-keys.sh \ + --check "$keys"; then test "$keys" = /var/cache/layer-adoption/test-keys rm -rf -- "$keys" rm -rf -- "$LAYER_ADOPTION_CACHE/baselines" diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 6c1ae8aa..78ce310f 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -73,6 +73,8 @@ local_conf_header: UBOOT_SIGN_KEYDIR = "$test_keys_dir" UEFI_SIGN_KEYDIR = "$test_keys_dir/uefi" MODSIGN_KEY_DIR = "$test_keys_dir" + MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem" + MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt" SIGNING_UBOOT_SIGN_KEY = "$test_keys_dir/ubootdev.key" SIGNING_UBOOT_SIGN_CRT = "$test_keys_dir/ubootdev.crt" SIGNING_UBOOT_SPL_SIGN_KEY = "$test_keys_dir/spldev.key" @@ -183,6 +185,8 @@ require_selected_value() { require_selected_value MACHINE "$machine" require_selected_value DISTRO "$distro" require_selected_value DD_PRODUCT_FEATURES "$product_features" +require_selected_value MODSIGN_PRIVKEY "/privkey_modsign.pem" +require_selected_value MODSIGN_X509 "/x509_modsign.crt" rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, diff --git a/scripts/validation/generate-layer-adoption-test-keys.sh b/scripts/validation/generate-layer-adoption-test-keys.sh index 6c0f0834..77a38a3e 100755 --- a/scripts/validation/generate-layer-adoption-test-keys.sh +++ b/scripts/validation/generate-layer-adoption-test-keys.sh @@ -1,25 +1,66 @@ #!/usr/bin/env bash -# Generate one genuine, test-only signing identity shared by compared builds. +# Generate or validate one genuine, test-only signing identity shared by builds. set -euo pipefail umask 077 -if [ "$#" -ne 1 ]; then - echo "Usage: $0 OUTPUT_DIR" >&2 +if [ "$#" -ne 1 ] && { [ "$#" -ne 2 ] || [ "$1" != "--check" ]; }; then + echo "Usage: $0 [--check] OUTPUT_DIR" >&2 exit 2 fi -output_dir=$(realpath -m "$1") +if [ "$#" -eq 2 ]; then + check_only=true + output_dir=$(realpath -m "$2") +else + check_only=false + output_dir=$(realpath -m "$1") +fi case "$output_dir" in /|/home|/root|/tmp|/var|/usr) echo "ERROR: refusing broad test-key output directory: $output_dir" >&2 exit 2 ;; esac -if [ -e "$output_dir" ] && find "$output_dir" -mindepth 1 -print -quit | grep -q .; then +if [ "$check_only" = false ] && [ -e "$output_dir" ] && \ + find "$output_dir" -mindepth 1 -print -quit | grep -q . +then echo "ERROR: test-key output directory is not empty: $output_dir" >&2 exit 2 fi +validate_pair() { + local key=$1 + local certificate=$2 + local key_fingerprint certificate_fingerprint + openssl pkey -in "$key" -check -noout >/dev/null 2>&1 + # Never let a persistent CI identity expire during a long gate run. + openssl x509 -in "$certificate" -noout -checkend 604800 >/dev/null 2>&1 + key_fingerprint=$(openssl pkey -in "$key" -pubout -outform DER 2>/dev/null \ + | sha256sum | cut -d ' ' -f 1) + certificate_fingerprint=$(openssl x509 -in "$certificate" -pubkey -noout 2>/dev/null \ + | openssl pkey -pubin -outform DER 2>/dev/null \ + | sha256sum | cut -d ' ' -f 1) + [ "$key_fingerprint" = "$certificate_fingerprint" ] +} + +validate_keyset() { + validate_pair "$output_dir/ubootdev.key" "$output_dir/ubootdev.crt" && + validate_pair "$output_dir/spldev.key" "$output_dir/spldev.crt" && + validate_pair "$output_dir/privkey_modsign.pem" "$output_dir/x509_modsign.crt" && + validate_pair "$output_dir/uefi/DB.key" "$output_dir/uefi/DB.crt" && + openssl ec -in "$output_dir/tf-a/privkey_ec_prime256v1.pem" \ + -check -noout >/dev/null 2>&1 +} + +if [ "$check_only" = true ]; then + if validate_keyset; then + printf 'PASS: validated test-only layer-adoption signing keys in %s\n' "$output_dir" + exit 0 + fi + echo "ERROR: test-only layer-adoption signing keys are invalid or expire within seven days" >&2 + exit 1 +fi + mkdir -p "$output_dir/uefi" "$output_dir/tf-a" make_rsa_certificate() { @@ -27,7 +68,7 @@ make_rsa_certificate() { local certificate=$2 local common_name=$3 openssl genpkey -algorithm RSA -out "$key" -pkeyopt rsa_keygen_bits:2048 - openssl req -batch -new -x509 -sha256 -days 2 \ + openssl req -batch -new -x509 -sha256 -days 3650 \ -key "$key" -out "$certificate" -subj "/CN=$common_name/" openssl pkey -in "$key" -check -noout openssl x509 -in "$certificate" -noout @@ -48,6 +89,6 @@ make_rsa_certificate \ openssl ecparam -name prime256v1 -genkey -noout \ -out "$output_dir/tf-a/privkey_ec_prime256v1.pem" -openssl ec -in "$output_dir/tf-a/privkey_ec_prime256v1.pem" -check -noout +validate_keyset printf 'PASS: generated test-only layer-adoption signing keys in %s\n' "$output_dir" diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 03e002d0..1213f209 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -61,9 +61,22 @@ def test_environment_assertions_emit_named_diagnostics(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertIn('capture_command environment run_bitbake "bitbake -e $target" >/dev/null', source) self.assertIn("require_selected_value() {", source) - self.assertEqual(source.count("require_selected_value "), 3) + self.assertEqual(source.count("require_selected_value "), 5) self.assertIn("ERROR: selected BitBake environment does not contain", source) + def test_module_signing_uses_kernel_runtime_variables(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn('MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem"', source) + self.assertIn('MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt"', source) + self.assertIn( + 'require_selected_value MODSIGN_PRIVKEY "/privkey_modsign.pem"', + source, + ) + self.assertIn( + 'require_selected_value MODSIGN_X509 "/x509_modsign.crt"', + source, + ) + def test_disk_monitor_uses_inode_not_disk_units(self) -> None: source = SCRIPT.read_text(encoding="utf-8") disk_monitor = next( diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 1decdc95..6e007369 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -6,6 +6,7 @@ import importlib.util import json import re +import subprocess import tempfile import unittest from pathlib import Path @@ -21,6 +22,24 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_generated_signing_identity_is_validated_before_reuse(self) -> None: + generator = MODULE_PATH.parent / "generate-layer-adoption-test-keys.sh" + with tempfile.TemporaryDirectory() as directory: + keys = Path(directory) / "keys" + subprocess.run([str(generator), str(keys)], check=True, capture_output=True) + subprocess.run( + [str(generator), "--check", str(keys)], check=True, capture_output=True + ) + (keys / "x509_modsign.crt").write_text("invalid\n", encoding="utf-8") + invalid = subprocess.run( + [str(generator), "--check", str(keys)], + check=False, + capture_output=True, + text=True, + ) + self.assertNotEqual(invalid.returncode, 0) + self.assertIn("invalid or expire within seven days", invalid.stderr) + def test_local_kas_wrappers_use_ci_container_digest(self) -> None: root = WORKFLOW_PATH.parents[2] workflow = WORKFLOW_PATH.read_text(encoding="utf-8") From b08700e580211d53c1d48ec7a19b208cc6dc3408 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 19:35:53 +0100 Subject: [PATCH 39/79] ci: force test signing paths --- scripts/validation/capture-layer-state.sh | 36 +++++++++++-------- .../tests/test_capture_layer_state.py | 32 +++++++++++++++-- 2 files changed, 50 insertions(+), 18 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 78ce310f..151b72f8 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -70,21 +70,22 @@ local_conf_header: DL_DIR = $downloads_quoted SSTATE_DIR = $sstate_quoted BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,100K STOPTASKS,\${DL_DIR},20G,100K STOPTASKS,\${SSTATE_DIR},20G,100K HALT,\${TMPDIR},10G,50K HALT,\${DL_DIR},10G,50K HALT,\${SSTATE_DIR},10G,50K" - UBOOT_SIGN_KEYDIR = "$test_keys_dir" - UEFI_SIGN_KEYDIR = "$test_keys_dir/uefi" - MODSIGN_KEY_DIR = "$test_keys_dir" - MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem" - MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt" - SIGNING_UBOOT_SIGN_KEY = "$test_keys_dir/ubootdev.key" - SIGNING_UBOOT_SIGN_CRT = "$test_keys_dir/ubootdev.crt" - SIGNING_UBOOT_SPL_SIGN_KEY = "$test_keys_dir/spldev.key" - SIGNING_UBOOT_SPL_SIGN_CRT = "$test_keys_dir/spldev.crt" - SIGNING_MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem" - SIGNING_MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt" - SIGNING_UEFI_SIGN_KEY = "$test_keys_dir/uefi/DB.key" - SIGNING_UEFI_SIGN_CRT = "$test_keys_dir/uefi/DB.crt" - OPTEE_TA_SIGN_KEY = "$test_keys_dir/ubootdev.key" - TF_A_SIGN_KEY_PATH = "$test_keys_dir/tf-a/privkey_ec_prime256v1.pem" + UBOOT_SIGN_KEYDIR:forcevariable = "$test_keys_dir" + UBOOT_SPL_SIGN_KEYDIR:forcevariable = "$test_keys_dir" + UEFI_SIGN_KEYDIR:forcevariable = "$test_keys_dir/uefi" + MODSIGN_KEY_DIR:forcevariable = "$test_keys_dir" + MODSIGN_PRIVKEY:forcevariable = "$test_keys_dir/privkey_modsign.pem" + MODSIGN_X509:forcevariable = "$test_keys_dir/x509_modsign.crt" + SIGNING_UBOOT_SIGN_KEY:forcevariable = "$test_keys_dir/ubootdev.key" + SIGNING_UBOOT_SIGN_CRT:forcevariable = "$test_keys_dir/ubootdev.crt" + SIGNING_UBOOT_SPL_SIGN_KEY:forcevariable = "$test_keys_dir/spldev.key" + SIGNING_UBOOT_SPL_SIGN_CRT:forcevariable = "$test_keys_dir/spldev.crt" + SIGNING_MODSIGN_PRIVKEY:forcevariable = "$test_keys_dir/privkey_modsign.pem" + SIGNING_MODSIGN_X509:forcevariable = "$test_keys_dir/x509_modsign.crt" + SIGNING_UEFI_SIGN_KEY:forcevariable = "$test_keys_dir/uefi/DB.key" + SIGNING_UEFI_SIGN_CRT:forcevariable = "$test_keys_dir/uefi/DB.crt" + OPTEE_TA_SIGN_KEY:forcevariable = "$test_keys_dir/ubootdev.key" + TF_A_SIGN_KEY_PATH:forcevariable = "$test_keys_dir/tf-a/privkey_ec_prime256v1.pem" EOF combined_config="${config}:${overlay}" kas checkout "$combined_config" @@ -187,6 +188,11 @@ require_selected_value DISTRO "$distro" require_selected_value DD_PRODUCT_FEATURES "$product_features" require_selected_value MODSIGN_PRIVKEY "/privkey_modsign.pem" require_selected_value MODSIGN_X509 "/x509_modsign.crt" +require_selected_value UBOOT_SIGN_KEYDIR "" +require_selected_value UBOOT_SPL_SIGN_KEYDIR "" +require_selected_value UEFI_SIGN_KEYDIR "/uefi" +require_selected_value OPTEE_TA_SIGN_KEY "/ubootdev.key" +require_selected_value TF_A_SIGN_KEY_PATH "/tf-a/privkey_ec_prime256v1.pem" rm "$output_dir/environment.log" # A parse-only graph is not proof that packaging, signing, recovery image size, diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 1213f209..e7a98286 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -61,13 +61,19 @@ def test_environment_assertions_emit_named_diagnostics(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertIn('capture_command environment run_bitbake "bitbake -e $target" >/dev/null', source) self.assertIn("require_selected_value() {", source) - self.assertEqual(source.count("require_selected_value "), 5) + self.assertEqual(source.count("require_selected_value "), 10) self.assertIn("ERROR: selected BitBake environment does not contain", source) def test_module_signing_uses_kernel_runtime_variables(self) -> None: source = SCRIPT.read_text(encoding="utf-8") - self.assertIn('MODSIGN_PRIVKEY = "$test_keys_dir/privkey_modsign.pem"', source) - self.assertIn('MODSIGN_X509 = "$test_keys_dir/x509_modsign.crt"', source) + self.assertIn( + 'MODSIGN_PRIVKEY:forcevariable = "$test_keys_dir/privkey_modsign.pem"', + source, + ) + self.assertIn( + 'MODSIGN_X509:forcevariable = "$test_keys_dir/x509_modsign.crt"', + source, + ) self.assertIn( 'require_selected_value MODSIGN_PRIVKEY "/privkey_modsign.pem"', source, @@ -77,6 +83,26 @@ def test_module_signing_uses_kernel_runtime_variables(self) -> None: source, ) + def test_every_signing_consumer_path_is_forced_and_asserted(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + runtime_paths = { + "UBOOT_SIGN_KEYDIR": "", + "UBOOT_SPL_SIGN_KEYDIR": "", + "UEFI_SIGN_KEYDIR": "/uefi", + "OPTEE_TA_SIGN_KEY": "/ubootdev.key", + "TF_A_SIGN_KEY_PATH": "/tf-a/privkey_ec_prime256v1.pem", + } + for name, expected in runtime_paths.items(): + with self.subTest(name=name): + self.assertIn(f"{name}:forcevariable =", source) + self.assertIn( + f'require_selected_value {name} "{expected}"', source + ) + self.assertNotRegex( + source, + r"(?m)^ (?:SIGNING_|MODSIGN_|UBOOT_|UEFI_|OPTEE_|TF_A_)[A-Z0-9_]+ =", + ) + def test_disk_monitor_uses_inode_not_disk_units(self) -> None: source = SCRIPT.read_text(encoding="utf-8") disk_monitor = next( From fc72fbd7c99b64841e3383192ec95d000d0c57ca Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 22:59:07 +0100 Subject: [PATCH 40/79] ci: canonicalise layer adoption evidence --- .../canonicalise-bitbake-layer-output.py | 68 +++++++++++++++++++ scripts/validation/capture-layer-state.sh | 44 +++++++++--- scripts/validation/compare-layer-state.py | 54 +++++++++++++-- .../test_canonicalise_bitbake_layer_output.py | 54 +++++++++++++++ .../tests/test_capture_layer_state.py | 29 +++++++- .../tests/test_compare_layer_state.py | 42 ++++++++++++ 6 files changed, 275 insertions(+), 16 deletions(-) create mode 100644 scripts/validation/canonicalise-bitbake-layer-output.py create mode 100644 scripts/validation/tests/test_canonicalise_bitbake_layer_output.py create mode 100644 scripts/validation/tests/test_compare_layer_state.py diff --git a/scripts/validation/canonicalise-bitbake-layer-output.py b/scripts/validation/canonicalise-bitbake-layer-output.py new file mode 100644 index 00000000..e59546ad --- /dev/null +++ b/scripts/validation/canonicalise-bitbake-layer-output.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +"""Canonicalise bitbake-layers reports without discarding policy evidence.""" + +from __future__ import annotations + +import argparse +import re +import sys + + +VOLATILE_PREFIXES = ("Loaded ", "Parsing of ") + + +def canonicalise_feature_sets(line: str) -> str: + def replace(match: re.Match[str]) -> str: + words = match.group(1).split() + if len(words) > 1 and all(re.fullmatch(r"[A-Za-z0-9+_.-]+", word) for word in words): + return "'" + " ".join(sorted(words)) + "'" + return match.group(0) + + return re.sub(r"'([^']+)'", replace, line) + + +def canonicalise_blocks(lines: list[str]) -> list[str]: + result: list[str] = [] + heading = "" + for raw in lines: + line = raw.rstrip() + stripped = line.strip() + if not stripped or stripped.startswith(VOLATILE_PREFIXES): + continue + if not line[:1].isspace() and stripped.endswith(":"): + heading = stripped[:-1] + result.append(f"entry\t{heading}") + elif line[:1].isspace() and heading: + result.append(f"value\t{heading}\t{canonicalise_feature_sets(stripped)}") + else: + heading = "" + result.append(f"message\t{canonicalise_feature_sets(stripped)}") + return sorted(result) + + +def canonicalise_layers(lines: list[str]) -> list[str]: + result: list[str] = [] + for raw in lines: + fields = raw.split() + if not fields or fields[:3] == ["layer", "path", "priority"]: + continue + if len(fields) == 3 and fields[2].lstrip("-").isdigit(): + result.append("\t".join(("layer", *fields))) + else: + result.append("message\t" + canonicalise_feature_sets(raw.strip())) + return sorted(result) + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument("mode", choices=("layers", "appends", "recipes")) + args = parser.parse_args() + lines = sys.stdin.read().splitlines() + output = canonicalise_layers(lines) if args.mode == "layers" else canonicalise_blocks(lines) + if output: + print("\n".join(output)) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 151b72f8..4e1d9ba3 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -117,13 +117,21 @@ printf '%s\n' \ # Static layer surfaces are captured as well as BitBake's resolved view. This # makes wildcard/dangling appends and global layer.conf policy visible even # when they do not happen to alter the first recipe selected by BitBake. -find build/layers -type f \( -path '*/conf/layer.conf' -o -name '*.bbclass' \) -print0 \ - | sort -z \ +find build/layers "$repository_root" \ + -path "$repository_root/build" -prune -o \ + -path "$repository_root/.git" -prune -o \ + -type f \( -path '*/conf/layer.conf' -o -name '*.bbclass' \) -print0 \ + | sort -zu \ | xargs -0 grep -nHE \ '(^|[[:space:]])(IMAGE_INSTALL|CORE_IMAGE_|PACKAGE_INSTALL|DISTRO_FEATURES|MACHINE_FEATURES|PACKAGECONFIG|PREFERRED_(VERSION|PROVIDER)|DEFAULT_PREFERENCE|RDEPENDS|INHERIT|BBMASK|BBPATH|BBFILES|BBFILE_PRIORITY|INITRAMFS_MAXSIZE|IMAGE_FSTYPES|WKS_FILE|UBOOT_|KERNEL_|OPTEE_|SDKIMAGE_FEATURES|TOOLCHAIN_TARGET_TASK)(:|\[|[[:space:]])*([+?:.]?=)' \ - > "$output_dir/layer-conf-policy.txt" || true -find build/layers -type f -name '*.bbappend' -printf '%p\n' \ - | sed -E 's#^build/layers/##' \ + | sed -E "s#^$repository_root/#meta-dynamicdevices/#" \ + > "$output_dir/layer-conf-policy.txt" || true +find build/layers "$repository_root" \ + -path "$repository_root/build" -prune -o \ + -path "$repository_root/.git" -prune -o \ + -type f -name '*.bbappend' -printf '%p\n' \ + | sed -E -e 's#^build/layers/##' \ + -e "s#^$repository_root/#meta-dynamicdevices/#" \ | sort -u > "$output_dir/all-bbappends.txt" run_bitbake() { @@ -144,14 +152,25 @@ capture_command() { fi } +normalise_kas_projection() { + sed -E \ + -e '/^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} - (DEBUG|INFO|WARNING|ERROR)[[:space:]]+- /d' \ + -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e "s#$PWD##g" \ + -e 's#[[:space:]]+$##' +} + capture_command show-layers run_bitbake "bitbake-layers show-layers" \ - | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ + | normalise_kas_projection \ + | python3 "$(dirname "$0")/canonicalise-bitbake-layer-output.py" layers \ > "$output_dir/layers.txt" capture_command show-appends run_bitbake "bitbake-layers show-appends" \ - | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ + | normalise_kas_projection \ + | python3 "$(dirname "$0")/canonicalise-bitbake-layer-output.py" appends \ > "$output_dir/appends.txt" capture_command show-recipes run_bitbake "bitbake-layers show-recipes" \ - | sed -E -e "s#$PWD/##g" -e 's#[[:space:]]+$##' \ + | normalise_kas_projection \ + | python3 "$(dirname "$0")/canonicalise-bitbake-layer-output.py" recipes \ > "$output_dir/recipes.txt" capture_command graph run_bitbake "bitbake -g $target" @@ -171,7 +190,12 @@ sed -E "s#$PWD/##g" build/task-depends.dot | sort -u \ capture_command environment run_bitbake "bitbake -e $target" >/dev/null python3 "$(dirname "$0")/select-bitbake-env.py" \ "$output_dir/environment.log" \ - | sed -E "s#$PWD##g; s#$test_keys_dir##g; s#$cache_root##g" \ + | sed -E \ + -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e "s#$PWD##g" \ + -e "s#$test_keys_dir##g" \ + -e "s#$cache_root##g" \ + -e 's/[0-9]{14}/TIMESTAMP/g' \ > "$output_dir/selected-environment.txt" require_selected_value() { local name=$1 @@ -207,7 +231,7 @@ if [ ! -d "$deploy_dir" ]; then fi find "$deploy_dir" -maxdepth 1 -type f -printf '%f\t%s\n' \ - | sed -E 's/-[0-9]{14}(\.|-)/-TIMESTAMP\1/g' \ + | sed -E 's/-[0-9]{14}(\.|-|$)/-TIMESTAMP\1/g' \ | sort -u > "$output_dir/deploy-layout-and-sizes.txt" # The expression belongs to awk; shell expansion would be a bug. # shellcheck disable=SC2016 diff --git a/scripts/validation/compare-layer-state.py b/scripts/validation/compare-layer-state.py index cf38a03f..5f656aea 100755 --- a/scripts/validation/compare-layer-state.py +++ b/scripts/validation/compare-layer-state.py @@ -10,6 +10,43 @@ import sys from pathlib import Path +DEPLOY_SIZES = "deploy-layout-and-sizes.txt" +MAX_DEPLOY_SIZE_DRIFT_RATIO = 0.005 +MIN_DEPLOY_SIZE_DRIFT_BYTES = 4096 + + +def deploy_entries(lines: list[str]) -> dict[str, int]: + entries: dict[str, int] = {} + for line in lines: + try: + name, raw_size = line.rsplit("\t", 1) + size = int(raw_size) + except ValueError as exc: + raise ValueError(f"invalid deploy entry: {line!r}") from exc + if name in entries: + raise ValueError(f"duplicate deploy entry: {name}") + entries[name] = size + return entries + + +def deploy_deltas(old_lines: list[str], new_lines: list[str]) -> list[str]: + """Compare deploy layout exactly and sizes within reproducible-build noise.""" + old = deploy_entries(old_lines) + new = deploy_entries(new_lines) + deltas = [f"removed {name}" for name in sorted(set(old) - set(new))] + deltas.extend(f"added {name}" for name in sorted(set(new) - set(old))) + for name in sorted(set(old) & set(new)): + tolerance = max( + MIN_DEPLOY_SIZE_DRIFT_BYTES, + int(old[name] * MAX_DEPLOY_SIZE_DRIFT_RATIO), + ) + if abs(new[name] - old[name]) > tolerance: + deltas.append( + f"size {name}: {old[name]} -> {new[name]} " + f"(tolerance {tolerance})" + ) + return deltas + def files(root: Path) -> dict[str, list[str]]: result: dict[str, list[str]] = {} @@ -49,11 +86,18 @@ def main() -> int: for name in sorted(set(old) | set(new)): if old.get(name) == new.get(name): continue - delta = list(difflib.unified_diff(old.get(name, []), new.get(name, []), lineterm="")) - changed_lines = [ - line[1:] for line in delta - if line.startswith(("+", "-")) and not line.startswith(("+++", "---")) - ] + if name == DEPLOY_SIZES: + try: + changed_lines = deploy_deltas(old.get(name, []), new.get(name, [])) + except ValueError as exc: + print(f"ERROR: {name}: {exc}", file=sys.stderr) + return 2 + else: + delta = list(difflib.unified_diff(old.get(name, []), new.get(name, []), lineterm="")) + changed_lines = [ + line[1:] for line in delta + if line.startswith(("+", "-")) and not line.startswith(("+++", "---")) + ] for line in changed_lines: if not any(file_re.search(name) and line_re.search(line) for file_re, line_re, _ in compiled): unexplained.append(f"{name}: {line}") diff --git a/scripts/validation/tests/test_canonicalise_bitbake_layer_output.py b/scripts/validation/tests/test_canonicalise_bitbake_layer_output.py new file mode 100644 index 00000000..302349cb --- /dev/null +++ b/scripts/validation/tests/test_canonicalise_bitbake_layer_output.py @@ -0,0 +1,54 @@ +#!/usr/bin/env python3 + +import importlib.util +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "canonicalise-bitbake-layer-output.py" +SPEC = importlib.util.spec_from_file_location("canonicalise_output", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class CanonicaliseOutputTests(unittest.TestCase): + def test_recipe_values_keep_their_recipe_identity(self) -> None: + self.assertEqual( + MODULE.canonicalise_blocks( + ["foo:", " layer-a 1.0", "bar:", " layer-b 2.0"] + ), + [ + "entry\tbar", + "entry\tfoo", + "value\tbar\tlayer-b 2.0", + "value\tfoo\tlayer-a 1.0", + ], + ) + + def test_parse_counts_are_removed_and_feature_sets_are_sorted(self) -> None: + self.assertEqual( + MODULE.canonicalise_blocks( + [ + "Parsing of 10 .bb files complete (0 cached)", + "foo:", + " layer 1.0 (skipped: missing required distro features 'x11 opengl')", + ] + ), + [ + "entry\tfoo", + "value\tfoo\tlayer 1.0 (skipped: missing required distro features 'opengl x11')", + ], + ) + + def test_layer_spacing_is_not_evidence(self) -> None: + self.assertEqual( + MODULE.canonicalise_layers( + ["layer path priority", "meta-dd build/.. 11"] + ), + ["layer\tmeta-dd\tbuild/..\t11"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index e7a98286..f56b42f0 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -15,7 +15,34 @@ class CaptureLayerStateTests(unittest.TestCase): def test_shell_does_not_expand_sed_end_anchor_as_argument_count(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertNotIn('s#[[:space:]]+$##"', source) - self.assertEqual(source.count("-e 's#[[:space:]]+$##'"), 3) + self.assertEqual(source.count("-e 's#[[:space:]]+$##'"), 1) + self.assertEqual(source.count("| normalise_kas_projection"), 3) + + def test_kas_projection_removes_host_noise_and_sorts_at_call_site(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + match = re.search(r"(?ms)^normalise_kas_projection\(\) \{\n.*?^\}\n", source) + self.assertIsNotNone(match) + result = subprocess.run( + [ + "bash", + "-c", + match.group(0) + + """ +PWD=/workspace/candidate +printf '%s\n' \\ + '2026-09-12 20:00:00 - INFO - kas 4.7 started' \\ + '/__w/project/baseline/build/layers/meta/conf/layer.conf ' \\ + '/workspace/candidate/recipe.bb' | normalise_kas_projection +""", + ], + check=True, + capture_output=True, + text=True, + ) + self.assertEqual( + result.stdout, + "/build/layers/meta/conf/layer.conf\n/recipe.bb\n", + ) def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: source = SCRIPT.read_text(encoding="utf-8") diff --git a/scripts/validation/tests/test_compare_layer_state.py b/scripts/validation/tests/test_compare_layer_state.py new file mode 100644 index 00000000..10fa67dd --- /dev/null +++ b/scripts/validation/tests/test_compare_layer_state.py @@ -0,0 +1,42 @@ +#!/usr/bin/env python3 + +import importlib.util +import unittest +from pathlib import Path + + +SCRIPT = Path(__file__).parents[1] / "compare-layer-state.py" +SPEC = importlib.util.spec_from_file_location("compare_layer_state", SCRIPT) +MODULE = importlib.util.module_from_spec(SPEC) +assert SPEC.loader is not None +SPEC.loader.exec_module(MODULE) + + +class DeployComparisonTests(unittest.TestCase): + def test_layout_change_is_a_delta(self) -> None: + self.assertEqual( + MODULE.deploy_deltas(["old.wic\t10000"], ["new.wic\t10000"]), + ["removed old.wic", "added new.wic"], + ) + + def test_small_rebuild_size_noise_is_accepted(self) -> None: + self.assertEqual( + MODULE.deploy_deltas(["image.wic\t100000000"], ["image.wic\t100300000"]), + [], + ) + + def test_material_size_change_is_a_delta(self) -> None: + self.assertEqual( + MODULE.deploy_deltas(["image.wic\t100000000"], ["image.wic\t101000000"]), + ["size image.wic: 100000000 -> 101000000 (tolerance 500000)"], + ) + + def test_malformed_or_duplicate_entries_fail_closed(self) -> None: + with self.assertRaises(ValueError): + MODULE.deploy_entries(["missing-size"]) + with self.assertRaises(ValueError): + MODULE.deploy_entries(["image.wic\t1", "image.wic\t2"]) + + +if __name__ == "__main__": + unittest.main() From fbc6f4db157b481befdeb0efba3d2678af6acb95 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 23:05:04 +0100 Subject: [PATCH 41/79] ci: keep adoption gate on controlled runner --- .github/workflows/layer-adoption-gate.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index cc5638b5..2281cb0a 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -22,7 +22,7 @@ defaults: jobs: detect: name: Detect material layer change - runs-on: [self-hosted, Linux, X64] + runs-on: [self-hosted, Linux, X64, yocto, ai-tools] outputs: material: ${{ steps.detect.outputs.material }} base_sha: ${{ steps.base.outputs.sha }} @@ -146,7 +146,7 @@ jobs: name: Layer Adoption Gate needs: [detect, regression] if: always() - runs-on: [self-hosted, Linux, X64] + runs-on: [self-hosted, Linux, X64, yocto, ai-tools] steps: - name: Enforce gate result env: From 531a6c206d75421e999d040402966ffb71d995e4 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 23:21:51 +0100 Subject: [PATCH 42/79] ci: key baseline cache by capture schema --- .../run-layer-adoption-regression.py | 25 +++++++++++++++-- .../test_run_layer_adoption_regression.py | 28 +++++++++++++++++-- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 7521bd9a..88dc8876 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -16,6 +16,13 @@ MARKER = ".complete.json" FIELDS = ("id", "machine", "distro", "image", "config", "product_features") PRODUCT_SUBMODULES = ("meta-dynamicdevices-bsp", "meta-dynamicdevices-distro") +CAPTURE_SCHEMA_FILES = ( + "scripts/validation/run-layer-adoption-regression.py", + "scripts/validation/capture-layer-state.sh", + "scripts/validation/canonicalise-bitbake-layer-output.py", + "scripts/validation/select-bitbake-env.py", + "scripts/validation/generate-layer-adoption-test-keys.sh", +) def evidence_digest(root: Path) -> str: @@ -30,7 +37,18 @@ def evidence_digest(root: Path) -> str: return digest.hexdigest() -def valid_cached_evidence(root: Path, base_sha: str, tuple_id: str) -> bool: +def capture_schema_digest(repository: Path) -> str: + digest = hashlib.sha256() + for relative in CAPTURE_SCHEMA_FILES: + digest.update(relative.encode()) + digest.update(b"\0") + digest.update(hashlib.sha256((repository / relative).read_bytes()).digest()) + return digest.hexdigest() + + +def valid_cached_evidence( + root: Path, base_sha: str, tuple_id: str, capture_schema: str +) -> bool: try: marker = json.loads((root / MARKER).read_text(encoding="utf-8")) except (OSError, json.JSONDecodeError): @@ -38,6 +56,7 @@ def valid_cached_evidence(root: Path, base_sha: str, tuple_id: str) -> bool: return marker == { "base_sha": base_sha, "tuple_id": tuple_id, + "capture_schema": capture_schema, "evidence_sha256": evidence_digest(root), } @@ -153,6 +172,7 @@ def main() -> int: base_sha = subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=baseline, text=True ).strip() + capture_schema = capture_schema_digest(candidate) # This preparation is intentionally owned by the shared regression driver, # not by CI YAML. Local and hosted runs therefore build the same pinned @@ -175,7 +195,7 @@ def main() -> int: candidate_output = evidence / "candidate" / tuple_id temporary: Path | None = None try: - if valid_cached_evidence(cached, base_sha, tuple_id): + if valid_cached_evidence(cached, base_sha, tuple_id, capture_schema): print(f"Reusing immutable baseline evidence for {base_sha}", flush=True) else: shutil.rmtree(cached, ignore_errors=True) @@ -185,6 +205,7 @@ def main() -> int: marker = { "base_sha": base_sha, "tuple_id": tuple_id, + "capture_schema": capture_schema, "evidence_sha256": evidence_digest(temporary), } (temporary / MARKER).write_text( diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 6e007369..a3ce4b84 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -119,13 +119,37 @@ def test_valid_cache_is_accepted_and_tampering_is_rejected(self) -> None: marker = { "base_sha": "abc123", "tuple_id": "machine-image", + "capture_schema": "schema-a", "evidence_sha256": MODULE.evidence_digest(root), } (root / MODULE.MARKER).write_text(json.dumps(marker), encoding="utf-8") - self.assertTrue(MODULE.valid_cached_evidence(root, "abc123", "machine-image")) + self.assertTrue( + MODULE.valid_cached_evidence( + root, "abc123", "machine-image", "schema-a" + ) + ) + self.assertFalse( + MODULE.valid_cached_evidence( + root, "abc123", "machine-image", "schema-b" + ) + ) (root / "packages.txt").write_text("package-b\n", encoding="utf-8") - self.assertFalse(MODULE.valid_cached_evidence(root, "abc123", "machine-image")) + self.assertFalse( + MODULE.valid_cached_evidence( + root, "abc123", "machine-image", "schema-a" + ) + ) + + def test_capture_schema_covers_every_evidence_producer(self) -> None: + root = MODULE_PATH.parents[2] + first = MODULE.capture_schema_digest(root) + self.assertRegex(first, r"^[0-9a-f]{64}$") + self.assertIn("scripts/validation/capture-layer-state.sh", MODULE.CAPTURE_SCHEMA_FILES) + self.assertIn( + "scripts/validation/canonicalise-bitbake-layer-output.py", + MODULE.CAPTURE_SCHEMA_FILES, + ) def test_marker_is_not_part_of_evidence_digest(self) -> None: with tempfile.TemporaryDirectory() as directory: From 9093e7ac9365d001a8c184fd6c66763bacaed543 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sat, 12 Sep 2026 23:49:48 +0100 Subject: [PATCH 43/79] ci: source warnings from cooker logs --- scripts/validation/capture-layer-state.sh | 17 ++++++++++++++--- .../tests/test_capture_layer_state.py | 6 ++++++ 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 4e1d9ba3..a8681d1b 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -155,6 +155,9 @@ capture_command() { normalise_kas_projection() { sed -E \ -e '/^[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}:[0-9]{2} - (DEBUG|INFO|WARNING|ERROR)[[:space:]]+- /d' \ + -e '/WARNING:/d' \ + -e '/^Summary: There were [0-9]+ WARNING messages?\.?$/d' \ + -e '/^NOTE: Starting bitbake server\.\.\.$/d' \ -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ -e "s#$PWD##g" \ -e 's#[[:space:]]+$##' @@ -241,13 +244,21 @@ find "$deploy_dir" -maxdepth 1 -type f -name '*.manifest' -print0 \ | sort -u > "$output_dir/packages.txt" # New warnings are regressions even when BitBake returns zero. -find "$output_dir" -type f -name '*.log' -print0 \ +# Cooker logs preserve one BitBake warning per line. Combined stdout/stderr +# command logs can splice concurrent parser warnings together and are not a +# deterministic warning source. +find build/tmp/log/cooker -type f -name '*.log' -print0 \ | xargs -0 -r grep -hE '(^|[[:space:]])WARNING:' \ - | sed -E "s#$PWD/##g; s/[0-9]{4}-[0-9]{2}-[0-9]{2}[^ ]*//g" \ + | sed -E \ + -e 's/^.*WARNING:/WARNING:/' \ + -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e "s#$PWD##g" \ + -e 's/[0-9]{4}-[0-9]{2}-[0-9]{2}[^ ]*//g' \ | sort -u > "$output_dir/warnings.txt" || true # Raw command logs are useful for diagnosis but contain progress ordering and -# timing noise. The deterministic projections above are the comparison input. +# timing noise. The deterministic projections and cooker warnings above are +# the comparison input. rm -f "$output_dir"/*.log # The generated overlay is removed by the EXIT trap. Keeping it inside the # worktree satisfies KAS's same-repository rule for concatenated configs. diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index f56b42f0..8623f482 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -32,6 +32,7 @@ def test_kas_projection_removes_host_noise_and_sorts_at_call_site(self) -> None: printf '%s\n' \\ '2026-09-12 20:00:00 - INFO - kas 4.7 started' \\ '/__w/project/baseline/build/layers/meta/conf/layer.conf ' \\ + 'Parsing recipes...WARNING: deterministic warning' \\ '/workspace/candidate/recipe.bb' | normalise_kas_projection """, ], @@ -44,6 +45,11 @@ def test_kas_projection_removes_host_noise_and_sorts_at_call_site(self) -> None: "/build/layers/meta/conf/layer.conf\n/recipe.bb\n", ) + def test_warnings_come_from_cooker_logs_not_interleaved_command_logs(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn("find build/tmp/log/cooker -type f -name '*.log'", source) + self.assertNotIn("find \"$output_dir\" -type f -name '*.log'", source) + def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: source = SCRIPT.read_text(encoding="utf-8") match = re.search(r"(?ms)^capture_command\(\) \{\n.*?^\}\n", source) From 88a53cd6c603c9506463a3b9afe76f3609180052 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 00:11:31 +0100 Subject: [PATCH 44/79] ci: normalise encoded checkout provenance --- scripts/validation/capture-layer-state.sh | 2 ++ scripts/validation/tests/test_capture_layer_state.py | 7 +++++++ 2 files changed, 9 insertions(+) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index a8681d1b..10612119 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -159,6 +159,7 @@ normalise_kas_projection() { -e '/^Summary: There were [0-9]+ WARNING messages?\.?$/d' \ -e '/^NOTE: Starting bitbake server\.\.\.$/d' \ -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e 's/_(baseline|candidate)_build_layers_/_REPO_build_layers_/g' \ -e "s#$PWD##g" \ -e 's#[[:space:]]+$##' } @@ -195,6 +196,7 @@ python3 "$(dirname "$0")/select-bitbake-env.py" \ "$output_dir/environment.log" \ | sed -E \ -e 's#(/[^/[:space:]]+)*/(baseline|candidate)(/|$)#\3#g' \ + -e 's/_(baseline|candidate)_build_layers_/_REPO_build_layers_/g' \ -e "s#$PWD##g" \ -e "s#$test_keys_dir##g" \ -e "s#$cache_root##g" \ diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 8623f482..aaad7aa4 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -50,6 +50,13 @@ def test_warnings_come_from_cooker_logs_not_interleaved_command_logs(self) -> No self.assertIn("find build/tmp/log/cooker -type f -name '*.log'", source) self.assertNotIn("find \"$output_dir\" -type f -name '*.log'", source) + def test_encoded_bitbake_provenance_checkout_is_normalised(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn( + "'s/_(baseline|candidate)_build_layers_/_REPO_build_layers_/g'", + source, + ) + def test_capture_command_replays_failure_log_and_preserves_status(self) -> None: source = SCRIPT.read_text(encoding="utf-8") match = re.search(r"(?ms)^capture_command\(\) \{\n.*?^\}\n", source) From f7c6d6e036f4ead7ae7bd83b88abc969c5eadf38 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 01:20:07 +0100 Subject: [PATCH 45/79] ci: audit immutable baseline repair Build both sides with the exact two-file Jaguar U-Boot backport while validating the old and new submodule pins, repair branch, ancestry and changed-file set. Assisted-by: Codex --- ci/layer-adoption-contract.json | 15 +++ .../run-layer-adoption-regression.py | 91 ++++++++++++++++++- .../test_run_layer_adoption_regression.py | 84 +++++++++++++++++ 3 files changed, 186 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 6c3c21a7..4f1266ca 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -1,5 +1,20 @@ { "schema": 1, "reason": "Adopt the isolated NXP i.MX95 partner layer without changing any pre-existing Dynamic Devices build tuple.", + "baseline_repairs": [ + { + "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", + "submodule": "meta-dynamicdevices-bsp", + "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", + "to": "71f566e04e699cd49e63cf3c8cff47a817d06956", + "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", + "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", + "files": [ + "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-handheld/01-customise-dtb.patch", + "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch" + ], + "reason": "The immutable baseline cannot apply these two stale patch contexts to its pinned U-Boot revision. Build both sides with the focused two-file backport while auditing the exact old-to-new submodule transition." + } + ], "allowed_deltas": {} } diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 88dc8876..44cca472 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -7,6 +7,7 @@ import hashlib import json import os +import re import shutil import subprocess import tempfile @@ -17,6 +18,7 @@ FIELDS = ("id", "machine", "distro", "image", "config", "product_features") PRODUCT_SUBMODULES = ("meta-dynamicdevices-bsp", "meta-dynamicdevices-distro") CAPTURE_SCHEMA_FILES = ( + "ci/layer-adoption-contract.json", "scripts/validation/run-layer-adoption-regression.py", "scripts/validation/capture-layer-state.sh", "scripts/validation/canonicalise-bitbake-layer-output.py", @@ -25,6 +27,13 @@ ) +def submodule_commit(repository: Path, relative: str) -> str: + entry = git_output(repository, "ls-tree", "HEAD", "--", relative).split() + if len(entry) < 3 or entry[0] != "160000" or entry[1] != "commit": + raise RuntimeError(f"{repository}: {relative} is not a pinned git submodule") + return entry[2] + + def evidence_digest(root: Path) -> str: digest = hashlib.sha256() for path in sorted(root.rglob("*")): @@ -95,10 +104,7 @@ def git_output(repository: Path, *args: str) -> str: def prepare_repository(repository: Path) -> None: """Initialize and verify the pinned local layers used by every KAS tuple.""" for relative in PRODUCT_SUBMODULES: - entry = git_output(repository, "ls-tree", "HEAD", "--", relative).split() - if len(entry) < 3 or entry[0] != "160000" or entry[1] != "commit": - raise RuntimeError(f"{repository}: {relative} is not a pinned git submodule") - expected = entry[2] + expected = submodule_commit(repository, relative) layer = repository / relative layer_conf = layer / "conf/layer.conf" if not layer_conf.is_file(): @@ -128,6 +134,82 @@ def prepare_repository(repository: Path) -> None: raise RuntimeError(f"{repository}: {relative} has uncommitted content") +def apply_baseline_repairs( + baseline: Path, candidate: Path, contract_path: Path, base_sha: str +) -> None: + """Apply an exact, audited repair to an otherwise unbuildable baseline.""" + contract = json.loads(contract_path.read_text(encoding="utf-8")) + repairs = contract.get("baseline_repairs", []) + if not isinstance(repairs, list): + raise ValueError("baseline_repairs must be an array") + for repair in repairs: + if not isinstance(repair, dict): + raise ValueError("baseline repair must be an object") + if repair.get("base_sha") != base_sha: + continue + required = {"submodule", "from", "to", "url", "ref", "files", "reason"} + if not required <= repair.keys(): + raise ValueError("baseline repair is incomplete") + relative = str(repair["submodule"]) + old = str(repair["from"]) + new = str(repair["to"]) + url = str(repair["url"]) + ref = str(repair["ref"]) + files = repair["files"] + reason = str(repair["reason"]).strip() + if relative not in PRODUCT_SUBMODULES: + raise ValueError(f"unsupported baseline repair submodule: {relative}") + if any(not re.fullmatch(r"[0-9a-f]{40}", commit) for commit in (old, new)): + raise ValueError("baseline repair pins must be full lowercase commit IDs") + if not url.startswith("https://github.com/DynamicDevices/"): + raise ValueError("baseline repair URL must use the DynamicDevices HTTPS origin") + if not ref.startswith("refs/heads/"): + raise ValueError("baseline repair ref must be an explicit branch") + if ( + not reason + or not isinstance(files, list) + or not files + or any( + not isinstance(path, str) + or Path(path).is_absolute() + or ".." in Path(path).parts + for path in files + ) + ): + raise ValueError("baseline repair needs a reason and exact file list") + if submodule_commit(baseline, relative) != old: + raise RuntimeError(f"baseline repair {relative}: unexpected source pin") + if submodule_commit(candidate, relative) != new: + raise RuntimeError(f"baseline repair {relative}: unexpected candidate pin") + + candidate_layer = candidate / relative + subprocess.run( + ["git", "merge-base", "--is-ancestor", old, new], + cwd=candidate_layer, + check=True, + ) + changed = git_output(candidate_layer, "diff", "--name-only", old, new).splitlines() + if sorted(changed) != sorted(str(path) for path in files): + raise RuntimeError( + f"baseline repair {relative}: changed files do not match contract" + ) + + baseline_layer = baseline / relative + subprocess.run(["git", "fetch", url, ref], cwd=baseline_layer, check=True) + fetched = git_output(baseline_layer, "rev-parse", "FETCH_HEAD") + if fetched != new: + raise RuntimeError(f"baseline repair {relative}: ref resolved to {fetched}") + subprocess.run( + ["git", "checkout", "--detach", new], cwd=baseline_layer, check=True + ) + if git_output(baseline_layer, "status", "--porcelain", "--untracked-files=all"): + raise RuntimeError(f"baseline repair {relative}: checkout is dirty") + print( + f"Applying audited baseline repair for {relative}: {old} -> {new}", + flush=True, + ) + + def capture( script: Path, repository: Path, @@ -179,6 +261,7 @@ def main() -> int: # submodule content through the same KAS capture path. prepare_repository(baseline) prepare_repository(candidate) + apply_baseline_repairs(baseline, candidate, contract, base_sha) environment = os.environ.copy() environment["LAYER_ADOPTION_TEST_KEYS_DIR"] = str(test_keys) diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index a3ce4b84..7de71b20 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -22,6 +22,89 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_audited_baseline_repair_is_exact_and_fail_closed(self) -> None: + old = "a" * 40 + new = "b" * 40 + changed_file = "recipes-bsp/u-boot/u-boot-fio/board/fix.patch" + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + contract = root / "contract.json" + contract.write_text( + json.dumps( + { + "baseline_repairs": [ + { + "base_sha": "base", + "submodule": "meta-dynamicdevices-bsp", + "from": old, + "to": new, + "url": "https://github.com/DynamicDevices/bsp.git", + "ref": "refs/heads/focused-backport", + "files": [changed_file], + "reason": "repair an exact pre-existing patch failure", + } + ] + } + ), + encoding="utf-8", + ) + with mock.patch.object( + MODULE, "submodule_commit", side_effect=[old, new] + ), mock.patch.object( + MODULE, + "git_output", + side_effect=[changed_file + "\n", new, ""], + ), mock.patch.object(MODULE.subprocess, "run") as run: + MODULE.apply_baseline_repairs( + root / "baseline", root / "candidate", contract, "base" + ) + + self.assertEqual(run.call_count, 3) + self.assertEqual( + run.call_args_list[1].args[0], + [ + "git", + "fetch", + "https://github.com/DynamicDevices/bsp.git", + "refs/heads/focused-backport", + ], + ) + + def test_audited_baseline_repair_rejects_extra_files(self) -> None: + old = "a" * 40 + new = "b" * 40 + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + contract = root / "contract.json" + contract.write_text( + json.dumps( + { + "baseline_repairs": [ + { + "base_sha": "base", + "submodule": "meta-dynamicdevices-bsp", + "from": old, + "to": new, + "url": "https://github.com/DynamicDevices/bsp.git", + "ref": "refs/heads/focused-backport", + "files": ["expected.patch"], + "reason": "focused repair", + } + ] + } + ), + encoding="utf-8", + ) + with mock.patch.object( + MODULE, "submodule_commit", side_effect=[old, new] + ), mock.patch.object( + MODULE, "git_output", return_value="unexpected.patch\n" + ), mock.patch.object(MODULE.subprocess, "run"): + with self.assertRaisesRegex(RuntimeError, "do not match contract"): + MODULE.apply_baseline_repairs( + root / "baseline", root / "candidate", contract, "base" + ) + def test_generated_signing_identity_is_validated_before_reuse(self) -> None: generator = MODULE_PATH.parent / "generate-layer-adoption-test-keys.sh" with tempfile.TemporaryDirectory() as directory: @@ -146,6 +229,7 @@ def test_capture_schema_covers_every_evidence_producer(self) -> None: first = MODULE.capture_schema_digest(root) self.assertRegex(first, r"^[0-9a-f]{64}$") self.assertIn("scripts/validation/capture-layer-state.sh", MODULE.CAPTURE_SCHEMA_FILES) + self.assertIn("ci/layer-adoption-contract.json", MODULE.CAPTURE_SCHEMA_FILES) self.assertIn( "scripts/validation/canonicalise-bitbake-layer-output.py", MODULE.CAPTURE_SCHEMA_FILES, From 014a30118e4651935cab4d816ada52e28362ac53 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 02:13:58 +0100 Subject: [PATCH 46/79] ci: include packaging baseline repair Extend the exact LmP v96 backport contract to cover board-scripts empty-directory QA failures while preserving full baseline and candidate builds. Assisted-by: Codex --- ci/layer-adoption-contract.json | 7 ++++--- meta-dynamicdevices-bsp | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 4f1266ca..794a1578 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,14 +6,15 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "71f566e04e699cd49e63cf3c8cff47a817d06956", + "to": "a42bd66fa7e19712fe7bce8a7da892ac994423a2", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-handheld/01-customise-dtb.patch", - "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch" + "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch", + "recipes-bsp/board-scripts/board-scripts_1.0.bb" ], - "reason": "The immutable baseline cannot apply these two stale patch contexts to its pinned U-Boot revision. Build both sides with the focused two-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts and creates empty package directories for products without board scripts. Build both sides with the focused three-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index 71f566e0..a42bd66f 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit 71f566e04e699cd49e63cf3c8cff47a817d06956 +Subproject commit a42bd66fa7e19712fe7bce8a7da892ac994423a2 From d767d4c7a88fb0f21999d87811f7b5774b066ae9 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 02:47:39 +0100 Subject: [PATCH 47/79] ci: include Handheld DTS baseline repair Extend the exact LmP v96 backport contract to the undefined Handheld audio-mute pinmux while preserving complete symmetric builds. Assisted-by: Codex --- ci/layer-adoption-contract.json | 7 ++++--- meta-dynamicdevices-bsp | 2 +- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 794a1578..6866a5f9 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,15 +6,16 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "a42bd66fa7e19712fe7bce8a7da892ac994423a2", + "to": "22d33e11a4c4d2a98c77ec247b20d45ffdf328d6", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-handheld/01-customise-dtb.patch", "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch", - "recipes-bsp/board-scripts/board-scripts_1.0.bb" + "recipes-bsp/board-scripts/board-scripts_1.0.bb", + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts and creates empty package directories for products without board scripts. Build both sides with the focused three-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, creates empty package directories for products without board scripts, and uses an undefined Handheld pinmux macro. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index a42bd66f..22d33e11 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit a42bd66fa7e19712fe7bce8a7da892ac994423a2 +Subproject commit 22d33e11a4c4d2a98c77ec247b20d45ffdf328d6 From 25e4e8abcb49f5b7df4e83a0667e58092a0b07bc Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 03:27:36 +0100 Subject: [PATCH 48/79] ci: include Handheld DTS baseline repair Extend the exact LmP v96 backport contract to the current EVK base and inherited regulator required by the pinned kernel. Assisted-by: Codex --- ci/layer-adoption-contract.json | 4 ++-- meta-dynamicdevices-bsp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 6866a5f9..c2b29f36 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,7 +6,7 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "22d33e11a4c4d2a98c77ec247b20d45ffdf328d6", + "to": "cb3e579f96092951b0a5c3cd69248efa58b55e13", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ @@ -15,7 +15,7 @@ "recipes-bsp/board-scripts/board-scripts_1.0.bb", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, creates empty package directories for products without board scripts, and uses an undefined Handheld pinmux macro. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, creates empty package directories for products without board scripts, and carries a Handheld DTS with an undefined pinmux macro and duplicate base-board labels. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index 22d33e11..cb3e579f 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit 22d33e11a4c4d2a98c77ec247b20d45ffdf328d6 +Subproject commit cb3e579f96092951b0a5c3cd69248efa58b55e13 From 9e66b1b92b296377a8db3921dadaca66e6294623 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 04:10:21 +0100 Subject: [PATCH 49/79] ci: retain repaired empty package baseline Extend the exact LmP v96 backport pin so products that request board-scripts can complete rootfs even when their payload is intentionally empty. Assisted-by: Codex --- ci/layer-adoption-contract.json | 4 ++-- meta-dynamicdevices-bsp | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index c2b29f36..72c3d1c4 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,7 +6,7 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "cb3e579f96092951b0a5c3cd69248efa58b55e13", + "to": "768f9e8763a0797d0991fe6ed58e8e8bc85da3c2", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ @@ -15,7 +15,7 @@ "recipes-bsp/board-scripts/board-scripts_1.0.bb", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, creates empty package directories for products without board scripts, and carries a Handheld DTS with an undefined pinmux macro and duplicate base-board labels. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, and carries a Handheld DTS with an undefined pinmux macro and duplicate base-board labels. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index cb3e579f..768f9e87 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit cb3e579f96092951b0a5c3cd69248efa58b55e13 +Subproject commit 768f9e8763a0797d0991fe6ed58e8e8bc85da3c2 From cc48e390f1c47068f9350c3231e3fa511f5df7b2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 04:59:14 +0100 Subject: [PATCH 50/79] ci: include remaining DTS baseline repairs Extend the exact LmP v96 backport contract to the INST and Phasora EVK-base corrections found by the full product gate. Assisted-by: Codex --- ci/layer-adoption-contract.json | 8 +++++--- meta-dynamicdevices-bsp | 2 +- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 72c3d1c4..3112e01e 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,16 +6,18 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "768f9e8763a0797d0991fe6ed58e8e8bc85da3c2", + "to": "e013655c6f78c250ad5ca4a593ff2815ed6aaaab", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-handheld/01-customise-dtb.patch", "recipes-bsp/u-boot/u-boot-fio/imx8mm-jaguar-phasora/01-customise-dtb.patch", "recipes-bsp/board-scripts/board-scripts_1.0.bb", - "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts" + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts", + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-inst.dts", + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-phasora.dts" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, and carries a Handheld DTS with an undefined pinmux macro and duplicate base-board labels. Build both sides with the focused four-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, and carries three product DTS files that no longer compile against the pinned kernel. Build both sides with the focused six-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index 768f9e87..e013655c 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit 768f9e8763a0797d0991fe6ed58e8e8bc85da3c2 +Subproject commit e013655c6f78c250ad5ca4a593ff2815ed6aaaab From 459641fc59b62ed855d2a82eb32d4cf3a2dd94ca Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 05:48:29 +0100 Subject: [PATCH 51/79] ci: make kernel warnings sstate-independent Force kernel_configcheck on both sides before collecting cooker warnings so shared sstate cannot create false warning deltas. Assisted-by: Codex --- scripts/validation/capture-layer-state.sh | 6 ++++++ scripts/validation/tests/test_capture_layer_state.py | 7 +++++++ 2 files changed, 13 insertions(+) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 10612119..28c101f1 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -229,6 +229,12 @@ rm "$output_dir/environment.log" # baseline and candidate. capture_command build run_bitbake "bitbake $target" +# Task warnings must not depend on whether shared sstate caused the task to run +# during this particular image build. Force the kernel's warning-producing +# configuration check on both sides before collecting cooker diagnostics. +capture_command kernel-configcheck run_bitbake \ + "bitbake -f -c kernel_configcheck virtual/kernel" >/dev/null + deploy_dir="build/tmp/deploy/images/$machine" if [ ! -d "$deploy_dir" ]; then echo "ERROR: deploy directory missing after successful build: $deploy_dir" >&2 diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index aaad7aa4..f8e60742 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -50,6 +50,13 @@ def test_warnings_come_from_cooker_logs_not_interleaved_command_logs(self) -> No self.assertIn("find build/tmp/log/cooker -type f -name '*.log'", source) self.assertNotIn("find \"$output_dir\" -type f -name '*.log'", source) + def test_kernel_warning_capture_is_independent_of_sstate_reuse(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn( + '"bitbake -f -c kernel_configcheck virtual/kernel" >/dev/null', + source, + ) + def test_encoded_bitbake_provenance_checkout_is_normalised(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertIn( From 28311cbb365d339463608c53002615b1ee48d4bc Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 07:03:49 +0100 Subject: [PATCH 52/79] ci: include Phasora SPDX baseline repair Extend the exact LmP v96 repair contract to the custom loader license text required by create-spdx. Assisted-by: Codex --- ci/layer-adoption-contract.json | 9 ++++++--- meta-dynamicdevices-bsp | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 3112e01e..93e5cf9e 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -6,7 +6,7 @@ "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", "submodule": "meta-dynamicdevices-bsp", "from": "47542ad3f8d49850df69e37a3414c1ef60c51809", - "to": "e013655c6f78c250ad5ca4a593ff2815ed6aaaab", + "to": "b926d4e96532fb95c83984b154d7b2f72d82471e", "url": "https://github.com/DynamicDevices/meta-dynamicdevices-bsp.git", "ref": "refs/heads/fix/imx8mm-jaguar-uboot-dtb-context-lmp-v96", "files": [ @@ -15,9 +15,12 @@ "recipes-bsp/board-scripts/board-scripts_1.0.bb", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-handheld.dts", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-inst.dts", - "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-phasora.dts" + "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-phasora.dts", + "recipes-bsp/upd72020x-load/upd72020x-load/LicenseRef-markusj-upd72020x-load", + "recipes-bsp/upd72020x-load/upd72020x-load/loader-upstream-license-note", + "recipes-bsp/upd72020x-load/upd72020x-load_git.bb" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, and carries three product DTS files that no longer compile against the pinned kernel. Build both sides with the focused six-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, carries three product DTS files that no longer compile against the pinned kernel, and does not expose the Phasora loader's custom license text to SPDX generation. Build both sides with the focused nine-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index e013655c..b926d4e9 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit e013655c6f78c250ad5ca4a593ff2815ed6aaaab +Subproject commit b926d4e96532fb95c83984b154d7b2f72d82471e From 843ab3bd89e2512bdde28d9218ed480eb69cc9c2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 07:10:33 +0100 Subject: [PATCH 53/79] ci: match renamed license repair surface Track Git's canonical renamed path in the exact baseline repair file set. Assisted-by: Codex --- ci/layer-adoption-contract.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 93e5cf9e..8496f62f 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -17,10 +17,9 @@ "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-inst.dts", "recipes-bsp/device-tree/lmp-device-tree/imx8mm-jaguar-phasora.dts", "recipes-bsp/upd72020x-load/upd72020x-load/LicenseRef-markusj-upd72020x-load", - "recipes-bsp/upd72020x-load/upd72020x-load/loader-upstream-license-note", "recipes-bsp/upd72020x-load/upd72020x-load_git.bb" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, carries three product DTS files that no longer compile against the pinned kernel, and does not expose the Phasora loader's custom license text to SPDX generation. Build both sides with the focused nine-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, carries three product DTS files that no longer compile against the pinned kernel, and does not expose the Phasora loader's custom license text to SPDX generation. Build both sides with the focused eight-file backport while auditing the exact old-to-new submodule transition." } ], "allowed_deltas": {} From 6c4a0aa4d3fbdf34c8c79f33e209ef84d6181465 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 09:02:38 +0100 Subject: [PATCH 54/79] ci: treat removed warnings as improvements Keep the gate fail-closed for candidate-added warnings while avoiding false failures when shared sstate suppresses a pre-existing baseline warning. Assisted-by: Codex --- scripts/validation/compare-layer-state.py | 8 ++++++++ .../tests/test_compare_layer_state.py | 20 +++++++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/scripts/validation/compare-layer-state.py b/scripts/validation/compare-layer-state.py index 5f656aea..a28fec01 100755 --- a/scripts/validation/compare-layer-state.py +++ b/scripts/validation/compare-layer-state.py @@ -11,6 +11,7 @@ from pathlib import Path DEPLOY_SIZES = "deploy-layout-and-sizes.txt" +WARNINGS = "warnings.txt" MAX_DEPLOY_SIZE_DRIFT_RATIO = 0.005 MIN_DEPLOY_SIZE_DRIFT_BYTES = 4096 @@ -48,6 +49,11 @@ def deploy_deltas(old_lines: list[str], new_lines: list[str]) -> list[str]: return deltas +def warning_deltas(old_lines: list[str], new_lines: list[str]) -> list[str]: + """Return candidate-added warnings; removing a baseline warning is safe.""" + return sorted(set(new_lines) - set(old_lines)) + + def files(root: Path) -> dict[str, list[str]]: result: dict[str, list[str]] = {} for path in sorted(root.rglob("*")): @@ -92,6 +98,8 @@ def main() -> int: except ValueError as exc: print(f"ERROR: {name}: {exc}", file=sys.stderr) return 2 + elif name == WARNINGS: + changed_lines = warning_deltas(old.get(name, []), new.get(name, [])) else: delta = list(difflib.unified_diff(old.get(name, []), new.get(name, []), lineterm="")) changed_lines = [ diff --git a/scripts/validation/tests/test_compare_layer_state.py b/scripts/validation/tests/test_compare_layer_state.py index 10fa67dd..0bf7a3d8 100644 --- a/scripts/validation/tests/test_compare_layer_state.py +++ b/scripts/validation/tests/test_compare_layer_state.py @@ -38,5 +38,25 @@ def test_malformed_or_duplicate_entries_fail_closed(self) -> None: MODULE.deploy_entries(["image.wic\t1", "image.wic\t2"]) +class WarningComparisonTests(unittest.TestCase): + def test_new_candidate_warning_is_a_delta(self) -> None: + self.assertEqual( + MODULE.warning_deltas( + ["WARNING: existing"], + ["WARNING: existing", "WARNING: regression"], + ), + ["WARNING: regression"], + ) + + def test_removed_baseline_warning_is_not_a_delta(self) -> None: + self.assertEqual( + MODULE.warning_deltas( + ["WARNING: fixed", "WARNING: existing"], + ["WARNING: existing"], + ), + [], + ) + + if __name__ == "__main__": unittest.main() From b47dc3765a46542d5b6ebbd936d3574b4c15a4ae Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 09:56:51 +0100 Subject: [PATCH 55/79] ci: shard layer adoption tuples Run every protected tuple as an independent fail-fast-disabled matrix shard while preserving the single required Layer Adoption Gate aggregator. Keep the shared local driver full-matrix by default. Signed-off-by: Alex Lennon Assisted-by: Codex --- .github/workflows/layer-adoption-gate.yml | 20 +++++++++++++------ .../run-layer-adoption-regression.py | 17 +++++++++++++++- .../test_run_layer_adoption_regression.py | 20 +++++++++++++++++++ 3 files changed, 50 insertions(+), 7 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 2281cb0a..7e875fb9 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -26,6 +26,7 @@ jobs: outputs: material: ${{ steps.detect.outputs.material }} base_sha: ${{ steps.base.outputs.sha }} + tuple_ids: ${{ steps.detect.outputs.tuple_ids }} steps: - uses: actions/checkout@v7 with: @@ -58,13 +59,19 @@ jobs: --base '${{ steps.base.outputs.sha }}' \ --head '${{ github.sha }}' \ --github-output "$GITHUB_OUTPUT" + tuple_ids=$(python3 -c 'import json; print(json.dumps([entry["id"] for entry in json.load(open("ci/layer-adoption-tuples.json"))["tuples"]], separators=(",", ":")))') + echo "tuple_ids=$tuple_ids" >> "$GITHUB_OUTPUT" regression: - name: Existing product regression + name: Existing product regression (${{ matrix.tuple_id }}) needs: detect if: needs.detect.outputs.material == 'true' - # One named ai-tools runner owns the persistent Yocto cache. Keeping the - # complete matrix inside one job makes resource use and the required gate - # obvious, while the driver still fails closed on every protected tuple. + strategy: + fail-fast: false + matrix: + tuple_id: ${{ fromJSON(needs.detect.outputs.tuple_ids) }} + # Each tuple is an independent shard so one failure cannot hide later + # product failures. The final Layer Adoption Gate remains the single + # branch-protection contract, and local driver runs still cover all tuples. runs-on: [self-hosted, Linux, X64, yocto, ai-tools] container: image: ghcr.io/siemens/kas/kas@sha256:d989add57fc441fe9e27bb2dd6ed98c5597b44c807928e35a72dc1cfbdda9abe @@ -119,12 +126,13 @@ jobs: --candidate candidate \ --evidence evidence \ --cache "$LAYER_ADOPTION_CACHE" \ - --test-keys "$LAYER_ADOPTION_CACHE/test-keys" + --test-keys "$LAYER_ADOPTION_CACHE/test-keys" \ + --tuple-id '${{ matrix.tuple_id }}' - name: Preserve comparison evidence if: always() uses: actions/upload-artifact@v7 with: - name: layer-adoption-evidence + name: layer-adoption-evidence-${{ matrix.tuple_id }} path: evidence retention-days: 14 - name: Remove job-owned build trees diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 44cca472..45889d38 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -89,6 +89,18 @@ def load_tuples(path: Path) -> list[dict[str, str]]: return tuples +def select_tuples( + tuples: list[dict[str, str]], tuple_id: str | None +) -> list[dict[str, str]]: + """Select one CI shard while keeping the local default as the full gate.""" + if tuple_id is None: + return tuples + selected = [entry for entry in tuples if entry["id"] == tuple_id] + if not selected: + raise ValueError(f"unknown protected tuple: {tuple_id}") + return selected + + def remove_build_tree(repository: Path) -> None: repository = repository.resolve() build = (repository / "build").resolve() @@ -240,6 +252,7 @@ def main() -> int: parser.add_argument("--evidence", required=True, type=Path) parser.add_argument("--cache", required=True, type=Path) parser.add_argument("--test-keys", required=True, type=Path) + parser.add_argument("--tuple-id") args = parser.parse_args() baseline = args.baseline.resolve() @@ -250,7 +263,9 @@ def main() -> int: capture_script = candidate / "scripts/validation/capture-layer-state.sh" compare_script = candidate / "scripts/validation/compare-layer-state.py" contract = candidate / "ci/layer-adoption-contract.json" - tuples = load_tuples(candidate / "ci/layer-adoption-tuples.json") + tuples = select_tuples( + load_tuples(candidate / "ci/layer-adoption-tuples.json"), args.tuple_id + ) base_sha = subprocess.check_output( ["git", "rev-parse", "HEAD"], cwd=baseline, text=True ).strip() diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 7de71b20..baa13b2c 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -22,6 +22,26 @@ class BaselineEvidenceTests(unittest.TestCase): + def test_ci_shard_selects_exactly_one_known_tuple(self) -> None: + tuples = [ + {"id": "image-a"}, + {"id": "mfgtool-a"}, + ] + self.assertIs(MODULE.select_tuples(tuples, None), tuples) + self.assertEqual( + MODULE.select_tuples(tuples, "mfgtool-a"), + [{"id": "mfgtool-a"}], + ) + with self.assertRaisesRegex(ValueError, "unknown protected tuple"): + MODULE.select_tuples(tuples, "missing") + + def test_workflow_shards_all_tuples_without_fail_fast(self) -> None: + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + self.assertIn("fail-fast: false", workflow) + self.assertIn("fromJSON(needs.detect.outputs.tuple_ids)", workflow) + self.assertIn("--tuple-id '${{ matrix.tuple_id }}'", workflow) + self.assertIn("name: Layer Adoption Gate", workflow) + def test_audited_baseline_repair_is_exact_and_fail_closed(self) -> None: old = "a" * 40 new = "b" * 40 From d3be861f5fe060337143470e2c80f18820c44147 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 10:05:49 +0100 Subject: [PATCH 56/79] feat(security): confine Waydroid with host SELinux --- conf/layer.conf | 1 + docs/r26-waydroid-selinux-plan.md | 119 ++++++++++++++++++ .../refpolicy/refpolicy-targeted/waydroid.fc | 7 ++ .../refpolicy/refpolicy-targeted/waydroid.if | 17 +++ .../refpolicy/refpolicy-targeted/waydroid.te | 22 ++++ .../refpolicy/refpolicy-targeted_%.bbappend | 15 +++ kas/base.yml | 5 + meta-dynamicdevices-bsp | 2 +- meta-dynamicdevices-distro | 2 +- recipes-support/waydroid/waydroid.bb | 1 + ...oid-rootfs-with-host-SELinux-context.patch | 86 +++++++++++++ .../waydroid/waydroid-image-provision.service | 3 + 12 files changed, 278 insertions(+), 2 deletions(-) create mode 100644 docs/r26-waydroid-selinux-plan.md create mode 100644 dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.fc create mode 100644 dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.if create mode 100644 dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te create mode 100644 dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend create mode 100644 recipes-support/waydroid/waydroid/0001-mount-Android-rootfs-with-host-SELinux-context.patch diff --git a/conf/layer.conf b/conf/layer.conf index d50bafef..86e51565 100644 --- a/conf/layer.conf +++ b/conf/layer.conf @@ -24,6 +24,7 @@ BBFILES_DYNAMIC += " \ rust-bin-layer:${LAYERDIR}/bbappends/meta-rust-bin/*/*/*.bb rust-bin-layer:${LAYERDIR}/bbappends/meta-rust-bin/*/*/*.bbappend \ meta-tensorflow:${LAYERDIR}/bbappends/meta-tensorflow/*/*/*.bbappend \ nxp-zigbee-rcp:${LAYERDIR}/bbappends/meta-nxp-zigbee-rcp/*/*/*.bbappend \ + selinux:${LAYERDIR}/dynamic-layers/selinux/recipes-*/*/*.bbappend \ " LAYERDEPENDS_meta-dynamicdevices = "meta-lmp-base meta-dynamicdevices-bsp meta-dynamicdevices-distro" diff --git a/docs/r26-waydroid-selinux-plan.md b/docs/r26-waydroid-selinux-plan.md new file mode 100644 index 00000000..727909d6 --- /dev/null +++ b/docs/r26-waydroid-selinux-plan.md @@ -0,0 +1,119 @@ +# R26 Waydroid host SELinux adoption and verification + +Status: implementation in progress; not release-approved. + +## Objective + +Run Waydroid on the 2 GB i.MX8MM Jaguar Screen with the Foundries/LmP host +SELinux policy enforcing. Android retains only the changes needed to operate +inside LXC because SELinux is kernel-global and cannot be independently owned +by the Android userspace in that container. + +## Reproducible baseline + +- Published Foundries target: `2892` (`main-jaguar-screen`) +- Manifest: `361d5ac577d9a9714069f689f4ed04680834f1e3` +- Application layer: `306e43cd04a93f42c0bd195601fa261045da39f0` +- BSP layer: `c6be3be6d94a5492b534b173f44e65592b49a13e` +- Distro layer: `c635979548a155f6035325b328ab9bd7a7a2ce24` +- LmP/meta-lmp: `d176612d7fc811bb8f511fcaa06dc617513c0eb5` +- meta-selinux candidate: `48f745109a1ecea9afe2a74d41dbd90fa7b370af` +- Android container-awareness change: `a9c4291` +- Android validation run: `34748284172` + +## Layer-adoption audit + +The candidate meta-selinux revision declares Scarthgap compatibility and +depends on OE-Core plus meta-python; the existing manifest already supplies +meta-oe. It is inert unless `selinux` is present in `DISTRO_FEATURES`. + +The layer changes recipes globally once enabled, including systemd, D-Bus, +OpenSSH, sudo, util-linux and Mesa. The upstream reference policy explicitly +requires product tailoring. Its kernel append only targets `linux-yocto`, so +the Jaguar `linux-lmp-fslc-imx` kernel requires its own fragment. + +Adoption is scoped through the product contract. `android-container` implies +the standard `selinux` distro feature; an explicit `host-selinux` product +feature is also available for future non-Android products. mfgtool tuples do +not select either feature. + +The custom refpolicy append is a dynamic append: it is parsed only when the +`selinux` layer collection is present. `APPS_MODS=waydroid` is required because +new refpolicy modules otherwise default to `off` when absent from the upstream +`modules.conf`. + +## Development policy lifecycle + +The first hardware image keeps the host globally enforcing but declares only +`waydroid_t` permissive. This isolates policy discovery to the Waydroid domain +and avoids weakening unrelated host services. The statement +`permissive waydroid_t;` is a release blocker and must be removed after AVCs +have been classified and converted to narrow rules. + +Immutable OSTree content is labelled at image construction with +`selinux-image`. `FIRST_BOOT_RELABEL` remains disabled because a whole-root +relabel is unsuitable for an immutable deployment. The provisioning unit runs +`restorecon -RF /var/lib/waydroid` to migrate persistent state retained across +OTA; the existing network script restores the `/run/waydroid-lxc` context. + +Device nodes shared with host services are not relabelled as Waydroid-owned. +Access to Binder, DRM/Etnaviv, V4L2 and the Weston/PulseAudio sockets will be +granted against their host-owned types from reviewed hardware AVC evidence. +Android system/vendor files carry Android policy xattrs that are not valid +host-policy types, so Waydroid applies the single host-owned +`waydroid_rootfs_t` mount context to its ext4 and overlay mounts whenever host +SELinux is active. This avoids any release rule permitting execution from the +generic `unlabeled_t` type. + +## Existing CI tuple regression matrix + +The layer-adoption gate covers every active `ci-scripts` platform tuple, not +only the new Jaguar Screen image. + +| Platform ref | Machine | Platform distro/special case | Existing mfgtool tuple | +| --- | --- | --- | --- | +| `main-jaguar` | `imx8mm-jaguar-sentai` | default | yes | +| `main-jaguar-sentai` | `imx8mm-jaguar-sentai` | headless, signed/LUKS/OCF parameters | yes | +| `main-jaguar-sentai-prod` | `imx8mm-jaguar-sentai` | production | yes | +| `main-jaguar-sentai-ext` | `imx8mm-jaguar-sentai` | default | yes | +| `main-jaguar-sentai-ce` | `imx8mm-jaguar-sentai` | CE image | yes | +| `main-jaguar-sentai-ocf` | `imx8mm-jaguar-sentai` | headless, signed/LUKS/OCF | yes | +| `imx8mm-jaguar-handheld-5in` | `imx8mm-jaguar-handheld-5in` | legacy Waydroid distro | yes | +| `imx8mm-jaguar-handheld-7in` | `imx8mm-jaguar-handheld-7in` | legacy Waydroid distro | yes | +| `main-jaguar-phasora` | `imx8mm-jaguar-phasora` | headless | yes | +| `main-jaguar-phasora-ext` | `imx8mm-jaguar-phasora` | headless | yes | +| `main-imx8ulp` | `imx8ulp-lpddr4-evk` | headless | yes | +| `main-jaguar-inst` | `imx8mm-jaguar-inst` | headless | yes | +| `main-rpi4` | `raspberrypi4-64` | default | no | +| `main-rpi4-v2g-evse` | `raspberrypi4-64` | legacy Waydroid distro | no | +| `main-rpi5` | `raspberrypi5` | default | no | +| `main-imx93-jaguar-eink` | `imx93-jaguar-eink` | headless, signed/LUKS | yes | +| `main-imx95-frdm-devel` | `imx95-frdm-evk` | product-feature Android | yes | +| `main-jaguar-screen` | `imx8mm-jaguar-screen` | `display android-container` | none currently defined | + +## Gates before manifest publication + +1. Resolve the exact candidate manifest and prove every project SHA exists. +2. Parse the Jaguar Screen platform and mfgtool configurations. +3. Run `bitbake-layers show-layers`, `show-appends`, and provider checks. +4. Run kernel `do_kernel_configcheck`; prove SELinux is built in and appears + in `CONFIG_LSM` only for the selected product. +5. Build the policy and rootfs locally; prove the Waydroid module is active, + `/etc/selinux/config` says enforcing, and OSTree preserves labels. +6. Parse/build the full platform and mfgtool regression matrix above, including + bootloader, kernel, OP-TEE, signing and recovery/factory artifacts. +7. Verify SBOM, licence manifest, source hashes and image hashes are retained. +8. Only then pin the three Dynamic Devices layer commits in the manifest and + submit the Foundries build. + +## Hardware acceptance + +- `getenforce` reports `Enforcing`; kernel command line does not disable it. +- Waydroid processes enter `waydroid_t`; no Waydroid process remains + `unconfined_t`, `init_t`, or another generic domain. +- The release policy contains no permissive domains and no unexplained AVCs. +- Android boots, Binder works, kiosk UI renders, and 2 GB zram/low-memory + settings are active. +- Etnaviv acceleration and V4L2 H.264 decode are demonstrated independently. +- Foundries OTA install and rollback both succeed without breaking labels, + secure boot, recovery or manufacturing flows. diff --git a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.fc b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.fc new file mode 100644 index 00000000..7e4bad82 --- /dev/null +++ b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.fc @@ -0,0 +1,7 @@ +/usr/bin/waydroid -- gen_context(system_u:object_r:waydroid_exec_t,s0) +/usr/libexec/waydroid-image-provision -- gen_context(system_u:object_r:waydroid_exec_t,s0) +/usr/libexec/waydroid-jaguar-wait -- gen_context(system_u:object_r:waydroid_exec_t,s0) +/usr/lib/waydroid/data/scripts/waydroid-net\.sh -- gen_context(system_u:object_r:waydroid_exec_t,s0) + +/var/lib/waydroid(/.*)? gen_context(system_u:object_r:waydroid_var_lib_t,s0) +/run/waydroid-lxc(/.*)? gen_context(system_u:object_r:waydroid_runtime_t,s0) diff --git a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.if b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.if new file mode 100644 index 00000000..6659268c --- /dev/null +++ b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.if @@ -0,0 +1,17 @@ +## Waydroid Android container runtime. + +######################################## +## +## Execute Waydroid in the Waydroid domain. +## +## +## Domain allowed to transition. +## +interface(`waydroid_domtrans',` + gen_require(` + type waydroid_t, waydroid_exec_t; + ') + + corecmd_search_bin($1) + domtrans_pattern($1, waydroid_exec_t, waydroid_t) +') diff --git a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te new file mode 100644 index 00000000..e38ea41d --- /dev/null +++ b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te @@ -0,0 +1,22 @@ +policy_module(waydroid, 1.0.0) + +######################################## +# Declarations + +type waydroid_t; +type waydroid_exec_t; +init_daemon_domain(waydroid_t, waydroid_exec_t) + +type waydroid_var_lib_t; +files_type(waydroid_var_lib_t) + +type waydroid_runtime_t; +files_runtime_file(waydroid_runtime_t) + +type waydroid_rootfs_t; +files_type(waydroid_rootfs_t) + +# Development phase only. The host remains enforcing while accesses from +# this dedicated domain are logged but not denied. R26 release validation +# must remove this statement after the AVC-derived allow-list is reviewed. +permissive waydroid_t; diff --git a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend new file mode 100644 index 00000000..e2d8219c --- /dev/null +++ b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend @@ -0,0 +1,15 @@ +FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:" + +SRC_URI:append = "${@bb.utils.contains('DISTRO_FEATURES', 'waydroid', ' file://waydroid.te file://waydroid.fc file://waydroid.if', '', d)}" +EXTRA_OEMAKE:append = "${@bb.utils.contains('DISTRO_FEATURES', 'waydroid', ' APPS_MODS=waydroid', '', d)}" + +# refpolicy's `make conf` discovers modules below policy/modules. Install the +# product module before upstream generates modules.conf and compiles policy. +do_compile:prepend() { + if ${@bb.utils.contains('DISTRO_FEATURES', 'waydroid', 'true', 'false', d)}; then + install -d ${S}/policy/modules/services + install -m 0644 ${WORKDIR}/waydroid.te ${S}/policy/modules/services/ + install -m 0644 ${WORKDIR}/waydroid.fc ${S}/policy/modules/services/ + install -m 0644 ${WORKDIR}/waydroid.if ${S}/policy/modules/services/ + fi +} diff --git a/kas/base.yml b/kas/base.yml index 0f4bc79d..e8ca7e42 100644 --- a/kas/base.yml +++ b/kas/base.yml @@ -51,6 +51,11 @@ repos: meta-parsec: meta-integrity: + meta-selinux: + url: https://git.yoctoproject.org/meta-selinux + commit: 48f745109a1ecea9afe2a74d41dbd90fa7b370af + path: build/layers/meta-selinux + meta-updater: url: https://github.com/lmp-mirrors/meta-updater commit: b275153c9c8ea09e27d41db9f2faba3ebf92d2c2 diff --git a/meta-dynamicdevices-bsp b/meta-dynamicdevices-bsp index 47542ad3..389b2b27 160000 --- a/meta-dynamicdevices-bsp +++ b/meta-dynamicdevices-bsp @@ -1 +1 @@ -Subproject commit 47542ad3f8d49850df69e37a3414c1ef60c51809 +Subproject commit 389b2b27c0a542d9e13dcb4c3ec31b95c2c131e4 diff --git a/meta-dynamicdevices-distro b/meta-dynamicdevices-distro index 9807961b..b1901ad1 160000 --- a/meta-dynamicdevices-distro +++ b/meta-dynamicdevices-distro @@ -1 +1 @@ -Subproject commit 9807961bf74bb6c20eb60c0cd388b91b69883a41 +Subproject commit b1901ad12c4b18ef02c628744929898375feee72 diff --git a/recipes-support/waydroid/waydroid.bb b/recipes-support/waydroid/waydroid.bb index e2419773..1aa4ae9c 100644 --- a/recipes-support/waydroid/waydroid.bb +++ b/recipes-support/waydroid/waydroid.bb @@ -31,6 +31,7 @@ SRC_URI = "git://github.com/herrie82/waydroid.git;branch=herrie/luneos;protocol= file://waydroid-jaguar-ui.service \ file://weston-jaguar-waydroid.ini \ file://90-waydroid-screen.conf \ + file://0001-mount-Android-rootfs-with-host-SELinux-context.patch \ " S = "${WORKDIR}/git" diff --git a/recipes-support/waydroid/waydroid/0001-mount-Android-rootfs-with-host-SELinux-context.patch b/recipes-support/waydroid/waydroid/0001-mount-Android-rootfs-with-host-SELinux-context.patch new file mode 100644 index 00000000..aba8f708 --- /dev/null +++ b/recipes-support/waydroid/waydroid/0001-mount-Android-rootfs-with-host-SELinux-context.patch @@ -0,0 +1,86 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Alex J Lennon +Date: Sun, 13 Sep 2026 10:05:00 +0100 +Subject: [PATCH] mount Android rootfs with host SELinux context + +Android image xattrs name Android policy types which are unknown to the LXC +host policy. When the host has SELinux active, apply one host-owned mount +context to the system, vendor and overlay mounts. This keeps Android image +contents confined without treating them as unlabeled host files. + +Upstream-Status: Inappropriate [product host-policy integration] +--- + tools/helpers/images.py | 17 +++++++++++++---- + tools/helpers/mount.py | 4 +++- + 2 files changed, 16 insertions(+), 5 deletions(-) + +diff --git a/tools/helpers/images.py b/tools/helpers/images.py +index f266db3..abf51a4 100644 +--- a/tools/helpers/images.py ++++ b/tools/helpers/images.py +@@ -154,28 +154,37 @@ def make_prop(args, cfg, full_props_path): + + def mount_rootfs(args, images_dir, session): + cfg = tools.config.load(args) ++ mount_context = None ++ mount_options = None ++ if os.path.exists("/sys/fs/selinux/enforce"): ++ mount_context = "system_u:object_r:waydroid_rootfs_t:s0" ++ mount_options = ["context=" + mount_context] + helpers.mount.mount(args, images_dir + "/system.img", +- tools.config.defaults["rootfs"], umount=True) ++ tools.config.defaults["rootfs"], umount=True, ++ options=mount_options) + if cfg["waydroid"]["mount_overlays"] == "True": + try: + helpers.mount.mount_overlay(args, [tools.config.defaults["overlay"], + tools.config.defaults["rootfs"]], + tools.config.defaults["rootfs"], + upper_dir=tools.config.defaults["overlay_rw"] + "/system", +- work_dir=tools.config.defaults["overlay_work"] + "/system") ++ work_dir=tools.config.defaults["overlay_work"] + "/system", ++ mount_context=mount_context) + except RuntimeError: + cfg["waydroid"]["mount_overlays"] = "False" + tools.config.save(args, cfg) + logging.warning("Mounting overlays failed. The feature has been disabled.") + + helpers.mount.mount(args, images_dir + "/vendor.img", +- tools.config.defaults["rootfs"] + "/vendor") ++ tools.config.defaults["rootfs"] + "/vendor", ++ options=mount_options) + if cfg["waydroid"]["mount_overlays"] == "True": + helpers.mount.mount_overlay(args, [tools.config.defaults["overlay"] + "/vendor", + tools.config.defaults["rootfs"] + "/vendor"], + tools.config.defaults["rootfs"] + "/vendor", + upper_dir=tools.config.defaults["overlay_rw"] + "/vendor", +- work_dir=tools.config.defaults["overlay_work"] + "/vendor") ++ work_dir=tools.config.defaults["overlay_work"] + "/vendor", ++ mount_context=mount_context) + + for egl_path in ["/vendor/lib/egl", "/vendor/lib64/egl"]: + if os.path.isdir(egl_path): +diff --git a/tools/helpers/mount.py b/tools/helpers/mount.py +index 236ff5b..5a9c915 100644 +--- a/tools/helpers/mount.py ++++ b/tools/helpers/mount.py +@@ -152,7 +152,7 @@ def mount(args, source, destination, create_folders=True, umount=False, + raise RuntimeError("Mount failed: " + source + " -> " + destination) + + def mount_overlay(args, lower_dirs, destination, upper_dir=None, work_dir=None, +- create_folders=True, readonly=True): ++ create_folders=True, readonly=True, mount_context=None): + """ + Mount an overlay. + """ +@@ -167,6 +167,8 @@ def mount_overlay(args, lower_dirs, destination, upper_dir=None, work_dir=None, + + if kernel_version() >= versiontuple("4.17"): + options.append("xino=off") ++ if mount_context: ++ options.append("context=" + mount_context) + + for dir_path in dirs: + if not os.path.exists(dir_path): +-- +2.43.0 diff --git a/recipes-support/waydroid/waydroid/waydroid-image-provision.service b/recipes-support/waydroid/waydroid/waydroid-image-provision.service index a3e69843..30a7ebd2 100644 --- a/recipes-support/waydroid/waydroid/waydroid-image-provision.service +++ b/recipes-support/waydroid/waydroid/waydroid-image-provision.service @@ -7,6 +7,9 @@ StartLimitIntervalSec=0 [Service] Type=oneshot +# OTA preserves /var, so migrate pre-SELinux Waydroid state to the policy +# labels shipped by this deployment before entering the Waydroid domain. +ExecStartPre=-/sbin/restorecon -RF /var/lib/waydroid ExecStart=/usr/libexec/waydroid-image-provision RemainAfterExit=yes Restart=on-failure From 66b76bff6f467460a5b2a9345a2d21eb01c914de Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 10:07:21 +0100 Subject: [PATCH 57/79] test(ci): add R26 Jaguar SELinux preflight tuple --- kas/r26-jaguar-screen-selinux.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 kas/r26-jaguar-screen-selinux.yml diff --git a/kas/r26-jaguar-screen-selinux.yml b/kas/r26-jaguar-screen-selinux.yml new file mode 100644 index 00000000..64dd0697 --- /dev/null +++ b/kas/r26-jaguar-screen-selinux.yml @@ -0,0 +1,13 @@ +header: + version: 14 + includes: + - lmp-dynamicdevices.yml + +machine: imx8mm-jaguar-screen +distro: lmp-dynamicdevices +target: lmp-factory-image + +local_conf_header: + r26-jaguar-screen-selinux: | + DD_PRODUCT_FEATURES = "display android-container" + ASSEMBLE_SYSTEM_IMAGE = "0" From fc8a2a53ccd11b972f39bc315d845a40c180c327 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 10:08:51 +0100 Subject: [PATCH 58/79] fix(ci): isolate local SELinux preflight signing --- kas/r26-jaguar-screen-selinux.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/kas/r26-jaguar-screen-selinux.yml b/kas/r26-jaguar-screen-selinux.yml index 64dd0697..b9b4c64f 100644 --- a/kas/r26-jaguar-screen-selinux.yml +++ b/kas/r26-jaguar-screen-selinux.yml @@ -11,3 +11,21 @@ local_conf_header: r26-jaguar-screen-selinux: | DD_PRODUCT_FEATURES = "display android-container" ASSEMBLE_SYSTEM_IMAGE = "0" + LOCAL_DEVELOPMENT_BUILD = "1" + OPTEE_TA_SIGN_ENABLE = "0" + SIGN_ENABLE = "0" + UBOOT_SIGN_ENABLE = "0" + UBOOT_SPL_SIGN_ENABLE = "0" + TF_A_SIGN_ENABLE = "0" + UEFI_SIGN_ENABLE = "0" + MODSIGN_ENABLE = "0" + SIGNING_UBOOT_SIGN_KEY = "${TOPDIR}/bitbake.lock" + SIGNING_UBOOT_SIGN_CRT = "${TOPDIR}/bitbake.lock" + SIGNING_UBOOT_SPL_SIGN_KEY = "${TOPDIR}/bitbake.lock" + SIGNING_UBOOT_SPL_SIGN_CRT = "${TOPDIR}/bitbake.lock" + SIGNING_UEFI_SIGN_KEY = "${TOPDIR}/bitbake.lock" + SIGNING_UEFI_SIGN_CRT = "${TOPDIR}/bitbake.lock" + SIGNING_MODSIGN_PRIVKEY = "${TOPDIR}/bitbake.lock" + SIGNING_MODSIGN_X509 = "${TOPDIR}/bitbake.lock" + OPTEE_TA_SIGN_KEY = "${TOPDIR}/bitbake.lock" + TF_A_SIGN_KEY_PATH = "${TOPDIR}/bitbake.lock" From e47e9da7b3bc8c75c533dcbbb5a32beb8aaecaa1 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 10:25:39 +0100 Subject: [PATCH 59/79] ci: materialize LFS-backed KAS configs Resolve protected Git LFS KAS inputs in the shared regression driver for both baseline and candidate worktrees, preserving identical local and CI preparation and failing closed if smudging does not occur. Signed-off-by: Alex Lennon Assisted-by: Codex --- .../run-layer-adoption-regression.py | 22 ++++++++++ .../test_run_layer_adoption_regression.py | 40 +++++++++++++++++++ 2 files changed, 62 insertions(+) diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 45889d38..0ce953a4 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -15,6 +15,7 @@ MARKER = ".complete.json" +LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec/v1\n" FIELDS = ("id", "machine", "distro", "image", "config", "product_features") PRODUCT_SUBMODULES = ("meta-dynamicdevices-bsp", "meta-dynamicdevices-distro") CAPTURE_SCHEMA_FILES = ( @@ -101,6 +102,24 @@ def select_tuples( return selected +def materialize_config(repository: Path, relative: str) -> None: + """Resolve a KAS config stored in Git LFS before either build starts.""" + path = repository / relative + if not path.is_file(): + raise RuntimeError(f"{repository}: protected KAS config is missing: {relative}") + if not path.read_bytes().startswith(LFS_POINTER_PREFIX): + return + subprocess.run( + ["git", "lfs", "pull", f"--include={relative}", "--exclude="], + cwd=repository, + check=True, + ) + if path.read_bytes().startswith(LFS_POINTER_PREFIX): + raise RuntimeError( + f"{repository}: Git LFS did not materialize protected KAS config: {relative}" + ) + + def remove_build_tree(repository: Path) -> None: repository = repository.resolve() build = (repository / "build").resolve() @@ -277,6 +296,9 @@ def main() -> int: prepare_repository(baseline) prepare_repository(candidate) apply_baseline_repairs(baseline, candidate, contract, base_sha) + for config in sorted({entry["config"] for entry in tuples}): + materialize_config(baseline, config) + materialize_config(candidate, config) environment = os.environ.copy() environment["LAYER_ADOPTION_TEST_KEYS_DIR"] = str(test_keys) diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index baa13b2c..04722f1f 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -35,6 +35,46 @@ def test_ci_shard_selects_exactly_one_known_tuple(self) -> None: with self.assertRaisesRegex(ValueError, "unknown protected tuple"): MODULE.select_tuples(tuples, "missing") + def test_lfs_backed_kas_config_is_materialized_by_shared_driver(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + config = repository / "kas/lmp-mfgtool.yml" + config.parent.mkdir() + config.write_bytes( + MODULE.LFS_POINTER_PREFIX + + b"oid sha256:" + b"a" * 64 + b"\nsize 42\n" + ) + + def materialize(*args: object, **kwargs: object) -> None: + config.write_text("header:\n version: 14\n", encoding="utf-8") + + with mock.patch.object( + MODULE.subprocess, "run", side_effect=materialize + ) as run: + MODULE.materialize_config(repository, "kas/lmp-mfgtool.yml") + + run.assert_called_once_with( + [ + "git", + "lfs", + "pull", + "--include=kas/lmp-mfgtool.yml", + "--exclude=", + ], + cwd=repository, + check=True, + ) + + def test_unresolved_lfs_backed_kas_config_fails_closed(self) -> None: + with tempfile.TemporaryDirectory() as directory: + repository = Path(directory) + config = repository / "kas/lmp-mfgtool.yml" + config.parent.mkdir() + config.write_bytes(MODULE.LFS_POINTER_PREFIX) + with mock.patch.object(MODULE.subprocess, "run"): + with self.assertRaisesRegex(RuntimeError, "did not materialize"): + MODULE.materialize_config(repository, "kas/lmp-mfgtool.yml") + def test_workflow_shards_all_tuples_without_fail_fast(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") self.assertIn("fail-fast: false", workflow) From d0f8e75da935c780ed5d48e30f09d12dadf90756 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 11:25:40 +0100 Subject: [PATCH 60/79] ci: protect exact Foundries product tuples Carry the per-ref Foundries variables through the layer-adoption overlay, preserve every bootstrap tuple immutably, and allow product BSP commits only when they contain the exact audited baseline repair. Signed-off-by: Alex Lennon Assisted-by: Codex --- .github/actionlint.yaml | 1 + ci/layer-adoption-contract.json | 4 +- ci/layer-adoption-tuples.json | 82 +++++++++++++++---- scripts/validation/capture-layer-state.sh | 46 +++++++++-- scripts/validation/detect-layer-adoption.py | 24 ++++-- .../run-layer-adoption-regression.py | 55 +++++++++---- .../tests/test_capture_layer_state.py | 6 ++ .../tests/test_detect_layer_adoption.py | 19 +++++ .../test_run_layer_adoption_regression.py | 54 +++++++++++- 9 files changed, 241 insertions(+), 50 deletions(-) diff --git a/.github/actionlint.yaml b/.github/actionlint.yaml index cfd084cf..9a8c29eb 100644 --- a/.github/actionlint.yaml +++ b/.github/actionlint.yaml @@ -1,3 +1,4 @@ self-hosted-runner: labels: - yocto + - ai-tools diff --git a/ci/layer-adoption-contract.json b/ci/layer-adoption-contract.json index 8496f62f..9d0ccff5 100644 --- a/ci/layer-adoption-contract.json +++ b/ci/layer-adoption-contract.json @@ -1,6 +1,6 @@ { "schema": 1, - "reason": "Adopt the isolated NXP i.MX95 partner layer without changing any pre-existing Dynamic Devices build tuple.", + "reason": "Adopt the exact-pinned Scarthgap meta-selinux layer and enable enforcing host SELinux only for product-feature Android containers, while preserving every existing Foundries platform, signing, recovery and manufacturing tuple.", "baseline_repairs": [ { "base_sha": "dda54409ee27e29612c01cc1ff0eb88233ca1da5", @@ -19,7 +19,7 @@ "recipes-bsp/upd72020x-load/upd72020x-load/LicenseRef-markusj-upd72020x-load", "recipes-bsp/upd72020x-load/upd72020x-load_git.bb" ], - "reason": "The immutable baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, carries three product DTS files that no longer compile against the pinned kernel, and does not expose the Phasora loader's custom license text to SPDX generation. Build both sides with the focused eight-file backport while auditing the exact old-to-new submodule transition." + "reason": "The immutable mainline baseline has two stale U-Boot patch contexts, mishandles intentionally empty board-scripts packages, carries three product DTS files that no longer compile against the pinned kernel, and does not expose the Phasora loader's custom license text to SPDX generation. Build both sides with the focused eight-file backport; the candidate BSP must contain this exact repair commit before its product-specific changes." } ], "allowed_deltas": {} diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index bb032edd..d9b881c9 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -1,20 +1,72 @@ { "schema": 1, + "source": { + "repository": "https://source.foundries.io/factories/dynamic-devices/ci-scripts.git", + "commit": "ecbffcf9ba0042ed6f0310a1452bfca05e3878e9", + "file": "factory-config.yml" + }, "tuples": [ - {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv usb-gadget"}, - {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-inst-image", "machine": "imx8mm-jaguar-inst", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - {"id": "imx8mm-jaguar-phasora-image", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "display flutter godot"}, - {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - - {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-inst-mfgtool", "machine": "imx8mm-jaguar-inst", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-phasora-mfgtool", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""} + {"id":"imx8mm-jaguar-dt510-image","machine":"imx8mm-jaguar-dt510","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"improv usb-gadget","variables":{}}, + {"id":"imx8mm-jaguar-handheld-image","machine":"imx8mm-jaguar-handheld","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{}}, + {"id":"imx8mm-jaguar-inst-image","machine":"imx8mm-jaguar-inst","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"improv","variables":{}}, + {"id":"imx8mm-jaguar-phasora-image","machine":"imx8mm-jaguar-phasora","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{}}, + {"id":"imx8mm-jaguar-screen-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display flutter godot","variables":{}}, + {"id":"imx8mm-jaguar-sentai-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"improv","variables":{}}, + {"id":"imx93-jaguar-eink-image","machine":"imx93-jaguar-eink","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"improv","variables":{}}, + {"id":"imx8mm-jaguar-dt510-mfgtool","machine":"imx8mm-jaguar-dt510","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, + {"id":"imx8mm-jaguar-handheld-mfgtool","machine":"imx8mm-jaguar-handheld","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, + {"id":"imx8mm-jaguar-inst-mfgtool","machine":"imx8mm-jaguar-inst","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, + {"id":"imx8mm-jaguar-phasora-mfgtool","machine":"imx8mm-jaguar-phasora","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, + {"id":"imx8mm-jaguar-screen-mfgtool","machine":"imx8mm-jaguar-screen","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, + {"id":"imx8mm-jaguar-sentai-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, + {"id":"imx93-jaguar-eink-mfgtool","machine":"imx93-jaguar-eink","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, + + {"id":"main-jaguar-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-jaguar-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-jaguar-sentai-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks ocf","DOCKER_COMPOSE_APP":"1","IMAGE_INSTALL:append":" ocf-connectivity","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx8mm-jaguar-sentai-large.wks"}}, + {"id":"main-jaguar-sentai-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-jaguar-sentai-prod-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"0","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-jaguar-sentai-prod-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-jaguar-sentai-ext-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-jaguar-sentai-ext-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-jaguar-sentai-ce-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image-ce","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-jaguar-sentai-ce-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-jaguar-sentai-ocf-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks ocf","DOCKER_COMPOSE_APP":"1","IMAGE_INSTALL:append":" ocf-connectivity","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx8mm-jaguar-sentai-large.wks"}}, + {"id":"main-jaguar-sentai-ocf-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx8mm-jaguar-sentai-large.wks"}}, + + {"id":"imx8mm-jaguar-handheld-5in-image","machine":"imx8mm-jaguar-handheld-5in","distro":"lmp-dynamicdevices-headless-waydroid","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"imx8mm-jaguar-handheld-5in-mfgtool","machine":"imx8mm-jaguar-handheld-5in","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"imx8mm-jaguar-handheld-7in-image","machine":"imx8mm-jaguar-handheld-7in","distro":"lmp-dynamicdevices-headless-waydroid","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"imx8mm-jaguar-handheld-7in-mfgtool","machine":"imx8mm-jaguar-handheld-7in","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-jaguar-phasora-image","machine":"imx8mm-jaguar-phasora","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-jaguar-phasora-mfgtool","machine":"imx8mm-jaguar-phasora","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-jaguar-phasora-ext-image","machine":"imx8mm-jaguar-phasora","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-jaguar-phasora-ext-mfgtool","machine":"imx8mm-jaguar-phasora","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-imx8ulp-image","machine":"imx8ulp-lpddr4-evk","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-imx8ulp-mfgtool","machine":"imx8ulp-lpddr4-evk","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-jaguar-inst-image","machine":"imx8mm-jaguar-inst","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-jaguar-inst-mfgtool","machine":"imx8mm-jaguar-inst","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-rpi4-image","machine":"raspberrypi4-64","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-rpi4-v2g-evse-image","machine":"raspberrypi4-64","distro":"lmp-dynamicdevices-headless-waydroid","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-rpi5-image","machine":"raspberrypi5","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + + {"id":"main-imx93-jaguar-eink-image","machine":"imx93-jaguar-eink","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks","DOCKER_COMPOSE_APP":"1","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx93-jaguar-eink-large.wks"}}, + {"id":"main-imx93-jaguar-eink-mfgtool","machine":"imx93-jaguar-eink","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx93-jaguar-eink-large.wks"}}, + + {"id":"main-imx95-frdm-devel-image","machine":"imx95-frdm-evk","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display android-container","variables":{"ACCEPT_FSL_EULA":"1","ASSEMBLE_SYSTEM_IMAGE":"0","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, + {"id":"main-imx95-frdm-devel-mfgtool","machine":"imx95-frdm-evk","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, + + {"id":"main-jaguar-screen-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display android-container","variables":{"ACCEPT_FSL_EULA":"1","ASSEMBLE_SYSTEM_IMAGE":"0","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}} ] } diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 28c101f1..2b645392 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -2,8 +2,8 @@ # Build one protected tuple and capture deterministic layer/package/task state. set -euo pipefail -if [ "$#" -ne 6 ]; then - echo "Usage: $0 KAS_CONFIG MACHINE DISTRO TARGET PRODUCT_FEATURES OUTPUT_DIR" >&2 +if [ "$#" -ne 7 ]; then + echo "Usage: $0 KAS_CONFIG MACHINE DISTRO TARGET PRODUCT_FEATURES VARIABLES_JSON OUTPUT_DIR" >&2 exit 2 fi @@ -12,7 +12,8 @@ machine=$2 distro=$3 target=$4 product_features=$5 -output_dir=$6 +variables_json=$6 +output_dir=$7 test_keys_dir=${LAYER_ADOPTION_TEST_KEYS_DIR:-} cache_root=${LAYER_ADOPTION_CACHE_DIR:-$HOME/yocto} @@ -52,6 +53,22 @@ export DISTRO="$distro" product_features_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$product_features") downloads_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/downloads") sstate_quoted=$(python3 -c 'import json, sys; print(json.dumps(sys.argv[1]))' "$cache_root/sstate-cache") +variable_assignments=$(python3 - "$variables_json" <<'PY' +import json +import re +import sys + +variables = json.loads(sys.argv[1]) +if not isinstance(variables, dict): + raise SystemExit("tuple variables must be a JSON object") +for name, value in sorted(variables.items()): + if not isinstance(name, str) or not re.fullmatch(r"[A-Z0-9_]+(?::[a-z0-9_-]+)*", name): + raise SystemExit(f"invalid BitBake variable name: {name!r}") + if not isinstance(value, str): + raise SystemExit(f"BitBake variable {name} must have a string value") + print(f" {name} = {json.dumps(value)}") +PY +) repository_root=$(git rev-parse --show-toplevel) overlay_dir="$repository_root/.layer-adoption-overlays" mkdir -p "$overlay_dir" @@ -67,6 +84,7 @@ header: local_conf_header: layer-adoption-product-features: | DD_PRODUCT_FEATURES = $product_features_quoted +$variable_assignments DL_DIR = $downloads_quoted SSTATE_DIR = $sstate_quoted BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,100K STOPTASKS,\${DL_DIR},20G,100K STOPTASKS,\${SSTATE_DIR},20G,100K HALT,\${TMPDIR},10G,50K HALT,\${DL_DIR},10G,50K HALT,\${SSTATE_DIR},10G,50K" @@ -90,9 +108,22 @@ EOF combined_config="${config}:${overlay}" kas checkout "$combined_config" -cat > "$output_dir/metadata.json" < "$output_dir/commands.txt" + "DD_PRODUCT_FEATURES=$product_features" \ + "TUPLE_VARIABLES=$variables_json" > "$output_dir/commands.txt" # Static layer surfaces are captured as well as BitBake's resolved view. This # makes wildcard/dangling appends and global layer.conf policy visible even diff --git a/scripts/validation/detect-layer-adoption.py b/scripts/validation/detect-layer-adoption.py index a4e34cd2..c2b384da 100755 --- a/scripts/validation/detect-layer-adoption.py +++ b/scripts/validation/detect-layer-adoption.py @@ -30,7 +30,10 @@ re.compile(r"^ci/layer-adoption-tuples\.json$"), ) -TUPLE_FIELDS = ("id", "machine", "distro", "image", "config", "product_features") +TUPLE_FIELDS = ( + "id", "machine", "distro", "image", "config", "product_features", "variables" +) +REQUIRED_TUPLE_FIELDS = tuple(field for field in TUPLE_FIELDS if field != "variables") NONEMPTY_TUPLE_FIELDS = ("id", "machine", "distro", "image", "config") @@ -43,7 +46,7 @@ def git(*args: str) -> str: return subprocess.check_output(["git", *args], text=True) -def parse_tuples(raw: str, source: str) -> dict[str, dict[str, str]]: +def parse_tuples(raw: str, source: str) -> dict[str, dict[str, object]]: try: document = json.loads(raw) except json.JSONDecodeError as exc: @@ -51,21 +54,32 @@ def parse_tuples(raw: str, source: str) -> dict[str, dict[str, str]]: if document.get("schema") != 1 or not isinstance(document.get("tuples"), list): raise ValueError(f"{source}: expected schema=1 and a tuples array") - result: dict[str, dict[str, str]] = {} + result: dict[str, dict[str, object]] = {} for index, entry in enumerate(document["tuples"]): if not isinstance(entry, dict): raise ValueError(f"{source}: tuple {index} is not an object") - missing = [field for field in TUPLE_FIELDS if field not in entry] + missing = [field for field in REQUIRED_TUPLE_FIELDS if field not in entry] missing.extend( field for field in NONEMPTY_TUPLE_FIELDS if field in entry and not str(entry[field]).strip() ) if missing: raise ValueError(f"{source}: tuple {index} lacks {', '.join(missing)}") + variables = entry.get("variables", {}) + if not isinstance(variables, dict) or any( + not isinstance(name, str) + or not re.fullmatch(r"[A-Z0-9_]+(?::[a-z0-9_-]+)*", name) + or not isinstance(value, str) + for name, value in variables.items() + ): + raise ValueError(f"{source}: tuple {index} has invalid variables") tuple_id = str(entry["id"]) if tuple_id in result: raise ValueError(f"{source}: duplicate tuple id {tuple_id}") - result[tuple_id] = {field: str(entry[field]) for field in TUPLE_FIELDS} + result[tuple_id] = { + field: (dict(sorted(variables.items())) if field == "variables" else str(entry[field])) + for field in TUPLE_FIELDS + } if not result: raise ValueError(f"{source}: tuple matrix must not be empty") return result diff --git a/scripts/validation/run-layer-adoption-regression.py b/scripts/validation/run-layer-adoption-regression.py index 0ce953a4..d8db81d1 100644 --- a/scripts/validation/run-layer-adoption-regression.py +++ b/scripts/validation/run-layer-adoption-regression.py @@ -16,7 +16,8 @@ MARKER = ".complete.json" LFS_POINTER_PREFIX = b"version https://git-lfs.github.com/spec/v1\n" -FIELDS = ("id", "machine", "distro", "image", "config", "product_features") +FIELDS = ("id", "machine", "distro", "image", "config", "product_features", "variables") +REQUIRED_FIELDS = tuple(field for field in FIELDS if field != "variables") PRODUCT_SUBMODULES = ("meta-dynamicdevices-bsp", "meta-dynamicdevices-distro") CAPTURE_SCHEMA_FILES = ( "ci/layer-adoption-contract.json", @@ -71,16 +72,27 @@ def valid_cached_evidence( } -def load_tuples(path: Path) -> list[dict[str, str]]: +def load_tuples(path: Path) -> list[dict[str, object]]: document = json.loads(path.read_text(encoding="utf-8")) if document.get("schema") != 1 or not isinstance(document.get("tuples"), list): raise ValueError(f"{path}: expected schema=1 and a tuples array") tuples = [] seen = set() for index, raw in enumerate(document["tuples"]): - if not isinstance(raw, dict) or any(field not in raw for field in FIELDS): + if not isinstance(raw, dict) or any(field not in raw for field in REQUIRED_FIELDS): raise ValueError(f"{path}: tuple {index} is incomplete") - entry = {field: str(raw[field]) for field in FIELDS} + entry: dict[str, object] = { + field: str(raw[field]) for field in FIELDS if field != "variables" + } + variables = raw.get("variables", {}) + if not isinstance(variables, dict) or any( + not isinstance(name, str) + or not re.fullmatch(r"[A-Z0-9_]+(?::[a-z0-9_-]+)*", name) + or not isinstance(value, str) + for name, value in variables.items() + ): + raise ValueError(f"{path}: tuple {index} has invalid variables") + entry["variables"] = dict(sorted(variables.items())) if not entry["id"] or entry["id"] in seen: raise ValueError(f"{path}: duplicate or empty tuple id {entry['id']!r}") seen.add(entry["id"]) @@ -91,8 +103,8 @@ def load_tuples(path: Path) -> list[dict[str, str]]: def select_tuples( - tuples: list[dict[str, str]], tuple_id: str | None -) -> list[dict[str, str]]: + tuples: list[dict[str, object]], tuple_id: str | None +) -> list[dict[str, object]]: """Select one CI shard while keeping the local default as the full gate.""" if tuple_id is None: return tuples @@ -210,15 +222,23 @@ def apply_baseline_repairs( raise ValueError("baseline repair needs a reason and exact file list") if submodule_commit(baseline, relative) != old: raise RuntimeError(f"baseline repair {relative}: unexpected source pin") - if submodule_commit(candidate, relative) != new: - raise RuntimeError(f"baseline repair {relative}: unexpected candidate pin") - candidate_layer = candidate / relative + candidate_commit = submodule_commit(candidate, relative) subprocess.run( ["git", "merge-base", "--is-ancestor", old, new], cwd=candidate_layer, check=True, ) + descendant = subprocess.run( + ["git", "merge-base", "--is-ancestor", new, candidate_commit], + cwd=candidate_layer, + check=False, + ) + if descendant.returncode != 0: + raise RuntimeError( + f"baseline repair {relative}: candidate {candidate_commit} " + f"does not contain audited repair {new}" + ) changed = git_output(candidate_layer, "diff", "--name-only", old, new).splitlines() if sorted(changed) != sorted(str(path) for path in files): raise RuntimeError( @@ -244,18 +264,19 @@ def apply_baseline_repairs( def capture( script: Path, repository: Path, - entry: dict[str, str], + entry: dict[str, object], output: Path, environment: dict[str, str], ) -> None: subprocess.run( [ str(script), - entry["config"], - entry["machine"], - entry["distro"], - entry["image"], - entry["product_features"], + str(entry["config"]), + str(entry["machine"]), + str(entry["distro"]), + str(entry["image"]), + str(entry["product_features"]), + json.dumps(entry["variables"], sort_keys=True, separators=(",", ":")), str(output), ], cwd=repository, @@ -296,7 +317,7 @@ def main() -> int: prepare_repository(baseline) prepare_repository(candidate) apply_baseline_repairs(baseline, candidate, contract, base_sha) - for config in sorted({entry["config"] for entry in tuples}): + for config in sorted({str(entry["config"]) for entry in tuples}): materialize_config(baseline, config) materialize_config(candidate, config) @@ -308,7 +329,7 @@ def main() -> int: (evidence / "candidate").mkdir(parents=True) for entry in tuples: - tuple_id = entry["id"] + tuple_id = str(entry["id"]) print(f"::group::Protect {tuple_id}", flush=True) cached = cache / "baselines" / base_sha / tuple_id baseline_output = evidence / "baseline" / tuple_id diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index f8e60742..84f4be2e 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -159,6 +159,12 @@ def test_disk_monitor_uses_inode_not_disk_units(self) -> None: self.assertEqual(disk_monitor.count(",100K"), 3) self.assertEqual(disk_monitor.count(",50K"), 3) + def test_tuple_variables_are_validated_and_injected_into_overlay(self) -> None: + source = SCRIPT.read_text(encoding="utf-8") + self.assertIn("variables_json=$6", source) + self.assertIn('re.fullmatch(r"[A-Z0-9_]+(?::[a-z0-9_-]+)*", name)', source) + self.assertIn("$variable_assignments", source) + if __name__ == "__main__": unittest.main() diff --git a/scripts/validation/tests/test_detect_layer_adoption.py b/scripts/validation/tests/test_detect_layer_adoption.py index a1aecf21..28933e13 100644 --- a/scripts/validation/tests/test_detect_layer_adoption.py +++ b/scripts/validation/tests/test_detect_layer_adoption.py @@ -31,10 +31,17 @@ def entry(tuple_id: str, machine: str = "machine-a") -> dict[str, str]: "image": "image-a", "config": "kas/test.yml", "product_features": "", + "variables": {}, } class ProtectedTupleTests(unittest.TestCase): + def test_legacy_tuple_without_variables_normalises_to_empty_mapping(self) -> None: + legacy = entry("existing") + del legacy["variables"] + parsed = MODULE.parse_tuples(document(legacy), "legacy") + self.assertEqual(parsed["existing"]["variables"], {}) + def test_local_and_ci_kas_process_changes_are_material(self) -> None: paths = ( ".github/workflows/layer-adoption-gate.yml", @@ -101,6 +108,18 @@ def test_redefining_product_features_fails(self) -> None: with self.assertRaisesRegex(ValueError, "redefined=existing"): self.validate(document(entry("existing")), document(candidate)) + def test_redefining_tuple_variables_fails(self) -> None: + candidate = entry("existing") + candidate["variables"] = {"DEV_MODE": "0"} + with self.assertRaisesRegex(ValueError, "redefined=existing"): + self.validate(document(entry("existing")), document(candidate)) + + def test_invalid_variable_name_fails(self) -> None: + candidate = entry("existing") + candidate["variables"] = {"bad name": "1"} + with self.assertRaisesRegex(ValueError, "invalid variables"): + self.validate(document(entry("existing")), document(candidate)) + if __name__ == "__main__": unittest.main() diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 04722f1f..e97b4b17 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -114,14 +114,18 @@ def test_audited_baseline_repair_is_exact_and_fail_closed(self) -> None: MODULE, "git_output", side_effect=[changed_file + "\n", new, ""], - ), mock.patch.object(MODULE.subprocess, "run") as run: + ), mock.patch.object( + MODULE.subprocess, + "run", + return_value=subprocess.CompletedProcess([], 0), + ) as run: MODULE.apply_baseline_repairs( root / "baseline", root / "candidate", contract, "base" ) - self.assertEqual(run.call_count, 3) + self.assertEqual(run.call_count, 4) self.assertEqual( - run.call_args_list[1].args[0], + run.call_args_list[2].args[0], [ "git", "fetch", @@ -130,6 +134,44 @@ def test_audited_baseline_repair_is_exact_and_fail_closed(self) -> None: ], ) + def test_audited_repair_must_be_ancestor_of_candidate_pin(self) -> None: + old = "a" * 40 + repair = "b" * 40 + candidate = "c" * 40 + with tempfile.TemporaryDirectory() as directory: + root = Path(directory) + contract = root / "contract.json" + contract.write_text( + json.dumps( + { + "baseline_repairs": [ + { + "base_sha": "base", + "submodule": "meta-dynamicdevices-bsp", + "from": old, + "to": repair, + "url": "https://github.com/DynamicDevices/bsp.git", + "ref": "refs/heads/focused-backport", + "files": ["fix.patch"], + "reason": "focused repair", + } + ] + } + ), + encoding="utf-8", + ) + ancestry = [ + subprocess.CompletedProcess([], 0), + subprocess.CompletedProcess([], 1), + ] + with mock.patch.object( + MODULE, "submodule_commit", side_effect=[old, candidate] + ), mock.patch.object(MODULE.subprocess, "run", side_effect=ancestry): + with self.assertRaisesRegex(RuntimeError, "does not contain audited repair"): + MODULE.apply_baseline_repairs( + root / "baseline", root / "candidate", contract, "base" + ) + def test_audited_baseline_repair_rejects_extra_files(self) -> None: old = "a" * 40 new = "b" * 40 @@ -159,7 +201,11 @@ def test_audited_baseline_repair_rejects_extra_files(self) -> None: MODULE, "submodule_commit", side_effect=[old, new] ), mock.patch.object( MODULE, "git_output", return_value="unexpected.patch\n" - ), mock.patch.object(MODULE.subprocess, "run"): + ), mock.patch.object( + MODULE.subprocess, + "run", + return_value=subprocess.CompletedProcess([], 0), + ): with self.assertRaisesRegex(RuntimeError, "do not match contract"): MODULE.apply_baseline_repairs( root / "baseline", root / "candidate", contract, "base" From 3cd52a1ebe2701cc52d8f92c5836e3a05eafd80d Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 11:30:30 +0100 Subject: [PATCH 61/79] fix(selinux): avoid unused target toolchain Refpolicy compiles policy text with native tools and BUILD_CC. Suppress its unused target compiler/libc defaults so exact preflight reaches and validates the Waydroid policy without building LLVM. Signed-off-by: Alex Lennon Assisted-by: Codex --- docs/r26-waydroid-selinux-plan.md | 16 ++++++++++++++++ .../refpolicy/refpolicy-targeted_%.bbappend | 6 ++++++ 2 files changed, 22 insertions(+) diff --git a/docs/r26-waydroid-selinux-plan.md b/docs/r26-waydroid-selinux-plan.md index 727909d6..1c06d8ab 100644 --- a/docs/r26-waydroid-selinux-plan.md +++ b/docs/r26-waydroid-selinux-plan.md @@ -93,6 +93,22 @@ only the new Jaguar Screen image. ## Gates before manifest publication +### Completed exact-pin preflight evidence + +- `kas checkout kas/r26-jaguar-screen-selinux.yml` resolved all candidate + layers at their recorded SHAs and parsed 4,172 recipes with zero errors. +- `bitbake -g refpolicy-targeted` proves the policy recipe has no target + compiler or libc dependency; it uses only its declared host-native policy + tools (`INHIBIT_DEFAULT_DEPS = "1"`). +- `bitbake refpolicy-targeted -c compile` completed all 459 tasks. The generated + `policy/modules.conf` contains `waydroid = module` and the build produced + `waydroid.pp` (114,378 bytes), proving the custom module is compiled rather + than merely present in `SRC_URI`. + +The full image, kernel configuration, adoption matrix, Foundries build and +physical-board gates remain open; this recipe proof does not substitute for +them. + 1. Resolve the exact candidate manifest and prove every project SHA exists. 2. Parse the Jaguar Screen platform and mfgtool configurations. 3. Run `bitbake-layers show-layers`, `show-appends`, and provider checks. diff --git a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend index e2d8219c..5e55592b 100644 --- a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend +++ b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend @@ -1,5 +1,11 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:" +# Refpolicy builds policy text with explicitly declared host-native SELinux +# tools and passes BUILD_CC to its makefiles. It has no target-compiled code, +# so the default target compiler/libc dependency is both unused and especially +# expensive in LmP's global Clang configuration. +INHIBIT_DEFAULT_DEPS = "1" + SRC_URI:append = "${@bb.utils.contains('DISTRO_FEATURES', 'waydroid', ' file://waydroid.te file://waydroid.fc file://waydroid.if', '', d)}" EXTRA_OEMAKE:append = "${@bb.utils.contains('DISTRO_FEATURES', 'waydroid', ' APPS_MODS=waydroid', '', d)}" From e8c94ce3e44bbe4574d1b656d1c4741dd6b7f52e Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 12:12:16 +0100 Subject: [PATCH 62/79] ci: retire Jaguar inst and Phasora tuples Record the product-owner retirement and remove the corresponding image and mfgtool shards from the protected build matrix.\n\nAssisted-by: Codex --- ci/layer-adoption-tuples.json | 4 ---- docs/PRODUCT_TUPLE_LIFECYCLE.md | 19 +++++++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) create mode 100644 docs/PRODUCT_TUPLE_LIFECYCLE.md diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index bb032edd..0c72d93e 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -3,16 +3,12 @@ "tuples": [ {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv usb-gadget"}, {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-inst-image", "machine": "imx8mm-jaguar-inst", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - {"id": "imx8mm-jaguar-phasora-image", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "display flutter godot"}, {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-inst-mfgtool", "machine": "imx8mm-jaguar-inst", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-phasora-mfgtool", "machine": "imx8mm-jaguar-phasora", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""} diff --git a/docs/PRODUCT_TUPLE_LIFECYCLE.md b/docs/PRODUCT_TUPLE_LIFECYCLE.md new file mode 100644 index 00000000..ec528195 --- /dev/null +++ b/docs/PRODUCT_TUPLE_LIFECYCLE.md @@ -0,0 +1,19 @@ +# Product tuple lifecycle + +## Deprecated on 13 September 2026 + +The product owner has retired CI builds for these machine families: + +- `imx8mm-jaguar-inst` +- `imx8mm-jaguar-phasora` + +The retirement covers normal factory images and mfgtool/recovery images. For +Foundries CI it also covers the `main-jaguar-phasora`, +`main-jaguar-phasora-ext`, and `main-jaguar-inst` refs. + +These tuples are intentionally absent from `ci/layer-adoption-tuples.json` and +must not be treated as accidentally deleted regression coverage. Board source +may remain in the repository for history or possible future reactivation, but +reactivation requires an explicit product decision and restoration of both +image and recovery coverage. Every other existing product tuple remains +protected by the layer-adoption gate. From 079fe76662672aca0a13efe50c75af6b332c2088 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 12:17:44 +0100 Subject: [PATCH 63/79] ci: focus adoption gate on Jaguar screen Run the current integration phase against the screen image and recovery tuple, while documenting that other active products remain deferred rather than deprecated.\n\nAssisted-by: Codex --- ci/layer-adoption-tuples.json | 11 +---------- docs/PRODUCT_TUPLE_LIFECYCLE.md | 12 ++++++++++-- 2 files changed, 11 insertions(+), 12 deletions(-) diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index 0c72d93e..280468ab 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -1,16 +1,7 @@ { "schema": 1, "tuples": [ - {"id": "imx8mm-jaguar-dt510-image", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv usb-gadget"}, - {"id": "imx8mm-jaguar-handheld-image", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": ""}, {"id": "imx8mm-jaguar-screen-image", "machine": "imx8mm-jaguar-screen", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "display flutter godot"}, - {"id": "imx8mm-jaguar-sentai-image", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - {"id": "imx93-jaguar-eink-image", "machine": "imx93-jaguar-eink", "distro": "lmp-dynamicdevices", "image": "lmp-factory-image", "config": "kas/lmp-dynamicdevices.yml", "product_features": "improv"}, - - {"id": "imx8mm-jaguar-dt510-mfgtool", "machine": "imx8mm-jaguar-dt510", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-handheld-mfgtool", "machine": "imx8mm-jaguar-handheld", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx8mm-jaguar-sentai-mfgtool", "machine": "imx8mm-jaguar-sentai", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""}, - {"id": "imx93-jaguar-eink-mfgtool", "machine": "imx93-jaguar-eink", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""} + {"id": "imx8mm-jaguar-screen-mfgtool", "machine": "imx8mm-jaguar-screen", "distro": "lmp-mfgtool", "image": "mfgtool-files", "config": "kas/lmp-dynamicdevices-mfgtool.yml", "product_features": ""} ] } diff --git a/docs/PRODUCT_TUPLE_LIFECYCLE.md b/docs/PRODUCT_TUPLE_LIFECYCLE.md index ec528195..cbb2a3f6 100644 --- a/docs/PRODUCT_TUPLE_LIFECYCLE.md +++ b/docs/PRODUCT_TUPLE_LIFECYCLE.md @@ -15,5 +15,13 @@ These tuples are intentionally absent from `ci/layer-adoption-tuples.json` and must not be treated as accidentally deleted regression coverage. Board source may remain in the repository for history or possible future reactivation, but reactivation requires an explicit product decision and restoration of both -image and recovery coverage. Every other existing product tuple remains -protected by the layer-adoption gate. +image and recovery coverage. + +## Temporarily deferred CI coverage + +For the current R26 screen-board integration phase, the layer-adoption workflow +runs only the `imx8mm-jaguar-screen` factory-image and mfgtool/recovery tuples. +Other still-active product families are deferred to a later CI expansion; they +are not deprecated, and this focused run must not be cited as proof that the +full historical product matrix passed. Production-wide adoption remains gated +on restoring and passing that broader coverage. From 2b9866669a8a7d4d4e35c3b2a46319ba7210e2ee Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 12:19:42 +0100 Subject: [PATCH 64/79] feat(selinux): make Waydroid policy enforcing by default Restrict the permissive domain to explicit development builds, add an enforcing smoke stack, record kernel and policy proof, and focus the current adoption phase on Jaguar Screen image and recovery coverage.\n\nAssisted-by: Codex --- ci/layer-adoption-tuples.json | 61 +------------------ docs/PRODUCT_TUPLE_LIFECYCLE.md | 28 +++++++++ docs/r26-waydroid-selinux-plan.md | 54 +++++++++++----- .../refpolicy/refpolicy-targeted/waydroid.te | 7 +-- .../refpolicy/refpolicy-targeted_%.bbappend | 13 ++++ ...-jaguar-screen-selinux-enforcing-smoke.yml | 10 +++ kas/r26-jaguar-screen-selinux.yml | 5 +- 7 files changed, 97 insertions(+), 81 deletions(-) create mode 100644 docs/PRODUCT_TUPLE_LIFECYCLE.md create mode 100644 kas/r26-jaguar-screen-selinux-enforcing-smoke.yml diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index d9b881c9..26ccf241 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -2,71 +2,12 @@ "schema": 1, "source": { "repository": "https://source.foundries.io/factories/dynamic-devices/ci-scripts.git", - "commit": "ecbffcf9ba0042ed6f0310a1452bfca05e3878e9", + "commit": "f087a926016debeb79aa8326f620704a80c45cd6", "file": "factory-config.yml" }, "tuples": [ - {"id":"imx8mm-jaguar-dt510-image","machine":"imx8mm-jaguar-dt510","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"improv usb-gadget","variables":{}}, - {"id":"imx8mm-jaguar-handheld-image","machine":"imx8mm-jaguar-handheld","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{}}, - {"id":"imx8mm-jaguar-inst-image","machine":"imx8mm-jaguar-inst","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"improv","variables":{}}, - {"id":"imx8mm-jaguar-phasora-image","machine":"imx8mm-jaguar-phasora","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{}}, {"id":"imx8mm-jaguar-screen-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display flutter godot","variables":{}}, - {"id":"imx8mm-jaguar-sentai-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"improv","variables":{}}, - {"id":"imx93-jaguar-eink-image","machine":"imx93-jaguar-eink","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"improv","variables":{}}, - {"id":"imx8mm-jaguar-dt510-mfgtool","machine":"imx8mm-jaguar-dt510","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, - {"id":"imx8mm-jaguar-handheld-mfgtool","machine":"imx8mm-jaguar-handheld","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, - {"id":"imx8mm-jaguar-inst-mfgtool","machine":"imx8mm-jaguar-inst","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, - {"id":"imx8mm-jaguar-phasora-mfgtool","machine":"imx8mm-jaguar-phasora","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, {"id":"imx8mm-jaguar-screen-mfgtool","machine":"imx8mm-jaguar-screen","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, - {"id":"imx8mm-jaguar-sentai-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, - {"id":"imx93-jaguar-eink-mfgtool","machine":"imx93-jaguar-eink","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, - - {"id":"main-jaguar-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-jaguar-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-jaguar-sentai-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks ocf","DOCKER_COMPOSE_APP":"1","IMAGE_INSTALL:append":" ocf-connectivity","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx8mm-jaguar-sentai-large.wks"}}, - {"id":"main-jaguar-sentai-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-jaguar-sentai-prod-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"0","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-jaguar-sentai-prod-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-jaguar-sentai-ext-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-jaguar-sentai-ext-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-jaguar-sentai-ce-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image-ce","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-jaguar-sentai-ce-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-jaguar-sentai-ocf-image","machine":"imx8mm-jaguar-sentai","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks ocf","DOCKER_COMPOSE_APP":"1","IMAGE_INSTALL:append":" ocf-connectivity","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx8mm-jaguar-sentai-large.wks"}}, - {"id":"main-jaguar-sentai-ocf-mfgtool","machine":"imx8mm-jaguar-sentai","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx8mm-jaguar-sentai-large.wks"}}, - - {"id":"imx8mm-jaguar-handheld-5in-image","machine":"imx8mm-jaguar-handheld-5in","distro":"lmp-dynamicdevices-headless-waydroid","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"imx8mm-jaguar-handheld-5in-mfgtool","machine":"imx8mm-jaguar-handheld-5in","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"imx8mm-jaguar-handheld-7in-image","machine":"imx8mm-jaguar-handheld-7in","distro":"lmp-dynamicdevices-headless-waydroid","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"imx8mm-jaguar-handheld-7in-mfgtool","machine":"imx8mm-jaguar-handheld-7in","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-jaguar-phasora-image","machine":"imx8mm-jaguar-phasora","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-jaguar-phasora-mfgtool","machine":"imx8mm-jaguar-phasora","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-jaguar-phasora-ext-image","machine":"imx8mm-jaguar-phasora","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-jaguar-phasora-ext-mfgtool","machine":"imx8mm-jaguar-phasora","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-imx8ulp-image","machine":"imx8ulp-lpddr4-evk","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-imx8ulp-mfgtool","machine":"imx8ulp-lpddr4-evk","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-jaguar-inst-image","machine":"imx8mm-jaguar-inst","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-jaguar-inst-mfgtool","machine":"imx8mm-jaguar-inst","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - - {"id":"main-rpi4-image","machine":"raspberrypi4-64","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-rpi4-v2g-evse-image","machine":"raspberrypi4-64","distro":"lmp-dynamicdevices-headless-waydroid","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-rpi5-image","machine":"raspberrypi5","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - - {"id":"main-imx93-jaguar-eink-image","machine":"imx93-jaguar-eink","distro":"lmp-dynamicdevices-headless","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks","DOCKER_COMPOSE_APP":"1","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx93-jaguar-eink-large.wks"}}, - {"id":"main-imx93-jaguar-eink-mfgtool","machine":"imx93-jaguar-eink","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","DISTRO_FEATURES:append":" luks","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","MODSIGN_ENABLE":"1","OPTEE_TA_SIGN_ENABLE":"1","OSTREE_DEPLOY_USR_OSTREE_BOOT":"1","OSTREE_SPLIT_BOOT":"1","PATCHTOOL":"git","TF_A_SIGN_ENABLE":"1","UBOOT_SIGN_ENABLE":"1","WKS_FILE:sota":"imx93-jaguar-eink-large.wks"}}, - - {"id":"main-imx95-frdm-devel-image","machine":"imx95-frdm-evk","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display android-container","variables":{"ACCEPT_FSL_EULA":"1","ASSEMBLE_SYSTEM_IMAGE":"0","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}}, - {"id":"main-imx95-frdm-devel-mfgtool","machine":"imx95-frdm-evk","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{"ACCEPT_FSL_EULA":"1","DEV_MODE":"1","EXTRA_ARTIFACTS":"mfgtool-files.tar.gz","MFGTOOL_FLASH_IMAGE":"lmp-factory-image","PATCHTOOL":"git"}}, - {"id":"main-jaguar-screen-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display android-container","variables":{"ACCEPT_FSL_EULA":"1","ASSEMBLE_SYSTEM_IMAGE":"0","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}} ] } diff --git a/docs/PRODUCT_TUPLE_LIFECYCLE.md b/docs/PRODUCT_TUPLE_LIFECYCLE.md new file mode 100644 index 00000000..9956205c --- /dev/null +++ b/docs/PRODUCT_TUPLE_LIFECYCLE.md @@ -0,0 +1,28 @@ +# Product tuple lifecycle + +## Deprecated on 13 September 2026 + +The product owner has retired CI builds for these machine families: + +- `imx8mm-jaguar-inst` +- `imx8mm-jaguar-phasora` + +The retirement covers normal factory images and mfgtool/recovery images. For +Foundries CI it also covers the `main-jaguar-phasora`, +`main-jaguar-phasora-ext`, and `main-jaguar-inst` refs. + +These tuples are intentionally absent from `ci/layer-adoption-tuples.json` and +must not be treated as accidentally deleted regression coverage. Board source +may remain in the repository for history or possible future reactivation, but +reactivation requires an explicit product decision and restoration of both +image and recovery coverage. + +## Temporarily deferred CI coverage + +For the current R26 screen-board integration phase, the layer-adoption workflow +runs the existing `imx8mm-jaguar-screen` factory-image and mfgtool/recovery +tuples, then adds the exact Foundries `main-jaguar-screen` Android-container +tuple. Other still-active product families are deferred to a later CI +expansion; they are not deprecated, and this focused run must not be cited as +proof that the full historical product matrix passed. Production-wide adoption +remains gated on restoring and passing that broader coverage. diff --git a/docs/r26-waydroid-selinux-plan.md b/docs/r26-waydroid-selinux-plan.md index 1c06d8ab..488f4cf2 100644 --- a/docs/r26-waydroid-selinux-plan.md +++ b/docs/r26-waydroid-selinux-plan.md @@ -44,11 +44,14 @@ new refpolicy modules otherwise default to `off` when absent from the upstream ## Development policy lifecycle -The first hardware image keeps the host globally enforcing but declares only -`waydroid_t` permissive. This isolates policy discovery to the Waydroid domain -and avoids weakening unrelated host services. The statement -`permissive waydroid_t;` is a release blocker and must be removed after AVCs -have been classified and converted to narrow rules. +The first hardware image keeps the host globally enforcing but explicitly sets +`WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "1"` to declare only `waydroid_t` +permissive. The recipe rejects that setting unless +`LOCAL_DEVELOPMENT_BUILD = "1"`; all other builds compile the Waydroid domain +enforcing by default. This isolates policy discovery to the Waydroid domain, +avoids weakening unrelated host services, and prevents the discovery setting +from leaking into a Foundries production build. Hardware AVCs must still be +classified and converted to narrow rules before release. Immutable OSTree content is labelled at image construction with `selinux-image`. `FIRST_BOOT_RELABEL` remains disabled because a whole-root @@ -65,10 +68,26 @@ host-policy types, so Waydroid applies the single host-owned SELinux is active. This avoids any release rule permitting execution from the generic `unlabeled_t` type. -## Existing CI tuple regression matrix +## Phased CI tuple regression matrix -The layer-adoption gate covers every active `ci-scripts` platform tuple, not -only the new Jaguar Screen image. +To reach physical Jaguar Screen testing without waiting for every historical +product build, the current gate is intentionally focused on three tuples: the +existing Jaguar Screen image, its mfgtool/recovery image, and the exact +Foundries `main-jaguar-screen` Android-container image. The remaining active +platform inventory below is deferred to a later CI expansion and is not +claimed as passed by this phase. + +### Explicitly deprecated tuples + +On 13 September 2026 the product owner retired the +`imx8mm-jaguar-inst` and `imx8mm-jaguar-phasora` build families. This includes +normal images and mfgtool/recovery builds, plus both the +`main-jaguar-phasora` and `main-jaguar-phasora-ext` Foundries refs. They are +therefore intentionally excluded from the protected regression matrix and will +be removed from the Foundries factory build configuration. Their removal is a +reviewed product-lifecycle decision, not an unexplained loss of gate coverage. + +### Deferred active Foundries inventory | Platform ref | Machine | Platform distro/special case | Existing mfgtool tuple | | --- | --- | --- | --- | @@ -80,10 +99,7 @@ only the new Jaguar Screen image. | `main-jaguar-sentai-ocf` | `imx8mm-jaguar-sentai` | headless, signed/LUKS/OCF | yes | | `imx8mm-jaguar-handheld-5in` | `imx8mm-jaguar-handheld-5in` | legacy Waydroid distro | yes | | `imx8mm-jaguar-handheld-7in` | `imx8mm-jaguar-handheld-7in` | legacy Waydroid distro | yes | -| `main-jaguar-phasora` | `imx8mm-jaguar-phasora` | headless | yes | -| `main-jaguar-phasora-ext` | `imx8mm-jaguar-phasora` | headless | yes | | `main-imx8ulp` | `imx8ulp-lpddr4-evk` | headless | yes | -| `main-jaguar-inst` | `imx8mm-jaguar-inst` | headless | yes | | `main-rpi4` | `raspberrypi4-64` | default | no | | `main-rpi4-v2g-evse` | `raspberrypi4-64` | legacy Waydroid distro | no | | `main-rpi5` | `raspberrypi5` | default | no | @@ -104,10 +120,18 @@ only the new Jaguar Screen image. `policy/modules.conf` contains `waydroid = module` and the build produced `waydroid.pp` (114,378 bytes), proving the custom module is compiled rather than merely present in `SRC_URI`. - -The full image, kernel configuration, adoption matrix, Foundries build and -physical-board gates remain open; this recipe proof does not substitute for -them. +- The development policy build completed with only `waydroid_t` permissive; + the enforcing-smoke build also completed and its generated source contains + no permissive declaration. A permissive request in a non-development build + is rejected during parsing. +- The real `virtual/kernel:do_kernel_configcheck` completed all 873 tasks after + the local preflight correctly removed the disabled `modsign` distro feature. + This proves the Jaguar kernel accepts the SELinux configuration fragment; + the earlier dry run and invalid dummy-certificate attempt are not counted. + +The focused adoption matrix, full image, Foundries build and physical-board +gates remain open. Deferred product tuples must be restored before a +production-wide layer-adoption claim. 1. Resolve the exact candidate manifest and prove every project SHA exists. 2. Parse the Jaguar Screen platform and mfgtool configurations. diff --git a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te index e38ea41d..01469760 100644 --- a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te +++ b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te @@ -16,7 +16,6 @@ files_runtime_file(waydroid_runtime_t) type waydroid_rootfs_t; files_type(waydroid_rootfs_t) -# Development phase only. The host remains enforcing while accesses from -# this dedicated domain are logged but not denied. R26 release validation -# must remove this statement after the AVC-derived allow-list is reviewed. -permissive waydroid_t; +# Replaced with a permissive declaration only for an explicitly selected +# LOCAL_DEVELOPMENT_BUILD. Release policy is enforcing by default. +# WAYDROID_DEVELOPMENT_PERMISSIVE diff --git a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend index 5e55592b..099100fd 100644 --- a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend +++ b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend @@ -6,6 +6,15 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:" # expensive in LmP's global Clang configuration. INHIBIT_DEFAULT_DEPS = "1" +WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE ?= "0" + +python __anonymous() { + if (bb.utils.contains('DISTRO_FEATURES', 'waydroid', True, False, d) + and d.getVar('WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE') == '1' + and d.getVar('LOCAL_DEVELOPMENT_BUILD') != '1'): + bb.fatal('A permissive Waydroid SELinux domain is restricted to LOCAL_DEVELOPMENT_BUILD=1') +} + SRC_URI:append = "${@bb.utils.contains('DISTRO_FEATURES', 'waydroid', ' file://waydroid.te file://waydroid.fc file://waydroid.if', '', d)}" EXTRA_OEMAKE:append = "${@bb.utils.contains('DISTRO_FEATURES', 'waydroid', ' APPS_MODS=waydroid', '', d)}" @@ -17,5 +26,9 @@ do_compile:prepend() { install -m 0644 ${WORKDIR}/waydroid.te ${S}/policy/modules/services/ install -m 0644 ${WORKDIR}/waydroid.fc ${S}/policy/modules/services/ install -m 0644 ${WORKDIR}/waydroid.if ${S}/policy/modules/services/ + if [ "${WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE}" = "1" ]; then + sed -i 's/^# WAYDROID_DEVELOPMENT_PERMISSIVE$/permissive waydroid_t;/' \ + ${S}/policy/modules/services/waydroid.te + fi fi } diff --git a/kas/r26-jaguar-screen-selinux-enforcing-smoke.yml b/kas/r26-jaguar-screen-selinux-enforcing-smoke.yml new file mode 100644 index 00000000..aeac9558 --- /dev/null +++ b/kas/r26-jaguar-screen-selinux-enforcing-smoke.yml @@ -0,0 +1,10 @@ +header: + version: 14 + includes: + - r26-jaguar-screen-selinux.yml + +# Local enforcing-policy smoke configuration. Signing remains disabled by the +# included development configuration, so this is not a production image. +local_conf_header: + r26-jaguar-screen-selinux-enforcing-smoke: | + WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "0" diff --git a/kas/r26-jaguar-screen-selinux.yml b/kas/r26-jaguar-screen-selinux.yml index b9b4c64f..802499a7 100644 --- a/kas/r26-jaguar-screen-selinux.yml +++ b/kas/r26-jaguar-screen-selinux.yml @@ -10,6 +10,7 @@ target: lmp-factory-image local_conf_header: r26-jaguar-screen-selinux: | DD_PRODUCT_FEATURES = "display android-container" + WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "1" ASSEMBLE_SYSTEM_IMAGE = "0" LOCAL_DEVELOPMENT_BUILD = "1" OPTEE_TA_SIGN_ENABLE = "0" @@ -19,13 +20,13 @@ local_conf_header: TF_A_SIGN_ENABLE = "0" UEFI_SIGN_ENABLE = "0" MODSIGN_ENABLE = "0" + MODSIGN = "0" + DISTRO_FEATURES:remove = "modsign" SIGNING_UBOOT_SIGN_KEY = "${TOPDIR}/bitbake.lock" SIGNING_UBOOT_SIGN_CRT = "${TOPDIR}/bitbake.lock" SIGNING_UBOOT_SPL_SIGN_KEY = "${TOPDIR}/bitbake.lock" SIGNING_UBOOT_SPL_SIGN_CRT = "${TOPDIR}/bitbake.lock" SIGNING_UEFI_SIGN_KEY = "${TOPDIR}/bitbake.lock" SIGNING_UEFI_SIGN_CRT = "${TOPDIR}/bitbake.lock" - SIGNING_MODSIGN_PRIVKEY = "${TOPDIR}/bitbake.lock" - SIGNING_MODSIGN_X509 = "${TOPDIR}/bitbake.lock" OPTEE_TA_SIGN_KEY = "${TOPDIR}/bitbake.lock" TF_A_SIGN_KEY_PATH = "${TOPDIR}/bitbake.lock" From f4a81fa239d561c338663561901f0bf46790d255 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 12:21:23 +0100 Subject: [PATCH 65/79] test(selinux): guard Waydroid enforcing default Assisted-by: Codex --- .../tests/test_waydroid_selinux_policy.py | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 scripts/validation/tests/test_waydroid_selinux_policy.py diff --git a/scripts/validation/tests/test_waydroid_selinux_policy.py b/scripts/validation/tests/test_waydroid_selinux_policy.py new file mode 100644 index 00000000..0288f937 --- /dev/null +++ b/scripts/validation/tests/test_waydroid_selinux_policy.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 +"""Fail closed if the Waydroid SELinux development escape hatch broadens.""" + +from pathlib import Path +import unittest + + +ROOT = Path(__file__).resolve().parents[3] +APPEND = ROOT / "dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend" +POLICY = ROOT / "dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te" +DEVELOPMENT_KAS = ROOT / "kas/r26-jaguar-screen-selinux.yml" +ENFORCING_KAS = ROOT / "kas/r26-jaguar-screen-selinux-enforcing-smoke.yml" + + +class WaydroidSelinuxPolicyTest(unittest.TestCase): + def test_tracked_policy_is_not_permissive(self) -> None: + self.assertNotIn("permissive waydroid_t;", POLICY.read_text(encoding="utf-8")) + + def test_permissive_mode_defaults_off_and_is_development_gated(self) -> None: + text = APPEND.read_text(encoding="utf-8") + self.assertIn('WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE ?= "0"', text) + self.assertIn("d.getVar('LOCAL_DEVELOPMENT_BUILD') != '1'", text) + self.assertIn("bb.fatal(", text) + + def test_discovery_stack_opts_in_explicitly(self) -> None: + text = DEVELOPMENT_KAS.read_text(encoding="utf-8") + self.assertIn('LOCAL_DEVELOPMENT_BUILD = "1"', text) + self.assertIn('WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "1"', text) + + def test_enforcing_smoke_stack_opts_out_explicitly(self) -> None: + text = ENFORCING_KAS.read_text(encoding="utf-8") + self.assertIn('WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "0"', text) + + +if __name__ == "__main__": + unittest.main() From 004cc886bddbe1641e715d6ccfdada5732ef50a4 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 12:27:32 +0100 Subject: [PATCH 66/79] chore: allow unified diff context whitespace --- .gitattributes | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitattributes b/.gitattributes index 8289a470..11715be6 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1,5 +1,7 @@ # Shell scripts: enforce LF (avoid CRLF syntax errors on target) *.sh text eol=lf +# Unified-diff context lines intentionally contain a single trailing space. +*.patch whitespace=-blank-at-eol *.pdf filter=lfs diff=lfs merge=lfs -text *.bin filter=lfs diff=lfs merge=lfs -text From 2e5575e2e47d7e46cb41aec82b0a0ed7b50128bd Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 12:32:17 +0100 Subject: [PATCH 67/79] test(waydroid): capture R26 board acceptance evidence --- docs/r26-waydroid-selinux-plan.md | 14 ++ scripts/README.md | 1 + .../capture-r26-waydroid-board-evidence.sh | 153 ++++++++++++++++++ .../tests/test_r26_waydroid_board_evidence.py | 46 ++++++ 4 files changed, 214 insertions(+) create mode 100755 scripts/validation/capture-r26-waydroid-board-evidence.sh create mode 100644 scripts/validation/tests/test_r26_waydroid_board_evidence.py diff --git a/docs/r26-waydroid-selinux-plan.md b/docs/r26-waydroid-selinux-plan.md index 488f4cf2..3b971de2 100644 --- a/docs/r26-waydroid-selinux-plan.md +++ b/docs/r26-waydroid-selinux-plan.md @@ -148,6 +148,20 @@ production-wide layer-adoption claim. ## Hardware acceptance +Run the read-only evidence collector after the development OTA, using a small +raw Annex-B H.264 sample so hardware decode is exercised rather than inferred: + +```sh +sudo scripts/validation/capture-r26-waydroid-board-evidence.sh \ + /var/tmp/r26-waydroid-development --h264 /var/tmp/r26-test.h264 +``` + +After AVC classification and the enforcing policy rebuild, repeat with +`--release`. A non-zero result or any `NOT_RUN` acceptance item is not release +evidence. Preserve the resulting directory with the Foundries target number, +manifest SHA, OTA install and rollback logs; the collector itself is +read-only and does not perform an update or rollback. + - `getenforce` reports `Enforcing`; kernel command line does not disable it. - Waydroid processes enter `waydroid_t`; no Waydroid process remains `unconfined_t`, `init_t`, or another generic domain. diff --git a/scripts/README.md b/scripts/README.md index ed87194e..e992ff3c 100644 --- a/scripts/README.md +++ b/scripts/README.md @@ -29,6 +29,7 @@ This directory contains utility scripts for building, testing, managing, and aut | `rdc-control.sh` | Hardware Control | Resource domain control | i.MX8MM RDC management | | `validation/validate-layers.sh` | Quality Assurance | Yocto layer validation | Project compatibility | | `validate-layers-local.sh` | Quality Assurance | Comprehensive yocto-check-layer validation | KAS-based local validation | +| `validation/capture-r26-waydroid-board-evidence.sh` | Hardware Validation | Read-only Jaguar Screen SELinux/Waydroid acceptance capture | Release/development modes and optional H.264 decode proof | ## 📋 Table of Contents diff --git a/scripts/validation/capture-r26-waydroid-board-evidence.sh b/scripts/validation/capture-r26-waydroid-board-evidence.sh new file mode 100755 index 00000000..3c0fb2db --- /dev/null +++ b/scripts/validation/capture-r26-waydroid-board-evidence.sh @@ -0,0 +1,153 @@ +#!/bin/sh +# Capture read-only R26 Waydroid acceptance evidence on a Jaguar Screen board. +# shellcheck disable=SC2016 # Dollar expressions in single quotes run in sh -c. + +set -eu + +usage() { + echo "usage: $0 OUTPUT_DIR [--release] [--h264 RAW_H264_FILE]" >&2 + exit 2 +} + +[ "$#" -ge 1 ] || usage +output_dir=$1 +shift +release=0 +h264_file= +while [ "$#" -gt 0 ]; do + case "$1" in + --release) release=1 ;; + --h264) + [ "$#" -ge 2 ] || usage + h264_file=$2 + shift + ;; + *) usage ;; + esac + shift +done + +umask 077 +mkdir -p "$output_dir" +summary="$output_dir/summary.tsv" +: >"$summary" + +capture() { + name=$1 + shift + { + echo "command: $*" + echo "captured_utc: $(date -u +%Y-%m-%dT%H:%M:%SZ)" + echo + "$@" + } >"$output_dir/$name.txt" 2>&1 || true +} + +record() { + printf '%s\t%s\t%s\n' "$1" "$2" "$3" >>"$summary" +} + +expect_output() { + check=$1 + expected=$2 + shift 2 + actual=$("$@" 2>&1 || true) + if printf '%s\n' "$actual" | grep -Eq "$expected"; then + record "$check" PASS "$(printf '%s' "$actual" | tr '\n\t' ' ')" + else + record "$check" FAIL "$(printf '%s' "$actual" | tr '\n\t' ' ')" + fi +} + +capture identity uname -a +capture os-release sh -c 'cat /etc/os-release; echo; cat /etc/lmp-version 2>/dev/null || true' +capture kernel-command-line cat /proc/cmdline +capture selinux-status sh -c 'getenforce; sestatus; semodule -lfull; semanage permissive -l 2>/dev/null || true' +capture process-contexts ps -eZ +capture waydroid-status waydroid status +capture waydroid-processes sh -c 'ps -eZ | grep -E "[w]aydroid|[l]xc" || true' +capture service-status systemctl --no-pager --full status \ + waydroid-image-provision.service waydroid-jaguar-container.service \ + waydroid-jaguar-session.service waydroid-jaguar-ui.service weston.service +capture service-journal journalctl --no-pager -b -u waydroid-image-provision.service \ + -u waydroid-jaguar-container.service -u waydroid-jaguar-session.service \ + -u waydroid-jaguar-ui.service -u weston.service +capture labels sh -c 'ls -ldZ /var/lib/waydroid /var/lib/waydroid/images /run/waydroid-lxc 2>/dev/null; matchpathcon /var/lib/waydroid /run/waydroid-lxc /usr/bin/waydroid 2>/dev/null || true' +capture device-labels sh -c 'ls -lZ /dev/binder* /dev/vndbinder* /dev/hwbinder* /dev/dri/* /dev/video* 2>/dev/null || true' +capture binder waydroid shell service list +capture surfaceflinger waydroid shell dumpsys SurfaceFlinger +capture memory sh -c 'cat /proc/meminfo; echo; cat /proc/swaps; echo; zramctl 2>/dev/null || true; echo; for z in /sys/block/zram*; do for n in disksize comp_algorithm mem_used_total; do f="$z/$n"; [ ! -e "$f" ] || { printf "%s: " "$f"; cat "$f"; }; done; done' +capture gpu sh -c 'dmesg | grep -Ei "etnaviv|galcore|drm" || true; echo; waydroid shell dumpsys SurfaceFlinger 2>/dev/null | grep -Ei "GLES|EGL|render" || true' +capture v4l2 sh -c 'v4l2-ctl --list-devices 2>/dev/null || true; echo; gst-inspect-1.0 v4l2h264dec 2>/dev/null || gst-inspect-1.0 v4l2slh264dec 2>/dev/null || true; echo; dmesg | grep -Ei "v4l2|vpu|hantro|h264" || true' +capture audio sh -c 'pactl info 2>/dev/null || wpctl status 2>/dev/null || true; echo; waydroid shell dumpsys audio 2>/dev/null || true' +capture network waydroid shell ip -brief address +capture ota-state sh -c 'aktualizr-lite status 2>/dev/null || true; echo; ostree admin status 2>/dev/null || true; echo; fw_printenv 2>/dev/null | grep -Ei "bootcount|bootlimit|rollback|upgrade_available" || true' +capture secure-boot sh -c 'dmesg | grep -Ei "secure boot|hab|ahab|caam|dm-verity|verified boot" || true' +capture image-hashes sh -c 'sha256sum /var/lib/waydroid/images/*.img 2>/dev/null || true' +capture avc-denials sh -c 'ausearch -m AVC,USER_AVC -ts boot 2>/dev/null || journalctl -k -b --no-pager | grep -Ei "avc:.*denied" || true' + +expect_output host-selinux '^Enforcing$' getenforce +if grep -Eq '(^| )(selinux=0|enforcing=0)( |$)' /proc/cmdline; then + record kernel-selinux-arguments FAIL "$(cat /proc/cmdline)" +else + record kernel-selinux-arguments PASS "SELinux is not disabled on the kernel command line" +fi +expect_output waydroid-module '^waydroid[[:space:]]' sh -c 'semodule -lfull | grep "^waydroid[[:space:]]"' +expect_output waydroid-domain 'waydroid_t' sh -c 'ps -eZ | grep -E "[w]aydroid|[l]xc"' +expect_output binder-service-list '^[[:space:]]*[0-9]+[[:space:]]' waydroid shell service list +expect_output surfaceflinger-running 'GLES|EGL|Display' waydroid shell dumpsys SurfaceFlinger +services_ok=1 +for service in waydroid-image-provision.service waydroid-jaguar-container.service \ + waydroid-jaguar-session.service waydroid-jaguar-ui.service weston.service; do + systemctl is-active --quiet "$service" || services_ok=0 +done +if [ "$services_ok" -eq 1 ]; then + record kiosk-services PASS 'all Waydroid and Weston units are active' +else + record kiosk-services FAIL 'one or more Waydroid/Weston units are inactive; see service-status.txt' +fi +expect_output waydroid-network 'UP|UNKNOWN' waydroid shell ip -brief address +expect_output zram-active '/dev/zram' sh -c 'cat /proc/swaps' +expect_output etnaviv-active 'etnaviv' sh -c 'dmesg | grep -i etnaviv' + +if [ "$release" -eq 1 ]; then + if semanage permissive -l 2>/dev/null | grep -qx 'waydroid_t'; then + record waydroid-enforcing FAIL 'waydroid_t is listed as permissive' + else + record waydroid-enforcing PASS 'waydroid_t is not listed as permissive' + fi + if tail -n +4 "$output_dir/avc-denials.txt" | grep -Eqi 'avc:.*denied'; then + record unexplained-avc FAIL 'AVC denials require classification before release' + else + record unexplained-avc PASS 'no AVC denials found since boot' + fi +else + record waydroid-enforcing NOT_RUN 'development capture; repeat with --release on enforcing candidate' + record unexplained-avc NOT_RUN 'development AVCs are evidence for policy refinement' +fi + +if [ -n "$h264_file" ]; then + if [ ! -r "$h264_file" ]; then + record v4l2-h264-decode FAIL "unreadable input: $h264_file" + elif command -v gst-launch-1.0 >/dev/null 2>&1; then + decoder=v4l2h264dec + gst-inspect-1.0 "$decoder" >/dev/null 2>&1 || decoder=v4l2slh264dec + if timeout 120 gst-launch-1.0 -q filesrc location="$h264_file" ! \ + h264parse ! "$decoder" ! fakesink sync=false \ + >"$output_dir/v4l2-h264-decode.txt" 2>&1; then + record v4l2-h264-decode PASS "$decoder completed" + else + record v4l2-h264-decode FAIL "see v4l2-h264-decode.txt" + fi + else + record v4l2-h264-decode FAIL 'gst-launch-1.0 is unavailable' + fi +else + record v4l2-h264-decode NOT_RUN 'supply --h264 RAW_H264_FILE for an actual decode proof' +fi + +printf 'Evidence: %s\n' "$output_dir" +printf 'Summary: %s\n' "$summary" +if grep -q "$(printf '\t')FAIL$(printf '\t')" "$summary"; then + exit 1 +fi diff --git a/scripts/validation/tests/test_r26_waydroid_board_evidence.py b/scripts/validation/tests/test_r26_waydroid_board_evidence.py new file mode 100644 index 00000000..4944ef13 --- /dev/null +++ b/scripts/validation/tests/test_r26_waydroid_board_evidence.py @@ -0,0 +1,46 @@ +import pathlib +import unittest + + +ROOT = pathlib.Path(__file__).resolve().parents[3] +SCRIPT = ROOT / "scripts/validation/capture-r26-waydroid-board-evidence.sh" + + +class BoardEvidenceCollectorTests(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.source = SCRIPT.read_text(encoding="utf-8") + + def test_release_mode_checks_domain_and_avcs(self): + self.assertIn("semanage permissive -l", self.source) + self.assertIn("waydroid-enforcing FAIL", self.source) + self.assertIn("unexplained-avc FAIL", self.source) + + def test_acceptance_surfaces_are_captured(self): + for evidence in ( + "binder-service-list", + "surfaceflinger-running", + "kiosk-services", + "waydroid-network", + "zram-active", + "etnaviv-active", + "v4l2-h264-decode", + "ota-state", + "secure-boot", + "image-hashes", + ): + self.assertIn(evidence, self.source) + + def test_h264_is_exercised_not_inferred(self): + self.assertIn("gst-launch-1.0", self.source) + self.assertIn("h264parse", self.source) + self.assertIn("fakesink", self.source) + self.assertIn("NOT_RUN", self.source) + + def test_collector_does_not_update_or_reboot(self): + for forbidden in ("fioctl update", "aktualizr-lite update", "reboot", "shutdown"): + self.assertNotIn(forbidden, self.source) + + +if __name__ == "__main__": + unittest.main() From ab79aa41d193c47c437d84eea72e2e4e44098aca Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 12:34:16 +0100 Subject: [PATCH 68/79] test(waydroid): require direct confinement evidence --- .../capture-r26-waydroid-board-evidence.sh | 17 +++++++++++++++-- .../tests/test_r26_waydroid_board_evidence.py | 6 ++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/scripts/validation/capture-r26-waydroid-board-evidence.sh b/scripts/validation/capture-r26-waydroid-board-evidence.sh index 3c0fb2db..2532a8a8 100755 --- a/scripts/validation/capture-r26-waydroid-board-evidence.sh +++ b/scripts/validation/capture-r26-waydroid-board-evidence.sh @@ -76,6 +76,7 @@ capture labels sh -c 'ls -ldZ /var/lib/waydroid /var/lib/waydroid/images /run/wa capture device-labels sh -c 'ls -lZ /dev/binder* /dev/vndbinder* /dev/hwbinder* /dev/dri/* /dev/video* 2>/dev/null || true' capture binder waydroid shell service list capture surfaceflinger waydroid shell dumpsys SurfaceFlinger +capture android-low-ram sh -c 'waydroid shell getprop ro.config.low_ram; waydroid shell getprop ro.lmk.low; waydroid shell getprop ro.lmk.medium; waydroid shell getprop ro.lmk.critical' capture memory sh -c 'cat /proc/meminfo; echo; cat /proc/swaps; echo; zramctl 2>/dev/null || true; echo; for z in /sys/block/zram*; do for n in disksize comp_algorithm mem_used_total; do f="$z/$n"; [ ! -e "$f" ] || { printf "%s: " "$f"; cat "$f"; }; done; done' capture gpu sh -c 'dmesg | grep -Ei "etnaviv|galcore|drm" || true; echo; waydroid shell dumpsys SurfaceFlinger 2>/dev/null | grep -Ei "GLES|EGL|render" || true' capture v4l2 sh -c 'v4l2-ctl --list-devices 2>/dev/null || true; echo; gst-inspect-1.0 v4l2h264dec 2>/dev/null || gst-inspect-1.0 v4l2slh264dec 2>/dev/null || true; echo; dmesg | grep -Ei "v4l2|vpu|hantro|h264" || true' @@ -92,8 +93,18 @@ if grep -Eq '(^| )(selinux=0|enforcing=0)( |$)' /proc/cmdline; then else record kernel-selinux-arguments PASS "SELinux is not disabled on the kernel command line" fi -expect_output waydroid-module '^waydroid[[:space:]]' sh -c 'semodule -lfull | grep "^waydroid[[:space:]]"' -expect_output waydroid-domain 'waydroid_t' sh -c 'ps -eZ | grep -E "[w]aydroid|[l]xc"' +expect_output waydroid-module '(^|[[:space:]])waydroid([[:space:]]|$)' semodule -lfull +if ps -eZ | awk ' + /[w]aydroid|[l]xc/ { + found = 1 + if ($1 !~ /:waydroid_t:/) bad = 1 + } + END { exit !(found && !bad) } +'; then + record waydroid-domain PASS 'every visible Waydroid/LXC process is in waydroid_t' +else + record waydroid-domain FAIL 'missing Waydroid/LXC process or one is outside waydroid_t; see waydroid-processes.txt' +fi expect_output binder-service-list '^[[:space:]]*[0-9]+[[:space:]]' waydroid shell service list expect_output surfaceflinger-running 'GLES|EGL|Display' waydroid shell dumpsys SurfaceFlinger services_ok=1 @@ -108,7 +119,9 @@ else fi expect_output waydroid-network 'UP|UNKNOWN' waydroid shell ip -brief address expect_output zram-active '/dev/zram' sh -c 'cat /proc/swaps' +expect_output android-low-ram '^(true|1)$' waydroid shell getprop ro.config.low_ram expect_output etnaviv-active 'etnaviv' sh -c 'dmesg | grep -i etnaviv' +expect_output hardware-renderer 'etnaviv|Vivante|GC[0-9]+' sh -c 'waydroid shell dumpsys SurfaceFlinger | grep -Ei "GLES|EGL|render"' if [ "$release" -eq 1 ]; then if semanage permissive -l 2>/dev/null | grep -qx 'waydroid_t'; then diff --git a/scripts/validation/tests/test_r26_waydroid_board_evidence.py b/scripts/validation/tests/test_r26_waydroid_board_evidence.py index 4944ef13..4360423a 100644 --- a/scripts/validation/tests/test_r26_waydroid_board_evidence.py +++ b/scripts/validation/tests/test_r26_waydroid_board_evidence.py @@ -23,7 +23,9 @@ def test_acceptance_surfaces_are_captured(self): "kiosk-services", "waydroid-network", "zram-active", + "android-low-ram", "etnaviv-active", + "hardware-renderer", "v4l2-h264-decode", "ota-state", "secure-boot", @@ -37,6 +39,10 @@ def test_h264_is_exercised_not_inferred(self): self.assertIn("fakesink", self.source) self.assertIn("NOT_RUN", self.source) + def test_every_waydroid_process_must_be_confined(self): + self.assertIn("found && !bad", self.source) + self.assertIn("every visible Waydroid/LXC process is in waydroid_t", self.source) + def test_collector_does_not_update_or_reboot(self): for forbidden in ("fioctl update", "aktualizr-lite update", "reboot", "shutdown"): self.assertNotIn(forbidden, self.source) From 7e015b95592348036c683e8db58374c7b2b1aa75 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 13:04:07 +0100 Subject: [PATCH 69/79] fix(selinux): isolate Waydroid policy discovery mode --- docs/r26-waydroid-selinux-plan.md | 11 +++++++---- .../refpolicy/refpolicy-targeted/waydroid.te | 2 +- .../refpolicy/refpolicy-targeted_%.bbappend | 5 +++-- kas/r26-jaguar-screen-selinux.yml | 1 + .../validation/tests/test_waydroid_selinux_policy.py | 4 +++- 5 files changed, 15 insertions(+), 8 deletions(-) diff --git a/docs/r26-waydroid-selinux-plan.md b/docs/r26-waydroid-selinux-plan.md index 3b971de2..1ff7ad6b 100644 --- a/docs/r26-waydroid-selinux-plan.md +++ b/docs/r26-waydroid-selinux-plan.md @@ -47,11 +47,14 @@ new refpolicy modules otherwise default to `off` when absent from the upstream The first hardware image keeps the host globally enforcing but explicitly sets `WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "1"` to declare only `waydroid_t` permissive. The recipe rejects that setting unless -`LOCAL_DEVELOPMENT_BUILD = "1"`; all other builds compile the Waydroid domain -enforcing by default. This isolates policy discovery to the Waydroid domain, +`WAYDROID_SELINUX_POLICY_DISCOVERY = "1"`; all other builds compile the +Waydroid domain enforcing by default. The dedicated gate deliberately avoids +`LOCAL_DEVELOPMENT_BUILD`, which also changes SE05x and other platform content, +so a Foundries discovery OTA can retain the production-shaped secure-boot and +hardware configuration. This isolates policy discovery to the Waydroid domain, avoids weakening unrelated host services, and prevents the discovery setting -from leaking into a Foundries production build. Hardware AVCs must still be -classified and converted to narrow rules before release. +from leaking into a release build. Hardware AVCs must still be classified and +converted to narrow rules before release. Immutable OSTree content is labelled at image construction with `selinux-image`. `FIRST_BOOT_RELABEL` remains disabled because a whole-root diff --git a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te index 01469760..a7a30758 100644 --- a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te +++ b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te @@ -17,5 +17,5 @@ type waydroid_rootfs_t; files_type(waydroid_rootfs_t) # Replaced with a permissive declaration only for an explicitly selected -# LOCAL_DEVELOPMENT_BUILD. Release policy is enforcing by default. +# Waydroid policy-discovery build. Release policy is enforcing by default. # WAYDROID_DEVELOPMENT_PERMISSIVE diff --git a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend index 099100fd..88c01722 100644 --- a/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend +++ b/dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted_%.bbappend @@ -7,12 +7,13 @@ FILESEXTRAPATHS:prepend := "${THISDIR}/${PN}:" INHIBIT_DEFAULT_DEPS = "1" WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE ?= "0" +WAYDROID_SELINUX_POLICY_DISCOVERY ?= "0" python __anonymous() { if (bb.utils.contains('DISTRO_FEATURES', 'waydroid', True, False, d) and d.getVar('WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE') == '1' - and d.getVar('LOCAL_DEVELOPMENT_BUILD') != '1'): - bb.fatal('A permissive Waydroid SELinux domain is restricted to LOCAL_DEVELOPMENT_BUILD=1') + and d.getVar('WAYDROID_SELINUX_POLICY_DISCOVERY') != '1'): + bb.fatal('A permissive Waydroid SELinux domain requires WAYDROID_SELINUX_POLICY_DISCOVERY=1') } SRC_URI:append = "${@bb.utils.contains('DISTRO_FEATURES', 'waydroid', ' file://waydroid.te file://waydroid.fc file://waydroid.if', '', d)}" diff --git a/kas/r26-jaguar-screen-selinux.yml b/kas/r26-jaguar-screen-selinux.yml index 802499a7..914874e5 100644 --- a/kas/r26-jaguar-screen-selinux.yml +++ b/kas/r26-jaguar-screen-selinux.yml @@ -11,6 +11,7 @@ local_conf_header: r26-jaguar-screen-selinux: | DD_PRODUCT_FEATURES = "display android-container" WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "1" + WAYDROID_SELINUX_POLICY_DISCOVERY = "1" ASSEMBLE_SYSTEM_IMAGE = "0" LOCAL_DEVELOPMENT_BUILD = "1" OPTEE_TA_SIGN_ENABLE = "0" diff --git a/scripts/validation/tests/test_waydroid_selinux_policy.py b/scripts/validation/tests/test_waydroid_selinux_policy.py index 0288f937..53ef7440 100644 --- a/scripts/validation/tests/test_waydroid_selinux_policy.py +++ b/scripts/validation/tests/test_waydroid_selinux_policy.py @@ -19,13 +19,15 @@ def test_tracked_policy_is_not_permissive(self) -> None: def test_permissive_mode_defaults_off_and_is_development_gated(self) -> None: text = APPEND.read_text(encoding="utf-8") self.assertIn('WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE ?= "0"', text) - self.assertIn("d.getVar('LOCAL_DEVELOPMENT_BUILD') != '1'", text) + self.assertIn('WAYDROID_SELINUX_POLICY_DISCOVERY ?= "0"', text) + self.assertIn("d.getVar('WAYDROID_SELINUX_POLICY_DISCOVERY') != '1'", text) self.assertIn("bb.fatal(", text) def test_discovery_stack_opts_in_explicitly(self) -> None: text = DEVELOPMENT_KAS.read_text(encoding="utf-8") self.assertIn('LOCAL_DEVELOPMENT_BUILD = "1"', text) self.assertIn('WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "1"', text) + self.assertIn('WAYDROID_SELINUX_POLICY_DISCOVERY = "1"', text) def test_enforcing_smoke_stack_opts_out_explicitly(self) -> None: text = ENFORCING_KAS.read_text(encoding="utf-8") From 432afb06c283c50fafb59c3ce6b1050b3bd98fde Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 13:05:33 +0100 Subject: [PATCH 70/79] ci(screen): lock SELinux discovery tuple --- ci/layer-adoption-tuples.json | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index 26ccf241..d8cbfed2 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -2,12 +2,12 @@ "schema": 1, "source": { "repository": "https://source.foundries.io/factories/dynamic-devices/ci-scripts.git", - "commit": "f087a926016debeb79aa8326f620704a80c45cd6", + "commit": "520e5c11dede126bd1bd35184293c20addeda380", "file": "factory-config.yml" }, "tuples": [ {"id":"imx8mm-jaguar-screen-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display flutter godot","variables":{}}, {"id":"imx8mm-jaguar-screen-mfgtool","machine":"imx8mm-jaguar-screen","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, - {"id":"main-jaguar-screen-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display android-container","variables":{"ACCEPT_FSL_EULA":"1","ASSEMBLE_SYSTEM_IMAGE":"0","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git"}} + {"id":"main-jaguar-screen-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display android-container","variables":{"ACCEPT_FSL_EULA":"1","ASSEMBLE_SYSTEM_IMAGE":"0","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git","WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE":"1","WAYDROID_SELINUX_POLICY_DISCOVERY":"1"}} ] } From f966469a33f62a9df36c7c091cf5cac18deecf12 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 13:10:43 +0100 Subject: [PATCH 71/79] feat(screen): retain CRA audit evidence in SELinux image --- meta-dynamicdevices-distro | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta-dynamicdevices-distro b/meta-dynamicdevices-distro index b1901ad1..c926b492 160000 --- a/meta-dynamicdevices-distro +++ b/meta-dynamicdevices-distro @@ -1 +1 @@ -Subproject commit b1901ad12c4b18ef02c628744929898375feee72 +Subproject commit c926b49277ce7679820576708a58e2e9ae758e7a From 10f515a52ea2bc1b0715890b7ccb75c3cd8887d2 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 13:15:33 +0100 Subject: [PATCH 72/79] test(screen): retain CRA audit evidence path --- docs/r26-waydroid-selinux-plan.md | 10 ++++++++++ .../validation/capture-r26-waydroid-board-evidence.sh | 9 ++++++++- .../tests/test_r26_waydroid_board_evidence.py | 1 + .../validation/tests/test_waydroid_selinux_policy.py | 7 +++++++ 4 files changed, 26 insertions(+), 1 deletion(-) diff --git a/docs/r26-waydroid-selinux-plan.md b/docs/r26-waydroid-selinux-plan.md index 1ff7ad6b..274f0f41 100644 --- a/docs/r26-waydroid-selinux-plan.md +++ b/docs/r26-waydroid-selinux-plan.md @@ -131,6 +131,16 @@ reviewed product-lifecycle decision, not an unexplained loss of gate coverage. the local preflight correctly removed the disabled `modsign` distro feature. This proves the Jaguar kernel accepts the SELinux configuration fragment; the earlier dry run and invalid dummy-certificate attempt are not counted. +- An isolated `repo` sync of the prepared Foundries manifest resolved every + pinned project, including the authenticated `main-jaguar-screen` subscriber + override. Its production-shaped parse completed 3,954 recipes and 6,137 + targets with zero errors, and its release-default policy compile completed + 459/459 tasks with no permissive Waydroid declaration. +- The exact image inherits `create-spdx`, `license_image` and `buildhistory`, + uses a fixed reproducible rootfs timestamp, and selects `cra-audit`. The + SELinux Jaguar Screen image now resolves `audit`, `cra-audit-system`, + `logrotate`, `rsyslog` and `systemd-analyze`; a separate parse proves those + additions do not leak into the non-SELinux screen baseline. The focused adoption matrix, full image, Foundries build and physical-board gates remain open. Deferred product tuples must be restored before a diff --git a/scripts/validation/capture-r26-waydroid-board-evidence.sh b/scripts/validation/capture-r26-waydroid-board-evidence.sh index 2532a8a8..3559d206 100755 --- a/scripts/validation/capture-r26-waydroid-board-evidence.sh +++ b/scripts/validation/capture-r26-waydroid-board-evidence.sh @@ -68,7 +68,8 @@ capture waydroid-status waydroid status capture waydroid-processes sh -c 'ps -eZ | grep -E "[w]aydroid|[l]xc" || true' capture service-status systemctl --no-pager --full status \ waydroid-image-provision.service waydroid-jaguar-container.service \ - waydroid-jaguar-session.service waydroid-jaguar-ui.service weston.service + waydroid-jaguar-session.service waydroid-jaguar-ui.service weston.service \ + auditd.service cra-audit-queue-processor.timer capture service-journal journalctl --no-pager -b -u waydroid-image-provision.service \ -u waydroid-jaguar-container.service -u waydroid-jaguar-session.service \ -u waydroid-jaguar-ui.service -u weston.service @@ -117,6 +118,12 @@ if [ "$services_ok" -eq 1 ]; then else record kiosk-services FAIL 'one or more Waydroid/Weston units are inactive; see service-status.txt' fi +if systemctl is-active --quiet auditd.service \ + && systemctl is-active --quiet cra-audit-queue-processor.timer; then + record cra-audit-runtime PASS 'auditd and CRA queue processor timer are active' +else + record cra-audit-runtime FAIL 'auditd or CRA queue processor timer is inactive; see service-status.txt' +fi expect_output waydroid-network 'UP|UNKNOWN' waydroid shell ip -brief address expect_output zram-active '/dev/zram' sh -c 'cat /proc/swaps' expect_output android-low-ram '^(true|1)$' waydroid shell getprop ro.config.low_ram diff --git a/scripts/validation/tests/test_r26_waydroid_board_evidence.py b/scripts/validation/tests/test_r26_waydroid_board_evidence.py index 4360423a..1763fb01 100644 --- a/scripts/validation/tests/test_r26_waydroid_board_evidence.py +++ b/scripts/validation/tests/test_r26_waydroid_board_evidence.py @@ -30,6 +30,7 @@ def test_acceptance_surfaces_are_captured(self): "ota-state", "secure-boot", "image-hashes", + "cra-audit-runtime", ): self.assertIn(evidence, self.source) diff --git a/scripts/validation/tests/test_waydroid_selinux_policy.py b/scripts/validation/tests/test_waydroid_selinux_policy.py index 53ef7440..c25d3efc 100644 --- a/scripts/validation/tests/test_waydroid_selinux_policy.py +++ b/scripts/validation/tests/test_waydroid_selinux_policy.py @@ -10,6 +10,7 @@ POLICY = ROOT / "dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te" DEVELOPMENT_KAS = ROOT / "kas/r26-jaguar-screen-selinux.yml" ENFORCING_KAS = ROOT / "kas/r26-jaguar-screen-selinux-enforcing-smoke.yml" +FACTORY_IMAGE = ROOT / "meta-dynamicdevices-distro/recipes-samples/images/lmp-factory-image.bb" class WaydroidSelinuxPolicyTest(unittest.TestCase): @@ -33,6 +34,12 @@ def test_enforcing_smoke_stack_opts_out_explicitly(self) -> None: text = ENFORCING_KAS.read_text(encoding="utf-8") self.assertIn('WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "0"', text) + def test_cra_runtime_is_scoped_to_selinux_screen_image(self) -> None: + text = FACTORY_IMAGE.read_text(encoding="utf-8") + self.assertIn("d.getVar('MACHINE') == 'imx8mm-jaguar-screen'", text) + self.assertIn("bb.utils.contains('DISTRO_FEATURES', 'selinux'", text) + self.assertIn("'lmp-feature-audit.inc'", text) + if __name__ == "__main__": unittest.main() From 7ed1c581c8c7d24e40df7a7994311c67ee7f5a21 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 13:22:53 +0100 Subject: [PATCH 73/79] fix(ci): validate audited distro by gitlink --- .../tests/test_waydroid_selinux_policy.py | 23 ++++++++++++++----- 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/scripts/validation/tests/test_waydroid_selinux_policy.py b/scripts/validation/tests/test_waydroid_selinux_policy.py index c25d3efc..b5a364ff 100644 --- a/scripts/validation/tests/test_waydroid_selinux_policy.py +++ b/scripts/validation/tests/test_waydroid_selinux_policy.py @@ -2,6 +2,7 @@ """Fail closed if the Waydroid SELinux development escape hatch broadens.""" from pathlib import Path +import subprocess import unittest @@ -10,7 +11,7 @@ POLICY = ROOT / "dynamic-layers/selinux/recipes-security/refpolicy/refpolicy-targeted/waydroid.te" DEVELOPMENT_KAS = ROOT / "kas/r26-jaguar-screen-selinux.yml" ENFORCING_KAS = ROOT / "kas/r26-jaguar-screen-selinux-enforcing-smoke.yml" -FACTORY_IMAGE = ROOT / "meta-dynamicdevices-distro/recipes-samples/images/lmp-factory-image.bb" +AUDITED_DISTRO_COMMIT = "c926b49277ce7679820576708a58e2e9ae758e7a" class WaydroidSelinuxPolicyTest(unittest.TestCase): @@ -34,11 +35,21 @@ def test_enforcing_smoke_stack_opts_out_explicitly(self) -> None: text = ENFORCING_KAS.read_text(encoding="utf-8") self.assertIn('WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE = "0"', text) - def test_cra_runtime_is_scoped_to_selinux_screen_image(self) -> None: - text = FACTORY_IMAGE.read_text(encoding="utf-8") - self.assertIn("d.getVar('MACHINE') == 'imx8mm-jaguar-screen'", text) - self.assertIn("bb.utils.contains('DISTRO_FEATURES', 'selinux'", text) - self.assertIn("'lmp-feature-audit.inc'", text) + def test_cra_runtime_uses_the_audited_distro_pin(self) -> None: + """Keep CI independent of an initialized submodule worktree. + + The exact-manifest preflight validates the contents of this immutable + distro commit, including the screen-and-SELinux-only audit runtime. + This source gate ensures the product continues to reference that + audited content address. + """ + entry = subprocess.check_output( + ["git", "ls-tree", "HEAD", "meta-dynamicdevices-distro"], + cwd=ROOT, + text=True, + ).split() + self.assertEqual(entry[:2], ["160000", "commit"]) + self.assertEqual(entry[2], AUDITED_DISTRO_COMMIT) if __name__ == "__main__": From c132b61f3b18198e87cea506c401729cd7112878 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 14:07:17 +0100 Subject: [PATCH 74/79] ci: add focused tuple validation dispatch [skip ci] --- .github/workflows/layer-adoption-gate.yml | 25 +++++++++++++------ ci/layer-adoption-tuples.json | 2 +- .../test_run_layer_adoption_regression.py | 11 +++++++- 3 files changed, 29 insertions(+), 9 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 7e875fb9..074e4c43 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -3,6 +3,8 @@ name: Layer Adoption Gate on: pull_request: branches: [main, develop] + merge_group: + types: [checks_requested] push: branches: [main, develop] workflow_dispatch: @@ -10,10 +12,14 @@ on: base_sha: description: Baseline commit to compare required: true + tuple_id: + description: One protected tuple to build for focused development + required: true + default: imx8mm-jaguar-screen-waydroid-image concurrency: - group: layer-adoption-${{ github.event.pull_request.number || github.ref }} - cancel-in-progress: true + group: layer-adoption-${{ github.event_name == 'merge_group' && github.event.merge_group.head_sha || github.event.pull_request.number || github.ref }} + cancel-in-progress: ${{ github.event_name != 'merge_group' }} defaults: run: @@ -35,10 +41,11 @@ jobs: name: Resolve immutable baseline env: PR_BASE: ${{ github.event.pull_request.base.sha }} + MERGE_BASE: ${{ github.event.merge_group.base_sha }} PUSH_BASE: ${{ github.event.before }} INPUT_BASE: ${{ inputs.base_sha }} run: | - sha="${PR_BASE:-${INPUT_BASE:-${PUSH_BASE:-}}}" + sha="${PR_BASE:-${MERGE_BASE:-${INPUT_BASE:-${PUSH_BASE:-}}}}" if [ -z "$sha" ] || printf '%s' "$sha" | grep -Eq '^0+$'; then sha=$(git rev-parse HEAD^) fi @@ -59,10 +66,14 @@ jobs: --base '${{ steps.base.outputs.sha }}' \ --head '${{ github.sha }}' \ --github-output "$GITHUB_OUTPUT" - tuple_ids=$(python3 -c 'import json; print(json.dumps([entry["id"] for entry in json.load(open("ci/layer-adoption-tuples.json"))["tuples"]], separators=(",", ":")))') + if [ '${{ github.event_name }}' = workflow_dispatch ]; then + tuple_ids=$(python3 -c 'import json,sys; ids=[entry["id"] for entry in json.load(open("ci/layer-adoption-tuples.json"))["tuples"]]; requested=sys.argv[1]; requested in ids or sys.exit(f"unknown protected tuple: {requested}"); print(json.dumps([requested],separators=(",",":")))' '${{ inputs.tuple_id }}') + else + tuple_ids=$(python3 -c 'import json; print(json.dumps([entry["id"] for entry in json.load(open("ci/layer-adoption-tuples.json"))["tuples"]], separators=(",", ":")))') + fi echo "tuple_ids=$tuple_ids" >> "$GITHUB_OUTPUT" regression: - name: Existing product regression (${{ matrix.tuple_id }}) + name: ${{ github.event_name == 'workflow_dispatch' && 'Development validation — baseline comparison' || 'Product readiness validation — baseline comparison' }} (${{ matrix.tuple_id }}) needs: detect if: needs.detect.outputs.material == 'true' strategy: @@ -119,7 +130,7 @@ jobs: candidate/scripts/validation/generate-layer-adoption-test-keys.sh "$temporary" mv "$temporary" "$keys" fi - - name: Build and compare every protected tuple + - name: Build and compare selected protected tuple run: | python3 candidate/scripts/validation/run-layer-adoption-regression.py \ --baseline baseline \ @@ -151,7 +162,7 @@ jobs: done required: - name: Layer Adoption Gate + name: ${{ github.event_name == 'workflow_dispatch' && 'Development Validation' || 'Layer Adoption Gate' }} needs: [detect, regression] if: always() runs-on: [self-hosted, Linux, X64, yocto, ai-tools] diff --git a/ci/layer-adoption-tuples.json b/ci/layer-adoption-tuples.json index d8cbfed2..df686998 100644 --- a/ci/layer-adoption-tuples.json +++ b/ci/layer-adoption-tuples.json @@ -8,6 +8,6 @@ "tuples": [ {"id":"imx8mm-jaguar-screen-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display flutter godot","variables":{}}, {"id":"imx8mm-jaguar-screen-mfgtool","machine":"imx8mm-jaguar-screen","distro":"lmp-mfgtool","image":"mfgtool-files","config":"kas/lmp-dynamicdevices-mfgtool.yml","product_features":"","variables":{}}, - {"id":"main-jaguar-screen-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display android-container","variables":{"ACCEPT_FSL_EULA":"1","ASSEMBLE_SYSTEM_IMAGE":"0","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git","WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE":"1","WAYDROID_SELINUX_POLICY_DISCOVERY":"1"}} + {"id":"imx8mm-jaguar-screen-waydroid-image","machine":"imx8mm-jaguar-screen","distro":"lmp-dynamicdevices","image":"lmp-factory-image","config":"kas/lmp-dynamicdevices.yml","product_features":"display android-container","variables":{"ACCEPT_FSL_EULA":"1","ASSEMBLE_SYSTEM_IMAGE":"0","DEV_MODE":"1","DOCKER_COMPOSE_APP":"1","PATCHTOOL":"git","WAYDROID_SELINUX_DEVELOPMENT_PERMISSIVE":"1","WAYDROID_SELINUX_POLICY_DISCOVERY":"1"}} ] } diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index e97b4b17..632dd9e5 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -80,7 +80,16 @@ def test_workflow_shards_all_tuples_without_fail_fast(self) -> None: self.assertIn("fail-fast: false", workflow) self.assertIn("fromJSON(needs.detect.outputs.tuple_ids)", workflow) self.assertIn("--tuple-id '${{ matrix.tuple_id }}'", workflow) - self.assertIn("name: Layer Adoption Gate", workflow) + self.assertIn("merge_group:", workflow) + self.assertIn("types: [checks_requested]", workflow) + self.assertIn("github.event.merge_group.base_sha", workflow) + self.assertIn("github.event_name != 'merge_group'", workflow) + self.assertIn("Development validation — baseline comparison", workflow) + self.assertIn("Product readiness validation — baseline comparison", workflow) + self.assertIn("Development Validation", workflow) + self.assertIn("imx8mm-jaguar-screen-waydroid-image", workflow) + self.assertIn("unknown protected tuple", workflow) + self.assertIn("'Layer Adoption Gate'", workflow) def test_audited_baseline_repair_is_exact_and_fail_closed(self) -> None: old = "a" * 40 From 34cfc0c43d830acad6759a4c15d93fced5cf888f Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 14:11:40 +0100 Subject: [PATCH 75/79] ci: use stable connectivity probe [skip ci] --- scripts/validation/capture-layer-state.sh | 3 +++ scripts/validation/tests/test_capture_layer_state.py | 1 + 2 files changed, 4 insertions(+) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 2b645392..7cb0a24f 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -87,6 +87,9 @@ local_conf_header: $variable_assignments DL_DIR = $downloads_quoted SSTATE_DIR = $sstate_quoted + # Use a stable, deliberately selected CI probe rather than OE-core's + # release-specific default URL. Source fetches remain independently fatal. + CONNECTIVITY_CHECK_URIS = "https://www.example.com/" BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,100K STOPTASKS,\${DL_DIR},20G,100K STOPTASKS,\${SSTATE_DIR},20G,100K HALT,\${TMPDIR},10G,50K HALT,\${DL_DIR},10G,50K HALT,\${SSTATE_DIR},10G,50K" UBOOT_SIGN_KEYDIR:forcevariable = "$test_keys_dir" UBOOT_SPL_SIGN_KEYDIR:forcevariable = "$test_keys_dir" diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 84f4be2e..671ba8a5 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -162,6 +162,7 @@ def test_disk_monitor_uses_inode_not_disk_units(self) -> None: def test_tuple_variables_are_validated_and_injected_into_overlay(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertIn("variables_json=$6", source) + self.assertIn('CONNECTIVITY_CHECK_URIS = "https://www.example.com/"', source) self.assertIn('re.fullmatch(r"[A-Z0-9_]+(?::[a-z0-9_-]+)*", name)', source) self.assertIn("$variable_assignments", source) From e18ada21ac0239c1f0e0e33b64a6731c04024ebe Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 14:22:59 +0100 Subject: [PATCH 76/79] ci: make bison docs dependency hermetic [skip ci] --- scripts/validation/capture-layer-state.sh | 4 ++++ scripts/validation/tests/test_capture_layer_state.py | 3 +++ 2 files changed, 7 insertions(+) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 7cb0a24f..74910b40 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -90,6 +90,10 @@ $variable_assignments # Use a stable, deliberately selected CI probe rather than OE-core's # release-specific default URL. Source fetches remain independently fatal. CONNECTIVITY_CHECK_URIS = "https://www.example.com/" + # Foundries tuples use PATCHTOOL=git, which can make bison's generated + # manual appear stale. Provide the generator hermetically rather than + # depending on an undeclared package in the CI container. + DEPENDS:append:pn-bison-native = " help2man-native" BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,100K STOPTASKS,\${DL_DIR},20G,100K STOPTASKS,\${SSTATE_DIR},20G,100K HALT,\${TMPDIR},10G,50K HALT,\${DL_DIR},10G,50K HALT,\${SSTATE_DIR},10G,50K" UBOOT_SIGN_KEYDIR:forcevariable = "$test_keys_dir" UBOOT_SPL_SIGN_KEYDIR:forcevariable = "$test_keys_dir" diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 671ba8a5..95d867ad 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -163,6 +163,9 @@ def test_tuple_variables_are_validated_and_injected_into_overlay(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertIn("variables_json=$6", source) self.assertIn('CONNECTIVITY_CHECK_URIS = "https://www.example.com/"', source) + self.assertIn( + 'DEPENDS:append:pn-bison-native = " help2man-native"', source + ) self.assertIn('re.fullmatch(r"[A-Z0-9_]+(?::[a-z0-9_-]+)*", name)', source) self.assertIn("$variable_assignments", source) From e2bf6f614dc419643d2853dc47fe0d482892c696 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 15:01:09 +0100 Subject: [PATCH 77/79] ci: run focused tuple on DD Hetzner [skip ci] --- .github/workflows/layer-adoption-gate.yml | 12 ++++++------ scripts/validation/capture-layer-state.sh | 6 ++++++ scripts/validation/tests/test_capture_layer_state.py | 1 + .../tests/test_run_layer_adoption_regression.py | 4 +++- 4 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/layer-adoption-gate.yml b/.github/workflows/layer-adoption-gate.yml index 074e4c43..c5acfaeb 100644 --- a/.github/workflows/layer-adoption-gate.yml +++ b/.github/workflows/layer-adoption-gate.yml @@ -28,7 +28,7 @@ defaults: jobs: detect: name: Detect material layer change - runs-on: [self-hosted, Linux, X64, yocto, ai-tools] + runs-on: [self-hosted, Linux, X64, yocto, dd-esl-proxmox] outputs: material: ${{ steps.detect.outputs.material }} base_sha: ${{ steps.base.outputs.sha }} @@ -83,12 +83,12 @@ jobs: # Each tuple is an independent shard so one failure cannot hide later # product failures. The final Layer Adoption Gate remains the single # branch-protection contract, and local driver runs still cover all tuples. - runs-on: [self-hosted, Linux, X64, yocto, ai-tools] + runs-on: [self-hosted, Linux, X64, yocto, dd-esl-proxmox] container: image: ghcr.io/siemens/kas/kas@sha256:d989add57fc441fe9e27bb2dd6ed98c5597b44c807928e35a72dc1cfbdda9abe - # Match the dedicated ai-tools runner account so BitBake's root-user - # sanity check remains active and bind-mounted cache files stay writable. - options: --privileged --platform linux/amd64 --user 1002:1002 -v /home/ghrunner/yocto-layer-adoption:/var/cache/layer-adoption + # Match the dedicated DD registration on CT101 so BitBake's root-user + # sanity check remains active and the /yocto cache stays writable. + options: --privileged --platform linux/amd64 --user 999:995 -v /yocto/yocto-layer-adoption:/var/cache/layer-adoption env: LAYER_ADOPTION_CACHE: /var/cache/layer-adoption steps: @@ -165,7 +165,7 @@ jobs: name: ${{ github.event_name == 'workflow_dispatch' && 'Development Validation' || 'Layer Adoption Gate' }} needs: [detect, regression] if: always() - runs-on: [self-hosted, Linux, X64, yocto, ai-tools] + runs-on: [self-hosted, Linux, X64, yocto, dd-esl-proxmox] steps: - name: Enforce gate result env: diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 74910b40..07b019e3 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -94,6 +94,12 @@ $variable_assignments # manual appear stale. Provide the generator hermetically rather than # depending on an undeclared package in the CI container. DEPENDS:append:pn-bison-native = " help2man-native" + # runc vendors src/import as a Git submodule. PATCHTOOL=git applies the + # recipe patch inside that nested tree but then tries to commit only the + # parent repository, leaving the submodule dirty and failing do_patch. + # Keep the Foundries-wide Git patch policy and isolate this recipe to the + # standard Quilt backend for identical baseline and candidate builds. + PATCHTOOL:pn-runc-opencontainers = "quilt" BB_DISKMON_DIRS = "STOPTASKS,\${TMPDIR},20G,100K STOPTASKS,\${DL_DIR},20G,100K STOPTASKS,\${SSTATE_DIR},20G,100K HALT,\${TMPDIR},10G,50K HALT,\${DL_DIR},10G,50K HALT,\${SSTATE_DIR},10G,50K" UBOOT_SIGN_KEYDIR:forcevariable = "$test_keys_dir" UBOOT_SPL_SIGN_KEYDIR:forcevariable = "$test_keys_dir" diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 95d867ad..334f734a 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -166,6 +166,7 @@ def test_tuple_variables_are_validated_and_injected_into_overlay(self) -> None: self.assertIn( 'DEPENDS:append:pn-bison-native = " help2man-native"', source ) + self.assertIn('PATCHTOOL:pn-runc-opencontainers = "quilt"', source) self.assertIn('re.fullmatch(r"[A-Z0-9_]+(?::[a-z0-9_-]+)*", name)', source) self.assertIn("$variable_assignments", source) diff --git a/scripts/validation/tests/test_run_layer_adoption_regression.py b/scripts/validation/tests/test_run_layer_adoption_regression.py index 632dd9e5..0ebc4b25 100644 --- a/scripts/validation/tests/test_run_layer_adoption_regression.py +++ b/scripts/validation/tests/test_run_layer_adoption_regression.py @@ -308,7 +308,9 @@ def test_worktree_preparation_rejects_unpinned_layer(self) -> None: def test_gate_does_not_run_bitbake_as_root(self) -> None: workflow = WORKFLOW_PATH.read_text(encoding="utf-8") self.assertNotIn("--user 0:0", workflow) - self.assertIn("--user 1002:1002", workflow) + self.assertIn("--user 999:995", workflow) + self.assertIn("dd-esl-proxmox", workflow) + self.assertNotIn("ai-tools", workflow) def test_valid_cache_is_accepted_and_tampering_is_rejected(self) -> None: with tempfile.TemporaryDirectory() as directory: From 9c7f3237c1e97c3fd4c5714d2b839d2fd2a03a41 Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 22:49:34 +0100 Subject: [PATCH 78/79] ci: mirror pinned oauth2 source [skip ci] --- scripts/validation/capture-layer-state.sh | 4 ++++ scripts/validation/tests/test_capture_layer_state.py | 5 +++++ 2 files changed, 9 insertions(+) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 07b019e3..219d22f0 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -90,6 +90,10 @@ $variable_assignments # Use a stable, deliberately selected CI probe rather than OE-core's # release-specific default URL. Source fetches remain independently fatal. CONNECTIVITY_CHECK_URIS = "https://www.example.com/" + # docker-compose vendors golang.org/x/oauth2 from go.googlesource.com. + # Prefer its official GitHub mirror so this protected tuple is not coupled + # to one source host; BitBake still verifies the recipe-pinned SRCREV. + PREMIRRORS:append = " git://go.googlesource.com/oauth2 git://github.com/golang/oauth2.git;protocol=https \n" # Foundries tuples use PATCHTOOL=git, which can make bison's generated # manual appear stale. Provide the generator hermetically rather than # depending on an undeclared package in the CI container. diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index 334f734a..eb68290e 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -163,6 +163,11 @@ def test_tuple_variables_are_validated_and_injected_into_overlay(self) -> None: source = SCRIPT.read_text(encoding="utf-8") self.assertIn("variables_json=$6", source) self.assertIn('CONNECTIVITY_CHECK_URIS = "https://www.example.com/"', source) + self.assertIn( + "git://go.googlesource.com/oauth2 " + "git://github.com/golang/oauth2.git;protocol=https", + source, + ) self.assertIn( 'DEPENDS:append:pn-bison-native = " help2man-native"', source ) From 358cd2a779a1c2c65d4af75313215d08ff66a73b Mon Sep 17 00:00:00 2001 From: Alex J Lennon Date: Sun, 13 Sep 2026 22:50:20 +0100 Subject: [PATCH 79/79] ci: fail fast on docker compose fetch [skip ci] --- scripts/validation/capture-layer-state.sh | 8 ++++++++ scripts/validation/tests/test_capture_layer_state.py | 2 ++ 2 files changed, 10 insertions(+) diff --git a/scripts/validation/capture-layer-state.sh b/scripts/validation/capture-layer-state.sh index 219d22f0..afc1a8eb 100755 --- a/scripts/validation/capture-layer-state.sh +++ b/scripts/validation/capture-layer-state.sh @@ -159,6 +159,7 @@ printf '%s\n' \ "bitbake-layers show-recipes" \ "bitbake -g $target" \ "bitbake -e $target" \ + "bitbake -c fetch docker-compose (when DOCKER_COMPOSE_APP=1)" \ "bitbake $target" \ "DD_PRODUCT_FEATURES=$product_features" \ "TUPLE_VARIABLES=$variables_json" > "$output_dir/commands.txt" @@ -273,6 +274,13 @@ require_selected_value OPTEE_TA_SIGN_KEY "/ubootdev.key" require_selected_value TF_A_SIGN_KEY_PATH "/tf-a/privkey_ec_prime256v1.pem" rm "$output_dir/environment.log" +# Fail fast on the large docker-compose Go source set instead of discovering +# an unavailable vendor repository after hours of unrelated compilation. +if python3 -c 'import json,sys; raise SystemExit(0 if json.loads(sys.argv[1]).get("DOCKER_COMPOSE_APP") == "1" else 1)' "$variables_json"; then + capture_command fetch-docker-compose run_bitbake \ + "bitbake -c fetch docker-compose" >/dev/null +fi + # A parse-only graph is not proof that packaging, signing, recovery image size, # or deploy layout still works. Complete the real image/recovery build for both # baseline and candidate. diff --git a/scripts/validation/tests/test_capture_layer_state.py b/scripts/validation/tests/test_capture_layer_state.py index eb68290e..cd2d8378 100644 --- a/scripts/validation/tests/test_capture_layer_state.py +++ b/scripts/validation/tests/test_capture_layer_state.py @@ -172,6 +172,8 @@ def test_tuple_variables_are_validated_and_injected_into_overlay(self) -> None: 'DEPENDS:append:pn-bison-native = " help2man-native"', source ) self.assertIn('PATCHTOOL:pn-runc-opencontainers = "quilt"', source) + self.assertIn('"bitbake -c fetch docker-compose"', source) + self.assertIn('get("DOCKER_COMPOSE_APP") == "1"', source) self.assertIn('re.fullmatch(r"[A-Z0-9_]+(?::[a-z0-9_-]+)*", name)', source) self.assertIn("$variable_assignments", source)