From 3f46b2540f86cfbb58a24dd21afc58f546e1a58e Mon Sep 17 00:00:00 2001 From: jjackbrandt Date: Wed, 12 Aug 2026 20:24:20 +0000 Subject: [PATCH] ci: generate process replay refs on hosted runners --- .github/CODEOWNERS | 5 + .../ci/process_replay_active_inventory.json | 356 +++++++++ .github/ci/test_assemble_replay_release.py | 285 +++++++ .../test_validate_process_replay_release.py | 210 +++-- .github/ci/validate_process_replay_release.py | 71 +- .github/workflows/process_replay_refs.yaml | 737 +++++++++++------- .github/workflows/tests.yaml | 5 + tools/ci/assemble_replay_release.py | 292 +++++++ 8 files changed, 1622 insertions(+), 339 deletions(-) create mode 100644 .github/ci/process_replay_active_inventory.json create mode 100644 .github/ci/test_assemble_replay_release.py create mode 100755 tools/ci/assemble_replay_release.py diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index d68706cad19697..7eb1125dc70eea 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1,6 +1,10 @@ # Longitudinal safety gate and its protected test surface /.github/CODEOWNERS @daggerhashjack @SloPOS /.github/workflows/ @daggerhashjack +/.github/ci/process_replay_active_inventory.json @daggerhashjack +/.github/ci/validate_process_replay_release.py @daggerhashjack +/.github/ci/test_assemble_replay_release.py @daggerhashjack +/.github/ci/test_validate_process_replay_release.py @daggerhashjack /.github/workflows/tests.yaml @daggerhashjack @SloPOS /.github/ci/tesla_preap_longitudinal_mutations.py @daggerhashjack @SloPOS /.github/ci/test_tesla_preap_longitudinal_workflow.py @daggerhashjack @SloPOS @@ -9,4 +13,5 @@ /selfdrive/test/process_replay/ @daggerhashjack /third_party/acados/ @daggerhashjack /tools/ci/Dockerfile.replay @daggerhashjack +/tools/ci/assemble_replay_release.py @daggerhashjack /Dockerfile.replay @daggerhashjack diff --git a/.github/ci/process_replay_active_inventory.json b/.github/ci/process_replay_active_inventory.json new file mode 100644 index 00000000000000..d067aebc8f22b7 --- /dev/null +++ b/.github/ci/process_replay_active_inventory.json @@ -0,0 +1,356 @@ +{ + "allowed_processes": [ + "calibrationd", + "card", + "controlsd", + "dmonitoringd", + "lagd", + "locationd", + "paramsd", + "plannerd", + "radard", + "selfdrived", + "torqued", + "ubloxd" + ], + "cases": [ + { + "car_brand": "HYUNDAI", + "case_id": "hyundai", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd", + "selfdrived", + "radard", + "plannerd", + "calibrationd", + "dmonitoringd", + "locationd", + "paramsd", + "ubloxd", + "torqued" + ], + "source": "regenAA0FC4ED71E|2025-04-08--22-57-50--0", + "source_bytes": 11873030, + "source_sha256": "0b6734026f7c775f7fd648ef76130f9b9a672c5236c6a328eff31ec9cc3c5343" + }, + { + "car_brand": "HYUNDAI2", + "case_id": "hyundai2", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regenAFB9780D823|2025-04-08--23-00-34--0", + "source_bytes": 12527194, + "source_sha256": "4cc51a77bf8d128d5720fd692e5a0994c887ee35fdd949cc0ce54d4215e2b7a2" + }, + { + "car_brand": "TOYOTA", + "case_id": "toyota", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd", + "selfdrived", + "radard", + "plannerd", + "calibrationd", + "dmonitoringd", + "locationd", + "paramsd", + "ubloxd", + "torqued" + ], + "source": "regen218A4DCFAA1|2025-04-08--22-57-51--0", + "source_bytes": 10846201, + "source_sha256": "6c028c0d40328fc00fa12acf91dba30d9f152e89d99a7089aadac62f6c7d5d64" + }, + { + "car_brand": "TOYOTA3", + "case_id": "toyota3", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen1455E3B4BDF|2025-04-09--03-26-06--0", + "source_bytes": 9272538, + "source_sha256": "876100021b13ddd1a39efad5fdb4e7376ad5b298f64bf390b5f58742ffc6e429" + }, + { + "car_brand": "HONDA", + "case_id": "honda", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regenB328FF8BA0A|2025-04-08--22-57-45--0", + "source_bytes": 10090147, + "source_sha256": "8ea4ab5412a99bfc65fbe95cd169a54edb6bcb68f6fa20c97f2bc58ed4bba538" + }, + { + "car_brand": "HONDA2", + "case_id": "honda2", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen6170C8C9A35|2025-04-08--22-57-46--0", + "source_bytes": 10419375, + "source_sha256": "fc9ab2b4fd9702ceb8e3342186076dc6a85dbd6a3bd859ac6b965234bcc65c69" + }, + { + "car_brand": "CHRYSLER", + "case_id": "chrysler", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen5B28FC2A437|2025-04-08--23-04-24--0", + "source_bytes": 11951101, + "source_sha256": "a30f65eb1033afbd0cff70d965aa9c0a276cc5ceb73d28baa817430df178009a" + }, + { + "car_brand": "RAM", + "case_id": "ram", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regenBF81EA96E08|2025-04-08--23-06-54--0", + "source_bytes": 11420074, + "source_sha256": "cc1d8bb9729a71c3166e04343e436a67fbc1ff7107a30201a5d13ea9b45ea143" + }, + { + "car_brand": "SUBARU", + "case_id": "subaru", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen7366F13F6A1|2025-04-08--23-07-07--0", + "source_bytes": 9393878, + "source_sha256": "f501c1c99bae27ca710f40f723a1960ea4e898836d7edfb57ea6397591e5aff2" + }, + { + "car_brand": "GM", + "case_id": "gm", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen1271097D038|2025-04-09--03-26-00--0", + "source_bytes": 9846869, + "source_sha256": "6c10ce899270e4fc8350730e44cb39dfbda451670dd598798207ffc929baa69a" + }, + { + "car_brand": "NISSAN", + "case_id": "nissan", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen15D60604EAB|2025-04-08--23-06-59--0", + "source_bytes": 11775886, + "source_sha256": "58f3822418e86d8c4afca79ef9058602c8825fadaedf8b63477ed53a2b9ba748" + }, + { + "car_brand": "VOLKSWAGEN", + "case_id": "volkswagen", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen0F2F06C9539|2025-04-08--23-06-56--0", + "source_bytes": 10593610, + "source_sha256": "bca512375f61ade1af793227f4aa9137ba1cbd0067d3b92f1242f5907f84a0af" + }, + { + "car_brand": "MAZDA", + "case_id": "mazda", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regenACF84CCF482|2024-08-30--03-21-55--0", + "source_bytes": 12309353, + "source_sha256": "a84e9f84c39802f58a64031e7c56b362c0e745b5c216848ca2d561f5ae6d440a" + }, + { + "car_brand": "FORD", + "case_id": "ford", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen755D8CB1E1F|2025-04-08--23-13-43--0", + "source_bytes": 11315727, + "source_sha256": "786dfef08325d24e8205320862adc940706ff3475ae082ea104127fb5da62823" + }, + { + "car_brand": "RIVIAN", + "case_id": "rivian", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "regen5FCAC896BBE|2025-04-08--23-13-35--0", + "source_bytes": 11022997, + "source_sha256": "2f2574c057b7cb55fbb2e05c99c5f1b1d6d894222422c4b51d4363ad66e6c840" + }, + { + "car_brand": "TESLA", + "case_id": "tesla", + "custom_params": {}, + "executable": true, + "params_digest": "37517e5f3dc66819f61f5a7bb8ace1921282415f10551d2defa5c3eb0985b570", + "processes": [ + "card", + "controlsd", + "lagd" + ], + "source": "2c912ca5de3b1ee9|0000025d--6eb6bcbca4--4", + "source_bytes": 8287366, + "source_sha256": "ca3bfc37b118f0e1987c2177b77b7446d44fbc7806cc5fae71c2ed79d4776fff" + } + ], + "expected_cases": 16, + "expected_params_digest": "39222ea1947da902789761de487c2f308ebe66c6b2242c983a7cf9d9d64913d3", + "expected_sources_digest": "73a05665dbaaf10abae7f7566e53cfb2e90f30ef7aafa5f4ce035d9b71298ca7", + "expected_tasks": 66, + "inventory": "active", + "partition": { + "card": 16, + "controlsd": 16, + "lagd": 16, + "other": 18 + }, + "schema_version": 1, + "task_ids": [ + "hyundai:card", + "hyundai:controlsd", + "hyundai:lagd", + "hyundai:selfdrived", + "hyundai:radard", + "hyundai:plannerd", + "hyundai:calibrationd", + "hyundai:dmonitoringd", + "hyundai:locationd", + "hyundai:paramsd", + "hyundai:ubloxd", + "hyundai:torqued", + "hyundai2:card", + "hyundai2:controlsd", + "hyundai2:lagd", + "toyota:card", + "toyota:controlsd", + "toyota:lagd", + "toyota:selfdrived", + "toyota:radard", + "toyota:plannerd", + "toyota:calibrationd", + "toyota:dmonitoringd", + "toyota:locationd", + "toyota:paramsd", + "toyota:ubloxd", + "toyota:torqued", + "toyota3:card", + "toyota3:controlsd", + "toyota3:lagd", + "honda:card", + "honda:controlsd", + "honda:lagd", + "honda2:card", + "honda2:controlsd", + "honda2:lagd", + "chrysler:card", + "chrysler:controlsd", + "chrysler:lagd", + "ram:card", + "ram:controlsd", + "ram:lagd", + "subaru:card", + "subaru:controlsd", + "subaru:lagd", + "gm:card", + "gm:controlsd", + "gm:lagd", + "nissan:card", + "nissan:controlsd", + "nissan:lagd", + "volkswagen:card", + "volkswagen:controlsd", + "volkswagen:lagd", + "mazda:card", + "mazda:controlsd", + "mazda:lagd", + "ford:card", + "ford:controlsd", + "ford:lagd", + "rivian:card", + "rivian:controlsd", + "rivian:lagd", + "tesla:card", + "tesla:controlsd", + "tesla:lagd" + ] +} diff --git a/.github/ci/test_assemble_replay_release.py b/.github/ci/test_assemble_replay_release.py new file mode 100644 index 00000000000000..63ca8da17cb6d0 --- /dev/null +++ b/.github/ci/test_assemble_replay_release.py @@ -0,0 +1,285 @@ +"""Focused tests for active process-replay release assembly. + +Covers a valid 66-ref assembly and fail-closed rejects (missing/extra task, +invalid digest, executable=false). Uses local synthetic fakedata only — no +network downloads. +""" + +from __future__ import annotations + +import hashlib +import importlib.util +import json +import subprocess +import sys +from pathlib import Path + +CI_DIR = Path(__file__).resolve().parent +REPO_ROOT = CI_DIR.parents[1] +ASSEMBLE_PATH = REPO_ROOT / "tools" / "ci" / "assemble_replay_release.py" +ACTIVE_INVENTORY = CI_DIR / "process_replay_active_inventory.json" + +SOURCE_SHA = "a" * 40 +OPENDBC_SHA = "b" * 40 +PREV_TAG = "process-replay/v1/" + ("c" * 40) + "-" + ("d" * 12) + + +def _load_assemble(): + spec = importlib.util.spec_from_file_location("assemble_replay_release", ASSEMBLE_PATH) + assert spec is not None and spec.loader is not None + mod = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = mod + spec.loader.exec_module(mod) + return mod + + +assemble = _load_assemble() + + +def _canonical(obj: object) -> bytes: + return assemble.canonical_json(obj) + + +def _sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def _load_inventory_cases() -> list[dict]: + inventory = json.loads(ACTIVE_INVENTORY.read_text(encoding="utf-8")) + assert inventory["inventory"] == "active" + assert inventory["expected_cases"] == 16 + assert inventory["expected_tasks"] == 66 + cases = inventory["cases"] + assert isinstance(cases, list) and len(cases) == 16 + return [dict(c) for c in cases] + + +def _write_cases(path: Path, cases: list[dict]) -> None: + path.write_text(json.dumps(cases, indent=2, sort_keys=True) + "\n", encoding="utf-8") + + +def _seed_fakedata(fakedata: Path, cases: list[dict], *, skip_task: str | None = None, extra_name: str | None = None) -> list[str]: + fakedata.mkdir(parents=True, exist_ok=True) + written: list[str] = [] + for case in cases: + for proc in case["processes"]: + task_id = f"{case['case_id']}:{proc}" + if skip_task is not None and task_id == skip_task: + continue + name = f"{case['case_id']}__{proc}__{SOURCE_SHA}.zst" + (fakedata / name).write_bytes(f"{task_id}\n".encode()) + written.append(task_id) + if extra_name is not None: + (fakedata / extra_name).write_bytes(b"extra-ref\n") + return written + + +def _run_assemble(cases_json: Path, fakedata: Path, release: Path) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(ASSEMBLE_PATH), + "--operation", + "generate-active", + "--source-sha", + SOURCE_SHA, + "--fakedata-dir", + str(fakedata), + "--release-dir", + str(release), + "--opendbc-sha", + OPENDBC_SHA, + "--cases-json", + str(cases_json), + "--previous-accepted-tag", + PREV_TAG, + ], + check=False, + capture_output=True, + text=True, + ) + + +def test_active_inventory_matches_harness_case_record_contract(): + inventory = json.loads(ACTIVE_INVENTORY.read_text(encoding="utf-8")) + assert inventory["schema_version"] == 1 + assert inventory["inventory"] == "active" + assert inventory["expected_cases"] == 16 + assert inventory["expected_tasks"] == 66 + assert inventory["partition"] == {"card": 16, "controlsd": 16, "lagd": 16, "other": 18} + assert inventory["allowed_processes"] == sorted(inventory["allowed_processes"]) + + cases = inventory["cases"] + task_ids = inventory["task_ids"] + assert len(cases) == 16 + assert len(task_ids) == 66 + assert len(set(task_ids)) == 66 + + derived = [f"{c['case_id']}:{p}" for c in cases for p in c["processes"]] + assert derived == task_ids + assert all(c["executable"] is True for c in cases) + assert all(isinstance(c["source"], str) and c["source"] for c in cases) + assert all(isinstance(c["source_bytes"], int) and c["source_bytes"] > 0 for c in cases) + assert all(isinstance(c["source_sha256"], str) and len(c["source_sha256"]) == 64 for c in cases) + assert all(isinstance(c["params_digest"], str) and len(c["params_digest"]) == 64 for c in cases) + + sorted_cases = sorted(cases, key=lambda c: c["case_id"]) + sources_payload = [ + { + "case_id": c["case_id"], + "source": c["source"], + "source_bytes": c["source_bytes"], + "source_sha256": c["source_sha256"], + } + for c in sorted_cases + ] + params_payload = [{"case_id": c["case_id"], "params_digest": c["params_digest"]} for c in sorted_cases] + assert inventory["expected_sources_digest"] == _sha256(_canonical(sources_payload)) + assert inventory["expected_params_digest"] == _sha256(_canonical(params_payload)) + + counts = {"card": 0, "controlsd": 0, "lagd": 0, "other": 0} + for task_id in task_ids: + proc = task_id.split(":", 1)[1] + counts[proc if proc in counts else "other"] += 1 + assert counts == inventory["partition"] + + +def test_assemble_valid_66_ref_release(tmp_path: Path): + cases = _load_inventory_cases() + cases_json = tmp_path / "cases.json" + fakedata = tmp_path / "fakedata" + release = tmp_path / "release" + _write_cases(cases_json, cases) + written = _seed_fakedata(fakedata, cases) + assert len(written) == 66 + + proc = _run_assemble(cases_json, fakedata, release) + assert proc.returncode == 0, proc.stderr + proc.stdout + + assert (release / "ref_commit").read_text(encoding="utf-8") == SOURCE_SHA + "\n" + manifest_bytes = (release / "manifest.json").read_bytes() + assert manifest_bytes == _canonical(json.loads(manifest_bytes.decode("utf-8"))) + + manifest = json.loads(manifest_bytes.decode("utf-8")) + assert manifest["schema_version"] == 1 + assert manifest["openpilot_sha"] == SOURCE_SHA + assert manifest["opendbc_sha"] == OPENDBC_SHA + assert manifest["previous_accepted_tag"] == PREV_TAG + assert len(manifest["cases"]) == 16 + assert len(manifest["refs"]) == 66 + + inventory = json.loads(ACTIVE_INVENTORY.read_text(encoding="utf-8")) + assert manifest["sources_digest"] == inventory["expected_sources_digest"] + assert manifest["params_digest"] == inventory["expected_params_digest"] + + by_id = {c["case_id"]: c for c in cases} + for cid, rec in by_id.items(): + assembled = next(c for c in manifest["cases"] if c["case_id"] == cid) + assert assembled["source"] == rec["source"] + assert assembled["source_sha256"] == rec["source_sha256"] + assert assembled["source_bytes"] == rec["source_bytes"] + assert assembled["params_digest"] == rec["params_digest"] + assert assembled["executable"] is True + + for task_id, entry in manifest["refs"].items(): + case_id, process = task_id.split(":", 1) + rec = by_id[case_id] + assert process in rec["processes"] + assert entry["filename"] == f"{case_id}__{process}__{SOURCE_SHA}.zst" + assert entry["source"] == rec["source"] + assert entry["source_sha256"] == rec["source_sha256"] + assert entry["source_bytes"] == rec["source_bytes"] + assert entry["params_digest"] == rec["params_digest"] + payload = (release / entry["filename"]).read_bytes() + assert entry["size"] == len(payload) + assert entry["sha256"] == _sha256(payload) + + zsts = sorted(p.name for p in release.iterdir() if p.name.endswith(".zst")) + assert len(zsts) == 66 + assert set(release.iterdir()) == {release / "ref_commit", release / "manifest.json"} | {release / n for n in zsts} + + +def test_assemble_rejects_missing_task(tmp_path: Path): + cases = _load_inventory_cases() + cases_json = tmp_path / "cases.json" + fakedata = tmp_path / "fakedata" + release = tmp_path / "release" + _write_cases(cases_json, cases) + victim = f"{cases[0]['case_id']}:{cases[0]['processes'][0]}" + _seed_fakedata(fakedata, cases, skip_task=victim) + + proc = _run_assemble(cases_json, fakedata, release) + assert proc.returncode != 0 + assert "expected 66 tasks" in (proc.stderr + proc.stdout) + + +def test_assemble_rejects_extra_task(tmp_path: Path): + cases = _load_inventory_cases() + cases_json = tmp_path / "cases.json" + fakedata = tmp_path / "fakedata" + release = tmp_path / "release" + _write_cases(cases_json, cases) + # hyundai2 is core-only; selfdrived is an extra process for that case. + extra = f"hyundai2__selfdrived__{SOURCE_SHA}.zst" + _seed_fakedata(fakedata, cases, extra_name=extra) + + proc = _run_assemble(cases_json, fakedata, release) + assert proc.returncode != 0 + combined = proc.stderr + proc.stdout + assert "process selfdrived not in case hyundai2 process set" in combined or "expected 66 tasks" in combined + + +def test_assemble_rejects_invalid_digest(tmp_path: Path): + cases = _load_inventory_cases() + cases[0]["params_digest"] = "not-a-digest" + cases_json = tmp_path / "cases.json" + fakedata = tmp_path / "fakedata" + release = tmp_path / "release" + _write_cases(cases_json, cases) + _seed_fakedata(fakedata, cases) + + proc = _run_assemble(cases_json, fakedata, release) + assert proc.returncode != 0 + assert "params_digest" in (proc.stderr + proc.stdout) + + +def test_assemble_rejects_executable_false(tmp_path: Path): + cases = _load_inventory_cases() + cases[0]["executable"] = False + cases_json = tmp_path / "cases.json" + fakedata = tmp_path / "fakedata" + release = tmp_path / "release" + _write_cases(cases_json, cases) + _seed_fakedata(fakedata, cases) + + proc = _run_assemble(cases_json, fakedata, release) + assert proc.returncode != 0 + assert "must be executable" in (proc.stderr + proc.stdout) + + +def test_assemble_rejects_generate_staged_operation(): + proc = subprocess.run( + [ + sys.executable, + str(ASSEMBLE_PATH), + "--operation", + "generate-staged", + "--source-sha", + SOURCE_SHA, + "--fakedata-dir", + ".", + "--release-dir", + ".", + "--opendbc-sha", + OPENDBC_SHA, + "--cases-json", + str(ACTIVE_INVENTORY), + "--previous-accepted-tag", + PREV_TAG, + ], + check=False, + capture_output=True, + text=True, + ) + assert proc.returncode != 0 + assert "generate-staged" in (proc.stderr + proc.stdout) or "invalid choice" in (proc.stderr + proc.stdout).lower() diff --git a/.github/ci/test_validate_process_replay_release.py b/.github/ci/test_validate_process_replay_release.py index 7eb03dd33b3196..3a2a0726690e4c 100644 --- a/.github/ci/test_validate_process_replay_release.py +++ b/.github/ci/test_validate_process_replay_release.py @@ -33,31 +33,38 @@ def _sha256(data: bytes) -> str: return hashlib.sha256(data).hexdigest() -def _build_cases(executable: bool = True, pin_sources: bool = True) -> list[dict]: - cases = [] - # 16 active-like full/partial + 2 pending-style ids => 18 cases / 78 tasks - # card/controlsd/lagd for all 18 = 54; remaining 24 "other" on first 2 full cases (12 each). - # Use exact allowed process set from staged inventory contract. - allowed = [ +def _allowed_processes() -> list[str]: + return sorted({ "calibrationd", "card", "controlsd", "dmonitoringd", "lagd", "locationd", "paramsd", "plannerd", "radard", "selfdrived", "torqued", "ubloxd", - ] + }) + + +def _build_cases( + inventory: str = "staged", + *, + executable: bool = True, + pin_sources: bool = True, +) -> list[dict]: + """Synthetic cases matching exact active (16/66) or staged (18/78) contracts.""" + allowed = _allowed_processes() other = [p for p in allowed if p not in {"card", "controlsd", "lagd"}] assert len(other) == 9 - # two full cases with core+all other (2*(3+9)=24 other tasks), sixteen core-only (16*3=48), total 72? - - # Need 78 = 18*3 core + 24 other => other tasks across cases must total 24. - # 2 full cases with all 9 other would be 18 other; need 24 => use 2 cases with 9 other + 2 cases with 3 other? + if inventory == "active": + # 16 cases / 66 tasks: two full (9 other each => 18), fourteen core-only. + n_cases = 16 + extra_plan = {0: other, 1: other} + elif inventory == "staged": + # 18 cases / 78 tasks: two full (9 other) + two partial (3 other) => 24 other. + n_cases = 18 + extra_plan = {0: other, 1: other, 2: other[:3], 3: other[:3]} + else: + raise ValueError(f"unknown inventory {inventory!r}") - # Simpler: first case has 12 procs? allowed only has 9 other. - # 9+9+3+3 = 24 other with four cases carrying extras. - for idx in range(18): + cases = [] + for idx in range(n_cases): cid = f"case{idx:02d}" - procs = ["card", "controlsd", "lagd"] - if idx < 2: - procs = procs + other # 9 other each => 18 - elif idx < 4: - procs = procs + other[:3] # 3 other each => 6; total other 24 + procs = ["card", "controlsd", "lagd"] + list(extra_plan.get(idx, [])) source = f"https://example.test/rlogs/{cid}.zst" if executable else "" case = { "case_id": cid, @@ -74,17 +81,18 @@ def _build_cases(executable: bool = True, pin_sources: bool = True) -> list[dict return cases -def _inventory_from_cases(cases: list[dict]) -> dict: - allowed = sorted({ - "calibrationd", "card", "controlsd", "dmonitoringd", "lagd", "locationd", - "paramsd", "plannerd", "radard", "selfdrived", "torqued", "ubloxd", - }) +def _inventory_from_cases(cases: list[dict], inventory: str = "staged") -> dict: + contract = v.INVENTORY_CONTRACTS[inventory] + expected_cases = contract["expected_cases"] + expected_tasks = contract["expected_tasks"] + partition = dict(contract["partition"]) + allowed = _allowed_processes() task_ids = [] for case in cases: for proc in case["processes"]: task_ids.append(f"{case['case_id']}:{proc}") - assert len(cases) == 18 - assert len(task_ids) == 78 + assert len(cases) == expected_cases + assert len(task_ids) == expected_tasks sorted_cases = sorted(cases, key=lambda c: c["case_id"]) sources_payload = [ { @@ -98,10 +106,10 @@ def _inventory_from_cases(cases: list[dict]) -> dict: params_payload = [{"case_id": c["case_id"], "params_digest": c["params_digest"]} for c in sorted_cases] return { "schema_version": 1, - "inventory": "staged", - "expected_cases": 18, - "expected_tasks": 78, - "partition": {"card": 18, "controlsd": 18, "lagd": 18, "other": 24}, + "inventory": inventory, + "expected_cases": expected_cases, + "expected_tasks": expected_tasks, + "partition": partition, "allowed_processes": allowed, "cases": cases, "task_ids": task_ids, @@ -110,7 +118,13 @@ def _inventory_from_cases(cases: list[dict]) -> dict: } -def _write_release(release: Path, cases: list[dict], *, opendbc_sha: str = OPENDBC_SHA) -> str: +def _write_release( + release: Path, + cases: list[dict], + *, + opendbc_sha: str = OPENDBC_SHA, + previous_accepted_tag: str = "", +) -> str: release.mkdir(parents=True, exist_ok=True) (release / "ref_commit").write_text(SOURCE_SHA + "\n", encoding="utf-8") refs = {} @@ -124,6 +138,10 @@ def _write_release(release: Path, cases: list[dict], *, opendbc_sha: str = OPEND "filename": filename, "size": len(payload), "sha256": _sha256(payload), + "source": case["source"], + "source_sha256": case["source_sha256"], + "source_bytes": case["source_bytes"], + "params_digest": case["params_digest"], } sorted_cases = sorted((v.case_identity(c) for c in cases), key=lambda c: c["case_id"]) sources_payload = [ @@ -144,7 +162,7 @@ def _write_release(release: Path, cases: list[dict], *, opendbc_sha: str = OPEND "opendbc_sha": opendbc_sha, "sources_digest": _sha256(_canonical(sources_payload)), "params_digest": _sha256(_canonical(params_payload)), - "previous_accepted_tag": "", + "previous_accepted_tag": previous_accepted_tag, } raw = _canonical(manifest) (release / "manifest.json").write_bytes(raw) @@ -199,6 +217,91 @@ def test_happy_path_accepts_fully_pinned_inventory(tmp_path: Path): assert result["release_tag"] == f"process-replay/v1/{SOURCE_SHA}-{manifest_sha[:12]}" +def test_valid_active_inventory_loads(tmp_path: Path): + cases = _build_cases("active", executable=True, pin_sources=True) + inventory = _inventory_from_cases(cases, inventory="active") + inv_path = tmp_path / "active_inventory.json" + inv_path.write_bytes(_canonical(inventory)) + trusted = v.load_trusted_inventory(inv_path) + assert trusted["inventory"] == "active" + assert trusted["expected_cases"] == 16 + assert trusted["expected_tasks"] == 66 + assert trusted["partition"] == {"card": 16, "controlsd": 16, "lagd": 16, "other": 18} + + +def test_active_declared_count_or_partition_mismatch_rejected(tmp_path: Path): + cases = _build_cases("active") + inventory = _inventory_from_cases(cases, inventory="active") + + bad_counts = dict(inventory) + bad_counts["expected_cases"] = 18 + bad_counts["expected_tasks"] = 78 + path_counts = tmp_path / "bad_counts.json" + path_counts.write_bytes(_canonical(bad_counts)) + with pytest.raises(SystemExit, match="must declare 16 cases / 66 tasks for inventory=active"): + v.load_trusted_inventory(path_counts) + + bad_partition = dict(inventory) + bad_partition["partition"] = {"card": 18, "controlsd": 18, "lagd": 18, "other": 24} + path_part = tmp_path / "bad_partition.json" + path_part.write_bytes(_canonical(bad_partition)) + with pytest.raises(SystemExit, match="partition mismatch for inventory=active"): + v.load_trusted_inventory(path_part) + + cross = dict(inventory) + cross["inventory"] = "staged" + path_cross = tmp_path / "cross_label.json" + path_cross.write_bytes(_canonical(cross)) + with pytest.raises(SystemExit, match="must declare 18 cases / 78 tasks for inventory=staged"): + v.load_trusted_inventory(path_cross) + + +def test_archive_previous_accepted_tag_identity_accepted(tmp_path: Path): + cases = _build_cases("staged", executable=True, pin_sources=True) + inventory = _inventory_from_cases(cases, inventory="staged") + inv_path = tmp_path / "inventory.json" + inv_path.write_bytes(_canonical(inventory)) + archive_tag = f"process-replay/archive/{'f' * 40}" + release = tmp_path / "release" + manifest_sha = _write_release(release, cases, previous_accepted_tag=archive_tag) + trusted = v.load_trusted_inventory(inv_path) + result = v.validate_release( + release, + trusted=trusted, + source_sha=SOURCE_SHA, + operation="publish", + expected_manifest_sha256=manifest_sha, + rollback_tag=None, + expected_opendbc_sha=OPENDBC_SHA, + ) + assert result["release_manifest_sha256"] == manifest_sha + + +def test_ref_identity_fields_must_match_trusted_case(tmp_path: Path): + cases = _build_cases("staged", executable=True, pin_sources=True) + inventory = _inventory_from_cases(cases, inventory="staged") + inv_path = tmp_path / "inventory.json" + inv_path.write_bytes(_canonical(inventory)) + release = tmp_path / "release" + _write_release(release, cases) + manifest = json.loads((release / "manifest.json").read_text(encoding="utf-8")) + victim = next(iter(manifest["refs"])) + manifest["refs"][victim]["source_sha256"] = "e" * 64 + tampered = _canonical(manifest) + (release / "manifest.json").write_bytes(tampered) + trusted = v.load_trusted_inventory(inv_path) + with pytest.raises(SystemExit, match="ref source_sha256 diverges from trusted case"): + v.validate_release( + release, + trusted=trusted, + source_sha=SOURCE_SHA, + operation="publish", + expected_manifest_sha256=_sha256(tampered), + rollback_tag=None, + expected_opendbc_sha=OPENDBC_SHA, + ) + + def test_opendbc_gitlink_mismatch_is_rejected(tmp_path: Path): cases = _build_cases() inventory = _inventory_from_cases(cases) @@ -275,35 +378,46 @@ def test_non_executable_inventory_rejected(): v.require_publishable_inventory(inventory) -def test_workflow_authenticates_controller_run_and_artifact_name(): +def test_workflow_generates_only_current_active_inventory(): workflow = WORKFLOW_PATH.read_text(encoding="utf-8") - assert 'EXPECTED_ARTIFACT_NAME="nap-replay-${CONTROLLER_RUN_ID}"' in workflow - assert 'repos/${CONTROLLER_REPO}/actions/runs/${CONTROLLER_RUN_ID}' in workflow - assert '.github/workflows/replay.yaml' in workflow - assert 'nap-replay' in workflow - assert "RUN_STATUS" in workflow and "RUN_CONCLUSION" in workflow - assert "RUN_HEAD_SHA" in workflow - assert "RUN_EVENT" in workflow - assert "META_NAME" in workflow + assert "generate-active" in workflow + assert "generate-staged" not in workflow + assert "CONTROLLER_REPO" not in workflow + assert "NAP_CI_CONTROLLER_ARTIFACT_READ_TOKEN" not in workflow + assert 'EXPECTED_TASKS: "66"' in workflow + assert 'github.ref == \'refs/heads/nap-dev\'' in workflow + assert 'EXPECTED_WORKFLOW_REF="${SOURCE_REPO}/.github/workflows/process_replay_refs.yaml@refs/heads/nap-dev"' in workflow -def test_workflow_rejects_duplicate_and_unsafe_zip_members(): +def test_workflow_rejects_unsafe_downloaded_artifact_layouts(): workflow = WORKFLOW_PATH.read_text(encoding="utf-8") - assert "duplicate zip member" in workflow - assert "non-regular zip member rejected" in workflow - assert "external_attr" in workflow - assert "O_EXCL" in workflow + assert "unexpected candidate top-level entry" in workflow + assert "path traversal in candidate" in workflow + assert "symlink forbidden in candidate" in workflow + assert "non-regular candidate member" in workflow + assert "nested release path forbidden" in workflow -def test_workflow_binds_handoff_review_proof_and_opendbc_gitlink(): +def test_workflow_binds_same_run_review_proof_and_opendbc_gitlink(): workflow = WORKFLOW_PATH.read_text(encoding="utf-8") assert "handoff-meta.json" in workflow assert "compare" in workflow and "results.json" in workflow - assert "pass count must be 78" in workflow or "pass', -1)) != 78" in workflow + assert "review proof pass count must be {expected_tasks}" in workflow + assert "workflow_run_id" in workflow + assert "GITHUB_RUN_ID" in workflow assert "HEAD:opendbc_repo" in workflow assert "--expected-opendbc-sha" in workflow +def test_workflow_preserves_same_source_no_op_and_atomic_publication(): + workflow = WORKFLOW_PATH.read_text(encoding="utf-8") + assert 'CURRENT_REF_COMMIT}" == "${SOURCE_SHA}' in workflow + assert "manifest.json?ref=${PREVIOUS_HEAD}" in workflow + assert "previous_accepted_tag" in workflow + assert "git push --atomic" in workflow + assert '--force-with-lease="refs/heads/${ARTIFACTS_BRANCH}:${CAPTURED_HEAD}"' in workflow + + def test_zip_extractor_snippet_rejects_symlink_member(tmp_path: Path): """Exercise the same mode checks the workflow embeds before extraction.""" zpath = tmp_path / "cand.zip" diff --git a/.github/ci/validate_process_replay_release.py b/.github/ci/validate_process_replay_release.py index 8f3ca1fd8e1bc6..8b664736722660 100755 --- a/.github/ci/validate_process_replay_release.py +++ b/.github/ci/validate_process_replay_release.py @@ -1,8 +1,8 @@ #!/usr/bin/env python3 """Fail-closed validation of a process-replay release tree. -Trusted staged inventory is loaded from the protected nap-dev checkout -(GITHUB_SHA). Controller-supplied inventory is never authoritative. +Trusted inventory is loaded from the protected nap-dev checkout (GITHUB_SHA) +under the exact active or staged contract declared by the inventory itself. """ from __future__ import annotations @@ -37,14 +37,29 @@ "custom_params", "executable", ) -REQUIRED_REF_KEYS = ("filename", "size", "sha256") +REQUIRED_REF_KEYS = ("filename", "size", "sha256", "source", "source_sha256", "source_bytes", "params_digest") CASE_ID_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") PROC_RE = re.compile(r"^[a-z0-9_]+$") SHA40_RE = re.compile(r"^[0-9a-f]{40}$") SHA64_RE = re.compile(r"^[0-9a-f]{64}$") TAG_RE = re.compile(r"^process-replay/v1/([0-9a-f]{40})-([0-9a-f]{12})$") +ARCHIVE_TAG_RE = re.compile(r"^process-replay/archive/([0-9a-f]{40})$") CORE_PROCS = frozenset({"card", "controlsd", "lagd"}) +# Exact trusted inventory contracts. Counts, partitions, and labels are fail-closed. +INVENTORY_CONTRACTS: dict[str, dict[str, Any]] = { + "active": { + "expected_cases": 16, + "expected_tasks": 66, + "partition": {"card": 16, "controlsd": 16, "lagd": 16, "other": 18}, + }, + "staged": { + "expected_cases": 18, + "expected_tasks": 78, + "partition": {"card": 18, "controlsd": 18, "lagd": 18, "other": 24}, + }, +} + def die(msg: str) -> Never: raise SystemExit(f"validate_process_replay_release: {msg}") @@ -66,20 +81,25 @@ def load_trusted_inventory(path: Path) -> dict[str, Any]: die("trusted inventory must be an object") if inventory.get("schema_version") != 1: die("trusted inventory schema_version must be 1") - if inventory.get("inventory") != "staged": - die("trusted inventory must declare inventory=staged") - if inventory.get("expected_cases") != 18 or inventory.get("expected_tasks") != 78: - die("trusted inventory must declare 18 cases / 78 tasks") + label = inventory.get("inventory") + contract = INVENTORY_CONTRACTS.get(label) if isinstance(label, str) else None + if contract is None: + die(f"trusted inventory must declare inventory as one of {sorted(INVENTORY_CONTRACTS)}") + expected_cases = contract["expected_cases"] + expected_tasks = contract["expected_tasks"] + expected_partition = contract["partition"] + if inventory.get("expected_cases") != expected_cases or inventory.get("expected_tasks") != expected_tasks: + die(f"trusted inventory must declare {expected_cases} cases / {expected_tasks} tasks for inventory={label}") partition = inventory.get("partition") - if partition != {"card": 18, "controlsd": 18, "lagd": 18, "other": 24}: - die(f"trusted inventory partition mismatch: {partition}") + if partition != expected_partition: + die(f"trusted inventory partition mismatch for inventory={label}: {partition}") cases = inventory.get("cases") task_ids = inventory.get("task_ids") allowed = inventory.get("allowed_processes") - if not isinstance(cases, list) or len(cases) != 18: - die("trusted inventory cases must be a list of 18") - if not isinstance(task_ids, list) or len(task_ids) != 78: - die("trusted inventory task_ids must be a list of 78") + if not isinstance(cases, list) or len(cases) != expected_cases: + die(f"trusted inventory cases must be a list of {expected_cases}") + if not isinstance(task_ids, list) or len(task_ids) != expected_tasks: + die(f"trusted inventory task_ids must be a list of {expected_tasks}") if not isinstance(allowed, list) or not all(isinstance(p, str) for p in allowed): die("trusted inventory allowed_processes must be a string list") if list(allowed) != sorted(allowed): @@ -106,7 +126,7 @@ def load_trusted_inventory(path: Path) -> dict[str, Any]: derived.append(f"{cid}:{proc}") if derived != task_ids: die("trusted task_ids do not match derived case/process inventory") - if len(set(task_ids)) != 78: + if len(set(task_ids)) != expected_tasks: die("trusted task_ids must be unique") sorted_cases = sorted(cases, key=lambda c: c["case_id"]) sources_payload = [ @@ -203,7 +223,7 @@ def validate_release( if operation not in {"publish", "rollback"}: die(f"unknown operation {operation}") - # Dormant publisher must not accept current pending inventory. + # Publication requires every trusted case to be executable with pinned source digests. require_publishable_inventory(trusted) expected_cases = int(trusted["expected_cases"]) @@ -261,8 +281,11 @@ def validate_release( if manifest["opendbc_sha"] != expected_opendbc_sha: die(f"manifest opendbc_sha {manifest['opendbc_sha']} != trusted gitlink {expected_opendbc_sha}") previous = manifest["previous_accepted_tag"] - if previous != "" and not (isinstance(previous, str) and TAG_RE.fullmatch(previous)): - die("previous_accepted_tag must be empty or process-replay/v1/<40-hex>-<12-hex>") + if previous != "" and not ( + isinstance(previous, str) + and (TAG_RE.fullmatch(previous) or ARCHIVE_TAG_RE.fullmatch(previous)) + ): + die("previous_accepted_tag must be empty, a process-replay/v1 release tag, or a process-replay/archive SHA tag") cases = manifest["cases"] refs = manifest["refs"] @@ -294,7 +317,7 @@ def validate_release( if case_ids != sorted(case_ids): die("cases must be sorted by case_id") if set(case_ids) != set(trusted_cases): - die("manifest case set must equal trusted staged inventory") + die("manifest case set must equal trusted inventory") sorted_cases = sorted((case_identity(c) for c in cases), key=lambda c: c["case_id"]) sources_payload = [ @@ -317,7 +340,7 @@ def validate_release( if list(refs.keys()) != sorted(refs.keys()): die("refs must be sorted by task key") if set(refs) != trusted_task_set: - die("manifest refs task set must equal trusted staged task inventory") + die("manifest refs task set must equal trusted task inventory") counts = {"card": 0, "controlsd": 0, "lagd": 0, "other": 0} name_re = re.compile(rf"^[a-z0-9][a-z0-9-]*__[a-z0-9_]+__{re.escape(source_sha)}\.zst$") @@ -333,6 +356,15 @@ def validate_release( die(f"unknown process rejected: {process}") if process not in trusted_cases[case_id]["processes"]: die(f"process {process} not in trusted case {case_id}") + trusted_case = trusted_cases[case_id] + if meta.get("source") != trusted_case["source"]: + die(f"ref source diverges from trusted case for {task_id}") + if meta.get("source_sha256") != trusted_case["source_sha256"]: + die(f"ref source_sha256 diverges from trusted case for {task_id}") + if meta.get("source_bytes") != trusted_case["source_bytes"]: + die(f"ref source_bytes diverges from trusted case for {task_id}") + if meta.get("params_digest") != trusted_case["params_digest"]: + die(f"ref params_digest diverges from trusted case for {task_id}") if process in CORE_PROCS: counts[process] += 1 else: @@ -375,6 +407,7 @@ def validate_release( "release_source_sha": source_sha, "release_manifest_sha256": manifest_sha, "release_tag": release_tag, + "release_previous_accepted_tag": previous, } diff --git a/.github/workflows/process_replay_refs.yaml b/.github/workflows/process_replay_refs.yaml index bc3ae3b481be05..d171bc4ad767f9 100644 --- a/.github/workflows/process_replay_refs.yaml +++ b/.github/workflows/process_replay_refs.yaml @@ -1,39 +1,19 @@ # GitHub-hosted reference publisher only. Never runs on self-hosted infrastructure. -# Candidate files are validated fail-closed from an immutable controller artifact -# against the trusted staged inventory checked out at GITHUB_SHA. -# CI_ARTIFACTS_TOKEN is exposed only in the final approved push step. +# generate-active builds and assembles refs on ubuntu-24.04; verify is fail-closed +# and execution-free against the trusted active inventory at GITHUB_SHA. +# The ci-artifacts write token is exposed only in the final push step. name: process replay refs on: workflow_dispatch: inputs: operation: - description: "publish a verified controller candidate, or rollback to a protected tag" + description: "generate-active refs on protected nap-dev, or rollback to a protected tag" type: choice options: - - publish + - generate-active - rollback - default: publish - controller_run_id: - description: "Private controller Actions run ID that uploaded the candidate" - required: false - type: string - artifact_id: - description: "Immutable controller artifact ID" - required: false - type: string - artifact_digest: - description: "Controller artifact digest (sha256:...)" - required: false - type: string - source_sha: - description: "Exact lowercase 40-hex nap-dev tip the candidate was generated from" - required: false - type: string - manifest_sha256: - description: "SHA-256 of release/manifest.json inside the candidate" - required: false - type: string + default: generate-active rollback_tag: description: "Protected tag process-replay/v1/- for rollback" required: false @@ -49,11 +29,12 @@ concurrency: env: SOURCE_REPO: NotAutopilot/openpilot - CONTROLLER_REPO: NotAutopilot/nap-ci-controller ARTIFACTS_REPO: NotAutopilot/ci-artifacts ARTIFACTS_BRANCH: process-replay - TRUSTED_INVENTORY_PATH: .github/ci/process_replay_staged_inventory.json + TRUSTED_INVENTORY_PATH: .github/ci/process_replay_active_inventory.json RELEASE_VALIDATOR: .github/ci/validate_process_replay_release.py + CANDIDATE_ARTIFACT_PREFIX: process-replay-candidate + EXPECTED_TASKS: "66" jobs: # workflow_dispatch is discoverable on the default branch (nap-release). @@ -68,42 +49,17 @@ jobs: env: GH_TOKEN: ${{ github.token }} OPERATION: ${{ inputs.operation }} - IN_CONTROLLER_RUN_ID: ${{ inputs.controller_run_id }} - IN_ARTIFACT_ID: ${{ inputs.artifact_id }} - IN_ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} - IN_SOURCE_SHA: ${{ inputs.source_sha }} - IN_MANIFEST_SHA256: ${{ inputs.manifest_sha256 }} IN_ROLLBACK_TAG: ${{ inputs.rollback_tag }} run: | set -euo pipefail case "${OPERATION}" in - publish|rollback) ;; + generate-active|rollback) ;; *) echo "Invalid operation" >&2 exit 1 ;; esac ARGS=(--ref nap-dev -f "operation=${OPERATION}") - if [[ -n "${IN_CONTROLLER_RUN_ID}" ]]; then - [[ "${IN_CONTROLLER_RUN_ID}" =~ ^[0-9]+$ ]] || { echo "controller_run_id must be numeric" >&2; exit 1; } - ARGS+=(-f "controller_run_id=${IN_CONTROLLER_RUN_ID}") - fi - if [[ -n "${IN_ARTIFACT_ID}" ]]; then - [[ "${IN_ARTIFACT_ID}" =~ ^[0-9]+$ ]] || { echo "artifact_id must be numeric" >&2; exit 1; } - ARGS+=(-f "artifact_id=${IN_ARTIFACT_ID}") - fi - if [[ -n "${IN_ARTIFACT_DIGEST}" ]]; then - [[ "${IN_ARTIFACT_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]] || { echo "artifact_digest invalid" >&2; exit 1; } - ARGS+=(-f "artifact_digest=${IN_ARTIFACT_DIGEST}") - fi - if [[ -n "${IN_SOURCE_SHA}" ]]; then - [[ "${IN_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]] || { echo "source_sha invalid" >&2; exit 1; } - ARGS+=(-f "source_sha=${IN_SOURCE_SHA}") - fi - if [[ -n "${IN_MANIFEST_SHA256}" ]]; then - [[ "${IN_MANIFEST_SHA256}" =~ ^[0-9a-f]{64}$ ]] || { echo "manifest_sha256 invalid" >&2; exit 1; } - ARGS+=(-f "manifest_sha256=${IN_MANIFEST_SHA256}") - fi if [[ -n "${IN_ROLLBACK_TAG}" ]]; then [[ "${IN_ROLLBACK_TAG}" =~ ^process-replay/v1/[0-9a-f]{40}-[0-9a-f]{12}$ ]] || { echo "rollback_tag invalid" >&2; exit 1; } ARGS+=(-f "rollback_tag=${IN_ROLLBACK_TAG}") @@ -119,22 +75,14 @@ jobs: outputs: operation: ${{ steps.validate.outputs.operation }} source_sha: ${{ steps.validate.outputs.source_sha }} - controller_run_id: ${{ steps.validate.outputs.controller_run_id }} - artifact_id: ${{ steps.validate.outputs.artifact_id }} - artifact_digest: ${{ steps.validate.outputs.artifact_digest }} - manifest_sha256: ${{ steps.validate.outputs.manifest_sha256 }} rollback_tag: ${{ steps.validate.outputs.rollback_tag }} + candidate_artifact: ${{ steps.validate.outputs.candidate_artifact }} steps: - name: Validate ref, sha, workflow_ref, and inputs id: validate env: GH_TOKEN: ${{ github.token }} OPERATION: ${{ inputs.operation }} - IN_SOURCE_SHA: ${{ inputs.source_sha }} - IN_CONTROLLER_RUN_ID: ${{ inputs.controller_run_id }} - IN_ARTIFACT_ID: ${{ inputs.artifact_id }} - IN_ARTIFACT_DIGEST: ${{ inputs.artifact_digest }} - IN_MANIFEST_SHA256: ${{ inputs.manifest_sha256 }} IN_ROLLBACK_TAG: ${{ inputs.rollback_tag }} run: | set -euo pipefail @@ -160,37 +108,8 @@ jobs: fi case "${OPERATION}" in - publish) - for value_name in IN_SOURCE_SHA IN_CONTROLLER_RUN_ID IN_ARTIFACT_ID IN_ARTIFACT_DIGEST IN_MANIFEST_SHA256; do - if [[ -z "${!value_name}" ]]; then - echo "publish requires ${value_name#IN_}" >&2 - exit 1 - fi - done - if [[ ! "${IN_SOURCE_SHA}" =~ ^[0-9a-f]{40}$ ]]; then - echo "source_sha must be exact lowercase 40-hex" >&2 - exit 1 - fi - if [[ "${IN_SOURCE_SHA}" != "${TIP}" ]]; then - echo "source_sha must equal current nap-dev tip; regenerate candidate" >&2 - exit 1 - fi - if [[ ! "${IN_CONTROLLER_RUN_ID}" =~ ^[0-9]+$ ]]; then - echo "controller_run_id must be numeric" >&2 - exit 1 - fi - if [[ ! "${IN_ARTIFACT_ID}" =~ ^[0-9]+$ ]]; then - echo "artifact_id must be numeric" >&2 - exit 1 - fi - if [[ ! "${IN_ARTIFACT_DIGEST}" =~ ^sha256:[0-9a-f]{64}$ ]]; then - echo "artifact_digest must be sha256:<64 lowercase hex>" >&2 - exit 1 - fi - if [[ ! "${IN_MANIFEST_SHA256}" =~ ^[0-9a-f]{64}$ ]]; then - echo "manifest_sha256 must be 64 lowercase hex" >&2 - exit 1 - fi + generate-active) + IN_SOURCE_SHA="${TIP}" ;; rollback) if [[ -z "${IN_ROLLBACK_TAG}" ]]; then @@ -212,17 +131,291 @@ jobs: { echo "operation=${OPERATION}" echo "source_sha=${IN_SOURCE_SHA}" - echo "controller_run_id=${IN_CONTROLLER_RUN_ID}" - echo "artifact_id=${IN_ARTIFACT_ID}" - echo "artifact_digest=${IN_ARTIFACT_DIGEST}" - echo "manifest_sha256=${IN_MANIFEST_SHA256}" echo "rollback_tag=${IN_ROLLBACK_TAG}" + echo "candidate_artifact=${CANDIDATE_ARTIFACT_PREFIX}-${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}" } >> "${GITHUB_OUTPUT}" + generate: + name: Generate active refs on GitHub-hosted runner + needs: gate-nap-dev + if: needs.gate-nap-dev.result == 'success' && needs.gate-nap-dev.outputs.operation == 'generate-active' + runs-on: ubuntu-24.04 + timeout-minutes: 120 + env: + CI: 1 + PYTHONPATH: ${{ github.workspace }} + SOURCE_SHA: ${{ needs.gate-nap-dev.outputs.source_sha }} + CANDIDATE_ARTIFACT: ${{ needs.gate-nap-dev.outputs.candidate_artifact }} + outputs: + source_sha: ${{ steps.prove.outputs.source_sha }} + manifest_sha256: ${{ steps.prove.outputs.manifest_sha256 }} + previous_accepted_tag: ${{ steps.prove.outputs.previous_accepted_tag }} + candidate_artifact: ${{ needs.gate-nap-dev.outputs.candidate_artifact }} + steps: + - name: Checkout exact GITHUB_SHA + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.0 + with: + ref: ${{ github.sha }} + submodules: true + persist-credentials: false + + - name: Reconfirm current tip before generation + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + TIP="$(gh api "repos/${SOURCE_REPO}/git/ref/heads/nap-dev" --jq '.object.sha')" + if [[ "${TIP}" != "${GITHUB_SHA}" || "${TIP}" != "${SOURCE_SHA}" ]]; then + echo "Refusing stale generate: tip=${TIP} sha=${GITHUB_SHA} source=${SOURCE_SHA}" >&2 + exit 1 + fi + HEAD_SHA="$(git rev-parse HEAD)" + if [[ "${HEAD_SHA}" != "${SOURCE_SHA}" ]]; then + echo "Checkout HEAD ${HEAD_SHA} != source_sha ${SOURCE_SHA}" >&2 + exit 1 + fi + + - name: Set up dependencies + uses: nick-fields/retry@7152eba30c6575329ac0576536151aca5a72780e + with: + timeout_minutes: 10 + max_attempts: 3 + command: ./tools/op.sh setup + + - name: Install video encoding deps + run: sudo apt-get install -y libva-dev libva-drm2 libdrm-dev + + - name: Build openpilot + run: scons -j$(nproc) + + - name: Generate, assemble, compare, and write proof + id: prove + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + REPLAY=selfdrive/test/process_replay/test_processes.py + ASSEMBLE=tools/ci/assemble_replay_release.py + test -x "${REPLAY}" -o -f "${REPLAY}" + test -f "${ASSEMBLE}" + + OPENDBC_SHA="$(git rev-parse HEAD:opendbc_repo)" + if [[ ! "${OPENDBC_SHA}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Failed to resolve opendbc_repo gitlink" >&2 + exit 1 + fi + + # Bind provenance to the current accepted head. A same-source rerun + # reuses the accepted manifest's predecessor so deterministic output + # remains byte-identical and publication becomes a no-op. + if ! gh api "repos/${ARTIFACTS_REPO}/git/ref/heads/${ARTIFACTS_BRANCH}" >/tmp/artifacts-head.json; then + echo "Missing ${ARTIFACTS_REPO}@${ARTIFACTS_BRANCH}; cannot resolve previous accepted identity" >&2 + exit 1 + fi + PREVIOUS_HEAD="$(jq -r '.object.sha' /tmp/artifacts-head.json)" + if [[ ! "${PREVIOUS_HEAD}" =~ ^[0-9a-f]{40}$ ]]; then + echo "Invalid process-replay head ${PREVIOUS_HEAD}" >&2 + exit 1 + fi + CURRENT_REF_COMMIT="$(gh api "repos/${ARTIFACTS_REPO}/contents/ref_commit?ref=${PREVIOUS_HEAD}" --jq '.content' | base64 -d | tr -d '\n')" + if [[ "${CURRENT_REF_COMMIT}" == "${SOURCE_SHA}" ]]; then + PREVIOUS_ACCEPTED_TAG="$(gh api "repos/${ARTIFACTS_REPO}/contents/manifest.json?ref=${PREVIOUS_HEAD}" --jq '.content' \ + | base64 -d | jq -r '.previous_accepted_tag')" + if [[ -z "${PREVIOUS_ACCEPTED_TAG}" || "${PREVIOUS_ACCEPTED_TAG}" == "null" ]]; then + echo "Current same-source manifest has no previous accepted identity" >&2 + exit 1 + fi + else + PREVIOUS_ACCEPTED_TAG="process-replay/archive/${PREVIOUS_HEAD}" + fi + + WORK="$(pwd)/.process-replay-generate" + export WORK OPENDBC_SHA PREVIOUS_HEAD PREVIOUS_ACCEPTED_TAG + rm -rf "${WORK}" + mkdir -p \ + "${WORK}/generate" \ + "${WORK}/compare" \ + "${WORK}/release" \ + "${WORK}/candidate/release" \ + "${WORK}/candidate/review/generate-active/compare" + + echo "Listing active cases" + python3 "${REPLAY}" --inventory active --list-cases > "${WORK}/cases.json" + python3 - <<'PY' + import json, os, pathlib + cases = json.loads(pathlib.Path(os.environ['WORK'], 'cases.json').read_text(encoding='utf-8')) + if not isinstance(cases, list) or len(cases) != 16: + raise SystemExit(f'expected 16 active cases, got {len(cases) if isinstance(cases, list) else type(cases)}') + print(f'listed {len(cases)} active cases') + PY + + echo "Generating refs with --update-refs" + python3 "${REPLAY}" \ + -j"$(nproc)" \ + --inventory active \ + --update-refs \ + --output-dir "${WORK}/generate" + + echo "Assembling release tree" + python3 "${ASSEMBLE}" \ + --operation generate-active \ + --source-sha "${SOURCE_SHA}" \ + --fakedata-dir selfdrive/test/process_replay/fakedata \ + --release-dir "${WORK}/release" \ + --opendbc-sha "${OPENDBC_SHA}" \ + --cases-json "${WORK}/cases.json" \ + --previous-accepted-tag "${PREVIOUS_ACCEPTED_TAG}" \ + | tee "${WORK}/assemble.log" + MANIFEST_SHA256="$(awk -F= '/^manifest_sha256=/{print $2; exit}' "${WORK}/assemble.log")" + if [[ ! "${MANIFEST_SHA256}" =~ ^[0-9a-f]{64}$ ]]; then + MANIFEST_SHA256="$(python3 - <<'PY' + import hashlib, pathlib, os + path = pathlib.Path(os.environ['WORK']) / 'release' / 'manifest.json' + print(hashlib.sha256(path.read_bytes()).hexdigest()) + PY + )" + fi + if [[ ! "${MANIFEST_SHA256}" =~ ^[0-9a-f]{64}$ ]]; then + echo "Failed to resolve manifest_sha256" >&2 + exit 1 + fi + export MANIFEST_SHA256 + + echo "Comparing all active tasks against assembled release" + python3 "${REPLAY}" \ + -j"$(nproc)" \ + --inventory active \ + --output-dir "${WORK}/compare" \ + --reference-dir "${WORK}/release" + + python3 - <<'PY' + import json, os, pathlib, re + work = pathlib.Path(os.environ['WORK']) + source_sha = os.environ['SOURCE_SHA'] + manifest_sha = os.environ['MANIFEST_SHA256'] + previous = os.environ['PREVIOUS_ACCEPTED_TAG'] + opendbc_sha = os.environ['OPENDBC_SHA'] + run_id = os.environ['GITHUB_RUN_ID'] + expected_tasks = int(os.environ['EXPECTED_TASKS']) + + results_path = work / 'compare' / 'results.json' + if not results_path.is_file() or results_path.is_symlink(): + raise SystemExit('missing compare results.json') + results = json.loads(results_path.read_text(encoding='utf-8')) + cases = json.loads((work / 'cases.json').read_text(encoding='utf-8')) + expected_ids = sorted( + f"{case['case_id']}:{process}" + for case in cases + for process in case['processes'] + ) + if results.get('schema_version') != 1 or results.get('inventory') != 'active': + raise SystemExit('compare proof requires schema_version=1 and inventory=active') + if results.get('tested_commit') != source_sha or results.get('ref_commit') != source_sha: + raise SystemExit('compare tested_commit/ref_commit must equal source_sha') + if results.get('expected_task_ids') != expected_ids: + raise SystemExit('compare expected_task_ids mismatch active inventory') + if results.get('completed_task_ids') != expected_ids or results.get('incomplete_task_ids') != []: + raise SystemExit('compare task completion proof is not exact') + records = results.get('results') + if not isinstance(records, list) or len(records) != expected_tasks: + raise SystemExit('compare results must contain exactly one record per active task') + record_ids = [record.get('task_id') for record in records if isinstance(record, dict)] + if sorted(record_ids) != expected_ids or len(set(record_ids)) != expected_tasks: + raise SystemExit('compare result task IDs must exactly match active inventory') + if any(record.get('status') != 'pass' for record in records): + raise SystemExit('compare result records must all pass') + counts = results.get('status_counts') or {} + if int(counts.get('pass', -1)) != expected_tasks: + raise SystemExit(f"compare pass count must be {expected_tasks}, got {counts.get('pass')}") + for key in ('diff', 'error', 'incomplete'): + if int(counts.get(key, -1)) != 0: + raise SystemExit(f'compare requires zero {key}, got {counts.get(key)}') + + release = work / 'release' + manifest_path = release / 'manifest.json' + actual = __import__('hashlib').sha256(manifest_path.read_bytes()).hexdigest() + if actual != manifest_sha: + raise SystemExit('manifest_sha256 drifted before proof write') + + # Reject unsafe release layout before packaging. + for path in sorted(release.rglob('*')): + rel = path.relative_to(release).as_posix() + if '..' in pathlib.Path(rel).parts or rel.startswith('/'): + raise SystemExit(f'unsafe release path: {rel}') + if path.is_symlink(): + raise SystemExit(f'symlink forbidden in release: {rel}') + + proof = { + 'schema_version': 1, + 'source_sha': source_sha, + 'opendbc_sha': opendbc_sha, + 'operation': 'generate-active', + 'manifest_sha256': manifest_sha, + 'accepted_head': os.environ['PREVIOUS_HEAD'], + 'previous_accepted_tag': previous, + 'workflow_run_id': str(run_id), + 'expected_tasks': expected_tasks, + 'status_counts': { + 'pass': expected_tasks, + 'diff': 0, + 'error': 0, + 'incomplete': 0, + }, + } + if not re.fullmatch(r'[0-9a-f]{40}', source_sha): + raise SystemExit('invalid source_sha in proof') + if not re.fullmatch(r'[0-9a-f]{64}', manifest_sha): + raise SystemExit('invalid manifest_sha256 in proof') + + candidate = work / 'candidate' + # Copy release tree files only (no nested dirs). + for entry in sorted(release.iterdir()): + if entry.is_symlink(): + raise SystemExit(f'symlink forbidden: {entry}') + if entry.is_dir(): + raise SystemExit(f'nested release directory forbidden: {entry.name}') + if entry.is_file(): + (candidate / 'release' / entry.name).write_bytes(entry.read_bytes()) + + (candidate / 'handoff-meta.json').write_text( + json.dumps(proof, indent=2, sort_keys=True) + '\n', encoding='utf-8' + ) + compare_dest = candidate / 'review' / 'generate-active' / 'compare' + compare_dest.mkdir(parents=True, exist_ok=True) + (compare_dest / 'results.json').write_bytes(results_path.read_bytes()) + if (work / 'compare' / 'diff.txt').is_file(): + (compare_dest / 'diff.txt').write_bytes((work / 'compare' / 'diff.txt').read_bytes()) + (candidate / 'review' / 'generate-active' / 'cases.json').write_bytes( + (work / 'cases.json').read_bytes() + ) + + out = pathlib.Path(os.environ['GITHUB_OUTPUT']) + with out.open('a', encoding='utf-8') as handle: + handle.write(f'source_sha={source_sha}\n') + handle.write(f'manifest_sha256={manifest_sha}\n') + handle.write(f'previous_accepted_tag={previous}\n') + print(f'proof bound source_sha={source_sha} manifest_sha256={manifest_sha} previous={previous}') + PY + + - name: Upload immutable same-run candidate artifact + uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: ${{ needs.gate-nap-dev.outputs.candidate_artifact }} + path: .process-replay-generate/candidate + if-no-files-found: error + overwrite: false + retention-days: 14 + verify: name: Verify candidate without execution - needs: gate-nap-dev - if: needs.gate-nap-dev.result == 'success' + needs: [gate-nap-dev, generate] + if: | + always() && + needs.gate-nap-dev.result == 'success' && + ( + needs.gate-nap-dev.outputs.operation == 'rollback' || + needs.generate.result == 'success' + ) runs-on: ubuntu-24.04 timeout-minutes: 20 # No environment: publication token must not be available here. @@ -230,6 +423,8 @@ jobs: release_source_sha: ${{ steps.validate.outputs.release_source_sha }} release_manifest_sha256: ${{ steps.validate.outputs.release_manifest_sha256 }} release_tag: ${{ steps.validate.outputs.release_tag }} + accepted_head: ${{ steps.proof.outputs.accepted_head }} + release_previous_accepted_tag: ${{ steps.validate.outputs.release_previous_accepted_tag }} steps: - name: Checkout trusted nap-dev inventory uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.0.0 @@ -241,119 +436,80 @@ jobs: id: fetch env: OPERATION: ${{ needs.gate-nap-dev.outputs.operation }} - CONTROLLER_RUN_ID: ${{ needs.gate-nap-dev.outputs.controller_run_id }} - ARTIFACT_ID: ${{ needs.gate-nap-dev.outputs.artifact_id }} - ARTIFACT_DIGEST: ${{ needs.gate-nap-dev.outputs.artifact_digest }} SOURCE_SHA: ${{ needs.gate-nap-dev.outputs.source_sha }} - MANIFEST_SHA256: ${{ needs.gate-nap-dev.outputs.manifest_sha256 }} ROLLBACK_TAG: ${{ needs.gate-nap-dev.outputs.rollback_tag }} - CONTROLLER_READ_TOKEN: ${{ secrets.NAP_CI_CONTROLLER_ARTIFACT_READ_TOKEN }} + CANDIDATE_ARTIFACT: ${{ needs.gate-nap-dev.outputs.candidate_artifact }} + GENERATE_MANIFEST_SHA256: ${{ needs.generate.outputs.manifest_sha256 }} GITHUB_READ_TOKEN: ${{ github.token }} run: | set -euo pipefail mkdir -p candidate - if [[ "${OPERATION}" == "publish" ]]; then - if [[ -z "${CONTROLLER_READ_TOKEN}" ]]; then - echo "Missing NAP_CI_CONTROLLER_ARTIFACT_READ_TOKEN" >&2 - exit 1 - fi - export GH_TOKEN="${CONTROLLER_READ_TOKEN}" - - EXPECTED_ARTIFACT_NAME="nap-replay-${CONTROLLER_RUN_ID}" - RUN_JSON="$(gh api "repos/${CONTROLLER_REPO}/actions/runs/${CONTROLLER_RUN_ID}")" - RUN_PATH="$(jq -r '.path // empty' <<<"${RUN_JSON}")" - RUN_NAME="$(jq -r '.name // empty' <<<"${RUN_JSON}")" - RUN_EVENT="$(jq -r '.event // empty' <<<"${RUN_JSON}")" - RUN_STATUS="$(jq -r '.status // empty' <<<"${RUN_JSON}")" - RUN_CONCLUSION="$(jq -r '.conclusion // empty' <<<"${RUN_JSON}")" - RUN_HEAD_SHA="$(jq -r '.head_sha // empty' <<<"${RUN_JSON}")" - if [[ "${RUN_PATH}" != ".github/workflows/replay.yaml" ]]; then - echo "Controller run path mismatch: ${RUN_PATH}" >&2 - exit 1 - fi - if [[ "${RUN_NAME}" != "nap-replay" ]]; then - echo "Controller run name mismatch: ${RUN_NAME}" >&2 - exit 1 - fi - if [[ "${RUN_EVENT}" != "workflow_dispatch" && "${RUN_EVENT}" != "repository_dispatch" ]]; then - echo "Controller run event must be workflow_dispatch or repository_dispatch, got ${RUN_EVENT}" >&2 + if [[ "${OPERATION}" == "generate-active" ]]; then + # Same-run artifact only; never trust foreign payloads. + if [[ -z "${CANDIDATE_ARTIFACT}" ]]; then + echo "Missing candidate artifact name" >&2 exit 1 fi - if [[ "${RUN_STATUS}" != "completed" || "${RUN_CONCLUSION}" != "success" ]]; then - echo "Controller run ${CONTROLLER_RUN_ID} is not a successful completion (${RUN_STATUS}/${RUN_CONCLUSION})" >&2 + if [[ ! "${CANDIDATE_ARTIFACT}" =~ ^process-replay-candidate-[0-9]+-[0-9]+$ ]]; then + echo "Unexpected candidate artifact name ${CANDIDATE_ARTIFACT}" >&2 exit 1 fi - if [[ ! "${RUN_HEAD_SHA}" =~ ^[0-9a-f]{40}$ ]]; then - echo "Controller run head_sha must be lowercase 40-hex" >&2 - exit 1 - fi - - META="$(gh api "repos/${CONTROLLER_REPO}/actions/artifacts/${ARTIFACT_ID}")" - META_DIGEST="$(jq -r '.digest // empty' <<<"${META}")" - META_RUN="$(jq -r '.workflow_run.id' <<<"${META}")" - META_EXPIRED="$(jq -r '.expired' <<<"${META}")" - META_NAME="$(jq -r '.name // empty' <<<"${META}")" - if [[ "${META_EXPIRED}" == "true" ]]; then - echo "Controller artifact expired" >&2 - exit 1 - fi - if [[ "${META_RUN}" != "${CONTROLLER_RUN_ID}" ]]; then - echo "Artifact run ID mismatch: ${META_RUN} != ${CONTROLLER_RUN_ID}" >&2 - exit 1 - fi - if [[ "${META_NAME}" != "${EXPECTED_ARTIFACT_NAME}" ]]; then - echo "Artifact name mismatch: ${META_NAME} != ${EXPECTED_ARTIFACT_NAME}" >&2 - exit 1 - fi - if [[ "${META_DIGEST}" != "${ARTIFACT_DIGEST}" ]]; then - echo "REST artifact digest mismatch" >&2 - echo "expected ${ARTIFACT_DIGEST}" >&2 - echo "actual ${META_DIGEST}" >&2 - exit 1 + else + # Rollback reads an archived tag tree with the job token only. + export GH_TOKEN="${GITHUB_READ_TOKEN}" + REF_SHA="$(gh api "repos/${ARTIFACTS_REPO}/git/ref/tags/${ROLLBACK_TAG}" --jq '.object.sha')" + OBJ_TYPE="$(gh api "repos/${ARTIFACTS_REPO}/git/ref/tags/${ROLLBACK_TAG}" --jq '.object.type')" + if [[ "${OBJ_TYPE}" == "tag" ]]; then + REF_SHA="$(gh api "repos/${ARTIFACTS_REPO}/git/tags/${REF_SHA}" --jq '.object.sha')" fi - gh api "repos/${CONTROLLER_REPO}/actions/artifacts/${ARTIFACT_ID}/zip" > /tmp/candidate.zip + mkdir -p candidate/release + TREE_SHA="$(gh api "repos/${ARTIFACTS_REPO}/git/commits/${REF_SHA}" --jq '.tree.sha')" + gh api "repos/${ARTIFACTS_REPO}/git/trees/${TREE_SHA}?recursive=1" > /tmp/tree.json python3 - <<'PY' - import os - import zipfile + import json, base64, urllib.request, os from pathlib import Path - - S_IFMT = 0o170000 - S_IFREG = 0o100000 - S_IFDIR = 0o040000 - - zf = zipfile.ZipFile('/tmp/candidate.zip') - seen: set[str] = set() - for info in zf.infolist(): - name = info.filename - parts = Path(name).parts - if name.startswith('/') or '..' in parts: - raise SystemExit(f'path traversal in artifact: {name}') - norm = Path(*parts).as_posix() if parts else name - key = norm.rstrip('/') if norm.endswith('/') else norm - if key in seen: - raise SystemExit(f'duplicate zip member: {norm}') - seen.add(key) - - mode = (info.external_attr >> 16) & 0xFFFF - ftype = mode & S_IFMT - is_dir = name.endswith('/') or info.is_dir() - if is_dir: - if ftype not in (0, S_IFDIR): - raise SystemExit(f'unsafe directory zip member type: {name}') + token = os.environ['GH_TOKEN'] + repo = os.environ['ARTIFACTS_REPO'] + tree = json.load(open('/tmp/tree.json')) + if tree.get('truncated'): + raise SystemExit('rollback tree truncated') + for entry in tree['tree']: + path = entry['path'] + if path.startswith('/') or '..' in Path(path).parts: + raise SystemExit(f'path traversal: {path}') + if entry['type'] != 'blob': continue - if ftype not in (0, S_IFREG): - raise SystemExit(f'non-regular zip member rejected: {name}') - - target = Path('candidate') / name + url = f'https://api.github.com/repos/{repo}/git/blobs/{entry["sha"]}' + req = urllib.request.Request(url, headers={ + 'Authorization': f'Bearer {token}', + 'Accept': 'application/vnd.github+json', + }) + with urllib.request.urlopen(req) as resp: + blob = json.load(resp) + data = base64.b64decode(blob['content']) + target = Path('candidate/release') / path target.parent.mkdir(parents=True, exist_ok=True) - fd = os.open(target, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o644) - with zf.open(info) as src, open(fd, 'wb') as dst: - dst.write(src.read()) + target.write_bytes(data) PY + fi - # Handoff/review proof: bind source SHA, hosted controller run, manifest SHA, - # and require green 78-pass compare results before environment approval. - python3 - <<'PY' + - name: Download same-run generate-active artifact + if: needs.gate-nap-dev.outputs.operation == 'generate-active' + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: ${{ needs.gate-nap-dev.outputs.candidate_artifact }} + path: candidate + + - name: Reject unsafe layouts and bind generate-active proof + id: proof + if: needs.gate-nap-dev.outputs.operation == 'generate-active' + env: + SOURCE_SHA: ${{ needs.gate-nap-dev.outputs.source_sha }} + MANIFEST_SHA256: ${{ needs.generate.outputs.manifest_sha256 }} + OPERATION: ${{ needs.gate-nap-dev.outputs.operation }} + run: | + set -euo pipefail + python3 - <<'PY' import hashlib import json import os @@ -362,87 +518,98 @@ jobs: source_sha = os.environ['SOURCE_SHA'] manifest_sha = os.environ['MANIFEST_SHA256'] - controller_run_id = os.environ['CONTROLLER_RUN_ID'] + operation = os.environ['OPERATION'] + workflow_run_id = os.environ['GITHUB_RUN_ID'] + expected_tasks = int(os.environ['EXPECTED_TASKS']) sha40 = re.compile(r'^[0-9a-f]{40}$') sha64 = re.compile(r'^[0-9a-f]{64}$') + if operation != 'generate-active': + raise SystemExit(f'expected generate-active, got {operation!r}') if not sha40.fullmatch(source_sha) or not sha64.fullmatch(manifest_sha): raise SystemExit('invalid source_sha or manifest_sha256 for proof binding') root = Path('candidate') + if root.is_symlink() or not root.is_dir(): + raise SystemExit('candidate root must be a real directory') + + # Fail closed on unsafe extracted layouts (symlinks / traversal / unexpected nesting). + allowed_top = {'release', 'handoff-meta.json', 'review'} + for child in root.iterdir(): + if child.name not in allowed_top: + raise SystemExit(f'unexpected candidate top-level entry: {child.name}') + for path in sorted(root.rglob('*')): + rel = path.relative_to(root).as_posix() + parts = Path(rel).parts + if rel.startswith('/') or '..' in parts: + raise SystemExit(f'path traversal in candidate: {rel}') + if path.is_symlink(): + raise SystemExit(f'symlink forbidden in candidate: {rel}') + if path.is_dir(): + continue + if not path.is_file(): + raise SystemExit(f'non-regular candidate member: {rel}') + if rel.startswith('release/') and '/' in rel[len('release/'):]: + raise SystemExit(f'nested release path forbidden: {rel}') + handoff = root / 'handoff-meta.json' if not handoff.is_file() or handoff.is_symlink(): - raise SystemExit('missing handoff-meta.json proof in controller artifact') + raise SystemExit('missing handoff-meta.json proof in candidate artifact') meta = json.loads(handoff.read_text(encoding='utf-8')) if meta.get('source_sha') != source_sha: raise SystemExit(f"handoff source_sha mismatch: {meta.get('source_sha')} != {source_sha}") if meta.get('manifest_sha256') != manifest_sha: - raise SystemExit('handoff manifest_sha256 mismatch against dispatch input') - if str(meta.get('controller_run_id')) != str(controller_run_id): - raise SystemExit('handoff controller_run_id mismatch against authenticated hosted run') - operation = meta.get('operation') - if operation not in {'generate-staged', 'generate-active'}: - raise SystemExit(f'publish requires generate-* controller operation, got {operation!r}') + raise SystemExit('handoff manifest_sha256 mismatch against generate output') + if meta.get('operation') != 'generate-active': + raise SystemExit(f"handoff operation must be generate-active, got {meta.get('operation')!r}") + if str(meta.get('workflow_run_id')) != workflow_run_id: + raise SystemExit('handoff workflow_run_id mismatch against current run') + if int(meta.get('expected_tasks', -1)) != expected_tasks: + raise SystemExit('handoff expected_tasks mismatch') + accepted_head = str(meta.get('accepted_head', '')) + if not sha40.fullmatch(accepted_head): + raise SystemExit('handoff accepted_head must be lowercase 40-hex') release_manifest = root / 'release' / 'manifest.json' if not release_manifest.is_file(): - raise SystemExit('candidate release/manifest.json missing after extraction') + raise SystemExit('candidate release/manifest.json missing after download') actual = hashlib.sha256(release_manifest.read_bytes()).hexdigest() if actual != manifest_sha: - raise SystemExit('release/manifest.json hash mismatch against handoff/dispatch proof') + raise SystemExit('release/manifest.json hash mismatch against handoff/generate proof') - results_path = root / 'review' / operation / 'compare' / 'results.json' + results_path = root / 'review' / 'generate-active' / 'compare' / 'results.json' if not results_path.is_file() or results_path.is_symlink(): raise SystemExit(f'missing review proof {results_path}') results = json.loads(results_path.read_text(encoding='utf-8')) + trusted = json.loads(Path(os.environ['TRUSTED_INVENTORY_PATH']).read_text(encoding='utf-8')) + expected_ids = trusted.get('task_ids') + if not isinstance(expected_ids, list) or len(expected_ids) != expected_tasks: + raise SystemExit('trusted active task inventory is invalid') + if results.get('schema_version') != 1 or results.get('inventory') != 'active': + raise SystemExit('review proof requires schema_version=1 and inventory=active') + if results.get('tested_commit') != source_sha or results.get('ref_commit') != source_sha: + raise SystemExit('review proof tested_commit/ref_commit must equal source_sha') + if results.get('expected_task_ids') != expected_ids: + raise SystemExit('review expected_task_ids mismatch trusted active inventory') + if results.get('completed_task_ids') != expected_ids or results.get('incomplete_task_ids') != []: + raise SystemExit('review task completion proof is not exact') + records = results.get('results') + if not isinstance(records, list) or len(records) != expected_tasks: + raise SystemExit('review results must contain exactly one record per active task') + record_ids = [record.get('task_id') for record in records if isinstance(record, dict)] + if sorted(record_ids) != sorted(expected_ids) or len(set(record_ids)) != expected_tasks: + raise SystemExit('review result task IDs must exactly match trusted active inventory') + if any(record.get('status') != 'pass' for record in records): + raise SystemExit('review result records must all pass') counts = results.get('status_counts') or {} - if int(counts.get('pass', -1)) != 78: - raise SystemExit(f"review proof pass count must be 78, got {counts.get('pass')}") + if int(counts.get('pass', -1)) != expected_tasks: + raise SystemExit(f"review proof pass count must be {expected_tasks}, got {counts.get('pass')}") for key in ('diff', 'error', 'incomplete'): if int(counts.get(key, -1)) != 0: raise SystemExit(f'review proof requires zero {key}, got {counts.get(key)}') - tested = results.get('tested_commit') - if tested not in (None, '', source_sha): - raise SystemExit('review proof tested_commit must equal source_sha when present') - print(f'authenticated controller run {controller_run_id} with green 78-pass review proof') - PY - else - # Rollback reads an archived tag tree with the job token only. - export GH_TOKEN="${GITHUB_READ_TOKEN}" - REF_SHA="$(gh api "repos/${ARTIFACTS_REPO}/git/ref/tags/${ROLLBACK_TAG}" --jq '.object.sha')" - OBJ_TYPE="$(gh api "repos/${ARTIFACTS_REPO}/git/ref/tags/${ROLLBACK_TAG}" --jq '.object.type')" - if [[ "${OBJ_TYPE}" == "tag" ]]; then - REF_SHA="$(gh api "repos/${ARTIFACTS_REPO}/git/tags/${REF_SHA}" --jq '.object.sha')" - fi - mkdir -p candidate/release - TREE_SHA="$(gh api "repos/${ARTIFACTS_REPO}/git/commits/${REF_SHA}" --jq '.tree.sha')" - gh api "repos/${ARTIFACTS_REPO}/git/trees/${TREE_SHA}?recursive=1" > /tmp/tree.json - python3 - <<'PY' - import json, base64, urllib.request, os - from pathlib import Path - token = os.environ['GH_TOKEN'] - repo = os.environ['ARTIFACTS_REPO'] - tree = json.load(open('/tmp/tree.json')) - if tree.get('truncated'): - raise SystemExit('rollback tree truncated') - for entry in tree['tree']: - path = entry['path'] - if path.startswith('/') or '..' in Path(path).parts: - raise SystemExit(f'path traversal: {path}') - if entry['type'] != 'blob': - continue - url = f'https://api.github.com/repos/{repo}/git/blobs/{entry["sha"]}' - req = urllib.request.Request(url, headers={ - 'Authorization': f'Bearer {token}', - 'Accept': 'application/vnd.github+json', - }) - with urllib.request.urlopen(req) as resp: - blob = json.load(resp) - data = base64.b64decode(blob['content']) - target = Path('candidate/release') / path - target.parent.mkdir(parents=True, exist_ok=True) - target.write_bytes(data) + with Path(os.environ['GITHUB_OUTPUT']).open('a', encoding='utf-8') as output: + output.write(f'accepted_head={accepted_head}\n') + print(f'bound same-run generate-active proof with {expected_tasks} green compares') PY - fi test -d candidate/release - name: Fail-closed release-tree validation @@ -450,13 +617,14 @@ jobs: env: OPERATION: ${{ needs.gate-nap-dev.outputs.operation }} SOURCE_SHA: ${{ needs.gate-nap-dev.outputs.source_sha }} - MANIFEST_SHA256: ${{ needs.gate-nap-dev.outputs.manifest_sha256 }} + MANIFEST_SHA256: ${{ needs.generate.outputs.manifest_sha256 }} ROLLBACK_TAG: ${{ needs.gate-nap-dev.outputs.rollback_tag }} GITHUB_READ_TOKEN: ${{ github.token }} run: | set -euo pipefail test -f "${TRUSTED_INVENTORY_PATH}" test -f "${RELEASE_VALIDATOR}" + test -d candidate/release HEAD_SHA="$(git rev-parse HEAD)" if [[ "${SOURCE_SHA}" == "${HEAD_SHA}" ]]; then EXPECTED_OPENDBC_SHA="$(git rev-parse HEAD:opendbc_repo)" @@ -470,11 +638,16 @@ jobs: echo "Failed to resolve trusted opendbc_repo gitlink for ${SOURCE_SHA}" >&2 exit 1 fi + VALIDATOR_OPERATION="${OPERATION}" + if [[ "${OPERATION}" == "generate-active" ]]; then + # Release-tree validator uses publish|rollback; workflow op remains generate-active. + VALIDATOR_OPERATION="publish" + fi python3 "${RELEASE_VALIDATOR}" \ --release-dir candidate/release \ --trusted-inventory "${TRUSTED_INVENTORY_PATH}" \ --source-sha "${SOURCE_SHA}" \ - --operation "${OPERATION}" \ + --operation "${VALIDATOR_OPERATION}" \ --expected-manifest-sha256 "${MANIFEST_SHA256}" \ --expected-opendbc-sha "${EXPECTED_OPENDBC_SHA}" \ --rollback-tag "${ROLLBACK_TAG}" @@ -482,14 +655,14 @@ jobs: - name: Upload verified same-run artifact uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 with: - name: process-replay-release-verified-${{ needs.gate-nap-dev.outputs.source_sha }} + name: process-replay-release-verified-${{ github.run_id }}-${{ github.run_attempt }} path: candidate/release if-no-files-found: error overwrite: false retention-days: 14 publish: - name: Approve and publish to ci-artifacts + name: Publish to ci-artifacts needs: [gate-nap-dev, verify] if: needs.verify.result == 'success' runs-on: ubuntu-24.04 @@ -505,7 +678,7 @@ jobs: - name: Download verified same-run artifact uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 with: - name: process-replay-release-verified-${{ needs.gate-nap-dev.outputs.source_sha }} + name: process-replay-release-verified-${{ github.run_id }}-${{ github.run_attempt }} path: release - name: Revalidate release tree before push @@ -532,11 +705,15 @@ jobs: exit 1 fi # Same strict validator as the no-environment verifier. + VALIDATOR_OPERATION="${OPERATION}" + if [[ "${OPERATION}" == "generate-active" ]]; then + VALIDATOR_OPERATION="publish" + fi python3 "${RELEASE_VALIDATOR}" \ --release-dir release \ --trusted-inventory "${TRUSTED_INVENTORY_PATH}" \ --source-sha "${SOURCE_SHA}" \ - --operation "${OPERATION}" \ + --operation "${VALIDATOR_OPERATION}" \ --expected-manifest-sha256 "${MANIFEST_SHA256}" \ --expected-opendbc-sha "${EXPECTED_OPENDBC_SHA}" \ --rollback-tag "${ROLLBACK_TAG}" @@ -545,6 +722,8 @@ jobs: id: head env: GH_TOKEN: ${{ github.token }} + EXPECTED_ACCEPTED_HEAD: ${{ needs.verify.outputs.accepted_head }} + OPERATION: ${{ needs.gate-nap-dev.outputs.operation }} run: | set -euo pipefail if gh api "repos/${ARTIFACTS_REPO}/git/ref/heads/${ARTIFACTS_BRANCH}" >/tmp/head.json 2>/dev/null; then @@ -558,13 +737,17 @@ jobs: echo "Invalid captured head ${CAPTURED}" >&2 exit 1 fi + if [[ "${OPERATION}" == "generate-active" && "${EXPECTED_ACCEPTED_HEAD}" != "${CAPTURED}" ]]; then + echo "process-replay moved after generation: expected=${EXPECTED_ACCEPTED_HEAD} current=${CAPTURED}" >&2 + exit 1 + fi echo "captured_head=${CAPTURED}" >> "${GITHUB_OUTPUT}" echo "Captured ${ARTIFACTS_BRANCH} head: ${CAPTURED:-}" - name: Push with force-with-lease, tag, and remote rehash env: - # CI_ARTIFACTS_TOKEN is intentionally only available in this final push step. - CI_ARTIFACTS_TOKEN: ${{ secrets.CI_ARTIFACTS_TOKEN }} + # Repository-scoped write token is intentionally available only in this final push step. + NAP_CI_ARTIFACTS_WRITE_TOKEN: ${{ secrets.NAP_CI_ARTIFACTS_WRITE_TOKEN }} JOB_TOKEN: ${{ github.token }} OPERATION: ${{ needs.gate-nap-dev.outputs.operation }} SOURCE_SHA: ${{ needs.verify.outputs.release_source_sha }} @@ -574,18 +757,18 @@ jobs: BRANCH_EXISTS: ${{ steps.head.outputs.branch_exists }} run: | set -euo pipefail - if [[ -z "${CI_ARTIFACTS_TOKEN}" ]]; then - echo "Missing CI_ARTIFACTS_TOKEN in process-replay-publish environment" >&2 + if [[ -z "${NAP_CI_ARTIFACTS_WRITE_TOKEN}" ]]; then + echo "Missing NAP_CI_ARTIFACTS_WRITE_TOKEN in process-replay-publish environment" >&2 exit 1 fi TIP="$(GH_TOKEN="${JOB_TOKEN}" gh api "repos/${SOURCE_REPO}/git/ref/heads/nap-dev" --jq '.object.sha')" - if [[ "${OPERATION}" == "publish" && "${TIP}" != "${SOURCE_SHA}" ]]; then + if [[ "${OPERATION}" == "generate-active" && "${TIP}" != "${SOURCE_SHA}" ]]; then echo "nap-dev moved to ${TIP}; refusing stale candidate" >&2 exit 1 fi - AUTH_URL="https://x-access-token:${CI_ARTIFACTS_TOKEN}@github.com/${ARTIFACTS_REPO}.git" + AUTH_URL="https://x-access-token:${NAP_CI_ARTIFACTS_WRITE_TOKEN}@github.com/${ARTIFACTS_REPO}.git" rm -rf /tmp/ci-artifacts if [[ "${BRANCH_EXISTS}" == "true" ]]; then git clone --depth 1 --branch "${ARTIFACTS_BRANCH}" "${AUTH_URL}" /tmp/ci-artifacts @@ -596,6 +779,7 @@ jobs: cd /tmp/ci-artifacts git config user.name "process-replay-publisher" git config user.email "process-replay-publisher@users.noreply.github.com" + EXTRA_REFS=() if [[ "${BRANCH_EXISTS}" == "true" ]]; then git checkout -B "${ARTIFACTS_BRANCH}" "origin/${ARTIFACTS_BRANCH}" @@ -604,9 +788,16 @@ jobs: echo "Captured head drifted before push: ${CURRENT} != ${CAPTURED_HEAD}" >&2 exit 1 fi - if ! git ls-remote --tags origin "refs/tags/process-replay/archive/${CAPTURED_HEAD}" | grep -q .; then - git tag "process-replay/archive/${CAPTURED_HEAD}" "${CAPTURED_HEAD}" - git push origin "refs/tags/process-replay/archive/${CAPTURED_HEAD}" + ARCHIVE_TAG="process-replay/archive/${CAPTURED_HEAD}" + if git ls-remote --tags origin "refs/tags/${ARCHIVE_TAG}" | grep -q .; then + git fetch origin "refs/tags/${ARCHIVE_TAG}:refs/tags/${ARCHIVE_TAG}" + if [[ "$(git rev-list -n 1 "refs/tags/${ARCHIVE_TAG}")" != "${CAPTURED_HEAD}" ]]; then + echo "Archive tag ${ARCHIVE_TAG} does not resolve to captured head" >&2 + exit 1 + fi + else + git tag "${ARCHIVE_TAG}" "${CAPTURED_HEAD}" + EXTRA_REFS+=("refs/tags/${ARCHIVE_TAG}") fi else git checkout --orphan "${ARTIFACTS_BRANCH}" @@ -635,19 +826,21 @@ jobs: fi else git tag -a "${RELEASE_TAG}" -m "process-replay release ${SOURCE_SHA}" - git push origin "refs/tags/${RELEASE_TAG}" + EXTRA_REFS+=("refs/tags/${RELEASE_TAG}") fi if [[ "${BRANCH_EXISTS}" == "true" ]]; then - git push --force-with-lease="refs/heads/${ARTIFACTS_BRANCH}:${CAPTURED_HEAD}" origin "HEAD:refs/heads/${ARTIFACTS_BRANCH}" + git push --atomic \ + --force-with-lease="refs/heads/${ARTIFACTS_BRANCH}:${CAPTURED_HEAD}" \ + origin "HEAD:refs/heads/${ARTIFACTS_BRANCH}" "${EXTRA_REFS[@]}" else - git push -u origin "HEAD:refs/heads/${ARTIFACTS_BRANCH}" + git push --atomic origin "HEAD:refs/heads/${ARTIFACTS_BRANCH}" "${EXTRA_REFS[@]}" fi echo "Published ${ARTIFACTS_BRANCH} at ${NEW_COMMIT} tag ${RELEASE_TAG}" fi - # Remote rehash audit by pinned commit SHA (same final step; same token scope). - export GH_TOKEN="${CI_ARTIFACTS_TOKEN}" + # ci-artifacts is public; the job token is sufficient for the remote rehash audit. + export GH_TOKEN="${JOB_TOKEN}" export NEW_COMMIT TREE="$(gh api "repos/${ARTIFACTS_REPO}/git/commits/${NEW_COMMIT}" --jq '.tree.sha')" gh api "repos/${ARTIFACTS_REPO}/git/trees/${TREE}?recursive=1" > /tmp/remote-tree.json @@ -656,7 +849,7 @@ jobs: token = os.environ['GH_TOKEN'] repo = os.environ['ARTIFACTS_REPO'] source_sha = os.environ['SOURCE_SHA'] - expected = 78 + expected = int(os.environ['EXPECTED_TASKS']) tree = json.load(open('/tmp/remote-tree.json')) if tree.get('truncated'): raise SystemExit('remote tree truncated') diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 5d4d00b28b0907..17b96f0e1bd181 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -146,6 +146,11 @@ jobs: uv pip install natsort - name: Build openpilot run: scons -j$(nproc) + - name: Validate process replay release contracts + run: | + pytest -q -n 0 \ + .github/ci/test_assemble_replay_release.py \ + .github/ci/test_validate_process_replay_release.py - name: Run unit tests timeout-minutes: ${{ contains(runner.name, 'nsc') && 2 || 20 }} run: | diff --git a/tools/ci/assemble_replay_release.py b/tools/ci/assemble_replay_release.py new file mode 100755 index 00000000000000..228f9e1d068de8 --- /dev/null +++ b/tools/ci/assemble_replay_release.py @@ -0,0 +1,292 @@ +#!/usr/bin/env python3 +"""Assemble a process-replay release/ tree from harness fakedata candidates. + +Layout (public consumer contract): + release/ref_commit + release/manifest.json (canonical sorted JSON + LF) + release/{case_id}__{process}__{source_sha}.zst + +This tool only assembles the active inventory (16 cases / 66 refs). Staged / +pending descriptors are rejected so non-executable sources cannot be published +through this path. + +Manifest is NOT final unless every case/ref identity carries harness +params_digest + source_sha256 (and aggregate sources_digest/params_digest), +plus openpilot/opendbc SHAs, previous_accepted_tag, filenames/sizes/hashes. + +results.json / timing / run metadata must stay outside release/. +""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import re +from pathlib import Path + +NAME_RE_TMPL = r"^[a-z0-9][a-z0-9-]*__[a-z0-9_]+__{sha}\.zst$" +CASE_RE = re.compile(r"^[a-z0-9][a-z0-9-]*$") +PROC_RE = re.compile(r"^[a-z0-9_]+$") +SHA_RE = re.compile(r"^[0-9a-f]{40}$") +DIGEST_RE = re.compile(r"^[0-9a-f]{64}$") +TAG_RE = re.compile(r"^[A-Za-z0-9][A-Za-z0-9._/-]{0,127}$") + +# Active inventory only: 16 cases / 66 tasks. +EXPECTED_CASES = 16 +EXPECTED_TASKS = 66 +EXPECTED_PARTITION = {"card": 16, "controlsd": 16, "lagd": 16, "other": 18} + + +def die(msg: str) -> None: + raise SystemExit(f"assemble_replay_release: {msg}") + + +def canonical_json(obj: object) -> bytes: + return (json.dumps(obj, sort_keys=True, separators=(",", ":"), ensure_ascii=False) + "\n").encode("utf-8") + + +def sha256_bytes(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def require_digest(label: str, value: object) -> str: + text = str(value or "") + if not DIGEST_RE.fullmatch(text): + die(f"{label} must be 64 lowercase hex (harness digest); omitted/invalid values reject the manifest") + return text + + +def validate_case_record(rec: dict) -> dict: + cid = str(rec.get("case_id", "")) + if not CASE_RE.fullmatch(cid): + die(f"invalid case_id {cid!r}") + for key in ( + "car_brand", + "source", + "processes", + "custom_params", + "params_digest", + "executable", + "source_sha256", + "source_bytes", + ): + if key not in rec: + die(f"case {cid} missing {key}") + if rec["executable"] is not True: + die(f"case {cid} must be executable for release assembly") + if not isinstance(rec["source"], str) or not rec["source"]: + die(f"case {cid} source must be non-empty string") + if not isinstance(rec["processes"], list) or not rec["processes"]: + die(f"case {cid} processes must be non-empty list") + if not isinstance(rec["custom_params"], dict): + die(f"case {cid} custom_params must be object") + params = require_digest(f"case {cid} params_digest", rec["params_digest"]) + source_sha256 = require_digest(f"case {cid} source_sha256", rec["source_sha256"]) + try: + source_bytes = int(rec["source_bytes"]) + except (TypeError, ValueError): + die(f"case {cid} source_bytes must be int") + if source_bytes <= 0: + die(f"case {cid} source_bytes must be > 0") + # Preserve harness record fields; normalize digests for identity. + out = dict(rec) + out["case_id"] = cid + out["params_digest"] = params + out["source_sha256"] = source_sha256 + out["source_bytes"] = source_bytes + return out + + +def main() -> int: + parser = argparse.ArgumentParser() + parser.add_argument( + "--operation", + required=True, + choices=("generate-active",), + help="Only generate-active is supported (staged publication is rejected)", + ) + parser.add_argument("--source-sha", required=True) + parser.add_argument("--fakedata-dir", required=True, type=Path) + parser.add_argument("--release-dir", required=True, type=Path) + parser.add_argument("--opendbc-sha", required=True) + parser.add_argument("--cases-json", required=True, type=Path, help="Harness --list-cases JSON for exact records") + parser.add_argument( + "--previous-accepted-tag", + required=True, + help="Immutable previous-accepted tag/ref identity (required provenance; no empty default)", + ) + args = parser.parse_args() + + source_sha = args.source_sha.strip() + if source_sha != source_sha.lower() or not SHA_RE.fullmatch(source_sha): + die("source_sha must be 40 lowercase hex") + opendbc_sha = args.opendbc_sha.strip() + if opendbc_sha != opendbc_sha.lower() or not SHA_RE.fullmatch(opendbc_sha): + die("opendbc_sha must be 40 lowercase hex") + + previous_accepted_tag = str(args.previous_accepted_tag).strip() + if not TAG_RE.fullmatch(previous_accepted_tag): + die("previous_accepted_tag missing or invalid") + + if not args.cases_json.is_file(): + die(f"cases json missing: {args.cases_json}") + case_records = json.loads(args.cases_json.read_text(encoding="utf-8")) + if not isinstance(case_records, list): + die("cases json must be a list") + if len(case_records) != EXPECTED_CASES: + die(f"cases json count {len(case_records)} != {EXPECTED_CASES}") + + by_id: dict[str, dict] = {} + for raw in case_records: + if not isinstance(raw, dict): + die("case record must be object") + rec = validate_case_record(raw) + cid = rec["case_id"] + if cid in by_id: + die(f"duplicate case_id {cid}") + by_id[cid] = rec + + fakedata = args.fakedata_dir + if not fakedata.is_dir(): + die(f"fakedata dir missing: {fakedata}") + + release = args.release_dir + if release.exists(): + for p in sorted(release.rglob("*"), reverse=True): + if p.is_symlink(): + die(f"symlink forbidden in release: {p}") + if p.is_file(): + p.unlink() + elif p.is_dir(): + p.rmdir() + release.mkdir(parents=True, exist_ok=True) + + name_re = re.compile(NAME_RE_TMPL.format(sha=re.escape(source_sha))) + refs: dict[str, dict[str, object]] = {} + case_ids: set[str] = set() + counts = {"card": 0, "controlsd": 0, "lagd": 0, "other": 0} + + for path in sorted(fakedata.glob(f"*__*__{source_sha}.zst")): + if path.is_symlink() or not path.is_file(): + die(f"invalid candidate: {path}") + name = path.name + if not name_re.fullmatch(name): + die(f"candidate name rejected: {name}") + case_id, process, _rest = name.split("__", 2) + if not CASE_RE.fullmatch(case_id) or not PROC_RE.fullmatch(process): + die(f"bad case/process in {name}") + if case_id not in by_id: + die(f"candidate case {case_id} not in harness case list") + rec = by_id[case_id] + if process not in set(rec["processes"]): + die(f"process {process} not in case {case_id} process set") + task_id = f"{case_id}:{process}" + if task_id in refs: + die(f"duplicate task {task_id}") + data = path.read_bytes() + digest = sha256_bytes(data) + dest = release / name + dest.write_bytes(data) + # Per-ref identity: filename/size/hash + harness source/params digests. + refs[task_id] = { + "filename": name, + "size": len(data), + "sha256": digest, + "source": rec["source"], + "source_sha256": rec["source_sha256"], + "source_bytes": rec["source_bytes"], + "params_digest": rec["params_digest"], + } + case_ids.add(case_id) + counts[process if process in counts else "other"] += 1 + + if len(refs) != EXPECTED_TASKS: + die(f"expected {EXPECTED_TASKS} tasks, found {len(refs)}") + if len(case_ids) != EXPECTED_CASES: + die(f"expected {EXPECTED_CASES} cases, found {len(case_ids)}") + if case_ids != set(by_id): + die(f"assembled cases {sorted(case_ids)} != harness cases {sorted(by_id)}") + if counts != EXPECTED_PARTITION: + die(f"partition mismatch: {counts} != {EXPECTED_PARTITION}") + + # Exact task identity set: every process in every case record must be present. + expected_task_ids = sorted( + f"{cid}:{proc}" for cid, rec in by_id.items() for proc in rec["processes"] + ) + if sorted(refs) != expected_task_ids: + die("assembled task identities do not match harness case process sets") + + ordered_cases = [by_id[cid] for cid in sorted(by_id)] + ordered_refs = {k: refs[k] for k in sorted(refs)} + + # Match validator/trusted-inventory aggregate digests exactly: sorted arrays of + # records (not case-id-keyed dicts). + sources_payload = [ + { + "case_id": cid, + "source": by_id[cid]["source"], + "source_bytes": by_id[cid]["source_bytes"], + "source_sha256": by_id[cid]["source_sha256"], + } + for cid in sorted(by_id) + ] + params_payload = [ + {"case_id": cid, "params_digest": by_id[cid]["params_digest"]} + for cid in sorted(by_id) + ] + sources_digest = sha256_bytes(canonical_json(sources_payload)) + params_digest_all = sha256_bytes(canonical_json(params_payload)) + + manifest: dict[str, object] = { + "cases": ordered_cases, + "openpilot_sha": source_sha, + "opendbc_sha": opendbc_sha, + "params_digest": params_digest_all, + "previous_accepted_tag": previous_accepted_tag, + "refs": ordered_refs, + "schema_version": 1, + "sources_digest": sources_digest, + } + + # Fail closed: every case/ref identity still carries digests after assembly. + for cid, rec in by_id.items(): + require_digest(f"assembled case {cid} params_digest", rec["params_digest"]) + require_digest(f"assembled case {cid} source_sha256", rec["source_sha256"]) + for tid, entry in ordered_refs.items(): + require_digest(f"assembled ref {tid} sha256", entry["sha256"]) + require_digest(f"assembled ref {tid} params_digest", entry["params_digest"]) + require_digest(f"assembled ref {tid} source_sha256", entry["source_sha256"]) + require_digest("manifest sources_digest", sources_digest) + require_digest("manifest params_digest", params_digest_all) + + manifest_bytes = canonical_json(manifest) + (release / "manifest.json").write_bytes(manifest_bytes) + (release / "ref_commit").write_text(source_sha + "\n", encoding="utf-8") + + rel_files = sorted(p.name for p in release.iterdir() if p.is_file()) + zsts = [n for n in rel_files if n.endswith(".zst")] + if "ref_commit" not in rel_files or "manifest.json" not in rel_files: + die("release missing ref_commit or manifest.json") + if len(zsts) != EXPECTED_TASKS: + die("zst count self-check failed") + extras = [n for n in rel_files if n not in {"ref_commit", "manifest.json"} and not n.endswith(".zst")] + if extras: + die(f"unexpected release files: {extras}") + + # Independent rebuild check: re-canonicalize must be byte-identical. + rebuild = canonical_json(json.loads(manifest_bytes.decode("utf-8"))) + if rebuild != manifest_bytes: + die("manifest failed independent byte-identical rebuild check") + + manifest_sha = sha256_bytes(manifest_bytes) + print(f"assembled release/ inventory=active tasks={len(zsts)} cases={len(case_ids)}") + print(f"sha={source_sha} previous_accepted_tag={previous_accepted_tag}") + print(f"manifest_sha256={manifest_sha}") + print(f"sources_digest={sources_digest}") + print(f"params_digest={params_digest_all}") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())